From f656979b96c74aa92dc2939fb6fd76b0443da644 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 18 Apr 2023 11:23:49 +0100 Subject: [PATCH 001/247] cargo new --bin capture --- .gitignore | 1 + Cargo.toml | 8 ++++++++ src/main.rs | 3 +++ 3 files changed, 12 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000..ea8c4bf7f35f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000000..7021e12acfb29 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "capture" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000000000..e7a11a969c037 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} From cdc04583071c0d1fc453e1da9138ec40396ddad9 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 18 Apr 2023 11:36:27 +0100 Subject: [PATCH 002/247] Add README.md --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000000000..cae8981ac9c48 --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +# capture + +This is a rewrite of [capture.py](https://github.com/PostHog/posthog/blob/master/posthog/api/capture.py), in Rust. + +## Why? + +Capture is very simple. It takes some JSON, checks a key in Redis, and then pushes onto Kafka. It's mostly IO bound. + +We currently use far too much compute to run this service, and it could be more efficient. This effort should not take too long to complete, but should massively reduce our CPU usage - and therefore spend. + +## How? + +I'm trying to ensure the rewrite at least vaguely resembles the Python version. This will both minimize accidental regressions, but also serve as a "rosetta stone" for engineers at PostHog who have not written Rust before. From 86ebcf95b22afe4b9c546e140b71262c44952456 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 18 Apr 2023 11:41:51 +0100 Subject: [PATCH 003/247] Add docker image and workflow --- .github/workflows/docker.yml | 61 ++++++++++++++++++++++++++++++++++++ Dockerfile | 26 +++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 .github/workflows/docker.yml create mode 100644 Dockerfile diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000000000..f6b8ebb7bb44a --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,61 @@ +name: Build docker image + +on: + push: + branches: + - 'main' + +permissions: + packages: write + +jobs: + build: + name: build and publish capture image + runs-on: ubuntu-latest + steps: + + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v4 + with: + context: ./ + file: ./Dockerfile + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64, linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: RUST_BACKTRACE=1 + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000..f9e1ce8920971 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM lukemathwalker/cargo-chef:latest-rust-1.68.0 AS chef +WORKDIR app + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder + +# Ensure working C compile setup (not installed by default in arm64 images) +RUN apt update && apt install build-essential -y + +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json + +COPY . . +RUN cargo build --release --bin capture + +FROM debian:bullseye-20230320-slim AS runtime + +WORKDIR app + +USER nobody + +COPY --from=builder /app/target/release/capture /usr/local/bin +ENTRYPOINT ["/usr/local/bin/capture"] From d03328a7056d19731c8bf2cc09d81aa972789ee8 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 18 Apr 2023 12:02:36 +0100 Subject: [PATCH 004/247] Add Rust workflow and lockfile --- .github/workflows/rust.yml | 96 ++++++++++++++++++++++++++++++++++++++ Cargo.lock | 7 +++ 2 files changed, 103 insertions(+) create mode 100644 .github/workflows/rust.yml create mode 100644 Cargo.lock diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000000000..97d1b9642c203 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,96 @@ +name: Rust + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Install rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }} + + - name: Run cargo build + run: cargo build --all --locked --release && strip target/release/capture + + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Install rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${ runner.os }-cargo-debug-${{ hashFiles('**/Cargo.lock') }} + + - name: Run cargo test + run: cargo test --all-features + + - name: Run cargo check + run: cargo check --all-features + + clippy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Install latest rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy + + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }} + + - name: Run clippy + run: cargo clippy -- -D warnings + + format: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Install latest rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: rustfmt + + - name: Format + run: cargo fmt -- --check diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000000..dcaf2fce1e801 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "capture" +version = "0.1.0" From bb556b5ec3bb6f3f2ac0a7822bf9fb87d1081ed9 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 18 Apr 2023 12:26:32 +0100 Subject: [PATCH 005/247] Add basic server (#3) * Add dependencies Tokio for async Axum for HTTP things Tracing for... tracing Serde for serialization * Setup simple hello world server * fmt --- Cargo.lock | 774 +++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 6 + src/capture.rs | 3 + src/main.rs | 24 +- 4 files changed, 805 insertions(+), 2 deletions(-) create mode 100644 src/capture.rs diff --git a/Cargo.lock b/Cargo.lock index dcaf2fce1e801..eef95c83bc083 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,780 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "axum" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b32c5ea3aabaf4deb5f5ced2d688ec0844c881c9e6c696a8b769a05fc691e62" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + [[package]] name = "capture" version = "0.1.0" +dependencies = [ + "axum", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.141" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "proc-macro2" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.160" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.160" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tokio" +version = "1.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" diff --git a/Cargo.toml b/Cargo.toml index 7021e12acfb29..fc2f48689b4b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,9 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +axum = "0.6.15" +tokio = { version = "1.0", features = ["full"] } +tracing = "0.1" +tracing-subscriber = "0.3" +serde = { version = "1.0.160", features = ["derive"] } +serde_json = "1.0.96" diff --git a/src/capture.rs b/src/capture.rs new file mode 100644 index 0000000000000..8f5f0d9cb154a --- /dev/null +++ b/src/capture.rs @@ -0,0 +1,3 @@ +pub async fn root() -> &'static str { + "Hello, World!" +} diff --git a/src/main.rs b/src/main.rs index e7a11a969c037..fefb0fabdbba8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,23 @@ -fn main() { - println!("Hello, world!"); +use axum::{routing::get, Router}; +use std::net::SocketAddr; + +mod capture; + +#[tokio::main] +async fn main() { + // initialize tracing + tracing_subscriber::fmt::init(); + + let app = Router::new().route("/", get(capture::root)); + + // run our app with hyper + // `axum::Server` is a re-export of `hyper::Server` + let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); + + tracing::debug!("listening on {}", addr); + + axum::Server::bind(&addr) + .serve(app.into_make_service()) + .await + .unwrap(); } From 585ac6654aaeb8ae955abf0434bf51908b89522b Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 16 May 2023 15:23:05 +0100 Subject: [PATCH 006/247] Add token shape checking (#4) * Add basic server (#3) * Add dependencies Tokio for async Axum for HTTP things Tracing for... tracing Serde for serialization * Setup simple hello world server * fmt * Server coming up, validating tokens * Spent ages trying to figure out the schema, which is currenly very fluid --- Cargo.lock | 545 +++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 7 + bin/send_event.sh | 2 + src/api.rs | 22 ++ src/capture.rs | 26 ++- src/main.rs | 14 +- src/token.rs | 99 +++++++++ src/utils.rs | 4 + 8 files changed, 714 insertions(+), 5 deletions(-) create mode 100755 bin/send_event.sh create mode 100644 src/api.rs create mode 100644 src/token.rs create mode 100644 src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index eef95c83bc083..1da53544d855d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +dependencies = [ + "memchr", +] + [[package]] name = "async-trait" version = "0.1.68" @@ -74,6 +83,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bumpalo" +version = "3.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" + [[package]] name = "bytes" version = "1.4.0" @@ -85,9 +100,14 @@ name = "capture" version = "0.1.0" dependencies = [ "axum", + "governor", + "mockall", "serde", "serde_json", + "time", "tokio", + "tower-http", + "tower_governor", "tracing", "tracing-subscriber", ] @@ -98,6 +118,55 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -113,6 +182,37 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "forwarded-header-value" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" +dependencies = [ + "nonempty", + "thiserror", +] + +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.28" @@ -120,6 +220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -128,24 +229,105 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + [[package]] name = "futures-task" version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "governor" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c390a940a5d157878dd057c78680a33ce3415bcd05b4799509ea44210914b4d5" +dependencies = [ + "cfg-if", + "dashmap", + "futures", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot", + "quanta", + "rand", + "smallvec", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hermit-abi" version = "0.2.6" @@ -177,6 +359,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + [[package]] name = "httparse" version = "1.8.0" @@ -212,12 +400,30 @@ dependencies = [ "want", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -249,6 +455,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "matchit" version = "0.7.0" @@ -275,10 +490,61 @@ checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", ] +[[package]] +name = "mockall" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" + +[[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -289,6 +555,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -372,6 +647,42 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "predicates" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "proc-macro2" version = "1.0.56" @@ -381,6 +692,22 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quanta" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" +dependencies = [ + "crossbeam-utils", + "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi 0.10.2+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.26" @@ -390,6 +717,45 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -399,6 +765,23 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + [[package]] name = "rustversion" version = "1.0.12" @@ -487,6 +870,15 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -531,6 +923,32 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "thread_local" version = "1.1.7" @@ -541,6 +959,22 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +dependencies = [ + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + [[package]] name = "tokio" version = "1.27.0" @@ -587,6 +1021,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower-layer" version = "0.3.2" @@ -599,6 +1052,26 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +[[package]] +name = "tower_governor" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6be418f6d18863291f0a7fa1da1de71495a19a54b5fb44969136f731a47e86" +dependencies = [ + "axum", + "forwarded-header-value", + "futures", + "futures-core", + "governor", + "http", + "pin-project", + "thiserror", + "tokio", + "tower", + "tower-layer", + "tracing", +] + [[package]] name = "tracing" version = "0.1.37" @@ -686,12 +1159,82 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index fc2f48689b4b5..c5b992c62c914 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,3 +12,10 @@ tracing = "0.1" tracing-subscriber = "0.3" serde = { version = "1.0.160", features = ["derive"] } serde_json = "1.0.96" +governor = "0.5.1" +tower_governor = "0.0.4" +time = "0.3.20" +tower-http = {version = "0.4.0", features = ["trace"]} + +[dev-dependencies] +mockall = "0.11.2" diff --git a/bin/send_event.sh b/bin/send_event.sh new file mode 100755 index 0000000000000..4885cf12db376 --- /dev/null +++ b/bin/send_event.sh @@ -0,0 +1,2 @@ +# Send an event to a test server +curl http://localhost:3000/capture -X POST -H "Content-Type: application/json" --data '{"token": "ferrisisbae"}' diff --git a/src/api.rs b/src/api.rs new file mode 100644 index 0000000000000..f647afacf0440 --- /dev/null +++ b/src/api.rs @@ -0,0 +1,22 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +// Define the API interface for capture here. +// This is used for serializing responses and deserializing requests + +// LATER ME +// Trying to figure out wtf the schema for this is. Turns out we have about +// a million special cases and properties all over the place + +Also - what are the possible types for a property value? Account for those. +#[derive(Debug, Deserialize, Serialize)] +pub struct CaptureRequest{ + #[serde(alias = "$token", alias = "api_key")] + pub token: String, + + pub event: String, + pub properties: HashMap +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct CaptureResponse{} diff --git a/src/capture.rs b/src/capture.rs index 8f5f0d9cb154a..db079d1fefa47 100644 --- a/src/capture.rs +++ b/src/capture.rs @@ -1,3 +1,25 @@ -pub async fn root() -> &'static str { - "Hello, World!" +use axum::{Json, http::StatusCode}; +use crate::{api::{CaptureRequest, CaptureResponse}, token}; + +/// A single event +/// Does not yet support everything the old method does - we expect to be able to deserialize the +/// entire POST body, and not keep checking for form attributes. +/// +/// TODO: Switch on this between two handlers. DO NOT branch in the code. +/// TODO: Add error responses in the same format as capture.py. Probs custom extractor. +pub async fn event(req: Json) -> Result, (StatusCode, String)> { + tracing::info!("new event of type {}", req.token); + + // I wanted to do some sort of middleware that pulled the token out of the headers, but... The + // token isn't usually in the headers, but in the body :( + // Could move token parsing into the middleware at some point + if let Err(invalid) = token::validate_token(req.token.as_str()){ + return Err((StatusCode::BAD_REQUEST, invalid.reason().to_string())); + } + + Ok(Json(CaptureResponse{})) } + +// A group of events! There is no limit here, though our HTTP stack will reject anything above +// 20mb. +pub async fn batch() -> &'static str {"No batching for you!"} diff --git a/src/main.rs b/src/main.rs index fefb0fabdbba8..422c588bdfba2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,24 @@ -use axum::{routing::get, Router}; +use axum::{ + routing::get, + Router, +}; + +use axum::{routing::get, routing::post, Router}; use std::net::SocketAddr; +use tower_http::trace::TraceLayer; mod capture; +mod utils; +mod api; +mod token; #[tokio::main] async fn main() { // initialize tracing tracing_subscriber::fmt::init(); - let app = Router::new().route("/", get(capture::root)); + let app = Router::new().route("/capture", post(capture::event)).route("/batch", post(capture::batch)).layer(TraceLayer::new_for_http()); + // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` diff --git a/src/token.rs b/src/token.rs new file mode 100644 index 0000000000000..133a3c44f8a11 --- /dev/null +++ b/src/token.rs @@ -0,0 +1,99 @@ +use std::error::Error; +use std::fmt::Display; + +/// Validate that a token is the correct shape + +#[derive(Debug, PartialEq)] +pub enum InvalidTokenReason { + IsEmpty, + + // ignoring for now, as serde and the type system save us but we need to error properly + IsNotString, + + IsTooLong, + IsNotAscii, + IsPersonalApiKey +} + +impl InvalidTokenReason { + pub fn reason(&self) -> &str { + match *self { + Self::IsEmpty => "empty", + Self::IsNotAscii => "not_ascii", + Self::IsNotString => "not_string", + Self::IsTooLong => "too_long", + Self::IsPersonalApiKey => "personal_api_key", + } + } +} + +impl Display for InvalidTokenReason{ + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.reason()) + } +} + +impl Error for InvalidTokenReason { + fn description(&self) -> &str { + self.reason() + } +} + +/// Check if a token is the right shape. It may not actually be a valid token! We don't validate +/// these at the edge yet. +pub fn validate_token(token: &str) -> Result<(), InvalidTokenReason> { + if token.is_empty() { + return Err(InvalidTokenReason::IsEmpty); + } + + if token.len() > 64 { + return Err(InvalidTokenReason::IsTooLong); + } + + if !token.is_ascii() { + return Err(InvalidTokenReason::IsNotAscii); + } + + if token.starts_with("phx_") { + return Err(InvalidTokenReason::IsPersonalApiKey); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::token::{validate_token, InvalidTokenReason}; + + #[test] + fn blocks_empty_tokens() { + let valid = validate_token(""); + + assert!(valid.is_err()); + assert_eq!(valid.unwrap_err(), InvalidTokenReason::IsEmpty); + } + + #[test] + fn blocks_too_long_tokens() { + let valid = validate_token("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); + + assert!(valid.is_err()); + assert_eq!(valid.unwrap_err(), InvalidTokenReason::IsTooLong); + } + + #[test] + fn blocks_invalid_ascii() { + let valid = validate_token("🦀"); + + assert!(valid.is_err()); + assert_eq!(valid.unwrap_err(), InvalidTokenReason::IsNotAscii); + } + + #[test] + fn blocks_personal_api_key() { + let valid = validate_token("phx_hellothere"); + + assert!(valid.is_err()); + assert_eq!(valid.unwrap_err(), InvalidTokenReason::IsPersonalApiKey); + } +} diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000000000..1dad9ba3dd46a --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,4 @@ +/// This is translated from posthog/utils.py +/// It might make more sense to do it as a middleware, but keep it here for now +/// as an example +pub fn cors_response(){} From be03ad45fcd92db93708da0e4b901fd1c3fe30f2 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 16 May 2023 15:51:41 +0100 Subject: [PATCH 007/247] Fix typo (#5) * Fix typo * Format * I really messed up rebasing on the plane huh * Clippy is now a happy bunny * Fmt --- src/api.rs | 11 ++++------- src/capture.rs | 19 +++++++++++++------ src/main.rs | 16 ++++++---------- src/token.rs | 44 ++++++++++++++++++++++---------------------- src/utils.rs | 4 ---- 5 files changed, 45 insertions(+), 49 deletions(-) delete mode 100644 src/utils.rs diff --git a/src/api.rs b/src/api.rs index f647afacf0440..82770f20ca0f1 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use serde::{Deserialize, Serialize}; // Define the API interface for capture here. // This is used for serializing responses and deserializing requests @@ -7,16 +5,15 @@ use serde::{Deserialize, Serialize}; // LATER ME // Trying to figure out wtf the schema for this is. Turns out we have about // a million special cases and properties all over the place - -Also - what are the possible types for a property value? Account for those. +// Also - what are the possible types for a property value? Account for those. #[derive(Debug, Deserialize, Serialize)] -pub struct CaptureRequest{ +pub struct CaptureRequest { #[serde(alias = "$token", alias = "api_key")] pub token: String, pub event: String, - pub properties: HashMap + // pub properties: HashMap, } #[derive(Debug, Deserialize, Serialize)] -pub struct CaptureResponse{} +pub struct CaptureResponse {} diff --git a/src/capture.rs b/src/capture.rs index db079d1fefa47..a77061089088e 100644 --- a/src/capture.rs +++ b/src/capture.rs @@ -1,5 +1,8 @@ -use axum::{Json, http::StatusCode}; -use crate::{api::{CaptureRequest, CaptureResponse}, token}; +use crate::{ + api::{CaptureRequest, CaptureResponse}, + token, +}; +use axum::{http::StatusCode, Json}; /// A single event /// Does not yet support everything the old method does - we expect to be able to deserialize the @@ -7,19 +10,23 @@ use crate::{api::{CaptureRequest, CaptureResponse}, token}; /// /// TODO: Switch on this between two handlers. DO NOT branch in the code. /// TODO: Add error responses in the same format as capture.py. Probs custom extractor. -pub async fn event(req: Json) -> Result, (StatusCode, String)> { +pub async fn event( + req: Json, +) -> Result, (StatusCode, String)> { tracing::info!("new event of type {}", req.token); // I wanted to do some sort of middleware that pulled the token out of the headers, but... The // token isn't usually in the headers, but in the body :( // Could move token parsing into the middleware at some point - if let Err(invalid) = token::validate_token(req.token.as_str()){ + if let Err(invalid) = token::validate_token(req.token.as_str()) { return Err((StatusCode::BAD_REQUEST, invalid.reason().to_string())); } - Ok(Json(CaptureResponse{})) + Ok(Json(CaptureResponse {})) } // A group of events! There is no limit here, though our HTTP stack will reject anything above // 20mb. -pub async fn batch() -> &'static str {"No batching for you!"} +pub async fn batch() -> &'static str { + "No batching for you!" +} diff --git a/src/main.rs b/src/main.rs index 422c588bdfba2..865a1b865f62c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,15 +1,9 @@ -use axum::{ - routing::get, - Router, -}; - -use axum::{routing::get, routing::post, Router}; +use axum::{routing::post, Router}; use std::net::SocketAddr; use tower_http::trace::TraceLayer; -mod capture; -mod utils; mod api; +mod capture; mod token; #[tokio::main] @@ -17,8 +11,10 @@ async fn main() { // initialize tracing tracing_subscriber::fmt::init(); - let app = Router::new().route("/capture", post(capture::event)).route("/batch", post(capture::batch)).layer(TraceLayer::new_for_http()); - + let app = Router::new() + .route("/capture", post(capture::event)) + .route("/batch", post(capture::batch)) + .layer(TraceLayer::new_for_http()); // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` diff --git a/src/token.rs b/src/token.rs index 133a3c44f8a11..7924cc9511485 100644 --- a/src/token.rs +++ b/src/token.rs @@ -5,29 +5,28 @@ use std::fmt::Display; #[derive(Debug, PartialEq)] pub enum InvalidTokenReason { - IsEmpty, + Empty, // ignoring for now, as serde and the type system save us but we need to error properly - IsNotString, - - IsTooLong, - IsNotAscii, - IsPersonalApiKey + // IsNotString, + TooLong, + NotAscii, + PersonalApiKey, } impl InvalidTokenReason { pub fn reason(&self) -> &str { match *self { - Self::IsEmpty => "empty", - Self::IsNotAscii => "not_ascii", - Self::IsNotString => "not_string", - Self::IsTooLong => "too_long", - Self::IsPersonalApiKey => "personal_api_key", + Self::Empty => "empty", + Self::NotAscii => "not_ascii", + // Self::IsNotString => "not_string", + Self::TooLong => "too_long", + Self::PersonalApiKey => "personal_api_key", } } } -impl Display for InvalidTokenReason{ +impl Display for InvalidTokenReason { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", self.reason()) } @@ -43,19 +42,19 @@ impl Error for InvalidTokenReason { /// these at the edge yet. pub fn validate_token(token: &str) -> Result<(), InvalidTokenReason> { if token.is_empty() { - return Err(InvalidTokenReason::IsEmpty); + return Err(InvalidTokenReason::Empty); } - + if token.len() > 64 { - return Err(InvalidTokenReason::IsTooLong); + return Err(InvalidTokenReason::TooLong); } if !token.is_ascii() { - return Err(InvalidTokenReason::IsNotAscii); + return Err(InvalidTokenReason::NotAscii); } if token.starts_with("phx_") { - return Err(InvalidTokenReason::IsPersonalApiKey); + return Err(InvalidTokenReason::PersonalApiKey); } Ok(()) @@ -70,15 +69,16 @@ mod tests { let valid = validate_token(""); assert!(valid.is_err()); - assert_eq!(valid.unwrap_err(), InvalidTokenReason::IsEmpty); + assert_eq!(valid.unwrap_err(), InvalidTokenReason::Empty); } #[test] fn blocks_too_long_tokens() { - let valid = validate_token("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); + let valid = + validate_token("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); assert!(valid.is_err()); - assert_eq!(valid.unwrap_err(), InvalidTokenReason::IsTooLong); + assert_eq!(valid.unwrap_err(), InvalidTokenReason::TooLong); } #[test] @@ -86,7 +86,7 @@ mod tests { let valid = validate_token("🦀"); assert!(valid.is_err()); - assert_eq!(valid.unwrap_err(), InvalidTokenReason::IsNotAscii); + assert_eq!(valid.unwrap_err(), InvalidTokenReason::NotAscii); } #[test] @@ -94,6 +94,6 @@ mod tests { let valid = validate_token("phx_hellothere"); assert!(valid.is_err()); - assert_eq!(valid.unwrap_err(), InvalidTokenReason::IsPersonalApiKey); + assert_eq!(valid.unwrap_err(), InvalidTokenReason::PersonalApiKey); } } diff --git a/src/utils.rs b/src/utils.rs deleted file mode 100644 index 1dad9ba3dd46a..0000000000000 --- a/src/utils.rs +++ /dev/null @@ -1,4 +0,0 @@ -/// This is translated from posthog/utils.py -/// It might make more sense to do it as a middleware, but keep it here for now -/// as an example -pub fn cors_response(){} From 96b9d59a7d8152eccd7f1ce240f88a88281c1869 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 22 May 2023 14:53:17 +0100 Subject: [PATCH 008/247] Add event decompress and deserialize (#6) * Add event decompress and deserialize * Try buildjet * Get api key from properties if not on event * fmt * fmt again * All events have the same token --- .github/workflows/rust.yml | 8 +-- Cargo.lock | 50 ++++++++++++++++ Cargo.toml | 4 ++ src/api.rs | 11 ++-- src/capture.rs | 118 ++++++++++++++++++++++++++++++++----- src/event.rs | 69 ++++++++++++++++++++++ src/main.rs | 1 + 7 files changed, 235 insertions(+), 26 deletions(-) create mode 100644 src/event.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 97d1b9642c203..dbb689dc8e72a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -11,7 +11,7 @@ env: jobs: build: - runs-on: ubuntu-latest + runs-on: buildjet-4vcpu-ubuntu-2204 steps: - uses: actions/checkout@v3 @@ -33,7 +33,7 @@ jobs: run: cargo build --all --locked --release && strip target/release/capture test: - runs-on: ubuntu-latest + runs-on: buildjet-4vcpu-ubuntu-2204 steps: - uses: actions/checkout@v3 @@ -58,7 +58,7 @@ jobs: run: cargo check --all-features clippy: - runs-on: ubuntu-latest + runs-on: buildjet-4vcpu-ubuntu-2204 steps: - uses: actions/checkout@v3 @@ -81,7 +81,7 @@ jobs: run: cargo clippy -- -D warnings format: - runs-on: ubuntu-latest + runs-on: buildjet-4vcpu-ubuntu-2204 steps: - uses: actions/checkout@v3 diff --git a/Cargo.lock b/Cargo.lock index 1da53544d855d..5b42fb031ec7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "1.0.1" @@ -11,6 +17,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "anyhow" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + [[package]] name = "async-trait" version = "0.1.68" @@ -77,6 +89,12 @@ dependencies = [ "tower-service", ] +[[package]] +name = "base64" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f1e31e207a6b8fb791a38ea3105e6cb541f55e4d029902d3039a4ad07cc4105" + [[package]] name = "bitflags" version = "1.3.2" @@ -99,7 +117,11 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" name = "capture" version = "0.1.0" dependencies = [ + "anyhow", "axum", + "base64", + "bytes", + "flate2", "governor", "mockall", "serde", @@ -118,6 +140,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-utils" version = "0.8.15" @@ -158,6 +189,16 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "flate2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "float-cmp" version = "0.9.0" @@ -482,6 +523,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.6" diff --git a/Cargo.toml b/Cargo.toml index c5b992c62c914..4de875b24442f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,10 @@ governor = "0.5.1" tower_governor = "0.0.4" time = "0.3.20" tower-http = {version = "0.4.0", features = ["trace"]} +bytes = "1" +anyhow = "1.0" +flate2 = "1.0" +base64 = "0.21.1" [dev-dependencies] mockall = "0.11.2" diff --git a/src/api.rs b/src/api.rs index 82770f20ca0f1..0b3b3ed3e248f 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,18 +1,15 @@ +use std::collections::HashMap; + use serde::{Deserialize, Serialize}; -// Define the API interface for capture here. -// This is used for serializing responses and deserializing requests +use serde_json::Value; -// LATER ME -// Trying to figure out wtf the schema for this is. Turns out we have about -// a million special cases and properties all over the place -// Also - what are the possible types for a property value? Account for those. #[derive(Debug, Deserialize, Serialize)] pub struct CaptureRequest { #[serde(alias = "$token", alias = "api_key")] pub token: String, pub event: String, - // pub properties: HashMap, + pub properties: HashMap, } #[derive(Debug, Deserialize, Serialize)] diff --git a/src/capture.rs b/src/capture.rs index a77061089088e..b13613e6d5bfb 100644 --- a/src/capture.rs +++ b/src/capture.rs @@ -1,32 +1,120 @@ +use std::collections::HashSet; + +use bytes::Bytes; + +use axum::{http::StatusCode, Json}; +// TODO: stream this instead +use axum::extract::Query; + use crate::{ - api::{CaptureRequest, CaptureResponse}, + api::CaptureResponse, + event::{Event, EventQuery}, token, }; -use axum::{http::StatusCode, Json}; -/// A single event -/// Does not yet support everything the old method does - we expect to be able to deserialize the -/// entire POST body, and not keep checking for form attributes. -/// -/// TODO: Switch on this between two handlers. DO NOT branch in the code. -/// TODO: Add error responses in the same format as capture.py. Probs custom extractor. pub async fn event( - req: Json, + meta: Query, + body: Bytes, ) -> Result, (StatusCode, String)> { - tracing::info!("new event of type {}", req.token); + let events = Event::from_bytes(&meta, body); + + let events = match events { + Ok(events) => events, + Err(_) => { + return Err(( + StatusCode::BAD_REQUEST, + String::from("Failed to decode event"), + )) + } + }; + + if events.is_empty() { + return Err((StatusCode::BAD_REQUEST, String::from("No events in batch"))); + } + + let processed = process_events(&events); - // I wanted to do some sort of middleware that pulled the token out of the headers, but... The - // token isn't usually in the headers, but in the body :( - // Could move token parsing into the middleware at some point - if let Err(invalid) = token::validate_token(req.token.as_str()) { - return Err((StatusCode::BAD_REQUEST, invalid.reason().to_string())); + if let Err(msg) = processed { + return Err((StatusCode::BAD_REQUEST, msg)); } Ok(Json(CaptureResponse {})) } +pub fn process_events(events: &[Event]) -> Result<(), String> { + let mut distinct_tokens = HashSet::new(); + + // 1. Tokens are all valid + for event in events { + let token = event.token.clone().unwrap_or_else(|| { + event + .properties + .get("token") + .map_or(String::new(), |t| String::from(t.as_str().unwrap())) + }); + + if let Err(invalid) = token::validate_token(token.as_str()) { + return Err(invalid.reason().to_string()); + } + + distinct_tokens.insert(token); + } + + if distinct_tokens.len() > 1 { + return Err(String::from("Number of distinct tokens in batch > 1")); + } + + Ok(()) +} + // A group of events! There is no limit here, though our HTTP stack will reject anything above // 20mb. pub async fn batch() -> &'static str { "No batching for you!" } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use serde_json::json; + + use super::process_events; + use crate::event::Event; + + #[test] + fn all_events_have_same_token() { + let events = vec![ + Event { + token: Some(String::from("hello")), + event: String::new(), + properties: HashMap::new(), + }, + Event { + token: None, + event: String::new(), + properties: HashMap::from([(String::from("token"), json!("hello"))]), + }, + ]; + + assert_eq!(process_events(&events).is_ok(), true); + } + + #[test] + fn all_events_have_different_token() { + let events = vec![ + Event { + token: Some(String::from("hello")), + event: String::new(), + properties: HashMap::new(), + }, + Event { + token: None, + event: String::new(), + properties: HashMap::from([(String::from("token"), json!("goodbye"))]), + }, + ]; + + assert_eq!(process_events(&events).is_err(), true); + } +} diff --git a/src/event.rs b/src/event.rs new file mode 100644 index 0000000000000..57a34289aed59 --- /dev/null +++ b/src/event.rs @@ -0,0 +1,69 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use anyhow::Result; +use bytes::{Buf, Bytes}; +use flate2::read::GzDecoder; + +#[derive(Deserialize, Default)] +pub enum Compression { + #[default] + #[serde(rename = "gzip-js")] + GzipJs, +} + +#[allow(dead_code)] // until they are used +#[derive(Deserialize, Default)] +pub struct EventQuery { + compression: Compression, + + #[serde(alias = "ver")] + version: String, + + #[serde(alias = "_")] + sent_at: i64, +} + +#[derive(Default, Debug, Deserialize, Serialize)] +pub struct Event { + #[serde(alias = "$token", alias = "api_key")] + pub token: Option, + + pub event: String, + pub properties: HashMap, +} + +impl Event { + /// We post up _at least one_ event, so when decompressiong and deserializing there + /// could be more than one. Hence this function has to return a Vec. + /// TODO: Use an axum extractor for this + pub fn from_bytes(_: &EventQuery, bytes: Bytes) -> Result> { + let d = GzDecoder::new(bytes.reader()); + let event = serde_json::from_reader(d)?; + + Ok(event) + } +} + +#[cfg(test)] +mod tests { + use base64::Engine as _; + use bytes::Bytes; + + use super::{Event, EventQuery}; + + #[test] + fn decode_bytes() { + let horrible_blob = "H4sIAAAAAAAAA31T207cMBD9lSrikSy+5bIrVX2g4oWWUlEqBEKRY08Sg4mD4+xCEf/e8XLZBSGeEp+ZOWOfmXPxkMAS+pAskp1BtmBBLiHZTQbvBvDBwJgsHpIdh5/kp1Rffp18OcMwAtUS/GhcjwFKZjSbkYjX3q1G8AgeGA+Nu4ughqVRUIX7ATDwHcbr4IYYUJP32LyavMVAF8Kw2NuzTknbuTEsSkIIHlvTf+vhLnzdizUxgslvs2JgkKHr5U1s8VS0dZ/NZSnlW7CVfTvhs7EG+vT0JJaMygP0VQem7bDTvBAbcGV06JAkIwTBpYHV4Hx4zS1FJH+FX7IFj7A1NbZZQR2b4GFbwFlWzFjETY/XCpXRiN538yt/S9mdnm7bSa+lDCY+kOalKDJGs/msZMVuos0YTK+e62hZciHqes7LnDcpoVmTg+TAaqnKMhWUaaa4TllBoCDpJn2uYK3k87xeyFjZFHWdzxmdq5Q0IstBzRXlDMiHbM/5kgnerKfs+tFZqHAolQflvDZ9W0Evawu6wveiENVoND4s+Ami2jBGZbayn/42g3xblizX4skp4FYMYfJQoSQf8DfSjrGBVMEsoWpArpMbK1vc8ItLDG1j1SDvrZM6muBxN/Eg7U1cVFw70KmyRl13bhqjYeBGGrtuFqWTSzzF/q8tRyvV9SfxHXQLoBuidXY0ekeF+KQnNCqgHXaIy7KJBncNERk6VUFhhB33j8zv5uhQ/rCTvbq9/9seH5Pj3Bf/TsuzYf9g2j+3h9N6yZ8Vfpmx4KSguSY5S0lOqc5LmgmhidoMmOaixoFvktFKOo9kK9Nrt3rPxViWk5RwIhtJykZzXohP2DjmZ08+bnH/4B1fkUnGSp2SMmNlIYTguS5ga//eERZZTSVeD8cWPTMGeTMgHSOMpyRLGftDyUKwBV9b6Dx5vPwPzQHjFwsFAAA="; + let decoded_horrible_blob = base64::engine::general_purpose::STANDARD + .decode(horrible_blob) + .unwrap(); + + let bytes = Bytes::from(decoded_horrible_blob); + let events = Event::from_bytes(&EventQuery::default(), bytes); + + assert_eq!(events.is_ok(), true); + } +} diff --git a/src/main.rs b/src/main.rs index 865a1b865f62c..52e1e9efa3dbd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ use tower_http::trace::TraceLayer; mod api; mod capture; +mod event; mod token; #[tokio::main] From 3271f8f1dfda351e98dd423d74911f947fa61b8c Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 22 May 2023 15:14:07 +0100 Subject: [PATCH 009/247] Expose router so we can test it (#8) --- src/main.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 52e1e9efa3dbd..08d1b775b8a56 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,15 +7,19 @@ mod capture; mod event; mod token; +pub fn router() -> Router { + Router::new() + .route("/capture", post(capture::event)) + .route("/batch", post(capture::batch)) + .layer(TraceLayer::new_for_http()) +} + #[tokio::main] async fn main() { // initialize tracing tracing_subscriber::fmt::init(); - let app = Router::new() - .route("/capture", post(capture::event)) - .route("/batch", post(capture::batch)) - .layer(TraceLayer::new_for_http()); + let app = router(); // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` From 605e9978cf76479d4d6d3eae71fc5d12dbcad3f3 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 22 May 2023 15:19:19 +0100 Subject: [PATCH 010/247] Make router a module (#9) --- src/main.rs | 12 ++---------- src/router.rs | 10 ++++++++++ 2 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 src/router.rs diff --git a/src/main.rs b/src/main.rs index 08d1b775b8a56..3fcd2574b7aef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,25 +1,17 @@ -use axum::{routing::post, Router}; use std::net::SocketAddr; -use tower_http::trace::TraceLayer; mod api; mod capture; mod event; +mod router; mod token; -pub fn router() -> Router { - Router::new() - .route("/capture", post(capture::event)) - .route("/batch", post(capture::batch)) - .layer(TraceLayer::new_for_http()) -} - #[tokio::main] async fn main() { // initialize tracing tracing_subscriber::fmt::init(); - let app = router(); + let app = router::router(); // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` diff --git a/src/router.rs b/src/router.rs new file mode 100644 index 0000000000000..db838ea0ca3f4 --- /dev/null +++ b/src/router.rs @@ -0,0 +1,10 @@ +use crate::capture; +use axum::{routing::post, Router}; +use tower_http::trace::TraceLayer; + +pub fn router() -> Router { + Router::new() + .route("/capture", post(capture::event)) + .route("/batch", post(capture::batch)) + .layer(TraceLayer::new_for_http()) +} From 0892ff6428544b0ac612d4b39edb68a97b80973c Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 22 May 2023 16:35:29 +0200 Subject: [PATCH 011/247] Add ProcessedEvent draft, export modules for testing (#10) * add django_compat test skeleton * add ProcessedEvent * remove tests * wip * fmt --- Cargo.lock | 11 +++++++++++ Cargo.toml | 3 ++- src/event.rs | 13 +++++++++++++ src/lib.rs | 5 +++++ 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 5b42fb031ec7b..ae3dd836bd117 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,6 +132,7 @@ dependencies = [ "tower_governor", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -1193,6 +1194,16 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +[[package]] +name = "uuid" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" +dependencies = [ + "getrandom", + "serde", +] + [[package]] name = "valuable" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4de875b24442f..74b6872d5eaf6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,11 +15,12 @@ serde_json = "1.0.96" governor = "0.5.1" tower_governor = "0.0.4" time = "0.3.20" -tower-http = {version = "0.4.0", features = ["trace"]} +tower-http = { version = "0.4.0", features = ["trace"] } bytes = "1" anyhow = "1.0" flate2 = "1.0" base64 = "0.21.1" +uuid = { version = "1.3.3", features = ["serde", "v4"] } [dev-dependencies] mockall = "0.11.2" diff --git a/src/event.rs b/src/event.rs index 57a34289aed59..deed63578e00b 100644 --- a/src/event.rs +++ b/src/event.rs @@ -6,6 +6,7 @@ use serde_json::Value; use anyhow::Result; use bytes::{Buf, Bytes}; use flate2::read::GzDecoder; +use uuid::Uuid; #[derive(Deserialize, Default)] pub enum Compression { @@ -47,6 +48,18 @@ impl Event { } } +#[derive(Default, Debug, Deserialize, Serialize)] +pub struct ProcessedEvent { + uuid: Uuid, + distinct_id: String, + ip: String, + site_url: String, + data: String, + now: String, + sent_at: String, + token: String, +} + #[cfg(test)] mod tests { use base64::Engine as _; diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000000000..b07a9eb167f1d --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,5 @@ +mod api; +mod capture; +pub mod event; +pub mod router; +mod token; From c6f5aa3d58cfa4d59dfd81fc8d04e781ec1771b8 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 22 May 2023 17:31:41 +0200 Subject: [PATCH 012/247] Add django_compat test skeleton (#7) --- Cargo.lock | 225 ++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/api.rs | 9 +- src/capture.rs | 11 +- src/event.rs | 29 +++-- src/router.rs | 9 +- tests/django_compat.rs | 57 ++++++++++ tests/requests_dump.jsonl | 21 ++++ 8 files changed, 344 insertions(+), 18 deletions(-) create mode 100644 tests/django_compat.rs create mode 100644 tests/requests_dump.jsonl diff --git a/Cargo.lock b/Cargo.lock index ae3dd836bd117..98ca254f338bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,6 +89,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-test-helper" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d349b3174ceac58442ea1f768233c817e59447c0343be2584fca9f0ed71d3a" +dependencies = [ + "axum", + "bytes", + "http", + "http-body", + "hyper", + "reqwest", + "serde", + "tokio", + "tower", + "tower-service", +] + [[package]] name = "base64" version = "0.21.1" @@ -119,6 +137,7 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", + "axum-test-helper", "base64", "bytes", "flate2", @@ -190,6 +209,15 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "encoding_rs" +version = "0.8.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +dependencies = [ + "cfg-if", +] + [[package]] name = "flate2" version = "1.0.26" @@ -364,6 +392,25 @@ dependencies = [ "smallvec", ] +[[package]] +name = "h2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -429,6 +476,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "httparse", @@ -442,6 +490,32 @@ dependencies = [ "want", ] +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" + [[package]] name = "itertools" version = "0.10.5" @@ -524,6 +598,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -833,6 +917,43 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +[[package]] +name = "reqwest" +version = "0.11.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "ipnet", + "js-sys", + "log", + "mime", + "mime_guess", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -1026,6 +1147,21 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.27.0" @@ -1056,6 +1192,20 @@ dependencies = [ "syn 2.0.15", ] +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -1188,12 +1338,47 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + [[package]] name = "unicode-ident" version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "uuid" version = "1.3.3" @@ -1210,6 +1395,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "want" version = "0.3.0" @@ -1257,6 +1448,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.84" @@ -1286,6 +1489,19 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +[[package]] +name = "wasm-streams" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.61" @@ -1383,3 +1599,12 @@ name = "windows_x86_64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/Cargo.toml b/Cargo.toml index 74b6872d5eaf6..5f73604d06415 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,4 +23,5 @@ base64 = "0.21.1" uuid = { version = "1.3.3", features = ["serde", "v4"] } [dev-dependencies] +axum-test-helper = "0.2.0" mockall = "0.11.2" diff --git a/src/api.rs b/src/api.rs index 0b3b3ed3e248f..b3a18e696c105 100644 --- a/src/api.rs +++ b/src/api.rs @@ -13,4 +13,11 @@ pub struct CaptureRequest { } #[derive(Debug, Deserialize, Serialize)] -pub struct CaptureResponse {} +pub enum CaptureResponseCode { + Ok = 1, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct CaptureResponse { + pub status: CaptureResponseCode, +} diff --git a/src/capture.rs b/src/capture.rs index b13613e6d5bfb..42b2718b29799 100644 --- a/src/capture.rs +++ b/src/capture.rs @@ -6,6 +6,7 @@ use axum::{http::StatusCode, Json}; // TODO: stream this instead use axum::extract::Query; +use crate::api::CaptureResponseCode; use crate::{ api::CaptureResponse, event::{Event, EventQuery}, @@ -38,7 +39,9 @@ pub async fn event( return Err((StatusCode::BAD_REQUEST, msg)); } - Ok(Json(CaptureResponse {})) + Ok(Json(CaptureResponse { + status: CaptureResponseCode::Ok, + })) } pub fn process_events(events: &[Event]) -> Result<(), String> { @@ -67,12 +70,6 @@ pub fn process_events(events: &[Event]) -> Result<(), String> { Ok(()) } -// A group of events! There is no limit here, though our HTTP stack will reject anything above -// 20mb. -pub async fn batch() -> &'static str { - "No batching for you!" -} - #[cfg(test)] mod tests { use std::collections::HashMap; diff --git a/src/event.rs b/src/event.rs index deed63578e00b..04e6c8ee571b2 100644 --- a/src/event.rs +++ b/src/event.rs @@ -18,13 +18,13 @@ pub enum Compression { #[allow(dead_code)] // until they are used #[derive(Deserialize, Default)] pub struct EventQuery { - compression: Compression, + compression: Option, #[serde(alias = "ver")] - version: String, + version: Option, #[serde(alias = "_")] - sent_at: i64, + sent_at: Option, } #[derive(Default, Debug, Deserialize, Serialize)] @@ -40,11 +40,14 @@ impl Event { /// We post up _at least one_ event, so when decompressiong and deserializing there /// could be more than one. Hence this function has to return a Vec. /// TODO: Use an axum extractor for this - pub fn from_bytes(_: &EventQuery, bytes: Bytes) -> Result> { - let d = GzDecoder::new(bytes.reader()); - let event = serde_json::from_reader(d)?; - - Ok(event) + pub fn from_bytes(query: &EventQuery, bytes: Bytes) -> Result> { + match query.compression { + Some(Compression::GzipJs) => { + let d = GzDecoder::new(bytes.reader()); + Ok(serde_json::from_reader(d)?) + } + None => Ok(serde_json::from_reader(bytes.reader())?), + } } } @@ -62,6 +65,7 @@ pub struct ProcessedEvent { #[cfg(test)] mod tests { + use super::Compression; use base64::Engine as _; use bytes::Bytes; @@ -75,7 +79,14 @@ mod tests { .unwrap(); let bytes = Bytes::from(decoded_horrible_blob); - let events = Event::from_bytes(&EventQuery::default(), bytes); + let events = Event::from_bytes( + &EventQuery { + compression: Some(Compression::GzipJs), + version: None, + sent_at: None, + }, + bytes, + ); assert_eq!(events.is_ok(), true); } diff --git a/src/router.rs b/src/router.rs index db838ea0ca3f4..52478b634d59e 100644 --- a/src/router.rs +++ b/src/router.rs @@ -4,7 +4,14 @@ use tower_http::trace::TraceLayer; pub fn router() -> Router { Router::new() + // TODO: use NormalizePathLayer::trim_trailing_slash .route("/capture", post(capture::event)) - .route("/batch", post(capture::batch)) + .route("/capture/", post(capture::event)) + .route("/batch", post(capture::event)) + .route("/batch/", post(capture::event)) + .route("/e", post(capture::event)) + .route("/e/", post(capture::event)) + .route("/engage", post(capture::event)) + .route("/engage/", post(capture::event)) .layer(TraceLayer::new_for_http()) } diff --git a/tests/django_compat.rs b/tests/django_compat.rs new file mode 100644 index 0000000000000..f76fea0ba8a24 --- /dev/null +++ b/tests/django_compat.rs @@ -0,0 +1,57 @@ +use axum::http::StatusCode; +use axum_test_helper::TestClient; +use capture::event::ProcessedEvent; +use capture::router::router; +use serde::Deserialize; +use serde_json::Value; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use time::OffsetDateTime; + +/* + "path": request.get_full_path(), + "method": request.method, + "content-encoding": request.META.get("content-encoding", ""), + "ip": request.META.get("HTTP_X_FORWARDED_FOR", request.META.get("REMOTE_ADDR")), + "now": now.isoformat(), + "body": base64.b64encode(request.body).decode(encoding="ascii"), + "output": [], +*/ + +#[derive(Debug, Deserialize)] +struct RequestDump { + path: String, + method: String, + #[serde(alias = "content-encoding")] + content_encoding: String, + ip: String, + now: String, + body: String, + output: Vec, +} + +static REQUESTS_DUMP_FILE_NAME: &str = "tests/requests_dump.jsonl"; + +#[ignore] +#[tokio::test] +async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { + let file = File::open(REQUESTS_DUMP_FILE_NAME)?; + let reader = BufReader::new(file); + for line in reader.lines() { + let request: RequestDump = serde_json::from_str(&line?)?; + + if request.path.starts_with("/s") { + println!("Skipping {} dump", &request.path); + continue; + } + + println!("{:?}", &request); + // TODO: massage data + + let app = router(); + let client = TestClient::new(app); + let res = client.post("/e/").send().await; + assert_eq!(res.status(), StatusCode::OK, "{}", res.text().await); + } + Ok(()) +} diff --git a/tests/requests_dump.jsonl b/tests/requests_dump.jsonl new file mode 100644 index 0000000000000..ee8ba3d5e8e50 --- /dev/null +++ b/tests/requests_dump.jsonl @@ -0,0 +1,21 @@ +{"path":"/s/?compression=gzip-js&ip=1&_=1684752074919&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:01.993704+00:00","body":"H4sIAAAAAAAAA+2Xb0vcMBjAv0oJvphgvfxrmvbV2M0xcOgNdI6pHLkmtdXa1iT1dhO/+9KxTdlgR+WmHJdXpcmTp3maX35NT++AulW1BSnYMrVoTdFYsANa3bRK21IZkN499EylsKJvsYtWgZTtgF8NbdVdlLXLovVczUZZU5umUq9Rn0ssqkbIPqpyz6pckNK60a7LapG5PKegqadfCz390R68Kqxt09GoajJRFY2xKceEjowVtsxGWdHVV+He2/H45PjwZPfSpBGKY5Yisu0yHupRS3ebWishF/0IlRWivlBPSMoQTAnbBuePSjgFZ+CNkMH7o6NJ0I/tTBrA4AyA8/t7V095rVzrdQtSxDiNIwxjlGDuuraMMqZ0dZYuDUCcU+L6mIQMh5AhJBlHEaUSZiFFWOKMyBAxOoM8fAhGiEjmqtyal7Vs5n/mwjhiMIQEilxAnktCYvqPbMTFR/0qNFeqX7q2yKYqLmllxgflx/xgX3youuryZvHpYjKBE6bjb8f8czt+142/VPudGylLY8s6s78nQiidzRLCGclDiKKcKUEUnomM84eJ4BiqGPYT+RmeZAoC946aPDfKoYgT7l7Zi5A5F7p+DOatqLonwYN5iqL/QGRC/yZyr77pVKdkkIuychet3L2xQe42k1ZWL4KyDhiEcAmnied0XTjdJIOSBDFP5iAyk8TtZS/QlQqU8iUGdZz6L/26cLphAvVgDgXTn0BXLVBMlwoUQ8/pQE5jL9BnEKgHczCY7szuBbpSgSbL/uEdp8hzuiacbphAPZhDwYy8QF/gBIo9p2vC6YYJ1IM5FEzqBbpigSK8XKDEczqY0/Pv4oS7EYgfAAA=","output":[{"uuid":"0188430d-40cb-0000-82bb-5cab205fc9d8","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-40ca-0000-c581-b136488335da\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPpGa2QC/+2XXS8DURCG35+y6RUJarfVD1fiI3HHBXGBSFVRSqu2aMRfxzNn28YNkdhoxWRydqczc+Zjz7tvtm+vh3pWQamG6qmFtqpIFS1wLehMDTyNkdXieupooAu1dTuyFtRHHtl7qqKa6uK559rBsqYYf5arR54h1i73sw8ZLe6B1Znka4WMXdZ4b4reIPe4v0PuVudET7rEd/JhT6Q5bCnSI7aIWM0m+zvYu/SWYq8pUUllvPdhwpSJmqH/S+a71bUWtaVNbSAH2tcO1yVdEb2qFaaqIhX0mCzzkz53qF+kbplY669PXzbtcFKlFSo08F2g/06vFdYyegnNej3+4kTsyR6x1oMt0rb2kF20ce1ByBuRMQqRlu8FGZ9UWzdMlkXfUCPLa13UmKFKRwl7q1jqaLXRzlnD4CMR/ZD/cwQ+hHMa/OI52vMyzK38GczVyfFdzG2R9y48T1uGvnOi2uF07JdNlvmtJ/Nm77vZ7XyG6IYJw85ykJ/jsz6j+HSO/A8cWQJ/ZnWOdI7MmyPL9P9zlswQWnOWdJacMkv6l6SzZP4smVA1H5Y0zVnSWXKaLDm7GHSW/MssWc/lH3eG0NhZ0llyyiwZO0s6S870t2TiLOksOWWWTJwlnSVzZ8k4YCwfliyx81jvS12b9+QeAAA=\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752071928, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752071929, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752073916, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752073918, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752073918, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752073920, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752073920, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752073921, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752073921, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752073922, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752073922, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752073923, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2988}","now":"2023-05-22T10:42:01.993704+00:00","sent_at":"2023-05-22T10:41:14.919000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684752098970&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:01.995810+00:00","body":"H4sIAAAAAAAAA+2Z207bMBiAXyWyuBgSoT7FcXI1rWOaxASdBGMaoMqNHRIISbAdug7x7nOmbaBNWteqCJX6Kor9+48PXz458ekdULeqtiAFW6YWrSkaC3ZAq5tWaVsqA9K7h5qxFFb0JXbWKpCyHfCroK26i7J2WbSeqskga2rTVOo16nOJWdUI2UdV7lmVC1JaN9pVWS0yl+cUNPX4a6HHP8qDV4W1bToYVE0mqqIxNuWY0IGxwpbZICu6+ircezscnhwfnuxemjRCccxSRLZdxkM9aOluU2sl5KxvobJC1BdqiaQMwZSwbXD+aAin4Ay8ETJ4f3Q0Cvq2nUkDGJwBcH5/78ZTXitXet2CFDFO4wjDJEri2FVtGWVM6cZZujQAcU4JjBGTkOEQMoQk4yiiVMIspAhLnBEZIkYnkIcPwQgRydwot6ZlLZvpn7kwjhgMIYEiF5DnkpCY/iMbcfFRvwrNleqXri2ysYpLWpnhQfkxP9gXH6quuryZfboYjeCI6fjbMf/cDt91wy/VfudaytLYss7s744QSieThHBG8hCiKGdKEIUnIuP8oSM4hiqGfUd+hieZgsDNUZPnRjkUcZKg+51nIXMqdP0YzFtRdUvBg3mKoicgMqF/E7lX33SqUzLIRVm5i1bu3tggdy+TVlbPgrIOGIRwDqfcc7oop/CZON0kg8YJJZ7MhchEEEfeoCs2KOVzFOpApR7UdQF1wxTqyVyYTDdlXqErVWgybxfqQHU+8KCuB6gbplBP5tqQ+XIV+j+7UOZBXRRU94XpFfr0CvVkLkwm9gpdsUIxna9Q/89+bUDdMIV6Mhcm058mrVqhCM9XqD9OWgLU8++WAqIwkh8AAA==","output":[{"uuid":"0188430d-40cd-0000-7103-c248d33d28b5","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-40cc-0000-fcd3-63fc3328fd1f\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPpGa2QC/+1ZXU/CUAw9P2XhSROVr30wn4xK4hs+aHxAQhAQ0MEmDJAY/7p6egeEF40JC8PYNHcrbe9pu9udLOHzo4435BBjgQhdaqew4OKI1xw6aNHTWlolLkKAKXoYYLS05jCmzLn3AXm0EdIz4TWg5QxF+hOsiDgLWkPeOxuIEjfjCtZ4XYMYcq32xtRbxF7VV+dd8jTxij59zY09Fg5oiykRY/MUydnm/oD2kLXFtFdQQhk2vRPTYcyO2qb+Pvsb4RnHqOISF5Q73KLG6wmeGH0Kh115FJd6kSiH6zprzJ9nXpuxUt+YdUm3i3WWrsnQoq9HfTe1ulwF6mVqUmvjhxORJ3vPdW5sFq5wQ7mmtso9NbgWES0TKXjvlNVJDTBkZ0n0kDkSXKmiwh48VlTiXp933zzFZOe+zeCcEWOD//0Ezsw5TXd4jiViyMw5f2bmfGL8duaqxH0xz1OWTN8jowbmdOSXdJb4pSbxJu+72OV8FtRlJmR2Cka2n8/Kns6ncuR/4EjPvD9l5UjlyNQ50mb927NkMqG2sqSyZMYsaStLKkumzpJ+Kt+SyYQ6ypLKkhmzpKMsqSy519+SrrKksmTGLOkqSypLps6SJWZNiyU9ZUllyYxZUv+7UZZMnyWLZsbSYUn596aBL0Ml6tzkHgAA\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752095977, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752095978, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752097943, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752097944, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752097944, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752097945, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752097945, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752097946, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752097946, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752097947, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752097947, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752097948, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2991}","now":"2023-05-22T10:42:01.995810+00:00","sent_at":"2023-05-22T10:41:38.970000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684752114005&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:02.050911+00:00","body":"H4sIAAAAAAAAA+2Y3WrbMBSAX8WIXqxQN/qzLPtqLOsYdLQZtOtYW4JiybVb13YluVlW8u6Tx7aUDZYlZIwQXRlLR8c60ucPocsnoB5VbUEK9kwtWlM0FhyAVjet0rZUBqRPi56xFFb0LXbWKpCyA/Cjoa26m7J2WbSeqskga2rTVOol6nOJWdUI2UdV7luVC1JaN9p1WS0yl+cSNPX4c6HH39qDF4W1bToYVE0mqqIxNuWY0IGxwpbZICu6+i48ej0cXpyfXhzemjRCccxSRPZdxlM9aOlhU2sl5KwfobJC1DdqjaQMwZSwfXD9rIRLcAVeCRm8PTsbBf3YzqQBDK4AuJ7PXT3lvXKt9y1IEeM0jjBCCELouvaMMqZ0dZYuDUCcUwJjxCRkOIQMIck4iiiVMAspwhJnRIaI0Qnk4SIYISKZq3JvWtaymf6aC+OIwRASKHIBeS4JiekfshEXH/W70NypfuvaIhuruKSVGZ6U7/OTY/Gu6qrbh9mHm9EIjpiOv5zzj+3wTTf8VB13bqQsjS3rzP6cCKF0MkkIZyQPIYpypgRReCIyzhcTwTFUMewn8j08yRQEbo2aPDfKoUggxPOD/0LmVOj6OZiPourWggfzFEX/gMiE/k7kUf3QqU7JIBdl5R5auXdjg9z9TFpZPQvKOmCOwyWcIs/ptnC6SwbFCfdkrkYmgtgtmTfoRg2KErzEoY5UJwRP6naQumMO9WSuTKY7uHuHbtShybJjqAOVeFC3BdQdU6gnc1UyUeIVumGFUr5codSDui2g7phCPZlbQ6ZXqAd1K0DdMYVGnsxVyeReoZu+DP2Lu1AP6hqgXn8FVjpm4JMfAAA=","output":[{"uuid":"0188430d-40cc-0004-71c9-b5d5762c946c","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-40cc-0003-1759-ec61e7e52da5\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPpGa2QC/+1Yy07CUBA9n9Kw0kTFlvJcGZXEHS40LpCQCghoeQgFJMZfV8/cSwkbiYmVR5xMbjudmTsP7ukJ6edHFW9IIcIcQ7SoleAghyNeU2gioCdYWCVuiBATtNFFf2FNYUSZce8D0mhgQM+Y15CWM7j021xD5pnTOuC9uZJR4qZc4TJfy2QccMV7I+oBc8f9VXmXOnW8okNffWWPgwPaIsqQsWmK1Gxwf0j7gL1FtBfgIQOf3rGZMOJEDdN/h/P18YxjlHGJC8odblHh9QRPjC4hy6nylBx1l1kOl31WWD/Nuj5jpb8R+5Jp58sqLVMhoK9NfTO95rhOqWeoSa+1NSciv+w917mxObjCDeWaWlx7YvI6zOiYSMn3TolPqoseJ7PRPdaweaWLAmfIsyOPT67pScTu3DUMzhgxMvm/R+DUnNNkg+foMYdgLrs3mCsyx08xV2beF/N7yhL0PTKqa05HnmQy65eexGvfd7HL+cypCyYEOxZbv8enu6P4VI78Dxzp8f0p7CwGlSP3mSNd1vUS4EmLUU95UnlyyzzpKU8qTybOk8VE/k1ahGaUJZUlt8ySGWVJZcnEWdJn/0mxpK8sqSy5ZZb0lSWVJZUllSWVJddgMKssqSz5B18mk/suKQit4QuIGGIM5h4AAA==\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752111000, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752111001, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752112981, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752112982, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752112982, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752112983, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752112983, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752112984, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752112984, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752112984, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752112985, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752112985, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 3002}","now":"2023-05-22T10:42:02.050911+00:00","sent_at":"2023-05-22T10:41:54.005000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684752101977&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:02.048295+00:00","body":"H4sIAAAAAAAAA+2Ub0vcMBjAv0oJvphgvfxrmvbV2M0xcOgNdI6pHLkmtdWa1CT1dhO/+1LZpmzgcXKbDPuq9MnzPM2T/Po7vgHqWmkPcrDhtGhdZTzYAq01rbK+Vg7kN/crUym86CN+0SqQsy3wM9A23VmtQxdr52o2Kox2plGvUd9LLBojZJ/VhG81IUlZa2xY8lYUoc8xMHr6tbLTu3j0qvK+zUejxhSiqYzzOceEjpwXvi5GRdXpi3jn7Xh8dLh/tH3u8gSlKcsR2Qwd9+2opdtGWyXkoq9QRSX0mXpCU4ZgTtgmOH0wwjE4AW+EjN4fHEyivrZzeQSjEwBOb2/DPPWlCtHLFuSIcZomGGY8S5OwtOGUc3WYsw5tAOKcEpgiJiHDMWQIScZRQqmERUwRlrggMkaMziCP75MRIpKFKTfmtZZm/nsvjBMGY0igKAXkpSQkpY90IyE/6W/BXKj+6tqqmKq0po0b79Ufy71d8aHpmvOrxaezyQROmE2/HfLP7fhdN/7S7HahUtbO17rwvzZCKJ3NMsIZKWOIkpIpQRSeiYLz+43gFKoU9hv5kZ4VCoJwRqYsnQooEgjh7dazkDkXVj8E81o03ZPgwTxHyV8gMqN/ErmjrzrVKRmVom7Cw6rw7nxUhp/JKm8XUa0jFg51Cads4HQlTnGWZc/E6csxKIIwY0EHA5krkIlC6WDQNRuU8kcVegcqGkBdFVQ6KPRfKBQPZK5KJhkUumaFYrpcoQOo/w2og0IHMh8nMxzZoNC1KhTh5QoNPhhAXRXU0+/5Yx7EDBUAAA==","output":[{"uuid":"0188430d-40cc-0002-a8da-245e5d68b7b7","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-40cc-0001-801c-85fd950326c8\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPpGa2QC/+2XQU8CQQyF30/ZcNJEhV1wEU9GJfGGB40HJAYRFUVBWFRi/Ovq11lBLhoTCCKZNLPbaTuv7U55Ce9vVb0oo0RDddVE21agWGs8M7pQHU/902pxXbU10JVauv+0ZtRDnjh7rqwa6uDp82xj2VGIP8XqgjPE2uF9MYFocY+s9hiv6RA7rNHZBL0O9qi+Km/Lc6ZnXeM7mzgTaAVbgnSJzSKWs8H5NvYOtSXYtxQprwLevuswoaOGq/+a/u51q3WVta895ETHqvDc0A3R29qkqyISo4egrI7rrJA/S94CsVZfj7qs2+E4S9NlqOO7Qp9PrTErh55Hs1prP9yIfdlT1q6zBTrQEXKINso9cLgBiIGLNLxXZHRTLd3RWRp9R44U16rYoociFUWcLbEruV16ctFm8ImInsP/fgIf3T0N5niPERg2c5v/ZuZKYPx25srgPrjvacum75Kolrsd21lnqd9qMm/6eze73c8Q3WbCZifnZPr5jBd0Pj1HLjtHhm6CS26WPUd6jpw1RxaofzqW/JrQ0LOkZ8k/ZsnIs6RnyZmzZETWWbFk5FnSs6RnSc+SS8eSofvPPBuWzHOypg9uMIiYmBQAAA==\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752098975, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752098976, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752100960, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752100961, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752100962, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752100962, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752100962, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752100963, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 3000}","now":"2023-05-22T10:42:02.048295+00:00","sent_at":"2023-05-22T10:41:41.977000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684752123024&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:03.027729+00:00","body":"H4sIAAAAAAAAA+2UXU/VMBjHv8rScCEJ4/RtXbcr4xFjgoFjAmIEctKzdqxQ2tF2HI+E725nVIgmEoiGC8/Vsuflv+e//voc3wB1rWwENdgIVvShcxFsgd67XvmoVQD1zV1mLkUUYySuegVqtgV+BHoznGmbVLxfqsWkcTY4o16iUUusjBNyrDLpWyYVKe+dT6noRZN0joGz88+dn3+LZy+6GPt6MjGuEaZzIdYcEzoJUUTdTJpusBf5zuvp9Ohw/2j7PNQFKktWI7KZFPf9pKfbznol5GrsUE0n7Jl6gihDsCZsE5zes3AMTsArIbO3BwezbOwdQp3B7ASA09vb5EdfqhS97EGNGKdlgRGGEJOU2ggqBJ186iQDEOeUwBIxCRnOIUNIMo4KSiVscoqwxA2ROWJ0AXl+V4wQkSy53FhqK93yVy2MCwZzSKBoBeStJKSkf1Ajqb4YT8FdqPHo+q6Zq1JTE6Z7+n27tyvemcGcX60+nM1mcMZ8+eWQf+ynb4bpJ7M7pE6pQ9S2iT8HIZQuFhXhjLQ5REXLlCAKL0TD+d0guISqhOMg38urRkGQ/pFr26ASiriqytutZyFzKby9D+a1MMOT4MG8RsU/ILKivxO5Y68GNSiZtUKb9PAqvYeYtekyeRX9KtM2YxDCBzgt1pw+llP2TJz+TxsUVbxak/koMlGysN6gf3mDVg+tUFRVaA3q40E9/QqvsD61hgoAAA==","output":[{"uuid":"0188430d-4494-0001-7d1f-eae0ac044b9f","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-4494-0000-2fde-272e9466a2a0\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPtGa2QC/+2Uy04CUQyG/0eZsNJE5X5dGZXEHS40LtCQERDQ4SIMIDG8uvr1DBA2GhMJwUianun08redds7He1VviinUTAM1kUrylNMRZ0wN+Vj8hdb8Bgo0Vksd9RbamIbQlNgHxVVXH8uIM0BzqiT2CGsAzgxtn2djDdH8JnCwwms6xD68jA2RfbCX9VV5Wp6aXtXGVluL8XSALoQG+MYhy1knPkDfp7YQfUEppZXBOnIdhnRUd/W36a+nZx2rrAudQ7e6UYXzRE94l5SlqzyUQ06Ccriqs0L+OHkz+Fp9Q+qybmerLE2XwcfWQt5OrTk4gZxGslrvv5mIfdk7+MzpPF3qGrpCWuYeO1wPRM95Gt4cWk6qoy6dRd5dckS4VkWBHvJUlOItRXzC9RVF7toOTvEYOvyvN3Di5jTe4hxTYNjOZf/MzhXB+OnOlcF9cd/T2LbvEa+Om469WWeR3Woya/S/m97mM0O2nbDdSTj6/X5md3Q/93fkf7gjk/w/BXh/R+7vyE3fkcWN3JK2oUXOOfGfu7RZCEwKAAA=\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752120023, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752120025, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752121989, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752121991, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2997}","now":"2023-05-22T10:42:03.027729+00:00","sent_at":"2023-05-22T10:42:03.024000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684751932581&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:04.993934+00:00","body":"H4sIAAAAAAAAA+2UX0/VMBjGv8rScCEJ47Rr13W7Mh4xJhg4JiBGICc9a8cKpR1tx/FI+O52RoVookIkhLCrZe+fZ++7/vocXgF5KU0AFVjzhne+tQFsgM7ZTrqgpAfV1U1mLnjgQySsOgkqugF+BDrdnygTVZxbysWktsZbLV+iQYuvtOViqNLxWzoWSeesi6ngeB11DoE188+tm3+LJy/aELpqMtG25rq1PlQsw2TiAw+qntRtb87SrdfT6cH+7sHmqa9yVBS0Qng9Ku66SUc2rXGSi9XQIeuWmxN5D1GKYIXpOji+tcIhOAKvuEje7u3NkqG391UCkyMAjq+v4z7qXMboeQcqRBkpclRmZc5QTK156b2Ke6ooAxBjBMMCUQFplkKKkKAM5YQIWKcEZSKrsUgRJQvI0ptihLCgccu1pTLCLn/VyrKcwhRiyBsOWSMwLsgf1HCsz4dTsGdyOLqureeyUET76Y563+xs83e616cXqw8nsxmcUVd82Wcfu+mbfvpJb/exUygflKnDz0EwIYtFiRnFTQpR3lDJscwWvGbsZpCsgLKAwyDfy8taQhD/kW0aLyOKWVmW1xuPQuaSO3MbzEuu+3vBk7EK5Q9AZEl+J3LLXPSylyJpuNLx4WR89yFp4mVyMrhVokxSUgjhX0DNRlDvCip7JFCfmYWOZD4ZMkcLHUF9EqA+MwvFI5l3JbMYLfQ/W+g/OOjI6T04Pf4KKs0GxMkPAAA=","output":[{"uuid":"0188430d-4c42-0001-5c53-b092a3eae230","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-4c42-0000-208e-16cab7bc0a1c\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPxGa2QC/+2US08CUQyFz0+ZsNJERV7DY2VUEne40LhAQhAQUF7CABLjX1e/3gHCRmIiIRgnzZ0pbe9pOy3n86OsN8UUaK6hmmgFefJ1xDOmhmp4agurxQ3V1UQtddRfWGMaITPuPiiuugZ4xjy7WM6UwB9iDcGZYx3wbqwhWtyU013hNR3igLO8G6DXwF7WV+Zteap6VRtfde2OpwNsATIkNo5Yzjr3u9gH1BZgzymplNJ4x67DgI7qrv42/fX1rGMVdakL5E63KvE80RPRBWXoKov46AlQDld1lsgfJ2+aWKtvRF3W7XyVpeky1PC10HdTq885RU+hWa2VDROxL3vPOXc2T1e6Qa7RlrknDtcD0XORhveOLCfVUY/OwugeOUJcqyJHD1lXUZ6O8mg59PDmvu3gjIiRw/9+A6duTpMdzjEJhu1c5s/sXB6Mn+5cEdwX9z3t2PY9EtVx07Ff1lnot5rMG/7fzW7zmaPbTnhk9clu8vsNTe7phkYs+X9YMhmxZMSSEUtGLBmx5IYdTEUsGbHk1llyexxp+1nRF6KoW4VyDwAA\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684751929581, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751929582, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684751929582, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751929582, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684751929583, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751929583, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2999}","now":"2023-05-22T10:42:04.993934+00:00","sent_at":"2023-05-22T10:38:52.581000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684751935588&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:04.995438+00:00","body":"H4sIAAAAAAAAA+2WYU/VMBSG/8rS8EESxm3Xruv2yXjFmGDgmoAYgdz0rh0rlHa0Hdcr4b/bGRWiiQSCGnSflp2evjunffquh1dAXkoTQAXWvOGdb20AG6BztpMuKOlBdXUzMhc88CESVp0EFd0A3wKd7k+UiSrOLeViUlvjrZbP0aDFV9pyMWTp+C0dk6Rz1sWh4HgddQ6BNfOPrZt/iSfP2hC6ajLRtua6tT5ULMNk4gMPqp7UbW/O0q2X0+nB/u7B5qmvclQUtEJ4PSruuklHNq1xkovVMEPWLTcn8gGiFMEK03VwfKuFQ3AEXnCRvN7bmyXD3N5XCUyOADi+vo79qHMZo+cdqBBlpMhRibOc0Ti05qX3KvapogxAjBEMC0QFpFkKKUKCMpQTImCdEpSJrMYiRZQsIEtvkhHCgsYu15bKCLv8USvLcgpTiCFvOGSNwLggv1DDMT8fdsGeyWHruraey0IR7ac76m2zs83f6F6fXqzencxmcEZd8Wmfve+mr/rpB73dx5lC+aBMHb4XgglZLErMKG5SiPKGSo5ltuA1YzeFZAWUBRwK+Zpe1hKCuEa2abyMKGII4fXGXyFzyZ25DeYl1/2D4MlYhfLfQGRJfiZyy1z0spciabjS8eFkfPchaeJhcjK4VaJMUtK4qneAWoygPhVQ/zMLHcl8MmT+uxaKstFCHxvUrCzL0UL/hIWykcz7khkP82ihj2qhd19Cy/jfGjm9L6fHnwGF59KmyQ8AAA==","output":[{"uuid":"0188430d-4c44-0001-b56a-cc6e82a03b21","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-4c44-0000-1533-2d9329721e64\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPxGa2QC/+2WS08CUQyFz0+ZsNJERV7DY2VUEne40LhAQxAQUF7CABLDX1e/3gHCRmMiIaiT5t4pbe/p6dxOw/tbWa+KKdBMAzXQCvLk64A9prqqeKoLq8UN1NFYTbXVW1hjGiJTzt4rrpr6eEbsHSwnSuAPsQbgzLD2edbXEC1uwuqs8BoOsc9ang3Qq2Av+ZV5Wp6KXtTCV1k742kPW4AMiI0jlrPG+Q72PtwC7DkllVIa78hVGFBRzfFvUV9PTzpUUec6Q250rRL7kR6JLihDVVnER0+Asr/iWSJ/nLxpYo3fEF5W7WyVpeEyVPE10bfD1Wcdo6fQjOvdFzdib/aWdepsni50hVyiLXOPHa4HouciDW+OLG+qrS6VhdFdcoS4xiJHDVnHKA+XJFoOa3hy13pwSsTQ4X/egRN3T+Mt3mMSDOu5zK/puTwY3+25IrjP7n3asu57IKrtbsd+WWWh3ziZN/zezW73M0O3nvDI6pPd5Ocdmt3RDo2m5P+ZktloSkZTcuNTMgH7aEpGU/KvTMlcNCWjKbnxKbmpf5J5MOac/wAjkc3Fcg8AAA==\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684751932586, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751932587, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684751932587, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751932587, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684751932588, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751932590, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 3000}","now":"2023-05-22T10:42:04.995438+00:00","sent_at":"2023-05-22T10:38:55.588000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684752077923&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:04.997482+00:00","body":"H4sIAAAAAAAAA+2XW0vcQBiG/0oYvKhg3Dllcrgq3VoKFt2C1lKVZTYzMdFxJs5M3G7F/95JabvSQmVlW6nmKuQ7vPm+zJMXcnwD5LXUHhRgw2neutp4sAVaa1ppfSMdKG6WmangnvcRv2glKNgW+BFoVXfW6KBi7VzORqXRzij5EvVafKEMF32VCs9SoUhaa2xIecvLoHMMjJ5+ru30Wzx6UXvfFqORMiVXtXG+yDChI+e5b8pRWXf6It55PR4fHe4fbZ+7IkFpygpENoPivh21dNtoK7lY9B2yrLk+kw8QZQgWhG2C0zsrHIMT8IqL6O3BwSTqeztXRDA6AeD09jbs01zKEL1sQYFYRtMEw5TmOAupDSeda8KeTZABKMsogSliAjIcQ4aQYBlKKBWwjCnCApdExIjRGcziZTFCRLCw5ca80cLMf9XCOGEwhgTyisOsEoSk9A9qJNQn/SmYC9kfXVuXU5k2VLnxXvO+2tvl71Snzq8WH84mEzhhNv1ymH1sx2+68Se124VO0Tjf6NL/HIRQOpvlJGOkiiFKKiY5kXjGyyxbDoJTKFPYD/K9PC8lBOEdmapyMqCI8zy53XoUMufc6rtgXnPVPQgenBUo+QtE5vR3Inf0VSc7KaKKNypcrAz3zkdV+Jis9HYRNTpiEMKB03VzSh+J0+fkoCxHZCBzJTIRRHBw0DU7aH6fhQZQgx8MoK4EKswHC/0XFjqQ+d+Q+XQtFNPBQp8QqM/MQsNP6UDmamSmg4Wu2UIRvt9C2QDq6qCefgX0iVnZDBUAAA==","output":[{"uuid":"0188430d-4c46-0001-9d63-c33d1ad5a8bc","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-4c46-0000-7b0c-6e3bf34bf14a\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPxGa2QC/+2USU8CQRCF30+ZcNJEZRGG5WRcEm940HhAYhBRUDZhECfGv65+1QPEi8bDxLVT6Z6iqvpVVXfxXp4belRGkWKN1UGrKVCoDfaMLtXC01pYLW6svma6Vk/DhTWjCTLn7IWyamuEZ8rex7KjPP4EawxOjHXE9/INosXds/orvI5DHLGWZyP0FtjL+hp8Lc+5HtTFd/7mTKA1bBEyJjaLWM425/vYR9QWYa+ooG0V8U5dhxEdtV39Xfob6labOtC+9pBTnajOvqUbomsq0VUZCdHzoKyv6qyTP0veIrFW34S6rNt4laXjMrTwXaN/Ta0hK4e+jWa1Nj94EbvZM9auswU61DFyhLbMPXO4AYiBizS8J2T5Uj0N6CyJHpAjwbUqKvRQpqICZ8voVbTK4uRPm8E5EROH//4E3rt3mn3hO9p92cyVfs3MVcH47MwdgHvn7tOWTd8VUT33OvbLOkv8VpN5k/+72e19YnSbCZudnJO/O5+eI/8DR4bMoN2e50jPkWlzZDUVlkwmtOhZ0rPkN7Nk0bOkZ8nUWbJAVs+SniX/CkuWPEt6lkydJfNuxtJhyZCTTb0CdWfxy5gUAAA=\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752074928, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752074928, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752076913, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752076914, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752076914, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752076914, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752076915, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752076916, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2995}","now":"2023-05-22T10:42:04.997482+00:00","sent_at":"2023-05-22T10:41:17.923000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684752104985&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:04.999014+00:00","body":"H4sIAAAAAAAAA+2ZbUvcMBzAv0oJvphgvTw1Tftq7OYYOPQGOsdUjlyT2mpta5J6u4nffenYdrKBxx0nclxelSb//JuHX3+E5PwBqHtVW5CCHVOL1hSNBXug1U2rtC2VAenDvGYshRV9iZ21CqRsD/wpaKvuqqxdFq2najLImto0lXqL+lxiVjVC9lGV+1blgpTWjXZVVovM5TkHTT3+Xujxr/LgTWFtmw4GVZOJqmiMTTkmdGCssGU2yIquvgkP3g+HZ6fHZ/vXJo1QHLMUkV2X8VgPWrrf1FoJOetbqKwQ9ZVaISlDMCVsF1w+GcI5uADvhAw+npyMgr5tZ9IABhcAXD4+uvGUt8qV3rYgRYzTOMIIooTHrmrHKGNKN87SpQGIc0pgjJiEDIeQISQZRxGlEmYhRVjijMgQMTqBPJwHI0Qkc6PcmZa1bKb/5sI4YjCEBIpcQJ5LQmL6TDbi4qN+FZob1S9dW2RjFZe0MsOj8nN+dCg+VV11fTf7cjUawRHT8Y9T/rUdfuiG36rDzrWUpbFlndm/HSGUTiYJ4YzkIURRzpQgCk9Exvm8IziGKoZ9R36HJ5mCwM1Rk+dGORRxktDHvVchcyp0/RTMe1F1K8GDeYqiFyAyof8TeVDfdapTMshFWbmHVu7d2CB3P5NWVs+Csg4YhHABp4nndFM43SaDkoQhT+ZSZCKI3ZR5g67VoJQvUKgDFXtQNwXULVMo8WQuSSZy+yGv0LUqNFm0C3Wgui2VB3UzQN0yhUaezGXJdEcfXqFrVSjCixXKPKibAuqWKdSTuTSZbsq8QteqUEwXK9Sf2S8NqtseeYW+vEK5J3NZMv1t0mso1F8nrQDq5U9Nk6Owkh8AAA==","output":[{"uuid":"0188430d-4c49-0001-a3b5-5cbaee834ee0","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-4c49-0000-e27d-3ad69e097d43\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAP1Ga2QC/+2ZTU8iQRCG358y4bQmrsgAA3gyfiTe9LBmD0oIIgsoCuKgks3+9dWneoB4WWPiRDBb6fRQVFXXB108IeH575l+q6BUM43VRdpRpESbPAu6VBtLe641v7GGmqqngW7n2oImrEfOXqiojkZY7nkO0eyqhD2LNSbODO2I18tXEc3vgT1cxuuGiCP24myK3Cb2or4zXi1PS0/qY2u9OhPpG7qUNca3yLKcHc4P0Y+oLUVfV6yyKljvQ4cpHXVC/X36u9W1vutQB9pn/dSpjnlu6QrvHVXpqsZKkEtE2VjWeUz+Inkr+Fp9E+qybmfLLN2QoY2th/w5tSbsbeQyktXafONG7JM9Z+8FXaQj/WCdIC1yT0PciIhR8LR4f1iLmxrohs4y7xtyZHGtijo91KgoDvWU1EBTm59ctxl8xGMS4v97Ah/CPU0/8R5jYtjMVb/MzDWI8d6ZOyTuXfg8bdv0/cJrEG7H3llnmd1qMmv2fTe93c8M2WbCZmc7rI/PZ2NN59MZ+T8wssz8WWXOSGdk3oysUP/HKZlNaOyUdEqumJJlp6RTMndKNnL5LZlNaMUp6ZRcMSWrTkmnZO6ULFF9XpRMnJJOyRVTMnFKOiVzp2RM1rwoWXNKOiVXTMm6U9IpudaUtH9vmnoBGOaMiuQeAAA=\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752101987, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752101989, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752103961, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752103962, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752103963, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752103964, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752103965, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752103966, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752103966, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752103967, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752103968, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752103969, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2994}","now":"2023-05-22T10:42:04.999014+00:00","sent_at":"2023-05-22T10:41:44.985000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} +{"path":"/e/?ip=1&_=1684752184509&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:04.525363+00:00","body":"ZGF0YT1leUpsZG1WdWRDSTZJaVJ3WVdkbGRtbGxkeUlzSW5CeWIzQmxjblJwWlhNaU9uc2lKRzl6SWpvaVRXRmpJRTlUSUZnaUxDSWtiM05mZG1WeWMybHZiaUk2SWpFd0xqRTFMakFpTENJa1luSnZkM05sY2lJNklrWnBjbVZtYjNnaUxDSWtaR1YyYVdObFgzUjVjR1VpT2lKRVpYTnJkRzl3SWl3aUpHTjFjbkpsYm5SZmRYSnNJam9pYUhSMGNEb3ZMMnh2WTJGc2FHOXpkRG80TURBd0x5SXNJaVJvYjNOMElqb2liRzlqWVd4b2IzTjBPamd3TURBaUxDSWtjR0YwYUc1aGJXVWlPaUl2SWl3aUpHSnliM2R6WlhKZmRtVnljMmx2YmlJNk1URXpMQ0lrWW5KdmQzTmxjbDlzWVc1bmRXRm5aU0k2SW1WdUxWVlRJaXdpSkhOamNtVmxibDlvWldsbmFIUWlPamszTkN3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRVd01Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPRFF6TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFME16TXNJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpVM0xqSWlMQ0lrYVc1elpYSjBYMmxrSWpvaWMzRXhhWEppT1dRME0ySmhZbXBxTWlJc0lpUjBhVzFsSWpveE5qZzBOelV5TVRnMExqVXdPU3dpWkdsemRHbHVZM1JmYVdRaU9pSXhPRGcwTXpCbE16UmlOVE5qWXkwd1pETmpObU0zTURrME5ERTFaRGd0TkRFeVpESmpNMlF0TVRZMFlqQTRMVEU0T0RRek1HVXpOR0kyWkRNNElpd2lKR1JsZG1salpWOXBaQ0k2SWpFNE9EUXpNR1V6TkdJMU0yTmpMVEJrTTJNMll6Y3dPVFEwTVRWa09DMDBNVEprTW1NelpDMHhOalJpTURndE1UZzRORE13WlRNMFlqWmtNemdpTENJa2NtVm1aWEp5WlhJaU9pSm9kSFJ3T2k4dmJHOWpZV3hvYjNOME9qZ3dNREF2YzJsbmJuVndJaXdpSkhKbFptVnljbWx1WjE5a2IyMWhhVzRpT2lKc2IyTmhiR2h2YzNRNk9EQXdNQ0lzSW5SdmEyVnVJam9pY0doalgycElZMFIxTjIwelduWnVTVzV3YTJKNGJVcHlTMFZpZVVwMWEzbEJXa042ZVV0bFREQnpWSGhDTTJzaUxDSWtjMlZ6YzJsdmJsOXBaQ0k2SWpFNE9EUXpNR1V6TkdJM04yWXdMVEF6TXpCa05XUXlPRE00WmpObE9DMDBNVEprTW1NelpDMHhOalJpTURndE1UZzRORE13WlRNMFlqZ3lPRE16SWl3aUpIZHBibVJ2ZDE5cFpDSTZJakU0T0RRek1HVXpOR0k1TXpBMU5pMHdORGRoT1RCaE1HSXdNVE5pTkRndE5ERXlaREpqTTJRdE1UWTBZakE0TFRFNE9EUXpNR1V6TkdKaE1qVmxOaUlzSWlSd1lXZGxkbWxsZDE5cFpDSTZJakU0T0RRek1HVXpOR0ppTVRZM1lpMHdOekJrTVRrMVpqVTVNak0yTmpndE5ERXlaREpqTTJRdE1UWTBZakE0TFRFNE9EUXpNR1V6TkdKak1qazRNQ0o5TENKMGFXMWxjM1JoYlhBaU9pSXlNREl6TFRBMUxUSXlWREV3T2pRek9qQTBMalV3T1ZvaWZRJTNEJTNE","output":[{"uuid":"0188430e-34ce-0000-0e70-879b9262bb17","distinct_id":"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"sq1irb9d43babjj2\", \"$time\": 1684752184.509, \"distinct_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"timestamp\": \"2023-05-22T10:43:04.509Z\"}","now":"2023-05-22T10:43:04.525363+00:00","sent_at":"2023-05-22T10:43:04.509000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?ip=1&_=1684752184509&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:04.515071+00:00","body":"ZGF0YT1leUpsZG1WdWRDSTZJaVJ2Y0hSZmFXNGlMQ0p3Y205d1pYSjBhV1Z6SWpwN0lpUnZjeUk2SWsxaFl5QlBVeUJZSWl3aUpHOXpYM1psY25OcGIyNGlPaUl4TUM0eE5TNHdJaXdpSkdKeWIzZHpaWElpT2lKR2FYSmxabTk0SWl3aUpHUmxkbWxqWlY5MGVYQmxJam9pUkdWemEzUnZjQ0lzSWlSamRYSnlaVzUwWDNWeWJDSTZJbWgwZEhBNkx5OXNiMk5oYkdodmMzUTZPREF3TUM4aUxDSWthRzl6ZENJNklteHZZMkZzYUc5emREbzRNREF3SWl3aUpIQmhkR2h1WVcxbElqb2lMeUlzSWlSaWNtOTNjMlZ5WDNabGNuTnBiMjRpT2pFeE15d2lKR0p5YjNkelpYSmZiR0Z1WjNWaFoyVWlPaUpsYmkxVlV5SXNJaVJ6WTNKbFpXNWZhR1ZwWjJoMElqbzVOelFzSWlSelkzSmxaVzVmZDJsa2RHZ2lPakUxTURBc0lpUjJhV1YzY0c5eWRGOW9aV2xuYUhRaU9qZzBNeXdpSkhacFpYZHdiM0owWDNkcFpIUm9Jam94TkRNekxDSWtiR2xpSWpvaWQyVmlJaXdpSkd4cFlsOTJaWEp6YVc5dUlqb2lNUzQxTnk0eUlpd2lKR2x1YzJWeWRGOXBaQ0k2SWpsbmVYTnpkM1JtZEdFNE1IVjNkbmdpTENJa2RHbHRaU0k2TVRZNE5EYzFNakU0TkM0MU1Ea3NJbVJwYzNScGJtTjBYMmxrSWpvaU1UZzRORE13WlRNMFlqVXpZMk10TUdRell6WmpOekE1TkRReE5XUTRMVFF4TW1ReVl6TmtMVEUyTkdJd09DMHhPRGcwTXpCbE16UmlObVF6T0NJc0lpUmtaWFpwWTJWZmFXUWlPaUl4T0RnME16QmxNelJpTlROall5MHdaRE5qTm1NM01EazBOREUxWkRndE5ERXlaREpqTTJRdE1UWTBZakE0TFRFNE9EUXpNR1V6TkdJMlpETTRJaXdpSkhKbFptVnljbVZ5SWpvaWFIUjBjRG92TDJ4dlkyRnNhRzl6ZERvNE1EQXdMM05wWjI1MWNDSXNJaVJ5WldabGNuSnBibWRmWkc5dFlXbHVJam9pYkc5allXeG9iM04wT2pnd01EQWlMQ0owYjJ0bGJpSTZJbkJvWTE5cVNHTkVkVGR0TTFwMmJrbHVjR3RpZUcxS2NrdEZZbmxLZFd0NVFWcERlbmxMWlV3d2MxUjRRak5ySWl3aUpITmxjM05wYjI1ZmFXUWlPaUl4T0RnME16QmxNelJpTnpkbU1DMHdNek13WkRWa01qZ3pPR1l6WlRndE5ERXlaREpqTTJRdE1UWTBZakE0TFRFNE9EUXpNR1V6TkdJNE1qZ3pNeUlzSWlSM2FXNWtiM2RmYVdRaU9pSXhPRGcwTXpCbE16UmlPVE13TlRZdE1EUTNZVGt3WVRCaU1ERXpZalE0TFRReE1tUXlZek5rTFRFMk5HSXdPQzB4T0RnME16QmxNelJpWVRJMVpUWWlMQ0lrY0dGblpYWnBaWGRmYVdRaU9pSXhPRGcwTXpCbE16UmlZakUyTjJJdE1EY3daREU1TldZMU9USXpOalk0TFRReE1tUXlZek5rTFRFMk5HSXdPQzB4T0RnME16QmxNelJpWXpJNU9EQWlmU3dpZEdsdFpYTjBZVzF3SWpvaU1qQXlNeTB3TlMweU1sUXhNRG8wTXpvd05DNDFNRGxhSW4wJTNE","output":[{"uuid":"0188430e-34c8-0000-5a19-0714f9887930","distinct_id":"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$opt_in\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"9gysswtfta80uwvx\", \"$time\": 1684752184.509, \"distinct_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"timestamp\": \"2023-05-22T10:43:04.509Z\"}","now":"2023-05-22T10:43:04.515071+00:00","sent_at":"2023-05-22T10:43:04.509000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?ip=1&_=1684752184514&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:04.529988+00:00","body":"ZGF0YT1leUpsZG1WdWRDSTZJaVJwWkdWdWRHbG1lU0lzSW5CeWIzQmxjblJwWlhNaU9uc2lKRzl6SWpvaVRXRmpJRTlUSUZnaUxDSWtiM05mZG1WeWMybHZiaUk2SWpFd0xqRTFMakFpTENJa1luSnZkM05sY2lJNklrWnBjbVZtYjNnaUxDSWtaR1YyYVdObFgzUjVjR1VpT2lKRVpYTnJkRzl3SWl3aUpHTjFjbkpsYm5SZmRYSnNJam9pYUhSMGNEb3ZMMnh2WTJGc2FHOXpkRG80TURBd0x5SXNJaVJvYjNOMElqb2liRzlqWVd4b2IzTjBPamd3TURBaUxDSWtjR0YwYUc1aGJXVWlPaUl2SWl3aUpHSnliM2R6WlhKZmRtVnljMmx2YmlJNk1URXpMQ0lrWW5KdmQzTmxjbDlzWVc1bmRXRm5aU0k2SW1WdUxWVlRJaXdpSkhOamNtVmxibDlvWldsbmFIUWlPamszTkN3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRVd01Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPRFF6TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFME16TXNJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpVM0xqSWlMQ0lrYVc1elpYSjBYMmxrSWpvaWVXVnViamx0YVhRNE4yNXNaR1J4YUNJc0lpUjBhVzFsSWpveE5qZzBOelV5TVRnMExqVXhOQ3dpWkdsemRHbHVZM1JmYVdRaU9pSm5VVVJJYkhCNGNVTklaMVZyYURWek0weFBWbFZSTlVGalRuUXhhMjFXYkRCNGRrOWhhakp6WmpadElpd2lKR1JsZG1salpWOXBaQ0k2SWpFNE9EUXpNR1V6TkdJMU0yTmpMVEJrTTJNMll6Y3dPVFEwTVRWa09DMDBNVEprTW1NelpDMHhOalJpTURndE1UZzRORE13WlRNMFlqWmtNemdpTENJa2RYTmxjbDlwWkNJNkltZFJSRWhzY0hoeFEwaG5WV3RvTlhNelRFOVdWVkUxUVdOT2RERnJiVlpzTUhoMlQyRnFNbk5tTm0waUxDSWtjbVZtWlhKeVpYSWlPaUpvZEhSd09pOHZiRzlqWVd4b2IzTjBPamd3TURBdmMybG5iblZ3SWl3aUpISmxabVZ5Y21sdVoxOWtiMjFoYVc0aU9pSnNiMk5oYkdodmMzUTZPREF3TUNJc0lpUmhibTl1WDJScGMzUnBibU4wWDJsa0lqb2lNVGc0TkRNd1pUTTBZalV6WTJNdE1HUXpZelpqTnpBNU5EUXhOV1E0TFRReE1tUXlZek5rTFRFMk5HSXdPQzB4T0RnME16QmxNelJpTm1Rek9DSXNJblJ2YTJWdUlqb2ljR2hqWDJwSVkwUjFOMjB6V25adVNXNXdhMko0YlVweVMwVmllVXAxYTNsQldrTjZlVXRsVERCelZIaENNMnNpTENJa2MyVnpjMmx2Ymw5cFpDSTZJakU0T0RRek1HVXpOR0kzTjJZd0xUQXpNekJrTldReU9ETTRaak5sT0MwME1USmtNbU16WkMweE5qUmlNRGd0TVRnNE5ETXdaVE0wWWpneU9ETXpJaXdpSkhkcGJtUnZkMTlwWkNJNklqRTRPRFF6TUdVek5HSTVNekExTmkwd05EZGhPVEJoTUdJd01UTmlORGd0TkRFeVpESmpNMlF0TVRZMFlqQTRMVEU0T0RRek1HVXpOR0poTWpWbE5pSXNJaVJ3WVdkbGRtbGxkMTlwWkNJNklqRTRPRFF6TUdVek5HSmlNVFkzWWkwd056QmtNVGsxWmpVNU1qTTJOamd0TkRFeVpESmpNMlF0TVRZMFlqQTRMVEU0T0RRek1HVXpOR0pqTWprNE1DSjlMQ0lrYzJWMElqcDdmU3dpSkhObGRGOXZibU5sSWpwN2ZTd2lkR2x0WlhOMFlXMXdJam9pTWpBeU15MHdOUzB5TWxReE1EbzBNem93TkM0MU1UUmFJbjAlM0Q=","output":[{"uuid":"0188430e-34d2-0000-99e4-4cb75151aa19","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$identify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"yenn9mit87nlddqh\", \"$time\": 1684752184.514, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$anon_distinct_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"$set\": {}, \"$set_once\": {}, \"timestamp\": \"2023-05-22T10:43:04.514Z\"}","now":"2023-05-22T10:43:04.529988+00:00","sent_at":"2023-05-22T10:43:04.514000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/engage/?ip=1&_=1684752184515&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:04.531659+00:00","body":"ZGF0YT1leUlrYzJWMElqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR0p5YjNkelpYSmZkbVZ5YzJsdmJpSTZNVEV6TENJa2NtVm1aWEp5WlhJaU9pSm9kSFJ3T2k4dmJHOWpZV3hvYjNOME9qZ3dNREF2YzJsbmJuVndJaXdpSkhKbFptVnljbWx1WjE5a2IyMWhhVzRpT2lKc2IyTmhiR2h2YzNRNk9EQXdNQ0lzSW1WdFlXbHNJam9pZUdGMmFXVnlRSEJ2YzNSb2IyY3VZMjl0SW4wc0lpUjBiMnRsYmlJNkluQm9ZMTlxU0dORWRUZHRNMXAyYmtsdWNHdGllRzFLY2t0RllubEtkV3Q1UVZwRGVubExaVXd3YzFSNFFqTnJJaXdpSkdScGMzUnBibU4wWDJsa0lqb2laMUZFU0d4d2VIRkRTR2RWYTJnMWN6Tk1UMVpWVVRWQlkwNTBNV3R0Vm13d2VIWlBZV295YzJZMmJTSXNJaVJrWlhacFkyVmZhV1FpT2lJeE9EZzBNekJsTXpSaU5UTmpZeTB3WkROak5tTTNNRGswTkRFMVpEZ3ROREV5WkRKak0yUXRNVFkwWWpBNExURTRPRFF6TUdVek5HSTJaRE00SWl3aUpIVnpaWEpmYVdRaU9pSm5VVVJJYkhCNGNVTklaMVZyYURWek0weFBWbFZSTlVGalRuUXhhMjFXYkRCNGRrOWhhakp6WmpadEluMCUzRA==","output":[{"uuid":"0188430e-34d4-0000-387d-e634d8ec56a6","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"$set\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$browser_version\": 113, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"email\": \"xavier@posthog.com\"}, \"$token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"event\": \"$identify\", \"properties\": {}}","now":"2023-05-22T10:43:04.531659+00:00","sent_at":"2023-05-22T10:43:04.515000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1684752187560&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:07.564872+00:00","body":"H4sIAAAAAAAAA+2cWXPbNhCA/4pHk0fTAQ/w8Fsat5MmadNM4kwnmYwGBJYSLZKgCR6iMvnvXVCybNmyY6d2a4/wZBELLBd7fAQP+Mu3EbRQ1KPD0bMEWN1UME4yNhlzlmUgRvujspIlVHUKanT4bfRM4p/RH4zvvfuw9zeKsWHcQqVSWaDAJgc2PSC6Pa5kp6DCxt/SChI5140C2pTDuO5LQMERqFktSy3gTVWhGeOmylAwrevy8PnzTKIVU6nqw5AQ8lz300fYYVOiBSWrpwXLtdqh4+r057bZtnuhOWPFpGET3R0K6/iDHqJ4BVCMp5BOpniSKPDOG7tU1FNUQgnBxjaFrpRVve4belr5uvmst+fq5iyN8TQdxPokeHDRXwc0OHB0e1qgWfU4FdgaCDuvMlVB77OyWQzj6lTPzfZDL6COHXoH1N4fiVTVacFXw+wQzSDgejF1ObeIcLnPAxJ5nk1FaHm2IxzuCsv2vZiE1nl3X7jhhejcjzKMOWBMdQJsD6dKJ0UzBH/ZNS0mYyFzlmrHXA3wxezEDgqU9qFVgYLawh+ZZDpdazkDraCc8vHJK37UBLn7uS1+L8pZPM9fV29+jfvXzax/8fnlon8Db4n6OP/FnQ3xX6q8PP8gSIhFXJcIKpzQDRMXbpp/iH1cra5LCyG7y9oil1DfIl7AIsJITGw39m5SxxwK/jLBJxgeuKIwtv0gtkhAhB3RhEaO6/s3KeROFJLR9/2RTBJ03ejQJZ73ff8CByaVbMpU4FGa9IYA/wcBqJ8vmCvjNJhy6bTTawhALyNg8v7oVVbOT1++mhzPplS5b999On5PX/A/a3uWf8rIvH3HThyV+Pm9F3yjvfoTVqRqLCCXY0yzE+DoyoRlClDhkIdDzq1FI7I6r+XaPlgYeWJRm/lWInzPp17oRwR0ct87fgZjztL2zJ51+wz62xq3HjMU37eR9hiivGkG191SxSrPjyBhTVbv/bW2B2cBqgYxXlXzypcrH6+PdS4tZKF1HH98OYxTNSt0YQ50LbQM3WhoeneaupGh6eOiaXFaNZ4/P+2p7bTaI4amd6Ap5nY1YUW6YPXSyetRTkSi5ShPgG1FEQ0Ch9nEo97DM3jDqOtAfJOF20B825GrytA1qLJGE1P/xNRmmr1M+9YhjmsRajnOR5sceu4hweS0vSCin7Era1masTjD9exyXYuR+fIVgcvmspB5r41aMlyNuWw0SrBQNqTnILnQw3D8oTh+dqZHhfDl5X5VAbdi+caIpwx137Hnee+0gpew6LJrlsiG6Y+b6bsGJvAAEjQvghB8ITyBqXS9OqAOScJNLJHNm/XH/tBuhwmVpdOs6xT3fZZRsNNrCGUQdc+IYk2NCVYOZYGe1assgSumCuM1Vngndm4c43XarpdgQw2t1mGbdVWyXj9e1NN4iHXtpUebJcqmcmJhrV8Rj3GlWEpMsvNbeUPQOxHUiaLNhZ0h6KMlaFE7UhVJRlm8yE9D/fjEENQQ9IcEhUKbbKG/8rJWhqGGobvLUN4mAhM+cjLRNrwbJrqFoZFhqGHoRYZOGZ9hPeCbdVYOETYIvWeEUoPQp4HQ/nTRxZNanIgiF12cbEeobwhqCLqxCmVV1luMc+SbtZIZkD4ASM0T0ScCUtX6cZPEzSmP27oVjQGpAeltlqJycjp8tGHIaci5m+SEXjJaJk0gRB/ldOCcIach5w/ImQCIGG/kLcUBvwQyCDUI3VmEFtKOe9qKYO57jasGX15FaGgYahi6wVBZxJJVAgdZrWMNn9Ebjt47R/3QcPRpcPSkBVGfCLsrw3nXzfQHsVs4GpkXSoajGxwtZA2xlDPzPv4B8Ek3XiatWbOHJNFXrj2FmDAAfRwATeOocCtoM+k7wNmQZ1cA6hPUbQC6ywA1FLwzBe0nsvVyh+kXtU3RErloQy93HJzcVvo5V/67haHfz9PvfF/gdRn5nwBy9WH7Rop47lBjuHMyy4dXPXr3usWzlM+mEv2OMkBAZoMdOqLrnZNra1SGTzhvkD/sPtS1Zy/tQd3u5nWn1aZTldZwEyl2cfPmv78OuO73r/8A7dHGd9pIAAA=","output":[{"uuid":"0188430e-40ad-0000-e548-fc5de1421f3b","distinct_id":"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"7d1mrlsrey6apuzb\", \"$time\": 1684752184.51, \"distinct_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"session-reset-on-load\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"offset\": 3044}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0001-b527-acee69cb829b","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"56mza3obi7hco2vh\", \"$time\": 1684752184.515, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\"}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"project\", \"$group_key\": \"0188430e-316e-0000-51a6-fd646548690e\", \"$group_set\": {\"id\": 1, \"uuid\": \"0188430e-316e-0000-51a6-fd646548690e\", \"name\": \"Default Project\", \"ingested_event\": false, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"offset\": 3039}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0002-e60d-ac4f19b25506","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"nqru46xqy512vurl\", \"$time\": 1684752184.515, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"organization\", \"$group_key\": \"0188430e-2909-0000-4de1-995772a10454\", \"$group_set\": {\"id\": \"0188430e-2909-0000-4de1-995772a10454\", \"name\": \"x\", \"slug\": \"x\", \"created_at\": \"2023-05-22T10:43:01.514795Z\", \"available_features\": [], \"taxonomy_set_events_count\": 0, \"taxonomy_set_properties_count\": 0, \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"offset\": 3039}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0003-e8e3-5965b1f00c95","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"621xmy2vdcpezwlh\", \"$time\": 1684752184.55, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 3004}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0004-b960-e71c6531c083","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"lihlwwsc66al5e1i\", \"$time\": 1684752184.555, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"posthog-3000\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2999}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0005-4907-567be483079e","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"nt2osnfl5abzmq8y\", \"$time\": 1684752184.555, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"enable-prompts\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2999}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0006-2957-27a39ce38568","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"cvfdres92ldvucw0\", \"$time\": 1684752184.559, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"hackathon-apm\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2995}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0007-9951-26b9cabcd2ab","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"yqzwbgtdjdnmdwbf\", \"$time\": 1684752184.56, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"early-access-feature\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2994}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0008-80c4-46772bf66fc5","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"sv6bufbuqcbvtvdu\", \"$time\": 1684752184.56, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"hogql\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2994}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0009-5175-65c70ceabd9d","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"eyoa5pfu7ddy9m5m\", \"$time\": 1684752184.56, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"feedback-scene\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2994}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-000a-5dc4-9d624d8dc3fa","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"no1by5vd7x64u3sx\", \"$time\": 1684752184.586, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"onboarding-v2-demo\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2968}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-000b-0f86-e81af913f30b","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"jvedtjd1wp8xwwkw\", \"$time\": 1684752184.599, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"notebooks\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2955}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-000c-fb88-5d95e7940f12","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"ingestion landing seen\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ib9n3revlo62eca3\", \"$time\": 1684752184.603, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2951}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-000d-fec3-693237c3318e","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"9vunv0ozv84m22me\", \"$time\": 1684752184.621, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"instance\", \"$group_key\": \"http://localhost:8000\", \"$group_set\": {\"site_url\": \"http://localhost:8000\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2933}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1684752190565&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:10.567715+00:00","body":"H4sIAAAAAAAAA+1YWXPbOAz+KxpNH6uE1GU5b027O712u51uOzvtdDSUCEmMZVGRKB/p9L8vKB+x5KNNkwdn6qfEOIiPIAAB+PLNhAkUyrwwn7BGyZiVqqnAfGqWlSyhUgJq8+Kb+UTiH/MvFhvvPhj/IRsJ4QSqWsgCGZScUe+MaHpUyWkNFRL/FBUkcqaJHCYihlDNS0DGC6hHSpaaETdVhebDpsqRkSlVXpyf5wgjz2StLgJCyLkoUqiVNoQKmoySXRHNKJnKCjbW53c1loBu0VLqbJBzVqQNS7UeFNbHD1qljiuAIsxApBlaGw7cW+JUcJXhIR4hSJwImJayUmvZwNWHr8kradfR5FxEaGYKkTaCPzY9eOYNzmxNFwXCUqHgSJ1AOvfoPOXFZEqLeXtRJfQlqR+4A8+mQXBGneFTkwu8cBEv9dL3L17m5ez6+cv04yjzauftu08f33vP4r8VHY0/5WQ2eceu7DrxxxvP06riia5DwHEjz4lji3An9uMBGbou9XhgudTmduxwi/puRALrVtznTqAPa7RXfwGFqEMOYxli4F1BjK5MWF4DHphWsinbKFyzTLK0aznUBwtDgFgeZb6VcN/1PTfwh0QHsaxSVogb1sbChpY9JMOFlsuBWsOhNxjYjBLXczWSolasiHVI7IxI8zui2siWEJ3Pohx4iFfHJw1rwVF5hZ/FSkwgTIC1wknOUrzNl6/I2qSFJZvnknF9UzRQorVMpp0QcZ02xypg+ViDQxHgVpyLeJRJ9DvyYMxE3uLQL8om+EtDW6OpcxaPDvCfYM4C5qRO4N3pWIu0aNrkXYhisoVcolmNcTsv2/KyyvwWapv3sJGPWiqHMcppv3wzFTpjmcraTpyzutZl6Is5jizbRMcxpaowbBko1FKfmoXKwjgTOYYeHql/yWRpeGEiVDDTwfPKiFlhMM4NZsSSg1EXoixBGUoa47mBUcabWJ3hM3ewcDHpoVGWDhc0T3aAQqbRsg4j6xmpS6ar1oaVt5gUxWWjlCzwdFkoXa63zO2UupPlqFVun2ztqb1IkLHxy7LKSoxZNe+TMY9UU+/j5qzCutsjJk2eW4uy2eNkrLZ0ZlkC77fwu3vriGWErW+x1z3GLuBdYhd2l9eC7pJuIXfpHcA6FnS03OFFtgIuyUF/TvUfK5b5wgf+djBoAWMlpQ37XcP4OesYtncavg/Sy0rwFP7B7+qBmN0l1DFr/xTOw2atacUwuaufMr8Wfrjbt4XxoO1W4qEMIv3Vqvt5vvZp1sboIRhGX8tY6tzvQZYRu89+G6gPa2jRpm1myT6rC8l1ntzzCfQrWhg8eNTK630SVpt8ZzT0BY19mh2I2FF2IDo/gvgBa9El08G9/C/E7ncuG411ScGiJThH/21hXAoYPdXV77XiL3jxzgoLZG2PWUnZKxp9t2x95yTXX6LlVyPDvkP3LW33fjAE6Xf0iZIjvCP2JVkcXr2MXzSDsfN5UrwqylE0G7+u3vwRzV83o/mzz89v5m/gLan/nV06bcuDj6D7nX6XPRgkxCKOQ7jH7cAJEgcOddkByjj6uKkouJz2Txs6xPMt4g7YkDASEepE7qHjmO2B/jjg+JTiEAD9A8EFSBDeEALwOXc5DgX7jwPPJkmg22OZJDVgB2G7duv/9ZC5snOcE+Y5pplKZNVORXcbNTuqj3nmdLLihmZpnsUQTHKvjd3tmdP1TjPnaeY8zpnz9yrSrg0JpRhgxLUZJI7nURI5zt4DXTshNIqcbpmmfqdMH/0u8FSpL0y3oqBEeQ1lNBgHs3Z236rUATltB0+V+kgr9T23g7tWVhHeAAp+VJurGMFDBRpUd4N5hEurFVa9NuptVe88cJ72Vad91WlfddpXnfZVp33VYxmFKPbNj2ljddvv3HUM2tB8zFOQmmV2dZVPy+ubpLC9691TED3tq05T0JFOQb9ZkfYCbg8xtJjtR2AnDh0CePtrtBcABUq7Ndr1v3/9H65UwSy8JgAA","output":[{"uuid":"0188430e-4c68-0000-2df4-40793d05388a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"vegy51ygdnvw1ny0\", \"$time\": 1684752188.139, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"p\", \"classes\": [\"mb-2\"], \"attr__class\": \"mb-2\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"I can add a code snippet to my product.\"}, {\"tag_name\": \"div\", \"classes\": [\"mt-4\", \"mb-0\"], \"attr__class\": \"mt-4 mb-0\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"button\", \"$el_text\": \"\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--has-side-icon\", \"mb-4\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--has-side-icon mb-4\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"mb-6\"], \"attr__class\": \"flex flex-col mb-6\", \"nth_child\": 4, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2421}","now":"2023-05-22T10:43:10.567715+00:00","sent_at":"2023-05-22T10:43:10.565000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-4c68-0001-eb2a-54453fb06eec","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/platform\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/platform\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"3hnz1hglhce8vl5k\", \"$time\": 1684752188.145, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e42ef110e-042aef35510b338-412d2c3d-164b08-188430e42f01bb3\"}, \"offset\": 2416}","now":"2023-05-22T10:43:10.567715+00:00","sent_at":"2023-05-22T10:43:10.565000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-4c68-0002-6ea4-3b0d05101aee","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/platform\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/platform\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"4r1etipqepb7m8xn\", \"$time\": 1684752188.809, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"backend\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"mb-6\"], \"attr__class\": \"flex flex-col mb-6\", \"nth_child\": 4, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e42ef110e-042aef35510b338-412d2c3d-164b08-188430e42f01bb3\"}, \"offset\": 1752}","now":"2023-05-22T10:43:10.567715+00:00","sent_at":"2023-05-22T10:43:10.565000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-4c68-0003-3fe5-c2dfb803996d","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/backend\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/backend\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"txh2rjlwpqzfn25q\", \"$time\": 1684752188.815, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e458d299-0a26be2f319ee58-412d2c3d-164b08-188430e458e1e11\"}, \"offset\": 1746}","now":"2023-05-22T10:43:10.567715+00:00","sent_at":"2023-05-22T10:43:10.565000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1684752193570&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:13.576236+00:00","body":"H4sIAAAAAAAAA+1ZbW/bNhD+K4ZR7FOVkJIoSxmKoWlXpC9bU3QthhaFQImUxFgSVYryS4r89x1ly7EcJ+2cFEhQf4p1L7yHx7sj7/L525BPeKmHR8NHtNEyppVuFB8+HlZKVlxpwevh0bfhIwl/hn/RePD2/eBfYAMhnHBVC1kCA6MDTA6QoUdKTmuugPhCKJ7ImSEyPhExD/W84sB4zuuxlpVhxI1SYD5sVA6MTOvq6PAwBxh5Jmt95COEDkWZ8lqDocOIxmNeMqNo2KDRFzWMiuqspIWxs11zCfASPcbOGjmnZdrQ1Ojz0vrw3qjUseK8DDMu0gysBiP3kjgVTGewCEEIiBPBp5VUeiXru2bxFbmTdh1DzkUEZqY8MkbgY92jB2R0YBu6KAGWDgUDql1MFEvjr8WEV+ek8Q1fC7NZ7PnuiNg4MIqPh0zAvst4qZa+e36SV7Ovz07SD+OM1M6btx8/vCNP4781HhcfczSbvKVndp14xdpptarYhx0g7rgRceLYQsyJvXiEAtfFhPmWi21mxw6zsOdGyLcuxT3mtPAa49QdUIg6ZLyQIcThGY/BkwnNaw4Lpko2VRuUK9YQLe1aDva4BZGALIKpZyXMcz3i+l6ATExLldJSnFMTEutadoCChZbLOLaCgIxGNsXIJa5BUtaalrGJiK0BOrwAVGvJE4LzaZRzFsLW4UTDWjBQ7vDTWIsJDxNOW+Ekpyns5vMXYK3TworOc0mZ2SkYqMBaJtNehLhOm3KK07ww4ECEMyvORTzOJPgdeLygIm9xmBOlE/gy0FZo6hwS4wb+I0hhDilq8nl7dtYiLZs2lxeikHMhk2DWYLyanm216QpBC7UtA3wtHY1UzguQM375NtTgjGVGR43WINNKhJrPzNmfnpwCIc5pXZta9Xn4BsKmPO4k174sq1KioGq+SYbz1U19HTenCsrBBjFp8txaZPMGJwbgXHFTa4rIsodwsFRrFXabXu1hQW2BA3ltjcE2zH1iH3Gf1+Ltky7R9ukd1kGLtIPEqKaW+QmwajiKWFuJggOYSjW2Fu4udRbGmcghr8niSybLDZKLx70zY2LSV4DK11PAWxXWDvRFZ/yUljy/dGjnug1+zxhE0w8YW6xX67kJ/WFBZwtnHQ0gbKvZ77ssuYb/WAmW8lO4VACzBI/DfXtlE9uEembtDbP2DmatqaIVXOs/ZH4lfHe7b8vCjbZbibsyCPSX3RPg2cqnWZsON8EYbGoNljq3O5Ak5+Y1dJ19w75jQ4s3Cvw0FCuW11tdSA5Wcrc7AnOKFgQPLNV5fZMEhS3fGg2bgoPrNG8sKs73IL6HS/mYmuBe/grh6TeXjcG6pFhWJhgD/13BuBQYbKh23yvFHby4Y+1qX1hKyo2i8b1aG0lm7rvlBZXBrWtu7fbpemMI4gvwiZbwpgb5KovDs5P4eTMqnE+T8mVZjaNZ8Uq9/jOav2rG86efnp3PX/M3qP5nduy0Fz4cgrntN9+Yo1GCLOQ4iBFm+46fOPymN6YPMo5ZbipKJqebqwUOIp6F3BENEEURwk7k3rQctQn3Fj1ECk9g3l/QJT6zA3gnUtuLuJ04OOCcXLseiHPMMTavQ5kkNYe3Cg6CkTmAVcvVGbrf/dZhlbXKu/VcnfZD7rsqUfqTs7pQfsGr2fRse9/l7/uufd91P/uuX6tUE0Ro7AI8HkXIdQl2Yxol164H4pHt8Y1S7fdL9YOZju2rdaG/TqMgcJoU5/rMlluqtX3gE29frvfl+n6W658wJjOdrCgbs9v9rOwuZmW36ez+d0dfmcmWlUB/t2140+Pesufc1Ng+zvu5A7OXUEpUAwVAlnU30wOv+1d3fkXSnI4pqPt53X5et5/X7ed1+3ndrzqvu4MmcITRQxrXgUGRzP+AbNOJVMWTZUv32+qfd092aw0X6z70rlDY5+m57+dTPJppPGv3e7Ur9CBz9l3hviu8j13hL1a/CYlt5FgIM8QTlCTcIS69tnwTwrANneR6+Ubk4st/4/nMb+ImAAA=","output":[{"uuid":"0188430e-582a-0000-2b14-ad7e278afb7b","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/backend\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/backend\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"2mvrdgcqmvepz5u8\", \"$time\": 1684752191.57, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"PHP\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"attr__data-attr\": \"select-framework-PHP\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"div\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"FrameworkPanel\"], \"attr__class\": \"FrameworkPanel\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e458d299-0a26be2f319ee58-412d2c3d-164b08-188430e458e1e11\"}, \"offset\": 1997}","now":"2023-05-22T10:43:13.576236+00:00","sent_at":"2023-05-22T10:43:13.570000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-582a-0001-7181-20c3ce2580fe","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/backend/php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/backend/php\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"pin8vjsmr8mepxwj\", \"$time\": 1684752191.58, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e505ac40-0ebb044514cabf8-412d2c3d-164b08-188430e505b26e1\"}, \"offset\": 1987}","now":"2023-05-22T10:43:13.576236+00:00","sent_at":"2023-05-22T10:43:13.570000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-582a-0002-bea7-d4b125bb0a9a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/backend/php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/backend/php\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"mtqwb993ug1ltj2o\", \"$time\": 1684752192.856, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"Continue\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"panel-footer\"], \"attr__class\": \"panel-footer\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 11, \"nth_of_type\": 5}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"InstructionsPanel\", \"mb-8\"], \"attr__class\": \"InstructionsPanel mb-8\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e505ac40-0ebb044514cabf8-412d2c3d-164b08-188430e505b26e1\"}, \"offset\": 710}","now":"2023-05-22T10:43:13.576236+00:00","sent_at":"2023-05-22T10:43:13.570000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-582a-0003-cd42-5039c46a0c3b","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/verify?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/verify\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"i2zgz88lw17xt1x0\", \"$time\": 1684752192.862, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e555c203-01d0ef0ffe354a-412d2c3d-164b08-188430e555d12ed\"}, \"offset\": 705}","now":"2023-05-22T10:43:13.576236+00:00","sent_at":"2023-05-22T10:43:13.570000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/engage/?ip=1&_=1684752196089&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:16.094175+00:00","body":"ZGF0YT1leUlrYzJWMElqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR0p5YjNkelpYSmZkbVZ5YzJsdmJpSTZNVEV6TENJa2NtVm1aWEp5WlhJaU9pSm9kSFJ3T2k4dmJHOWpZV3hvYjNOME9qZ3dNREF2YzJsbmJuVndJaXdpSkhKbFptVnljbWx1WjE5a2IyMWhhVzRpT2lKc2IyTmhiR2h2YzNRNk9EQXdNQ0lzSW1WdFlXbHNJam9pZUdGMmFXVnlRSEJ2YzNSb2IyY3VZMjl0SW4wc0lpUjBiMnRsYmlJNkluQm9ZMTlxU0dORWRUZHRNMXAyYmtsdWNHdGllRzFLY2t0RllubEtkV3Q1UVZwRGVubExaVXd3YzFSNFFqTnJJaXdpSkdScGMzUnBibU4wWDJsa0lqb2laMUZFU0d4d2VIRkRTR2RWYTJnMWN6Tk1UMVpWVVRWQlkwNTBNV3R0Vm13d2VIWlBZV295YzJZMmJTSXNJaVJrWlhacFkyVmZhV1FpT2lJeE9EZzBNekJsTXpSaU5UTmpZeTB3WkROak5tTTNNRGswTkRFMVpEZ3ROREV5WkRKak0yUXRNVFkwWWpBNExURTRPRFF6TUdVek5HSTJaRE00SWl3aUpIVnpaWEpmYVdRaU9pSm5VVVJJYkhCNGNVTklaMVZyYURWek0weFBWbFZSTlVGalRuUXhhMjFXYkRCNGRrOWhhakp6WmpadEluMCUzRA==","output":[{"uuid":"0188430e-61fe-0000-6f49-83e60118a598","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"$set\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$browser_version\": 113, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"email\": \"xavier@posthog.com\"}, \"$token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"event\": \"$identify\", \"properties\": {}}","now":"2023-05-22T10:43:16.094175+00:00","sent_at":"2023-05-22T10:43:16.089000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/engage/?ip=1&_=1684752196112&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:16.114065+00:00","body":"ZGF0YT1leUlrYzJWMElqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR0p5YjNkelpYSmZkbVZ5YzJsdmJpSTZNVEV6TENJa2NtVm1aWEp5WlhJaU9pSm9kSFJ3T2k4dmJHOWpZV3hvYjNOME9qZ3dNREF2YzJsbmJuVndJaXdpSkhKbFptVnljbWx1WjE5a2IyMWhhVzRpT2lKc2IyTmhiR2h2YzNRNk9EQXdNQ0lzSW1WdFlXbHNJam9pZUdGMmFXVnlRSEJ2YzNSb2IyY3VZMjl0SW4wc0lpUjBiMnRsYmlJNkluQm9ZMTlxU0dORWRUZHRNMXAyYmtsdWNHdGllRzFLY2t0RllubEtkV3Q1UVZwRGVubExaVXd3YzFSNFFqTnJJaXdpSkdScGMzUnBibU4wWDJsa0lqb2laMUZFU0d4d2VIRkRTR2RWYTJnMWN6Tk1UMVpWVVRWQlkwNTBNV3R0Vm13d2VIWlBZV295YzJZMmJTSXNJaVJrWlhacFkyVmZhV1FpT2lJeE9EZzBNekJsTXpSaU5UTmpZeTB3WkROak5tTTNNRGswTkRFMVpEZ3ROREV5WkRKak0yUXRNVFkwWWpBNExURTRPRFF6TUdVek5HSTJaRE00SWl3aUpIVnpaWEpmYVdRaU9pSm5VVVJJYkhCNGNVTklaMVZyYURWek0weFBWbFZSTlVGalRuUXhhMjFXYkRCNGRrOWhhakp6WmpadEluMCUzRA==","output":[{"uuid":"0188430e-6212-0000-ba3f-5cbd5ec58bd2","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"$set\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$browser_version\": 113, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"email\": \"xavier@posthog.com\"}, \"$token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"event\": \"$identify\", \"properties\": {}}","now":"2023-05-22T10:43:16.114065+00:00","sent_at":"2023-05-22T10:43:16.112000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1684752196576&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:16.582542+00:00","body":"H4sIAAAAAAAAA+1cbY/bNhL+K4ZR3Kdol3qX9hAUTdK7tM1dWiQpDgkKgRIpS2tJlCnKljfIf7+hLMuW37Kb3TvYLT/ZIofkkDN8HpJD6dPnMZ3TQoxvxt/hWrAIl6LmdPxsXHJWUi5SWo1vPo+/Y/Az/heORm/fjf4D2ZAQzCmvUlZAho6udPsKyfSQs0VFOST+I+U0Zo1MJHSeRjQQy5JCxitaTQUrZUZUcw7NBzXPICMRory5vs5AjSxhlbjxEELXaTGhlYCGrqHBNF5+X2ZYxIznz0McTWlB/hZznNMF49PnZdJWKwtDfcOKZEaJRVKAMGTu1bul/aZrum5uJWe4mNR4IovTQvvwThapIk5pESQ0nSTQqO9am8RFSkQCldgIQeI8pYuScdHLepasvE9eS1umTM7SEJpZ0FA2Ag/bw31lu1eGTE8LUEsEKYHUctb4TuHe8XBmorkjZL5IZV91x7Nc29B9Cwp6z8YkhX4XUVdu8tur11nZzF6+nnyYJnZlvnn7+4ff7B+ifwt9mv+eoWb+Ft8aVezkW7Zsi+oedAFR0wptM4o0RMzIiVzkW5ZuE0+zdIMYkUk03bFC5GkbcYeYnqyslqP6DVqkVUBozgLw0lsawVDGOKsoVDjhrC5bl+2zxqhrVzN1h2rgCUizdexoMXEsx7Y8x0fS4xmf4CK9w9IltksZPvJXpSxCdc33bdc1sI4s25KaFJXARSRd4qD7jr+AVltTK4DBx2FGSQBdB5MGVUqg8Fp/HIl0ToOY4lY4zvAEevPpD8jaTgtKvMwYJrKn0EAJrSVsMnARy2wnJKc4y6VyIEKJFmVpNE0YjDvk0RynWauHtCiew5NUrdemymCCncj/DiY4hQksZ/vhuVulk6Jup+RKFOZcQBg0K3Xcn54tFq1holW1BQm6NR+lVEZzkJPj8nksYDC6GV2VuAD5KMNVJWHr0/gN+EjxohaCFUEQsUJIqIOxxEJwSJCCUO6g1LNxIZIgStIMvBNalU8s7nRbaREI2kj/YnwkS6VFTUeLFAxRi9EKUqC7YP+BkmHbTtvZe1ZwrEeQsfWkaULCNeYSxwbp4J+irrSSp/mB3LjOMm2FPDs5EQwD5ZRsBqwzTN+Fo8M4OqjYMHWo1jBvo9QwvVdpYB17xzrGzpCTdD4cxB75tQyQkBbwrAGhaK37VfsO8hX5067yNWWkD3Qd2295O/MbmllVVomlnLTjHDerQb0ZwYQrm78/UvMXPCUT+ivw4YnJdUho0KzxUOsdqFFbcFzCcuVezffCT9f7FtBOtt1KPFWDkP7T2iVf9mOatNPmlBqj3VKjrszjDBJnVK7yjrUvs5+4odXyCv7KFC1ix1tdSY56uceZQFpRA+eBqtajvpukabA8PeQNu4KjYyUHKsJKcKCi+TUV38Fy4gWWzt39C2DVugRG2aRoWpISAuO3p2MnMNopun7uC37DKH4jdrVrQ87YDmjsDstuFSEjkuk62kpgvSDXG+2q+6QL6l9gTASDXYVcUCdRcPs6elW7uflxXvxUlNOwyX/mv/wYLn+up8sfPr68W/5C36DqffPCbJcqYAS5TtldHbtujDRkmojYxPBMLzbpqdWxBzKmrG6RFoQtdmvzTWQ7GrJc7COMQqSboXWqOmzY1FntfiaweKfDCm3bjgxkakgniMYojqlpW/hYdSBNdAMYGFadLI4rCusX3fdbt+x3kut2znQbWdWgUskW0O7T7yW3Kr/0DWWE9YYvk9AyxR3CM4kpBzaUvtpPqv3kee4n/2JAHumhAX815IUYE4uQ0LYM92iFIE90oPMhlHvtKmwN5T2und6gKpS/XJRHBmmanMzymZ5G1Sw6gvKwaFMwr2BewfyfBObRYMV+/rEfBef3g3Omz1Mj44i4NJ7OuTwq3YVz+8pDroJzBefnCeePjAIdCrC8ZHmZUSF7e79YypFgyelQSoZ5ezb70PgKHCKGmvHoMMvBSMqpKEur78MCL6NW08cd/z342BfCejSDgAs7GCIZ5D5OM8DsQQFLBVVUUEUFVVRQRQVVVFDl/75Jcx378FEcpxHjBJ6rESwFC2BlKKK2bRe9bVs0C4eXYRo3tnM7ncq5emDbpqtTOLVtO9Nt2xqle3QKWAlOCrJdg+sxBGauWEaDDPp8RASmtQQKaZpdEcUjD+eRwVlf1O2FwYeLFJbsImBFyHBrspGgOB9BKUCYyaguCQY5xSwXzSyTpGE5F+7tEi/ruJLhun1m8U1HMYtilnNllhaPvoJdIDjHWQ1qCF4rovgGonBMZ5soWp9r3X4knTGELWyVsIXcqJ4PH6yuYH+/cQNYXnQ+8lz6wb2xv7/LfdFg70fpjCzmIZrkbiVm7TH4PtjbcGNegb0C+3ME+67zAlfTCuZyLaEI3HWD/IMcOGrqciReAybJQEWfCVcTFQs8mAV0d3A1YOByYCxwW8UA58sAnp/EtjCMeYYXReW2vdxnAE/d2lUMcKYMMOhiewOA1UTDRQGoHrUx/10hOHOqSjhV2uikYP+hsG8PL/4q2L8s2J/YlS5EOA3r2ey21GU4aBf2nSuE1CmPgv0LgX1OZzVghdbqqbVvG6TRyj4K/p8e/p3B2c95vsKnMH+A+aZuZZM7YTYxZ3fZoj6M+TaopzBfYf45Yv5fC6cdnQAsE6gvtHyKTDcijnd81oA4+GpoD2Ba9wax3CyVmCmpEgKXQQz2Orc3OBRmDz/SVaG7BGXJ7cItxCRpPzq2j9nq7QwF2WcK2dtTEpfpdZhmmfwe0ty4htycQh+lq/zzx/fwSGremcqBM/zVuwhwrR7B7FPQ/1DoR8MDmnYWgV/Ca9nttwsV5J8p5Lt5nYCHmR7R04aidhj3IV+dyCvMP1fMX1lvjQZrA/bpU7q8vzW7Mi2kfR5LF4M7zXW9+hTB/aro4OEVjXGdidGvvT6r+3tgwg4kVxeBOp/cHA7B3Ltjhaziw/uXW64CoWV58FTIPDCeYqgHMpTlDT8DpRjqQhhqluKIYy+bU1EWUdjSyT5DKYJSBHURBDWwxzGWOm2cfZa6b8kOGSRGVVkt+UT+hUktr9IHWLqVgQz43J6tGcZ7Hd1Y5g2Caalbrm9/BNF+tNa27MwocMMKli+lUiuC629BAUQMcjdAuyWhWO6pWE7twy6S5ZpiUYdOkpnNHKFmOjvIcroOVwoVzSmaO3+aU/swxVCHGWr4wp9iqEthqGUjGub4hhnO5jM+PXR3VzIU1K0YSjHU+TOU2oipjdj/kOZs/8sf/wUlRV8AwWwAAA==","output":[{"uuid":"0188430e-63e7-0000-35f7-77eaf3880b39","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/verify?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/verify\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"pqx96n7zrbq30v6t\", \"$time\": 1684752194.578, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"or continue without verifying\"}, {\"tag_name\": \"button\", \"$el_text\": \"or continue without verifying\", \"classes\": [\"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-primary\", \"LemonButton--full-width\", \"LemonButton--centered\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--tertiary LemonButton--status-primary LemonButton--full-width LemonButton--centered\", \"nth_child\": 5, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"ingestion-listening-for-events\"], \"attr__class\": \"ingestion-listening-for-events\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"text-center\"], \"attr__class\": \"text-center\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e555c203-01d0ef0ffe354a-412d2c3d-164b08-188430e555d12ed\"}, \"offset\": 1993}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0001-3713-fae840c9d31f","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ca1xryhb43tz0aqr\", \"$time\": 1684752194.59, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 1982}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0002-9a6f-fdc96fab0d9d","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"ingestion continue without verifying\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"02dxxmdqmq1icsqc\", \"$time\": 1684752194.591, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 1980}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0003-c4bd-0cee63372ea6","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"o1vi2lr0d7efkvrd\", \"$time\": 1684752195.807, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"Complete\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"panel-footer\"], \"attr__class\": \"panel-footer\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 765}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0004-272b-0bdf0f14875d","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"ingestion recordings turned off\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"wxw6rpbifx56jkkv\", \"$time\": 1684752195.811, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"session_recording_opt_in\": false, \"capture_console_log_opt_in\": false, \"capture_performance_opt_in\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 760}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0005-82ae-014d11c490fb","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"completed_snippet_onboarding team setting updated\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ghxomrt7jyayufsg\", \"$time\": 1684752195.936, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"setting\": \"completed_snippet_onboarding\", \"value\": true, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 636}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0006-a1b6-c15b0e084f14","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"activation sidebar shown\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"9ciqdwvb0gm7stqn\", \"$time\": 1684752195.955, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"active_tasks_count\": 5, \"completed_tasks_count\": 2, \"completion_percent_count\": 29, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 617}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0007-2bd1-96f9150acdd3","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"89hf5t22vlawns70\", \"$time\": 1684752195.989, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"cloud-announcement\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 582}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0000-38d3-622f0b7db0b3","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"g5s1ttbkbuqqjp1f\", \"$time\": 1684752196.006, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"require-email-verification\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 566}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0001-26c9-ed81e4d2f77a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"314lgzt3xfrozlwu\", \"$time\": 1684752196.054, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 518}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0002-e82f-aba2ca288e21","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"client_request_failure\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ps0zh0lhjw7ntghy\", \"$time\": 1684752196.07, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"pathname\": \"/api/billing-v2/\", \"method\": \"GET\", \"duration\": 65, \"status\": 404, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 502}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0003-e240-08053caca9aa","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"7muhndo38d1ixe0p\", \"$time\": 1684752196.089, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"project\", \"$group_key\": \"0188430e-316e-0000-51a6-fd646548690e\", \"$group_set\": {\"id\": 1, \"uuid\": \"0188430e-316e-0000-51a6-fd646548690e\", \"name\": \"Default Project\", \"ingested_event\": true, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 483}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0004-ced5-5ad043dcd37a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"qiacra8lvetpncbm\", \"$time\": 1684752196.09, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"organization\", \"$group_key\": \"0188430e-2909-0000-4de1-995772a10454\", \"$group_set\": {\"id\": \"0188430e-2909-0000-4de1-995772a10454\", \"name\": \"x\", \"slug\": \"x\", \"created_at\": \"2023-05-22T10:43:01.514795Z\", \"available_features\": [], \"taxonomy_set_events_count\": 0, \"taxonomy_set_properties_count\": 0, \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 482}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0005-b2eb-104477bc52f8","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"xnwub6hl3xv00xkq\", \"$time\": 1684752196.112, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"project\", \"$group_key\": \"0188430e-316e-0000-51a6-fd646548690e\", \"$group_set\": {\"id\": 1, \"uuid\": \"0188430e-316e-0000-51a6-fd646548690e\", \"name\": \"Default Project\", \"ingested_event\": true, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 460}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0006-74b6-493aaf93af42","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"yxtxo6923bqvqrk0\", \"$time\": 1684752196.113, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"organization\", \"$group_key\": \"0188430e-2909-0000-4de1-995772a10454\", \"$group_set\": {\"id\": \"0188430e-2909-0000-4de1-995772a10454\", \"name\": \"x\", \"slug\": \"x\", \"created_at\": \"2023-05-22T10:43:01.514795Z\", \"available_features\": [], \"taxonomy_set_events_count\": 0, \"taxonomy_set_properties_count\": 0, \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 459}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1684752239786&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:59.790498+00:00","body":"H4sIAAAAAAAAA61We2/bNhD/KoaRPyeHessBhqFpNqRttqDIUgwNCoEiKYkxJWoS5UeLfvfd0YptOY+2QwEDhu75ux/vjrz7MhVLUZvp2fSE9kYz2pi+FdNfpk2rG9EaKbrp2ZfpiYa/6Z+UTa5vJv+AGgTpUrSd1DUoXDJzwxlBedbqVSdaEP4hW5HrNQq5WEomUrNpBCguRLcwukEF69sW0qd9q0BRGtOcnZ4qgKFK3ZmzhBByagF2v+k607Tlsi5SpqtGCSP4r6btEewJWkOAsScqGmrKmlaYdgh0AHJfgev6B2JF66KnBTqJ2rm9QZeOtULUaSlkUUKqeRzshSvJTQlBQkJAuJRi1ejW7GyTAIPvxA/WgY9iJTNIsxIZJoGPQ1ZnYTzzUC5rgGVSyUEaqzLM1T13g1AnUq1QbyRW6EZJEIee50ezOIHYXHZG1mzwK95fXKpm/e/ry+J2UYadf3X94fZ9+Ir9ZdxF9UGR9fKa3ntdHlUHR2Zd3QRKIMIPstBnzCHcZxGLyTwI3JAnTuB63GM+d9woyEji7M0j7icYrEdW/wcK2aVcVDqFZrwXDKjMqeoEBCxa3Te2M3eqKRnyOr4bCQfOnzihSyMn51EQhUESzQn2im4LWsvP1GxJ3nl5czLfegVcuM58HsaxR10ShAEiqTtDa4Yt8WSXTr8CqoMJSoF8minBUygdjjTtJAfnB/yUGbkUaS6oNc4VLaCau0+gOpSlDd0oTTlWCgkayFbqYtQigW/nrhVUVQgOTAR3mJJsUWrgHXSiolJZHHiidAlfCG2HplOULV7Qn8AcC5hTHOqnR7STRd3bgd6a4pByDWkR4+OhtIP4sA0sVLsLxME8opUSlR3YM9hSBsgY5rhbFmDOFO06XE530ytokfoNAz/gjxrTpqnVgu1e9aAZpm/qCmywrexhUEfCXCrcSbWukcOtDGf4HFYatM2ETLwAfjvdulI1phwYWq1Ws5U/g2479SxFFvUQWrPedgeYW453GtpK6pSSc4HMDdutNmXKSqlgfoAX/NL5wN6Wp9SINcKHJhwz1VAs/Jiq894YXaepfJ6ykcnLCI5yZtbTnvIe13MYQHHw5TgGbxzabo7lMHum75ymldUT2lo7ANPgNXakKWnnjKscmm6H8tnaJ08CG0vHsMa6PaixfAfph1jlcjkm8RUuELvDbmCxnFNbhO5EOlT26Fi/5fDz4eyOpDEO3mLN2sFpaTK4J74P38Af+k/Qe7L1/clQvwPLOCXc/KOUwbdS7qPsalN0o3vzOPVgMDk2fBGA/ySAHyZpi8Re063WRznhPfHy3GuOgzlMWQmrG1e/3aujON6jOMCB0Qu78JqSpfeX7KKPK//jsn5TN4tsXb1t3/2ebd72i82rj68/b96JK9L9vT737a0BFOOVcfxQieOcOMT3CQ+5l/hJ7ouXHioJ2PgYbiVrrlfH0eY+CSOHBDGdE0oy4vpZ8FI46oUC+xSenwW8o8Q4YOTywIs5xMuCuSB+zHiUPP+OAnN4vWQhPjF0nncCVqoPV8rXT/8B8mE6/L4LAAA=","output":[{"uuid":"0188430f-0caf-0000-dc00-030ac4424349","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"7lh5fljd145o8ilw\", \"$time\": 1684752236.783, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"svg\", \"classes\": [\"LemonIcon\"], \"attr__class\": \"LemonIcon\", \"attr__width\": \"1em\", \"attr__height\": \"1em\", \"attr__fill\": \"none\", \"attr__viewBox\": \"0 0 24 24\", \"attr__xmlns\": \"http://www.w3.org/2000/svg\", \"attr__focusable\": \"false\", \"attr__aria-hidden\": \"true\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"\"}, {\"tag_name\": \"span\", \"classes\": [\"LemonButton__icon\"], \"attr__class\": \"LemonButton__icon\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"button\", \"$el_text\": \"\", \"classes\": [\"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-primary\", \"LemonButton--no-content\", \"LemonButton--has-icon\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--tertiary LemonButton--status-primary LemonButton--no-content LemonButton--has-icon\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"ActivationSideBar__close_button\"], \"attr__class\": \"ActivationSideBar__close_button\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"ActivationSideBar__content\", \"pt-2\", \"px-4\", \"pb-16\"], \"attr__class\": \"ActivationSideBar__content pt-2 px-4 pb-16\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"ActivationSideBar\"], \"attr__class\": \"ActivationSideBar\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\"], \"attr__class\": \"SideBar SideBar__layout\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 3000}","now":"2023-05-22T10:43:59.790498+00:00","sent_at":"2023-05-22T10:43:59.786000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1684752248800&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:44:08.814768+00:00","body":"H4sIAAAAAAAAA51W607cOBR+lVHETwLOPUFarUrZFduyiyqWalVURY7tJGYSO7WdubTi3fc4E4bJMEDVX0zO7fvO8blw98NhCyaMc+Yc4d5IgjvTK+YcO52SHVOGM+2c/XCOJPxx/sZkdn0z+w/UIMgXTGkuBSg8dOJFJ8jKCyWXmikQ/skVK+XKCilbcMJys+4YKC6YnhvZWQXplQL4vFcNKGpjurPT0wZoNLXU5ixFCJ0OBPXvUhQSK8pFlRPZdg0zjP5mVG/JHllrCDD1tIoOm1rg1sKOgXZIPmXgecGOuMGi6nFlnZhwb2+siyaKMZHXjFc1QGVJ+CRccmpqCBIhBMIFZ8tOKrO1TUMbfCt+tA4DK254ATBLVlgQ+Nit6kmUnPhWzgXQMjmnIFUJWSA5/1bW82CFqt7qDbcZenEaJpHvh8kJQsmxQ7k2XJDRr/p0cdl0q2/vL6vbeR3p4Or68+2n6B35x3jz9nODVotrfO/rMm53nmxw9VJIAbEgLKKAEBfRgMQkQVkYehFN3dDzqU8C6npxWKDUfTKPaZDaYL2t6i+w4DqnrJU5NOM9I1DKEjeaQcBKyb4bOnOrctCI6wZezFx4f+RGHo7dksZhHIVpnCHbK1JVWPDv2GyKvPXyM5RtvELKPDfLoiTxsYfCKLRMhDZYENsSB7vUeQBWOxOUQ/Fx0TCaQ+rwpLnmFJwf+WNi+ILlJcODcdngCrK5+wqqXVne4XUjMbWZAkAHaLWsJi0SBsPcKYab1pIDE0Zd0nAyryXUHXSsxbwZeNgXxQv4stS2bHSDyfwV/RHMMYM5tUN9eEQ1r0Q/DPTG1A4plQBrOT4fymEQH7fBQHXYBWxnHq1Vw9phYM9gSxkoxjjHusMC7EmDtbbb6c65gh4R570xUuSwHISxGw1qiY1RILCG4HfQ6tgRps5JzRvoTkC1X7IcuW1Y5IatbH9dwUPMWqnYbFwkD8cTXsUQesjvZZ+XeINi58t1O8VbrNb7YmhC0+sdbbt27YS1K9c231PSY3G3nF4sxewQ7FQ4BZ1ZyNkj4KR+0X799ipE+WJagLEoz55qW6yNmGKDXftzq3LN0KGvokdvodv+dHHXuS+2zDOLCSCs7wlg8BbgDeyAc6xAOP7K4dSsZX8AeTSY7RtOCMAJ+gkCrzX4QYcNk82xkfKNpPdDFJLavhy7sIYRtitgOISTOP6zOFADI+fMroyuJvn9Jbnokzb4shB/iW5erNoP6uMfxfpDP1+/+/L++/oju0L639V5MGwPKLFdHfsHK0lK5KIgQDSifhqkZcBeO1gp2AQ23JILKpf70bIARbGLwgRnCKMCeUERvhYO+xGLN/+GVHBP2TRg7NHQTyjEK8KMoSAhNE5fvqdgDlesiOypkWWpGSwYL8nQw9f/AYmT89LGCQAA","output":[{"uuid":"0188430f-2fef-0000-25cb-b19f141897ba","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"r7cv0okqfhk3x0gu\", \"$time\": 1684752247.007, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Load more events\"}, {\"tag_name\": \"button\", \"$el_text\": \"Load more events\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"my-8\", \"mx-auto\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary my-8 mx-auto\", \"nth_child\": 5, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"events\"], \"attr__class\": \"events\", \"attr__data-attr\": \"events-table\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\"], \"attr__class\": \"SideBar SideBar__layout\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 1790}","now":"2023-05-22T10:44:08.814768+00:00","sent_at":"2023-05-22T10:44:08.800000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} From 8d57a3ab8ea7e1c82637874de6dbed9214dc5515 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 23 May 2023 11:59:30 +0100 Subject: [PATCH 013/247] Add processing sink for events (#11) * Add processing sink for events * fmt * Make tests async * Make clippy happy * Handle errors and batch events --- Cargo.lock | 1 + Cargo.toml | 1 + src/capture.rs | 86 +++++++++++++++++++++++++++++++++++++++++++------- src/event.rs | 32 ++++++++++++------- src/lib.rs | 6 ++-- src/main.rs | 1 + src/router.rs | 15 ++++++++- src/sink.rs | 31 ++++++++++++++++++ 8 files changed, 147 insertions(+), 26 deletions(-) create mode 100644 src/sink.rs diff --git a/Cargo.lock b/Cargo.lock index 98ca254f338bd..0b9ede977c52f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -136,6 +136,7 @@ name = "capture" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "axum", "axum-test-helper", "base64", diff --git a/Cargo.toml b/Cargo.toml index 5f73604d06415..dfd2756ce7317 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ anyhow = "1.0" flate2 = "1.0" base64 = "0.21.1" uuid = { version = "1.3.3", features = ["serde", "v4"] } +async-trait = "0.1.68" [dev-dependencies] axum-test-helper = "0.2.0" diff --git a/src/capture.rs b/src/capture.rs index 42b2718b29799..dc7a7fd960c8b 100644 --- a/src/capture.rs +++ b/src/capture.rs @@ -1,31 +1,40 @@ use std::collections::HashSet; +use std::sync::Arc; +use anyhow::Result; use bytes::Bytes; use axum::{http::StatusCode, Json}; // TODO: stream this instead -use axum::extract::Query; +use axum::extract::{Query, State}; +use uuid::Uuid; use crate::api::CaptureResponseCode; +use crate::event::ProcessedEvent; + use crate::{ api::CaptureResponse, event::{Event, EventQuery}, - token, + router, sink, token, }; pub async fn event( + state: State, meta: Query, body: Bytes, ) -> Result, (StatusCode, String)> { + tracing::debug!(len = body.len(), "new event request"); + let events = Event::from_bytes(&meta, body); let events = match events { Ok(events) => events, - Err(_) => { + Err(e) => { + tracing::error!("failed to decode event: {:?}", e); return Err(( StatusCode::BAD_REQUEST, String::from("Failed to decode event"), - )) + )); } }; @@ -33,7 +42,7 @@ pub async fn event( return Err((StatusCode::BAD_REQUEST, String::from("No events in batch"))); } - let processed = process_events(&events); + let processed = process_events(state.sink.clone(), &events).await; if let Err(msg) = processed { return Err((StatusCode::BAD_REQUEST, msg)); @@ -44,7 +53,24 @@ pub async fn event( })) } -pub fn process_events(events: &[Event]) -> Result<(), String> { +pub fn process_single_event(_event: &Event) -> Result { + // TODO: Put actual data in here and transform it properly + Ok(ProcessedEvent { + uuid: Uuid::new_v4(), + distinct_id: Uuid::new_v4().simple().to_string(), + ip: String::new(), + site_url: String::new(), + data: String::from("hallo I am some data 😊"), + now: String::new(), + sent_at: String::new(), + token: String::from("tokentokentoken"), + }) +} + +pub async fn process_events( + sink: Arc, + events: &[Event], +) -> Result<(), String> { let mut distinct_tokens = HashSet::new(); // 1. Tokens are all valid @@ -67,20 +93,50 @@ pub fn process_events(events: &[Event]) -> Result<(), String> { return Err(String::from("Number of distinct tokens in batch > 1")); } + let events: Vec = match events.iter().map(process_single_event).collect() { + Err(_) => return Err(String::from("Failed to process all events")), + Ok(events) => events, + }; + + if events.len() == 1 { + let sent = sink.send(events[0].clone()).await; + + if let Err(e) = sent { + tracing::error!("Failed to send event to sink: {:?}", e); + + return Err(String::from("Failed to send event to sink")); + } + } else { + let sent = sink.send_batch(&events).await; + + if let Err(e) = sent { + tracing::error!("Failed to send batch events to sink: {:?}", e); + + return Err(String::from("Failed to send batch events to sink")); + } + } + Ok(()) } #[cfg(test)] mod tests { + use crate::sink; use std::collections::HashMap; + use std::sync::Arc; use serde_json::json; use super::process_events; use crate::event::Event; + use crate::router::State; + + #[tokio::test] + async fn all_events_have_same_token() { + let state = State { + sink: Arc::new(sink::PrintSink {}), + }; - #[test] - fn all_events_have_same_token() { let events = vec![ Event { token: Some(String::from("hello")), @@ -94,11 +150,16 @@ mod tests { }, ]; - assert_eq!(process_events(&events).is_ok(), true); + let processed = process_events(state.sink, &events).await; + assert_eq!(processed.is_ok(), true); } - #[test] - fn all_events_have_different_token() { + #[tokio::test] + async fn all_events_have_different_token() { + let state = State { + sink: Arc::new(sink::PrintSink {}), + }; + let events = vec![ Event { token: Some(String::from("hello")), @@ -112,6 +173,7 @@ mod tests { }, ]; - assert_eq!(process_events(&events).is_err(), true); + let processed = process_events(state.sink, &events).await; + assert_eq!(processed.is_err(), true); } } diff --git a/src/event.rs b/src/event.rs index 04e6c8ee571b2..df6d200db90af 100644 --- a/src/event.rs +++ b/src/event.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::io::prelude::*; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -41,26 +42,35 @@ impl Event { /// could be more than one. Hence this function has to return a Vec. /// TODO: Use an axum extractor for this pub fn from_bytes(query: &EventQuery, bytes: Bytes) -> Result> { + tracing::debug!(len = bytes.len(), "decoding new event"); + match query.compression { Some(Compression::GzipJs) => { - let d = GzDecoder::new(bytes.reader()); - Ok(serde_json::from_reader(d)?) + let mut d = GzDecoder::new(bytes.reader()); + let mut s = String::new(); + d.read_to_string(&mut s)?; + + tracing::debug!(json = s, "decoded event data"); + + let event = serde_json::from_str(s.as_str())?; + + Ok(event) } None => Ok(serde_json::from_reader(bytes.reader())?), } } } -#[derive(Default, Debug, Deserialize, Serialize)] +#[derive(Clone, Default, Debug, Deserialize, Serialize)] pub struct ProcessedEvent { - uuid: Uuid, - distinct_id: String, - ip: String, - site_url: String, - data: String, - now: String, - sent_at: String, - token: String, + pub uuid: Uuid, + pub distinct_id: String, + pub ip: String, + pub site_url: String, + pub data: String, + pub now: String, + pub sent_at: String, + pub token: String, } #[cfg(test)] diff --git a/src/lib.rs b/src/lib.rs index b07a9eb167f1d..d8cf49f4b620a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,7 @@ -mod api; -mod capture; pub mod event; pub mod router; + +mod api; +mod capture; +mod sink; mod token; diff --git a/src/main.rs b/src/main.rs index 3fcd2574b7aef..85412e19adc35 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ mod api; mod capture; mod event; mod router; +mod sink; mod token; #[tokio::main] diff --git a/src/router.rs b/src/router.rs index 52478b634d59e..9af45c76454c3 100644 --- a/src/router.rs +++ b/src/router.rs @@ -1,8 +1,20 @@ -use crate::capture; +use std::sync::Arc; + use axum::{routing::post, Router}; use tower_http::trace::TraceLayer; +use crate::{capture, sink}; + +#[derive(Clone)] +pub struct State { + pub sink: Arc, +} + pub fn router() -> Router { + let state = State { + sink: Arc::new(sink::PrintSink {}), + }; + Router::new() // TODO: use NormalizePathLayer::trim_trailing_slash .route("/capture", post(capture::event)) @@ -14,4 +26,5 @@ pub fn router() -> Router { .route("/engage", post(capture::event)) .route("/engage/", post(capture::event)) .layer(TraceLayer::new_for_http()) + .with_state(state) } diff --git a/src/sink.rs b/src/sink.rs new file mode 100644 index 0000000000000..44592dcf7b409 --- /dev/null +++ b/src/sink.rs @@ -0,0 +1,31 @@ +use anyhow::Result; +use async_trait::async_trait; + +use crate::event::ProcessedEvent; + +#[async_trait] +pub trait EventSink { + async fn send(&self, event: ProcessedEvent) -> Result<()>; + async fn send_batch(&self, events: &[ProcessedEvent]) -> Result<()>; +} + +pub struct PrintSink {} + +#[async_trait] +impl EventSink for PrintSink { + async fn send(&self, event: ProcessedEvent) -> Result<()> { + tracing::info!("single event: {:?}", event); + + Ok(()) + } + async fn send_batch(&self, events: &[ProcessedEvent]) -> Result<()> { + let span = tracing::span!(tracing::Level::INFO, "batch of events"); + let _enter = span.enter(); + + for event in events { + tracing::info!("event: {:?}", event); + } + + Ok(()) + } +} From 63a960949f2d08e80e2c07358ff3881aea1f4c14 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 23 May 2023 13:53:46 +0100 Subject: [PATCH 014/247] Handle x-www-form-urlencoded requests to /e/ (#12) --- Cargo.lock | 1 + Cargo.toml | 1 + src/capture.rs | 22 +++++++++++++++++-- src/event.rs | 27 +++++++++++++++-------- tests/django_compat.rs | 45 ++++++++++++++++++++++----------------- tests/requests_dump.jsonl | 31 +++++++++------------------ 6 files changed, 75 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b9ede977c52f..2d2ffd6a7267f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,6 +146,7 @@ dependencies = [ "mockall", "serde", "serde_json", + "serde_urlencoded", "time", "tokio", "tower-http", diff --git a/Cargo.toml b/Cargo.toml index dfd2756ce7317..23cfe2d24d257 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ flate2 = "1.0" base64 = "0.21.1" uuid = { version = "1.3.3", features = ["serde", "v4"] } async-trait = "0.1.68" +serde_urlencoded = "0.7.1" [dev-dependencies] axum-test-helper = "0.2.0" diff --git a/src/capture.rs b/src/capture.rs index dc7a7fd960c8b..e8edaa005b4c9 100644 --- a/src/capture.rs +++ b/src/capture.rs @@ -1,4 +1,5 @@ use std::collections::HashSet; +use std::ops::Deref; use std::sync::Arc; use anyhow::Result; @@ -7,6 +8,8 @@ use bytes::Bytes; use axum::{http::StatusCode, Json}; // TODO: stream this instead use axum::extract::{Query, State}; +use axum::http::HeaderMap; +use base64::Engine; use uuid::Uuid; use crate::api::CaptureResponseCode; @@ -14,18 +17,31 @@ use crate::event::ProcessedEvent; use crate::{ api::CaptureResponse, - event::{Event, EventQuery}, + event::{Event, EventFormData, EventQuery}, router, sink, token, }; pub async fn event( state: State, meta: Query, + headers: HeaderMap, body: Bytes, ) -> Result, (StatusCode, String)> { tracing::debug!(len = body.len(), "new event request"); - let events = Event::from_bytes(&meta, body); + let events = match headers + .get("content-type") + .map_or("", |v| v.to_str().unwrap_or("")) + { + "application/x-www-form-urlencoded" => { + let input: EventFormData = serde_urlencoded::from_bytes(body.deref()).unwrap(); + let payload = base64::engine::general_purpose::STANDARD + .decode(input.data) + .unwrap(); + Event::from_bytes(&meta, payload.into()) + } + _ => Event::from_bytes(&meta, body), + }; let events = match events { Ok(events) => events, @@ -38,6 +54,8 @@ pub async fn event( } }; + println!("Got events {:?}", &events); + if events.is_empty() { return Err((StatusCode::BAD_REQUEST, String::from("No events in batch"))); } diff --git a/src/event.rs b/src/event.rs index df6d200db90af..c5f09db61cd4f 100644 --- a/src/event.rs +++ b/src/event.rs @@ -4,7 +4,7 @@ use std::io::prelude::*; use serde::{Deserialize, Serialize}; use serde_json::Value; -use anyhow::Result; +use anyhow::{anyhow, Result}; use bytes::{Buf, Bytes}; use flate2::read::GzDecoder; use uuid::Uuid; @@ -28,6 +28,11 @@ pub struct EventQuery { sent_at: Option, } +#[derive(Debug, Deserialize)] +pub struct EventFormData { + pub data: String, +} + #[derive(Default, Debug, Deserialize, Serialize)] pub struct Event { #[serde(alias = "$token", alias = "api_key")] @@ -44,20 +49,24 @@ impl Event { pub fn from_bytes(query: &EventQuery, bytes: Bytes) -> Result> { tracing::debug!(len = bytes.len(), "decoding new event"); - match query.compression { + let payload = match query.compression { Some(Compression::GzipJs) => { let mut d = GzDecoder::new(bytes.reader()); let mut s = String::new(); d.read_to_string(&mut s)?; - - tracing::debug!(json = s, "decoded event data"); - - let event = serde_json::from_str(s.as_str())?; - - Ok(event) + s } - None => Ok(serde_json::from_reader(bytes.reader())?), + None => String::from_utf8(bytes.into())?, + }; + + tracing::debug!(json = payload, "decoded event data"); + if let Ok(events) = serde_json::from_str::>(&payload) { + return Ok(events); + } + if let Ok(events) = serde_json::from_str::(&payload) { + return Ok(vec![events]); } + Err(anyhow!("unknown input shape")) } } diff --git a/tests/django_compat.rs b/tests/django_compat.rs index f76fea0ba8a24..24d526375f7bf 100644 --- a/tests/django_compat.rs +++ b/tests/django_compat.rs @@ -1,29 +1,19 @@ use axum::http::StatusCode; use axum_test_helper::TestClient; +use base64::engine::general_purpose; +use base64::Engine; use capture::event::ProcessedEvent; use capture::router::router; use serde::Deserialize; -use serde_json::Value; use std::fs::File; use std::io::{BufRead, BufReader}; -use time::OffsetDateTime; - -/* - "path": request.get_full_path(), - "method": request.method, - "content-encoding": request.META.get("content-encoding", ""), - "ip": request.META.get("HTTP_X_FORWARDED_FOR", request.META.get("REMOTE_ADDR")), - "now": now.isoformat(), - "body": base64.b64encode(request.body).decode(encoding="ascii"), - "output": [], -*/ #[derive(Debug, Deserialize)] struct RequestDump { path: String, method: String, - #[serde(alias = "content-encoding")] content_encoding: String, + content_type: String, ip: String, now: String, body: String, @@ -32,25 +22,40 @@ struct RequestDump { static REQUESTS_DUMP_FILE_NAME: &str = "tests/requests_dump.jsonl"; -#[ignore] #[tokio::test] async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let file = File::open(REQUESTS_DUMP_FILE_NAME)?; let reader = BufReader::new(file); + for line in reader.lines() { - let request: RequestDump = serde_json::from_str(&line?)?; + let case: RequestDump = serde_json::from_str(&line?)?; - if request.path.starts_with("/s") { - println!("Skipping {} dump", &request.path); + if !case.path.starts_with("/e/") { + println!("Skipping {} test case", &case.path); continue; } - println!("{:?}", &request); - // TODO: massage data + let raw_body = general_purpose::STANDARD.decode(&case.body)?; + assert_eq!( + case.method, "POST", + "update code to handle method {}", + case.method + ); let app = router(); let client = TestClient::new(app); - let res = client.post("/e/").send().await; + let mut req = client.post(&case.path).body(raw_body); + if !case.content_encoding.is_empty() { + req = req.header("Content-encoding", case.content_encoding); + } + if !case.content_type.is_empty() { + req = req.header("Content-type", case.content_type); + } + if !case.ip.is_empty() { + req = req.header("X-Forwarded-For", case.ip); + } + let res = req.send().await; + assert_eq!(res.status(), StatusCode::OK, "{}", res.text().await); } Ok(()) diff --git a/tests/requests_dump.jsonl b/tests/requests_dump.jsonl index ee8ba3d5e8e50..b62f1c61665a7 100644 --- a/tests/requests_dump.jsonl +++ b/tests/requests_dump.jsonl @@ -1,21 +1,10 @@ -{"path":"/s/?compression=gzip-js&ip=1&_=1684752074919&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:01.993704+00:00","body":"H4sIAAAAAAAAA+2Xb0vcMBjAv0oJvphgvfxrmvbV2M0xcOgNdI6pHLkmtdXa1iT1dhO/+9KxTdlgR+WmHJdXpcmTp3maX35NT++AulW1BSnYMrVoTdFYsANa3bRK21IZkN499EylsKJvsYtWgZTtgF8NbdVdlLXLovVczUZZU5umUq9Rn0ssqkbIPqpyz6pckNK60a7LapG5PKegqadfCz390R68Kqxt09GoajJRFY2xKceEjowVtsxGWdHVV+He2/H45PjwZPfSpBGKY5Yisu0yHupRS3ebWishF/0IlRWivlBPSMoQTAnbBuePSjgFZ+CNkMH7o6NJ0I/tTBrA4AyA8/t7V095rVzrdQtSxDiNIwxjlGDuuraMMqZ0dZYuDUCcU+L6mIQMh5AhJBlHEaUSZiFFWOKMyBAxOoM8fAhGiEjmqtyal7Vs5n/mwjhiMIQEilxAnktCYvqPbMTFR/0qNFeqX7q2yKYqLmllxgflx/xgX3youuryZvHpYjKBE6bjb8f8czt+142/VPudGylLY8s6s78nQiidzRLCGclDiKKcKUEUnomM84eJ4BiqGPYT+RmeZAoC946aPDfKoYgT7l7Zi5A5F7p+DOatqLonwYN5iqL/QGRC/yZyr77pVKdkkIuychet3L2xQe42k1ZWL4KyDhiEcAmnied0XTjdJIOSBDFP5iAyk8TtZS/QlQqU8iUGdZz6L/26cLphAvVgDgXTn0BXLVBMlwoUQ8/pQE5jL9BnEKgHczCY7szuBbpSgSbL/uEdp8hzuiacbphAPZhDwYy8QF/gBIo9p2vC6YYJ1IM5FEzqBbpigSK8XKDEczqY0/Pv4oS7EYgfAAA=","output":[{"uuid":"0188430d-40cb-0000-82bb-5cab205fc9d8","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-40ca-0000-c581-b136488335da\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPpGa2QC/+2XXS8DURCG35+y6RUJarfVD1fiI3HHBXGBSFVRSqu2aMRfxzNn28YNkdhoxWRydqczc+Zjz7tvtm+vh3pWQamG6qmFtqpIFS1wLehMDTyNkdXieupooAu1dTuyFtRHHtl7qqKa6uK559rBsqYYf5arR54h1i73sw8ZLe6B1Znka4WMXdZ4b4reIPe4v0PuVudET7rEd/JhT6Q5bCnSI7aIWM0m+zvYu/SWYq8pUUllvPdhwpSJmqH/S+a71bUWtaVNbSAH2tcO1yVdEb2qFaaqIhX0mCzzkz53qF+kbplY669PXzbtcFKlFSo08F2g/06vFdYyegnNej3+4kTsyR6x1oMt0rb2kF20ce1ByBuRMQqRlu8FGZ9UWzdMlkXfUCPLa13UmKFKRwl7q1jqaLXRzlnD4CMR/ZD/cwQ+hHMa/OI52vMyzK38GczVyfFdzG2R9y48T1uGvnOi2uF07JdNlvmtJ/Nm77vZ7XyG6IYJw85ykJ/jsz6j+HSO/A8cWQJ/ZnWOdI7MmyPL9P9zlswQWnOWdJacMkv6l6SzZP4smVA1H5Y0zVnSWXKaLDm7GHSW/MssWc/lH3eG0NhZ0llyyiwZO0s6S870t2TiLOksOWWWTJwlnSVzZ8k4YCwfliyx81jvS12b9+QeAAA=\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752071928, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752071929, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752073916, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752073918, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752073918, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752073920, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752073920, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752073921, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752073921, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752073922, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752073922, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752073923, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2988}","now":"2023-05-22T10:42:01.993704+00:00","sent_at":"2023-05-22T10:41:14.919000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684752098970&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:01.995810+00:00","body":"H4sIAAAAAAAAA+2Z207bMBiAXyWyuBgSoT7FcXI1rWOaxASdBGMaoMqNHRIISbAdug7x7nOmbaBNWteqCJX6Kor9+48PXz458ekdULeqtiAFW6YWrSkaC3ZAq5tWaVsqA9K7h5qxFFb0JXbWKpCyHfCroK26i7J2WbSeqskga2rTVOo16nOJWdUI2UdV7lmVC1JaN9pVWS0yl+cUNPX4a6HHP8qDV4W1bToYVE0mqqIxNuWY0IGxwpbZICu6+ircezscnhwfnuxemjRCccxSRLZdxkM9aOluU2sl5KxvobJC1BdqiaQMwZSwbXD+aAin4Ay8ETJ4f3Q0Cvq2nUkDGJwBcH5/78ZTXitXet2CFDFO4wjDJEri2FVtGWVM6cZZujQAcU4JjBGTkOEQMoQk4yiiVMIspAhLnBEZIkYnkIcPwQgRydwot6ZlLZvpn7kwjhgMIYEiF5DnkpCY/iMbcfFRvwrNleqXri2ysYpLWpnhQfkxP9gXH6quuryZfboYjeCI6fjbMf/cDt91wy/VfudaytLYss7s744QSieThHBG8hCiKGdKEIUnIuP8oSM4hiqGfUd+hieZgsDNUZPnRjkUcZKg+51nIXMqdP0YzFtRdUvBg3mKoicgMqF/E7lX33SqUzLIRVm5i1bu3tggdy+TVlbPgrIOGIRwDqfcc7oop/CZON0kg8YJJZ7MhchEEEfeoCs2KOVzFOpApR7UdQF1wxTqyVyYTDdlXqErVWgybxfqQHU+8KCuB6gbplBP5tqQ+XIV+j+7UOZBXRRU94XpFfr0CvVkLkwm9gpdsUIxna9Q/89+bUDdMIV6Mhcm058mrVqhCM9XqD9OWgLU8++WAqIwkh8AAA==","output":[{"uuid":"0188430d-40cd-0000-7103-c248d33d28b5","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-40cc-0000-fcd3-63fc3328fd1f\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPpGa2QC/+1ZXU/CUAw9P2XhSROVr30wn4xK4hs+aHxAQhAQ0MEmDJAY/7p6egeEF40JC8PYNHcrbe9pu9udLOHzo4435BBjgQhdaqew4OKI1xw6aNHTWlolLkKAKXoYYLS05jCmzLn3AXm0EdIz4TWg5QxF+hOsiDgLWkPeOxuIEjfjCtZ4XYMYcq32xtRbxF7VV+dd8jTxij59zY09Fg5oiykRY/MUydnm/oD2kLXFtFdQQhk2vRPTYcyO2qb+Pvsb4RnHqOISF5Q73KLG6wmeGH0Kh115FJd6kSiH6zprzJ9nXpuxUt+YdUm3i3WWrsnQoq9HfTe1ulwF6mVqUmvjhxORJ3vPdW5sFq5wQ7mmtso9NbgWES0TKXjvlNVJDTBkZ0n0kDkSXKmiwh48VlTiXp933zzFZOe+zeCcEWOD//0Ezsw5TXd4jiViyMw5f2bmfGL8duaqxH0xz1OWTN8jowbmdOSXdJb4pSbxJu+72OV8FtRlJmR2Cka2n8/Kns6ncuR/4EjPvD9l5UjlyNQ50mb927NkMqG2sqSyZMYsaStLKkumzpJ+Kt+SyYQ6ypLKkhmzpKMsqSy519+SrrKksmTGLOkqSypLps6SJWZNiyU9ZUllyYxZUv+7UZZMnyWLZsbSYUn596aBL0Ml6tzkHgAA\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752095977, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752095978, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752097943, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752097944, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752097944, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752097945, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752097945, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752097946, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752097946, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752097947, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752097947, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752097948, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2991}","now":"2023-05-22T10:42:01.995810+00:00","sent_at":"2023-05-22T10:41:38.970000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684752114005&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:02.050911+00:00","body":"H4sIAAAAAAAAA+2Y3WrbMBSAX8WIXqxQN/qzLPtqLOsYdLQZtOtYW4JiybVb13YluVlW8u6Tx7aUDZYlZIwQXRlLR8c60ucPocsnoB5VbUEK9kwtWlM0FhyAVjet0rZUBqRPi56xFFb0LXbWKpCyA/Cjoa26m7J2WbSeqskga2rTVOol6nOJWdUI2UdV7luVC1JaN9p1WS0yl+cSNPX4c6HH39qDF4W1bToYVE0mqqIxNuWY0IGxwpbZICu6+i48ej0cXpyfXhzemjRCccxSRPZdxlM9aOlhU2sl5KwfobJC1DdqjaQMwZSwfXD9rIRLcAVeCRm8PTsbBf3YzqQBDK4AuJ7PXT3lvXKt9y1IEeM0jjBCCELouvaMMqZ0dZYuDUCcUwJjxCRkOIQMIck4iiiVMAspwhJnRIaI0Qnk4SIYISKZq3JvWtaymf6aC+OIwRASKHIBeS4JiekfshEXH/W70NypfuvaIhuruKSVGZ6U7/OTY/Gu6qrbh9mHm9EIjpiOv5zzj+3wTTf8VB13bqQsjS3rzP6cCKF0MkkIZyQPIYpypgRReCIyzhcTwTFUMewn8j08yRQEbo2aPDfKoUggxPOD/0LmVOj6OZiPourWggfzFEX/gMiE/k7kUf3QqU7JIBdl5R5auXdjg9z9TFpZPQvKOmCOwyWcIs/ptnC6SwbFCfdkrkYmgtgtmTfoRg2KErzEoY5UJwRP6naQumMO9WSuTKY7uHuHbtShybJjqAOVeFC3BdQdU6gnc1UyUeIVumGFUr5codSDui2g7phCPZlbQ6ZXqAd1K0DdMYVGnsxVyeReoZu+DP2Lu1AP6hqgXn8FVjpm4JMfAAA=","output":[{"uuid":"0188430d-40cc-0004-71c9-b5d5762c946c","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-40cc-0003-1759-ec61e7e52da5\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPpGa2QC/+1Yy07CUBA9n9Kw0kTFlvJcGZXEHS40LpCQCghoeQgFJMZfV8/cSwkbiYmVR5xMbjudmTsP7ukJ6edHFW9IIcIcQ7SoleAghyNeU2gioCdYWCVuiBATtNFFf2FNYUSZce8D0mhgQM+Y15CWM7j021xD5pnTOuC9uZJR4qZc4TJfy2QccMV7I+oBc8f9VXmXOnW8okNffWWPgwPaIsqQsWmK1Gxwf0j7gL1FtBfgIQOf3rGZMOJEDdN/h/P18YxjlHGJC8odblHh9QRPjC4hy6nylBx1l1kOl31WWD/Nuj5jpb8R+5Jp58sqLVMhoK9NfTO95rhOqWeoSa+1NSciv+w917mxObjCDeWaWlx7YvI6zOiYSMn3TolPqoseJ7PRPdaweaWLAmfIsyOPT67pScTu3DUMzhgxMvm/R+DUnNNkg+foMYdgLrs3mCsyx08xV2beF/N7yhL0PTKqa05HnmQy65eexGvfd7HL+cypCyYEOxZbv8enu6P4VI78Dxzp8f0p7CwGlSP3mSNd1vUS4EmLUU95UnlyyzzpKU8qTybOk8VE/k1ahGaUJZUlt8ySGWVJZcnEWdJn/0mxpK8sqSy5ZZb0lSWVJZUllSWVJddgMKssqSz5B18mk/suKQit4QuIGGIM5h4AAA==\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752111000, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752111001, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752112981, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752112982, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752112982, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752112983, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752112983, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752112984, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752112984, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752112984, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752112985, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752112985, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 3002}","now":"2023-05-22T10:42:02.050911+00:00","sent_at":"2023-05-22T10:41:54.005000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684752101977&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:02.048295+00:00","body":"H4sIAAAAAAAAA+2Ub0vcMBjAv0oJvphgvfxrmvbV2M0xcOgNdI6pHLkmtdWa1CT1dhO/+1LZpmzgcXKbDPuq9MnzPM2T/Po7vgHqWmkPcrDhtGhdZTzYAq01rbK+Vg7kN/crUym86CN+0SqQsy3wM9A23VmtQxdr52o2Kox2plGvUd9LLBojZJ/VhG81IUlZa2xY8lYUoc8xMHr6tbLTu3j0qvK+zUejxhSiqYzzOceEjpwXvi5GRdXpi3jn7Xh8dLh/tH3u8gSlKcsR2Qwd9+2opdtGWyXkoq9QRSX0mXpCU4ZgTtgmOH0wwjE4AW+EjN4fHEyivrZzeQSjEwBOb2/DPPWlCtHLFuSIcZomGGY8S5OwtOGUc3WYsw5tAOKcEpgiJiHDMWQIScZRQqmERUwRlrggMkaMziCP75MRIpKFKTfmtZZm/nsvjBMGY0igKAXkpSQkpY90IyE/6W/BXKj+6tqqmKq0po0b79Ufy71d8aHpmvOrxaezyQROmE2/HfLP7fhdN/7S7HahUtbO17rwvzZCKJ3NMsIZKWOIkpIpQRSeiYLz+43gFKoU9hv5kZ4VCoJwRqYsnQooEgjh7dazkDkXVj8E81o03ZPgwTxHyV8gMqN/ErmjrzrVKRmVom7Cw6rw7nxUhp/JKm8XUa0jFg51Cads4HQlTnGWZc/E6csxKIIwY0EHA5krkIlC6WDQNRuU8kcVegcqGkBdFVQ6KPRfKBQPZK5KJhkUumaFYrpcoQOo/w2og0IHMh8nMxzZoNC1KhTh5QoNPhhAXRXU0+/5Yx7EDBUAAA==","output":[{"uuid":"0188430d-40cc-0002-a8da-245e5d68b7b7","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-40cc-0001-801c-85fd950326c8\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPpGa2QC/+2XQU8CQQyF30/ZcNJEhV1wEU9GJfGGB40HJAYRFUVBWFRi/Ovq11lBLhoTCCKZNLPbaTuv7U55Ce9vVb0oo0RDddVE21agWGs8M7pQHU/902pxXbU10JVauv+0ZtRDnjh7rqwa6uDp82xj2VGIP8XqgjPE2uF9MYFocY+s9hiv6RA7rNHZBL0O9qi+Km/Lc6ZnXeM7mzgTaAVbgnSJzSKWs8H5NvYOtSXYtxQprwLevuswoaOGq/+a/u51q3WVta895ETHqvDc0A3R29qkqyISo4egrI7rrJA/S94CsVZfj7qs2+E4S9NlqOO7Qp9PrTErh55Hs1prP9yIfdlT1q6zBTrQEXKINso9cLgBiIGLNLxXZHRTLd3RWRp9R44U16rYoociFUWcLbEruV16ctFm8ImInsP/fgIf3T0N5niPERg2c5v/ZuZKYPx25srgPrjvacum75Kolrsd21lnqd9qMm/6eze73c8Q3WbCZifnZPr5jBd0Pj1HLjtHhm6CS26WPUd6jpw1RxaofzqW/JrQ0LOkZ8k/ZsnIs6RnyZmzZETWWbFk5FnSs6RnSc+SS8eSofvPPBuWzHOypg9uMIiYmBQAAA==\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752098975, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752098976, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752100960, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752100961, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752100962, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752100962, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752100962, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752100963, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 3000}","now":"2023-05-22T10:42:02.048295+00:00","sent_at":"2023-05-22T10:41:41.977000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684752123024&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:03.027729+00:00","body":"H4sIAAAAAAAAA+2UXU/VMBjHv8rScCEJ4/RtXbcr4xFjgoFjAmIEctKzdqxQ2tF2HI+E725nVIgmEoiGC8/Vsuflv+e//voc3wB1rWwENdgIVvShcxFsgd67XvmoVQD1zV1mLkUUYySuegVqtgV+BHoznGmbVLxfqsWkcTY4o16iUUusjBNyrDLpWyYVKe+dT6noRZN0joGz88+dn3+LZy+6GPt6MjGuEaZzIdYcEzoJUUTdTJpusBf5zuvp9Ohw/2j7PNQFKktWI7KZFPf9pKfbznol5GrsUE0n7Jl6gihDsCZsE5zes3AMTsArIbO3BwezbOwdQp3B7ASA09vb5EdfqhS97EGNGKdlgRGGEJOU2ggqBJ186iQDEOeUwBIxCRnOIUNIMo4KSiVscoqwxA2ROWJ0AXl+V4wQkSy53FhqK93yVy2MCwZzSKBoBeStJKSkf1Ajqb4YT8FdqPHo+q6Zq1JTE6Z7+n27tyvemcGcX60+nM1mcMZ8+eWQf+ynb4bpJ7M7pE6pQ9S2iT8HIZQuFhXhjLQ5REXLlCAKL0TD+d0guISqhOMg38urRkGQ/pFr26ASiriqytutZyFzKby9D+a1MMOT4MG8RsU/ILKivxO5Y68GNSiZtUKb9PAqvYeYtekyeRX9KtM2YxDCBzgt1pw+llP2TJz+TxsUVbxak/koMlGysN6gf3mDVg+tUFRVaA3q40E9/QqvsD61hgoAAA==","output":[{"uuid":"0188430d-4494-0001-7d1f-eae0ac044b9f","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-4494-0000-2fde-272e9466a2a0\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPtGa2QC/+2Uy04CUQyG/0eZsNJE5X5dGZXEHS40LtCQERDQ4SIMIDG8uvr1DBA2GhMJwUianun08redds7He1VviinUTAM1kUrylNMRZ0wN+Vj8hdb8Bgo0Vksd9RbamIbQlNgHxVVXH8uIM0BzqiT2CGsAzgxtn2djDdH8JnCwwms6xD68jA2RfbCX9VV5Wp6aXtXGVluL8XSALoQG+MYhy1knPkDfp7YQfUEppZXBOnIdhnRUd/W36a+nZx2rrAudQ7e6UYXzRE94l5SlqzyUQ06Ccriqs0L+OHkz+Fp9Q+qybmerLE2XwcfWQt5OrTk4gZxGslrvv5mIfdk7+MzpPF3qGrpCWuYeO1wPRM95Gt4cWk6qoy6dRd5dckS4VkWBHvJUlOItRXzC9RVF7toOTvEYOvyvN3Di5jTe4hxTYNjOZf/MzhXB+OnOlcF9cd/T2LbvEa+Om469WWeR3Woya/S/m97mM0O2nbDdSTj6/X5md3Q/93fkf7gjk/w/BXh/R+7vyE3fkcWN3JK2oUXOOfGfu7RZCEwKAAA=\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752120023, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752120025, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752121989, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752121991, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2997}","now":"2023-05-22T10:42:03.027729+00:00","sent_at":"2023-05-22T10:42:03.024000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684751932581&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:04.993934+00:00","body":"H4sIAAAAAAAAA+2UX0/VMBjGv8rScCEJ47Rr13W7Mh4xJhg4JiBGICc9a8cKpR1tx/FI+O52RoVookIkhLCrZe+fZ++7/vocXgF5KU0AFVjzhne+tQFsgM7ZTrqgpAfV1U1mLnjgQySsOgkqugF+BDrdnygTVZxbysWktsZbLV+iQYuvtOViqNLxWzoWSeesi6ngeB11DoE188+tm3+LJy/aELpqMtG25rq1PlQsw2TiAw+qntRtb87SrdfT6cH+7sHmqa9yVBS0Qng9Ku66SUc2rXGSi9XQIeuWmxN5D1GKYIXpOji+tcIhOAKvuEje7u3NkqG391UCkyMAjq+v4z7qXMboeQcqRBkpclRmZc5QTK156b2Ke6ooAxBjBMMCUQFplkKKkKAM5YQIWKcEZSKrsUgRJQvI0ptihLCgccu1pTLCLn/VyrKcwhRiyBsOWSMwLsgf1HCsz4dTsGdyOLqureeyUET76Y563+xs83e616cXqw8nsxmcUVd82Wcfu+mbfvpJb/exUygflKnDz0EwIYtFiRnFTQpR3lDJscwWvGbsZpCsgLKAwyDfy8taQhD/kW0aLyOKWVmW1xuPQuaSO3MbzEuu+3vBk7EK5Q9AZEl+J3LLXPSylyJpuNLx4WR89yFp4mVyMrhVokxSUgjhX0DNRlDvCip7JFCfmYWOZD4ZMkcLHUF9EqA+MwvFI5l3JbMYLfQ/W+g/OOjI6T04Pf4KKs0GxMkPAAA=","output":[{"uuid":"0188430d-4c42-0001-5c53-b092a3eae230","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-4c42-0000-208e-16cab7bc0a1c\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPxGa2QC/+2US08CUQyFz0+ZsNJERV7DY2VUEne40LhAQhAQUF7CABLjX1e/3gHCRmIiIRgnzZ0pbe9pOy3n86OsN8UUaK6hmmgFefJ1xDOmhmp4agurxQ3V1UQtddRfWGMaITPuPiiuugZ4xjy7WM6UwB9iDcGZYx3wbqwhWtyU013hNR3igLO8G6DXwF7WV+Zteap6VRtfde2OpwNsATIkNo5Yzjr3u9gH1BZgzymplNJ4x67DgI7qrv42/fX1rGMVdakL5E63KvE80RPRBWXoKov46AlQDld1lsgfJ2+aWKtvRF3W7XyVpeky1PC10HdTq885RU+hWa2VDROxL3vPOXc2T1e6Qa7RlrknDtcD0XORhveOLCfVUY/OwugeOUJcqyJHD1lXUZ6O8mg59PDmvu3gjIiRw/9+A6duTpMdzjEJhu1c5s/sXB6Mn+5cEdwX9z3t2PY9EtVx07Ff1lnot5rMG/7fzW7zmaPbTnhk9clu8vsNTe7phkYs+X9YMhmxZMSSEUtGLBmx5IYdTEUsGbHk1llyexxp+1nRF6KoW4VyDwAA\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684751929581, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751929582, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684751929582, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751929582, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684751929583, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751929583, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2999}","now":"2023-05-22T10:42:04.993934+00:00","sent_at":"2023-05-22T10:38:52.581000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684751935588&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:04.995438+00:00","body":"H4sIAAAAAAAAA+2WYU/VMBSG/8rS8EESxm3Xruv2yXjFmGDgmoAYgdz0rh0rlHa0Hdcr4b/bGRWiiQSCGnSflp2evjunffquh1dAXkoTQAXWvOGdb20AG6BztpMuKOlBdXUzMhc88CESVp0EFd0A3wKd7k+UiSrOLeViUlvjrZbP0aDFV9pyMWTp+C0dk6Rz1sWh4HgddQ6BNfOPrZt/iSfP2hC6ajLRtua6tT5ULMNk4gMPqp7UbW/O0q2X0+nB/u7B5qmvclQUtEJ4PSruuklHNq1xkovVMEPWLTcn8gGiFMEK03VwfKuFQ3AEXnCRvN7bmyXD3N5XCUyOADi+vo79qHMZo+cdqBBlpMhRibOc0Ti05qX3KvapogxAjBEMC0QFpFkKKUKCMpQTImCdEpSJrMYiRZQsIEtvkhHCgsYu15bKCLv8USvLcgpTiCFvOGSNwLggv1DDMT8fdsGeyWHruraey0IR7ac76m2zs83f6F6fXqzencxmcEZd8Wmfve+mr/rpB73dx5lC+aBMHb4XgglZLErMKG5SiPKGSo5ltuA1YzeFZAWUBRwK+Zpe1hKCuEa2abyMKGII4fXGXyFzyZ25DeYl1/2D4MlYhfLfQGRJfiZyy1z0spciabjS8eFkfPchaeJhcjK4VaJMUtK4qneAWoygPhVQ/zMLHcl8MmT+uxaKstFCHxvUrCzL0UL/hIWykcz7khkP82ihj2qhd19Cy/jfGjm9L6fHnwGF59KmyQ8AAA==","output":[{"uuid":"0188430d-4c44-0001-b56a-cc6e82a03b21","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-4c44-0000-1533-2d9329721e64\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPxGa2QC/+2WS08CUQyFz0+ZsNJERV7DY2VUEne40LhAQxAQUF7CABLDX1e/3gHCRmMiIaiT5t4pbe/p6dxOw/tbWa+KKdBMAzXQCvLk64A9prqqeKoLq8UN1NFYTbXVW1hjGiJTzt4rrpr6eEbsHSwnSuAPsQbgzLD2edbXEC1uwuqs8BoOsc9ang3Qq2Av+ZV5Wp6KXtTCV1k742kPW4AMiI0jlrPG+Q72PtwC7DkllVIa78hVGFBRzfFvUV9PTzpUUec6Q250rRL7kR6JLihDVVnER0+Asr/iWSJ/nLxpYo3fEF5W7WyVpeEyVPE10bfD1Wcdo6fQjOvdFzdib/aWdepsni50hVyiLXOPHa4HouciDW+OLG+qrS6VhdFdcoS4xiJHDVnHKA+XJFoOa3hy13pwSsTQ4X/egRN3T+Mt3mMSDOu5zK/puTwY3+25IrjP7n3asu57IKrtbsd+WWWh3ziZN/zezW73M0O3nvDI6pPd5Ocdmt3RDo2m5P+ZktloSkZTcuNTMgH7aEpGU/KvTMlcNCWjKbnxKbmpf5J5MOac/wAjkc3Fcg8AAA==\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684751932586, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751932587, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684751932587, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751932587, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684751932588, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684751932590, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 3000}","now":"2023-05-22T10:42:04.995438+00:00","sent_at":"2023-05-22T10:38:55.588000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684752077923&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:04.997482+00:00","body":"H4sIAAAAAAAAA+2XW0vcQBiG/0oYvKhg3Dllcrgq3VoKFt2C1lKVZTYzMdFxJs5M3G7F/95JabvSQmVlW6nmKuQ7vPm+zJMXcnwD5LXUHhRgw2neutp4sAVaa1ppfSMdKG6WmangnvcRv2glKNgW+BFoVXfW6KBi7VzORqXRzij5EvVafKEMF32VCs9SoUhaa2xIecvLoHMMjJ5+ru30Wzx6UXvfFqORMiVXtXG+yDChI+e5b8pRWXf6It55PR4fHe4fbZ+7IkFpygpENoPivh21dNtoK7lY9B2yrLk+kw8QZQgWhG2C0zsrHIMT8IqL6O3BwSTqeztXRDA6AeD09jbs01zKEL1sQYFYRtMEw5TmOAupDSeda8KeTZABKMsogSliAjIcQ4aQYBlKKBWwjCnCApdExIjRGcziZTFCRLCw5ca80cLMf9XCOGEwhgTyisOsEoSk9A9qJNQn/SmYC9kfXVuXU5k2VLnxXvO+2tvl71Snzq8WH84mEzhhNv1ymH1sx2+68Se124VO0Tjf6NL/HIRQOpvlJGOkiiFKKiY5kXjGyyxbDoJTKFPYD/K9PC8lBOEdmapyMqCI8zy53XoUMufc6rtgXnPVPQgenBUo+QtE5vR3Inf0VSc7KaKKNypcrAz3zkdV+Jis9HYRNTpiEMKB03VzSh+J0+fkoCxHZCBzJTIRRHBw0DU7aH6fhQZQgx8MoK4EKswHC/0XFjqQ+d+Q+XQtFNPBQp8QqM/MQsNP6UDmamSmg4Wu2UIRvt9C2QDq6qCefgX0iVnZDBUAAA==","output":[{"uuid":"0188430d-4c46-0001-9d63-c33d1ad5a8bc","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-4c46-0000-7b0c-6e3bf34bf14a\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAPxGa2QC/+2USU8CQRCF30+ZcNJEZRGG5WRcEm940HhAYhBRUDZhECfGv65+1QPEi8bDxLVT6Z6iqvpVVXfxXp4belRGkWKN1UGrKVCoDfaMLtXC01pYLW6svma6Vk/DhTWjCTLn7IWyamuEZ8rex7KjPP4EawxOjHXE9/INosXds/orvI5DHLGWZyP0FtjL+hp8Lc+5HtTFd/7mTKA1bBEyJjaLWM425/vYR9QWYa+ooG0V8U5dhxEdtV39Xfob6labOtC+9pBTnajOvqUbomsq0VUZCdHzoKyv6qyTP0veIrFW34S6rNt4laXjMrTwXaN/Ta0hK4e+jWa1Nj94EbvZM9auswU61DFyhLbMPXO4AYiBizS8J2T5Uj0N6CyJHpAjwbUqKvRQpqICZ8voVbTK4uRPm8E5EROH//4E3rt3mn3hO9p92cyVfs3MVcH47MwdgHvn7tOWTd8VUT33OvbLOkv8VpN5k/+72e19YnSbCZudnJO/O5+eI/8DR4bMoN2e50jPkWlzZDUVlkwmtOhZ0rPkN7Nk0bOkZ8nUWbJAVs+SniX/CkuWPEt6lkydJfNuxtJhyZCTTb0CdWfxy5gUAAA=\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752074928, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752074928, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752076913, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752076914, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752076914, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752076914, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752076915, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752076916, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2995}","now":"2023-05-22T10:42:04.997482+00:00","sent_at":"2023-05-22T10:41:17.923000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684752104985&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:42:04.999014+00:00","body":"H4sIAAAAAAAAA+2ZbUvcMBzAv0oJvphgvTw1Tftq7OYYOPQGOsdUjlyT2mpta5J6u4nffenYdrKBxx0nclxelSb//JuHX3+E5PwBqHtVW5CCHVOL1hSNBXug1U2rtC2VAenDvGYshRV9iZ21CqRsD/wpaKvuqqxdFq2najLImto0lXqL+lxiVjVC9lGV+1blgpTWjXZVVovM5TkHTT3+Xujxr/LgTWFtmw4GVZOJqmiMTTkmdGCssGU2yIquvgkP3g+HZ6fHZ/vXJo1QHLMUkV2X8VgPWrrf1FoJOetbqKwQ9ZVaISlDMCVsF1w+GcI5uADvhAw+npyMgr5tZ9IABhcAXD4+uvGUt8qV3rYgRYzTOMIIooTHrmrHKGNKN87SpQGIc0pgjJiEDIeQISQZRxGlEmYhRVjijMgQMTqBPJwHI0Qkc6PcmZa1bKb/5sI4YjCEBIpcQJ5LQmL6TDbi4qN+FZob1S9dW2RjFZe0MsOj8nN+dCg+VV11fTf7cjUawRHT8Y9T/rUdfuiG36rDzrWUpbFlndm/HSGUTiYJ4YzkIURRzpQgCk9Exvm8IziGKoZ9R36HJ5mCwM1Rk+dGORRxktDHvVchcyp0/RTMe1F1K8GDeYqiFyAyof8TeVDfdapTMshFWbmHVu7d2CB3P5NWVs+Csg4YhHABp4nndFM43SaDkoQhT+ZSZCKI3ZR5g67VoJQvUKgDFXtQNwXULVMo8WQuSSZy+yGv0LUqNFm0C3Wgui2VB3UzQN0yhUaezGXJdEcfXqFrVSjCixXKPKibAuqWKdSTuTSZbsq8QteqUEwXK9Sf2S8NqtseeYW+vEK5J3NZMv1t0mso1F8nrQDq5U9Nk6Owkh8AAA==","output":[{"uuid":"0188430d-4c49-0001-a3b5-5cbaee834ee0","distinct_id":"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"0188430d-4c49-0000-e27d-3ad69e097d43\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAP1Ga2QC/+2ZTU8iQRCG358y4bQmrsgAA3gyfiTe9LBmD0oIIgsoCuKgks3+9dWneoB4WWPiRDBb6fRQVFXXB108IeH575l+q6BUM43VRdpRpESbPAu6VBtLe641v7GGmqqngW7n2oImrEfOXqiojkZY7nkO0eyqhD2LNSbODO2I18tXEc3vgT1cxuuGiCP24myK3Cb2or4zXi1PS0/qY2u9OhPpG7qUNca3yLKcHc4P0Y+oLUVfV6yyKljvQ4cpHXVC/X36u9W1vutQB9pn/dSpjnlu6QrvHVXpqsZKkEtE2VjWeUz+Inkr+Fp9E+qybmfLLN2QoY2th/w5tSbsbeQyktXafONG7JM9Z+8FXaQj/WCdIC1yT0PciIhR8LR4f1iLmxrohs4y7xtyZHGtijo91KgoDvWU1EBTm59ctxl8xGMS4v97Ah/CPU0/8R5jYtjMVb/MzDWI8d6ZOyTuXfg8bdv0/cJrEG7H3llnmd1qMmv2fTe93c8M2WbCZmc7rI/PZ2NN59MZ+T8wssz8WWXOSGdk3oysUP/HKZlNaOyUdEqumJJlp6RTMndKNnL5LZlNaMUp6ZRcMSWrTkmnZO6ULFF9XpRMnJJOyRVTMnFKOiVzp2RM1rwoWXNKOiVXTMm6U9IpudaUtH9vmnoBGOaMiuQeAAA=\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684752101987, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752101989, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752103961, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752103962, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752103963, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752103964, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752103965, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752103966, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752103966, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752103967, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}, {\"timestamp\": 1684752103968, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684752103969, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"warn\"}}}]}, \"$session_id\": \"188430716d062-0611d681544d0c-412d2c3d-164b08-188430716d113d6\", \"$window_id\": \"188430716d22560-030afa08fd3374-412d2c3d-164b08-188430716d32255\", \"token\": \"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu\", \"distinct_id\": \"188344bb93863f-015f6ea3e2bac88-412d2c3d-270e70-188344bb939ce0\"}, \"offset\": 2994}","now":"2023-05-22T10:42:04.999014+00:00","sent_at":"2023-05-22T10:41:44.985000+00:00","token":"phc_e7i4lsCNiQfNKaLluljqyVgPP0P6r7zU8XpCFuCZlKu"}]} -{"path":"/e/?ip=1&_=1684752184509&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:04.525363+00:00","body":"ZGF0YT1leUpsZG1WdWRDSTZJaVJ3WVdkbGRtbGxkeUlzSW5CeWIzQmxjblJwWlhNaU9uc2lKRzl6SWpvaVRXRmpJRTlUSUZnaUxDSWtiM05mZG1WeWMybHZiaUk2SWpFd0xqRTFMakFpTENJa1luSnZkM05sY2lJNklrWnBjbVZtYjNnaUxDSWtaR1YyYVdObFgzUjVjR1VpT2lKRVpYTnJkRzl3SWl3aUpHTjFjbkpsYm5SZmRYSnNJam9pYUhSMGNEb3ZMMnh2WTJGc2FHOXpkRG80TURBd0x5SXNJaVJvYjNOMElqb2liRzlqWVd4b2IzTjBPamd3TURBaUxDSWtjR0YwYUc1aGJXVWlPaUl2SWl3aUpHSnliM2R6WlhKZmRtVnljMmx2YmlJNk1URXpMQ0lrWW5KdmQzTmxjbDlzWVc1bmRXRm5aU0k2SW1WdUxWVlRJaXdpSkhOamNtVmxibDlvWldsbmFIUWlPamszTkN3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRVd01Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPRFF6TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFME16TXNJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpVM0xqSWlMQ0lrYVc1elpYSjBYMmxrSWpvaWMzRXhhWEppT1dRME0ySmhZbXBxTWlJc0lpUjBhVzFsSWpveE5qZzBOelV5TVRnMExqVXdPU3dpWkdsemRHbHVZM1JmYVdRaU9pSXhPRGcwTXpCbE16UmlOVE5qWXkwd1pETmpObU0zTURrME5ERTFaRGd0TkRFeVpESmpNMlF0TVRZMFlqQTRMVEU0T0RRek1HVXpOR0kyWkRNNElpd2lKR1JsZG1salpWOXBaQ0k2SWpFNE9EUXpNR1V6TkdJMU0yTmpMVEJrTTJNMll6Y3dPVFEwTVRWa09DMDBNVEprTW1NelpDMHhOalJpTURndE1UZzRORE13WlRNMFlqWmtNemdpTENJa2NtVm1aWEp5WlhJaU9pSm9kSFJ3T2k4dmJHOWpZV3hvYjNOME9qZ3dNREF2YzJsbmJuVndJaXdpSkhKbFptVnljbWx1WjE5a2IyMWhhVzRpT2lKc2IyTmhiR2h2YzNRNk9EQXdNQ0lzSW5SdmEyVnVJam9pY0doalgycElZMFIxTjIwelduWnVTVzV3YTJKNGJVcHlTMFZpZVVwMWEzbEJXa042ZVV0bFREQnpWSGhDTTJzaUxDSWtjMlZ6YzJsdmJsOXBaQ0k2SWpFNE9EUXpNR1V6TkdJM04yWXdMVEF6TXpCa05XUXlPRE00WmpObE9DMDBNVEprTW1NelpDMHhOalJpTURndE1UZzRORE13WlRNMFlqZ3lPRE16SWl3aUpIZHBibVJ2ZDE5cFpDSTZJakU0T0RRek1HVXpOR0k1TXpBMU5pMHdORGRoT1RCaE1HSXdNVE5pTkRndE5ERXlaREpqTTJRdE1UWTBZakE0TFRFNE9EUXpNR1V6TkdKaE1qVmxOaUlzSWlSd1lXZGxkbWxsZDE5cFpDSTZJakU0T0RRek1HVXpOR0ppTVRZM1lpMHdOekJrTVRrMVpqVTVNak0yTmpndE5ERXlaREpqTTJRdE1UWTBZakE0TFRFNE9EUXpNR1V6TkdKak1qazRNQ0o5TENKMGFXMWxjM1JoYlhBaU9pSXlNREl6TFRBMUxUSXlWREV3T2pRek9qQTBMalV3T1ZvaWZRJTNEJTNE","output":[{"uuid":"0188430e-34ce-0000-0e70-879b9262bb17","distinct_id":"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"sq1irb9d43babjj2\", \"$time\": 1684752184.509, \"distinct_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"timestamp\": \"2023-05-22T10:43:04.509Z\"}","now":"2023-05-22T10:43:04.525363+00:00","sent_at":"2023-05-22T10:43:04.509000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?ip=1&_=1684752184509&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:04.515071+00:00","body":"ZGF0YT1leUpsZG1WdWRDSTZJaVJ2Y0hSZmFXNGlMQ0p3Y205d1pYSjBhV1Z6SWpwN0lpUnZjeUk2SWsxaFl5QlBVeUJZSWl3aUpHOXpYM1psY25OcGIyNGlPaUl4TUM0eE5TNHdJaXdpSkdKeWIzZHpaWElpT2lKR2FYSmxabTk0SWl3aUpHUmxkbWxqWlY5MGVYQmxJam9pUkdWemEzUnZjQ0lzSWlSamRYSnlaVzUwWDNWeWJDSTZJbWgwZEhBNkx5OXNiMk5oYkdodmMzUTZPREF3TUM4aUxDSWthRzl6ZENJNklteHZZMkZzYUc5emREbzRNREF3SWl3aUpIQmhkR2h1WVcxbElqb2lMeUlzSWlSaWNtOTNjMlZ5WDNabGNuTnBiMjRpT2pFeE15d2lKR0p5YjNkelpYSmZiR0Z1WjNWaFoyVWlPaUpsYmkxVlV5SXNJaVJ6WTNKbFpXNWZhR1ZwWjJoMElqbzVOelFzSWlSelkzSmxaVzVmZDJsa2RHZ2lPakUxTURBc0lpUjJhV1YzY0c5eWRGOW9aV2xuYUhRaU9qZzBNeXdpSkhacFpYZHdiM0owWDNkcFpIUm9Jam94TkRNekxDSWtiR2xpSWpvaWQyVmlJaXdpSkd4cFlsOTJaWEp6YVc5dUlqb2lNUzQxTnk0eUlpd2lKR2x1YzJWeWRGOXBaQ0k2SWpsbmVYTnpkM1JtZEdFNE1IVjNkbmdpTENJa2RHbHRaU0k2TVRZNE5EYzFNakU0TkM0MU1Ea3NJbVJwYzNScGJtTjBYMmxrSWpvaU1UZzRORE13WlRNMFlqVXpZMk10TUdRell6WmpOekE1TkRReE5XUTRMVFF4TW1ReVl6TmtMVEUyTkdJd09DMHhPRGcwTXpCbE16UmlObVF6T0NJc0lpUmtaWFpwWTJWZmFXUWlPaUl4T0RnME16QmxNelJpTlROall5MHdaRE5qTm1NM01EazBOREUxWkRndE5ERXlaREpqTTJRdE1UWTBZakE0TFRFNE9EUXpNR1V6TkdJMlpETTRJaXdpSkhKbFptVnljbVZ5SWpvaWFIUjBjRG92TDJ4dlkyRnNhRzl6ZERvNE1EQXdMM05wWjI1MWNDSXNJaVJ5WldabGNuSnBibWRmWkc5dFlXbHVJam9pYkc5allXeG9iM04wT2pnd01EQWlMQ0owYjJ0bGJpSTZJbkJvWTE5cVNHTkVkVGR0TTFwMmJrbHVjR3RpZUcxS2NrdEZZbmxLZFd0NVFWcERlbmxMWlV3d2MxUjRRak5ySWl3aUpITmxjM05wYjI1ZmFXUWlPaUl4T0RnME16QmxNelJpTnpkbU1DMHdNek13WkRWa01qZ3pPR1l6WlRndE5ERXlaREpqTTJRdE1UWTBZakE0TFRFNE9EUXpNR1V6TkdJNE1qZ3pNeUlzSWlSM2FXNWtiM2RmYVdRaU9pSXhPRGcwTXpCbE16UmlPVE13TlRZdE1EUTNZVGt3WVRCaU1ERXpZalE0TFRReE1tUXlZek5rTFRFMk5HSXdPQzB4T0RnME16QmxNelJpWVRJMVpUWWlMQ0lrY0dGblpYWnBaWGRmYVdRaU9pSXhPRGcwTXpCbE16UmlZakUyTjJJdE1EY3daREU1TldZMU9USXpOalk0TFRReE1tUXlZek5rTFRFMk5HSXdPQzB4T0RnME16QmxNelJpWXpJNU9EQWlmU3dpZEdsdFpYTjBZVzF3SWpvaU1qQXlNeTB3TlMweU1sUXhNRG8wTXpvd05DNDFNRGxhSW4wJTNE","output":[{"uuid":"0188430e-34c8-0000-5a19-0714f9887930","distinct_id":"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$opt_in\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"9gysswtfta80uwvx\", \"$time\": 1684752184.509, \"distinct_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"timestamp\": \"2023-05-22T10:43:04.509Z\"}","now":"2023-05-22T10:43:04.515071+00:00","sent_at":"2023-05-22T10:43:04.509000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?ip=1&_=1684752184514&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:04.529988+00:00","body":"ZGF0YT1leUpsZG1WdWRDSTZJaVJwWkdWdWRHbG1lU0lzSW5CeWIzQmxjblJwWlhNaU9uc2lKRzl6SWpvaVRXRmpJRTlUSUZnaUxDSWtiM05mZG1WeWMybHZiaUk2SWpFd0xqRTFMakFpTENJa1luSnZkM05sY2lJNklrWnBjbVZtYjNnaUxDSWtaR1YyYVdObFgzUjVjR1VpT2lKRVpYTnJkRzl3SWl3aUpHTjFjbkpsYm5SZmRYSnNJam9pYUhSMGNEb3ZMMnh2WTJGc2FHOXpkRG80TURBd0x5SXNJaVJvYjNOMElqb2liRzlqWVd4b2IzTjBPamd3TURBaUxDSWtjR0YwYUc1aGJXVWlPaUl2SWl3aUpHSnliM2R6WlhKZmRtVnljMmx2YmlJNk1URXpMQ0lrWW5KdmQzTmxjbDlzWVc1bmRXRm5aU0k2SW1WdUxWVlRJaXdpSkhOamNtVmxibDlvWldsbmFIUWlPamszTkN3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRVd01Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPRFF6TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFME16TXNJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpVM0xqSWlMQ0lrYVc1elpYSjBYMmxrSWpvaWVXVnViamx0YVhRNE4yNXNaR1J4YUNJc0lpUjBhVzFsSWpveE5qZzBOelV5TVRnMExqVXhOQ3dpWkdsemRHbHVZM1JmYVdRaU9pSm5VVVJJYkhCNGNVTklaMVZyYURWek0weFBWbFZSTlVGalRuUXhhMjFXYkRCNGRrOWhhakp6WmpadElpd2lKR1JsZG1salpWOXBaQ0k2SWpFNE9EUXpNR1V6TkdJMU0yTmpMVEJrTTJNMll6Y3dPVFEwTVRWa09DMDBNVEprTW1NelpDMHhOalJpTURndE1UZzRORE13WlRNMFlqWmtNemdpTENJa2RYTmxjbDlwWkNJNkltZFJSRWhzY0hoeFEwaG5WV3RvTlhNelRFOVdWVkUxUVdOT2RERnJiVlpzTUhoMlQyRnFNbk5tTm0waUxDSWtjbVZtWlhKeVpYSWlPaUpvZEhSd09pOHZiRzlqWVd4b2IzTjBPamd3TURBdmMybG5iblZ3SWl3aUpISmxabVZ5Y21sdVoxOWtiMjFoYVc0aU9pSnNiMk5oYkdodmMzUTZPREF3TUNJc0lpUmhibTl1WDJScGMzUnBibU4wWDJsa0lqb2lNVGc0TkRNd1pUTTBZalV6WTJNdE1HUXpZelpqTnpBNU5EUXhOV1E0TFRReE1tUXlZek5rTFRFMk5HSXdPQzB4T0RnME16QmxNelJpTm1Rek9DSXNJblJ2YTJWdUlqb2ljR2hqWDJwSVkwUjFOMjB6V25adVNXNXdhMko0YlVweVMwVmllVXAxYTNsQldrTjZlVXRsVERCelZIaENNMnNpTENJa2MyVnpjMmx2Ymw5cFpDSTZJakU0T0RRek1HVXpOR0kzTjJZd0xUQXpNekJrTldReU9ETTRaak5sT0MwME1USmtNbU16WkMweE5qUmlNRGd0TVRnNE5ETXdaVE0wWWpneU9ETXpJaXdpSkhkcGJtUnZkMTlwWkNJNklqRTRPRFF6TUdVek5HSTVNekExTmkwd05EZGhPVEJoTUdJd01UTmlORGd0TkRFeVpESmpNMlF0TVRZMFlqQTRMVEU0T0RRek1HVXpOR0poTWpWbE5pSXNJaVJ3WVdkbGRtbGxkMTlwWkNJNklqRTRPRFF6TUdVek5HSmlNVFkzWWkwd056QmtNVGsxWmpVNU1qTTJOamd0TkRFeVpESmpNMlF0TVRZMFlqQTRMVEU0T0RRek1HVXpOR0pqTWprNE1DSjlMQ0lrYzJWMElqcDdmU3dpSkhObGRGOXZibU5sSWpwN2ZTd2lkR2x0WlhOMFlXMXdJam9pTWpBeU15MHdOUzB5TWxReE1EbzBNem93TkM0MU1UUmFJbjAlM0Q=","output":[{"uuid":"0188430e-34d2-0000-99e4-4cb75151aa19","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$identify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"yenn9mit87nlddqh\", \"$time\": 1684752184.514, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$anon_distinct_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"$set\": {}, \"$set_once\": {}, \"timestamp\": \"2023-05-22T10:43:04.514Z\"}","now":"2023-05-22T10:43:04.529988+00:00","sent_at":"2023-05-22T10:43:04.514000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/engage/?ip=1&_=1684752184515&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:04.531659+00:00","body":"ZGF0YT1leUlrYzJWMElqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR0p5YjNkelpYSmZkbVZ5YzJsdmJpSTZNVEV6TENJa2NtVm1aWEp5WlhJaU9pSm9kSFJ3T2k4dmJHOWpZV3hvYjNOME9qZ3dNREF2YzJsbmJuVndJaXdpSkhKbFptVnljbWx1WjE5a2IyMWhhVzRpT2lKc2IyTmhiR2h2YzNRNk9EQXdNQ0lzSW1WdFlXbHNJam9pZUdGMmFXVnlRSEJ2YzNSb2IyY3VZMjl0SW4wc0lpUjBiMnRsYmlJNkluQm9ZMTlxU0dORWRUZHRNMXAyYmtsdWNHdGllRzFLY2t0RllubEtkV3Q1UVZwRGVubExaVXd3YzFSNFFqTnJJaXdpSkdScGMzUnBibU4wWDJsa0lqb2laMUZFU0d4d2VIRkRTR2RWYTJnMWN6Tk1UMVpWVVRWQlkwNTBNV3R0Vm13d2VIWlBZV295YzJZMmJTSXNJaVJrWlhacFkyVmZhV1FpT2lJeE9EZzBNekJsTXpSaU5UTmpZeTB3WkROak5tTTNNRGswTkRFMVpEZ3ROREV5WkRKak0yUXRNVFkwWWpBNExURTRPRFF6TUdVek5HSTJaRE00SWl3aUpIVnpaWEpmYVdRaU9pSm5VVVJJYkhCNGNVTklaMVZyYURWek0weFBWbFZSTlVGalRuUXhhMjFXYkRCNGRrOWhhakp6WmpadEluMCUzRA==","output":[{"uuid":"0188430e-34d4-0000-387d-e634d8ec56a6","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"$set\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$browser_version\": 113, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"email\": \"xavier@posthog.com\"}, \"$token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"event\": \"$identify\", \"properties\": {}}","now":"2023-05-22T10:43:04.531659+00:00","sent_at":"2023-05-22T10:43:04.515000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?compression=gzip-js&ip=1&_=1684752187560&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:07.564872+00:00","body":"H4sIAAAAAAAAA+2cWXPbNhCA/4pHk0fTAQ/w8Fsat5MmadNM4kwnmYwGBJYSLZKgCR6iMvnvXVCybNmyY6d2a4/wZBELLBd7fAQP+Mu3EbRQ1KPD0bMEWN1UME4yNhlzlmUgRvujspIlVHUKanT4bfRM4p/RH4zvvfuw9zeKsWHcQqVSWaDAJgc2PSC6Pa5kp6DCxt/SChI5140C2pTDuO5LQMERqFktSy3gTVWhGeOmylAwrevy8PnzTKIVU6nqw5AQ8lz300fYYVOiBSWrpwXLtdqh4+r057bZtnuhOWPFpGET3R0K6/iDHqJ4BVCMp5BOpniSKPDOG7tU1FNUQgnBxjaFrpRVve4belr5uvmst+fq5iyN8TQdxPokeHDRXwc0OHB0e1qgWfU4FdgaCDuvMlVB77OyWQzj6lTPzfZDL6COHXoH1N4fiVTVacFXw+wQzSDgejF1ObeIcLnPAxJ5nk1FaHm2IxzuCsv2vZiE1nl3X7jhhejcjzKMOWBMdQJsD6dKJ0UzBH/ZNS0mYyFzlmrHXA3wxezEDgqU9qFVgYLawh+ZZDpdazkDraCc8vHJK37UBLn7uS1+L8pZPM9fV29+jfvXzax/8fnlon8Db4n6OP/FnQ3xX6q8PP8gSIhFXJcIKpzQDRMXbpp/iH1cra5LCyG7y9oil1DfIl7AIsJITGw39m5SxxwK/jLBJxgeuKIwtv0gtkhAhB3RhEaO6/s3KeROFJLR9/2RTBJ03ejQJZ73ff8CByaVbMpU4FGa9IYA/wcBqJ8vmCvjNJhy6bTTawhALyNg8v7oVVbOT1++mhzPplS5b999On5PX/A/a3uWf8rIvH3HThyV+Pm9F3yjvfoTVqRqLCCXY0yzE+DoyoRlClDhkIdDzq1FI7I6r+XaPlgYeWJRm/lWInzPp17oRwR0ct87fgZjztL2zJ51+wz62xq3HjMU37eR9hiivGkG191SxSrPjyBhTVbv/bW2B2cBqgYxXlXzypcrH6+PdS4tZKF1HH98OYxTNSt0YQ50LbQM3WhoeneaupGh6eOiaXFaNZ4/P+2p7bTaI4amd6Ap5nY1YUW6YPXSyetRTkSi5ShPgG1FEQ0Ch9nEo97DM3jDqOtAfJOF20B825GrytA1qLJGE1P/xNRmmr1M+9YhjmsRajnOR5sceu4hweS0vSCin7Era1masTjD9exyXYuR+fIVgcvmspB5r41aMlyNuWw0SrBQNqTnILnQw3D8oTh+dqZHhfDl5X5VAbdi+caIpwx137Hnee+0gpew6LJrlsiG6Y+b6bsGJvAAEjQvghB8ITyBqXS9OqAOScJNLJHNm/XH/tBuhwmVpdOs6xT3fZZRsNNrCGUQdc+IYk2NCVYOZYGe1assgSumCuM1Vngndm4c43XarpdgQw2t1mGbdVWyXj9e1NN4iHXtpUebJcqmcmJhrV8Rj3GlWEpMsvNbeUPQOxHUiaLNhZ0h6KMlaFE7UhVJRlm8yE9D/fjEENQQ9IcEhUKbbKG/8rJWhqGGobvLUN4mAhM+cjLRNrwbJrqFoZFhqGHoRYZOGZ9hPeCbdVYOETYIvWeEUoPQp4HQ/nTRxZNanIgiF12cbEeobwhqCLqxCmVV1luMc+SbtZIZkD4ASM0T0ScCUtX6cZPEzSmP27oVjQGpAeltlqJycjp8tGHIaci5m+SEXjJaJk0gRB/ldOCcIach5w/ImQCIGG/kLcUBvwQyCDUI3VmEFtKOe9qKYO57jasGX15FaGgYahi6wVBZxJJVAgdZrWMNn9Ebjt47R/3QcPRpcPSkBVGfCLsrw3nXzfQHsVs4GpkXSoajGxwtZA2xlDPzPv4B8Ek3XiatWbOHJNFXrj2FmDAAfRwATeOocCtoM+k7wNmQZ1cA6hPUbQC6ywA1FLwzBe0nsvVyh+kXtU3RErloQy93HJzcVvo5V/67haHfz9PvfF/gdRn5nwBy9WH7Rop47lBjuHMyy4dXPXr3usWzlM+mEv2OMkBAZoMdOqLrnZNra1SGTzhvkD/sPtS1Zy/tQd3u5nWn1aZTldZwEyl2cfPmv78OuO73r/8A7dHGd9pIAAA=","output":[{"uuid":"0188430e-40ad-0000-e548-fc5de1421f3b","distinct_id":"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"7d1mrlsrey6apuzb\", \"$time\": 1684752184.51, \"distinct_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"session-reset-on-load\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"offset\": 3044}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0001-b527-acee69cb829b","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"56mza3obi7hco2vh\", \"$time\": 1684752184.515, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\"}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"project\", \"$group_key\": \"0188430e-316e-0000-51a6-fd646548690e\", \"$group_set\": {\"id\": 1, \"uuid\": \"0188430e-316e-0000-51a6-fd646548690e\", \"name\": \"Default Project\", \"ingested_event\": false, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"offset\": 3039}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0002-e60d-ac4f19b25506","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"nqru46xqy512vurl\", \"$time\": 1684752184.515, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"organization\", \"$group_key\": \"0188430e-2909-0000-4de1-995772a10454\", \"$group_set\": {\"id\": \"0188430e-2909-0000-4de1-995772a10454\", \"name\": \"x\", \"slug\": \"x\", \"created_at\": \"2023-05-22T10:43:01.514795Z\", \"available_features\": [], \"taxonomy_set_events_count\": 0, \"taxonomy_set_properties_count\": 0, \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34bb167b-070d195f5923668-412d2c3d-164b08-188430e34bc2980\"}, \"offset\": 3039}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0003-e8e3-5965b1f00c95","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"621xmy2vdcpezwlh\", \"$time\": 1684752184.55, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 3004}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0004-b960-e71c6531c083","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"lihlwwsc66al5e1i\", \"$time\": 1684752184.555, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"posthog-3000\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2999}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0005-4907-567be483079e","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"nt2osnfl5abzmq8y\", \"$time\": 1684752184.555, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"enable-prompts\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2999}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0006-2957-27a39ce38568","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"cvfdres92ldvucw0\", \"$time\": 1684752184.559, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"hackathon-apm\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2995}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0007-9951-26b9cabcd2ab","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"yqzwbgtdjdnmdwbf\", \"$time\": 1684752184.56, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"early-access-feature\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2994}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0008-80c4-46772bf66fc5","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"sv6bufbuqcbvtvdu\", \"$time\": 1684752184.56, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"hogql\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2994}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-0009-5175-65c70ceabd9d","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"eyoa5pfu7ddy9m5m\", \"$time\": 1684752184.56, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"feedback-scene\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2994}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-000a-5dc4-9d624d8dc3fa","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"no1by5vd7x64u3sx\", \"$time\": 1684752184.586, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"onboarding-v2-demo\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2968}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-000b-0f86-e81af913f30b","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"jvedtjd1wp8xwwkw\", \"$time\": 1684752184.599, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"notebooks\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2955}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-000c-fb88-5d95e7940f12","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"ingestion landing seen\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ib9n3revlo62eca3\", \"$time\": 1684752184.603, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2951}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-40ad-000d-fec3-693237c3318e","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"9vunv0ozv84m22me\", \"$time\": 1684752184.621, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"instance\", \"$group_key\": \"http://localhost:8000\", \"$group_set\": {\"site_url\": \"http://localhost:8000\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2933}","now":"2023-05-22T10:43:07.564872+00:00","sent_at":"2023-05-22T10:43:07.560000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?compression=gzip-js&ip=1&_=1684752190565&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:10.567715+00:00","body":"H4sIAAAAAAAAA+1YWXPbOAz+KxpNH6uE1GU5b027O712u51uOzvtdDSUCEmMZVGRKB/p9L8vKB+x5KNNkwdn6qfEOIiPIAAB+PLNhAkUyrwwn7BGyZiVqqnAfGqWlSyhUgJq8+Kb+UTiH/MvFhvvPhj/IRsJ4QSqWsgCGZScUe+MaHpUyWkNFRL/FBUkcqaJHCYihlDNS0DGC6hHSpaaETdVhebDpsqRkSlVXpyf5wgjz2StLgJCyLkoUqiVNoQKmoySXRHNKJnKCjbW53c1loBu0VLqbJBzVqQNS7UeFNbHD1qljiuAIsxApBlaGw7cW+JUcJXhIR4hSJwImJayUmvZwNWHr8kradfR5FxEaGYKkTaCPzY9eOYNzmxNFwXCUqHgSJ1AOvfoPOXFZEqLeXtRJfQlqR+4A8+mQXBGneFTkwu8cBEv9dL3L17m5ez6+cv04yjzauftu08f33vP4r8VHY0/5WQ2eceu7DrxxxvP06riia5DwHEjz4lji3An9uMBGbou9XhgudTmduxwi/puRALrVtznTqAPa7RXfwGFqEMOYxli4F1BjK5MWF4DHphWsinbKFyzTLK0aznUBwtDgFgeZb6VcN/1PTfwh0QHsaxSVogb1sbChpY9JMOFlsuBWsOhNxjYjBLXczWSolasiHVI7IxI8zui2siWEJ3Pohx4iFfHJw1rwVF5hZ/FSkwgTIC1wknOUrzNl6/I2qSFJZvnknF9UzRQorVMpp0QcZ02xypg+ViDQxHgVpyLeJRJ9DvyYMxE3uLQL8om+EtDW6OpcxaPDvCfYM4C5qRO4N3pWIu0aNrkXYhisoVcolmNcTsv2/KyyvwWapv3sJGPWiqHMcppv3wzFTpjmcraTpyzutZl6Is5jizbRMcxpaowbBko1FKfmoXKwjgTOYYeHql/yWRpeGEiVDDTwfPKiFlhMM4NZsSSg1EXoixBGUoa47mBUcabWJ3hM3ewcDHpoVGWDhc0T3aAQqbRsg4j6xmpS6ar1oaVt5gUxWWjlCzwdFkoXa63zO2UupPlqFVun2ztqb1IkLHxy7LKSoxZNe+TMY9UU+/j5qzCutsjJk2eW4uy2eNkrLZ0ZlkC77fwu3vriGWErW+x1z3GLuBdYhd2l9eC7pJuIXfpHcA6FnS03OFFtgIuyUF/TvUfK5b5wgf+djBoAWMlpQ37XcP4OesYtncavg/Sy0rwFP7B7+qBmN0l1DFr/xTOw2atacUwuaufMr8Wfrjbt4XxoO1W4qEMIv3Vqvt5vvZp1sboIRhGX8tY6tzvQZYRu89+G6gPa2jRpm1myT6rC8l1ntzzCfQrWhg8eNTK630SVpt8ZzT0BY19mh2I2FF2IDo/gvgBa9El08G9/C/E7ncuG411ScGiJThH/21hXAoYPdXV77XiL3jxzgoLZG2PWUnZKxp9t2x95yTXX6LlVyPDvkP3LW33fjAE6Xf0iZIjvCP2JVkcXr2MXzSDsfN5UrwqylE0G7+u3vwRzV83o/mzz89v5m/gLan/nV06bcuDj6D7nX6XPRgkxCKOQ7jH7cAJEgcOddkByjj6uKkouJz2Txs6xPMt4g7YkDASEepE7qHjmO2B/jjg+JTiEAD9A8EFSBDeEALwOXc5DgX7jwPPJkmg22OZJDVgB2G7duv/9ZC5snOcE+Y5pplKZNVORXcbNTuqj3nmdLLihmZpnsUQTHKvjd3tmdP1TjPnaeY8zpnz9yrSrg0JpRhgxLUZJI7nURI5zt4DXTshNIqcbpmmfqdMH/0u8FSpL0y3oqBEeQ1lNBgHs3Z236rUATltB0+V+kgr9T23g7tWVhHeAAp+VJurGMFDBRpUd4N5hEurFVa9NuptVe88cJ72Vad91WlfddpXnfZVp33VYxmFKPbNj2ljddvv3HUM2tB8zFOQmmV2dZVPy+ubpLC9691TED3tq05T0JFOQb9ZkfYCbg8xtJjtR2AnDh0CePtrtBcABUq7Ndr1v3/9H65UwSy8JgAA","output":[{"uuid":"0188430e-4c68-0000-2df4-40793d05388a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"vegy51ygdnvw1ny0\", \"$time\": 1684752188.139, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"p\", \"classes\": [\"mb-2\"], \"attr__class\": \"mb-2\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"I can add a code snippet to my product.\"}, {\"tag_name\": \"div\", \"classes\": [\"mt-4\", \"mb-0\"], \"attr__class\": \"mt-4 mb-0\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"button\", \"$el_text\": \"\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--has-side-icon\", \"mb-4\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--has-side-icon mb-4\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"mb-6\"], \"attr__class\": \"flex flex-col mb-6\", \"nth_child\": 4, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e34e4eef-039e8e6dd4d53c-412d2c3d-164b08-188430e34e520f8\"}, \"offset\": 2421}","now":"2023-05-22T10:43:10.567715+00:00","sent_at":"2023-05-22T10:43:10.565000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-4c68-0001-eb2a-54453fb06eec","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/platform\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/platform\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"3hnz1hglhce8vl5k\", \"$time\": 1684752188.145, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e42ef110e-042aef35510b338-412d2c3d-164b08-188430e42f01bb3\"}, \"offset\": 2416}","now":"2023-05-22T10:43:10.567715+00:00","sent_at":"2023-05-22T10:43:10.565000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-4c68-0002-6ea4-3b0d05101aee","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/platform\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/platform\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"4r1etipqepb7m8xn\", \"$time\": 1684752188.809, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"backend\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"mb-6\"], \"attr__class\": \"flex flex-col mb-6\", \"nth_child\": 4, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e42ef110e-042aef35510b338-412d2c3d-164b08-188430e42f01bb3\"}, \"offset\": 1752}","now":"2023-05-22T10:43:10.567715+00:00","sent_at":"2023-05-22T10:43:10.565000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-4c68-0003-3fe5-c2dfb803996d","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/backend\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/backend\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"txh2rjlwpqzfn25q\", \"$time\": 1684752188.815, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e458d299-0a26be2f319ee58-412d2c3d-164b08-188430e458e1e11\"}, \"offset\": 1746}","now":"2023-05-22T10:43:10.567715+00:00","sent_at":"2023-05-22T10:43:10.565000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?compression=gzip-js&ip=1&_=1684752193570&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:13.576236+00:00","body":"H4sIAAAAAAAAA+1ZbW/bNhD+K4ZR7FOVkJIoSxmKoWlXpC9bU3QthhaFQImUxFgSVYryS4r89x1ly7EcJ+2cFEhQf4p1L7yHx7sj7/L525BPeKmHR8NHtNEyppVuFB8+HlZKVlxpwevh0bfhIwl/hn/RePD2/eBfYAMhnHBVC1kCA6MDTA6QoUdKTmuugPhCKJ7ImSEyPhExD/W84sB4zuuxlpVhxI1SYD5sVA6MTOvq6PAwBxh5Jmt95COEDkWZ8lqDocOIxmNeMqNo2KDRFzWMiuqspIWxs11zCfASPcbOGjmnZdrQ1Ojz0vrw3qjUseK8DDMu0gysBiP3kjgVTGewCEEIiBPBp5VUeiXru2bxFbmTdh1DzkUEZqY8MkbgY92jB2R0YBu6KAGWDgUDql1MFEvjr8WEV+ek8Q1fC7NZ7PnuiNg4MIqPh0zAvst4qZa+e36SV7Ovz07SD+OM1M6btx8/vCNP4781HhcfczSbvKVndp14xdpptarYhx0g7rgRceLYQsyJvXiEAtfFhPmWi21mxw6zsOdGyLcuxT3mtPAa49QdUIg6ZLyQIcThGY/BkwnNaw4Lpko2VRuUK9YQLe1aDva4BZGALIKpZyXMcz3i+l6ATExLldJSnFMTEutadoCChZbLOLaCgIxGNsXIJa5BUtaalrGJiK0BOrwAVGvJE4LzaZRzFsLW4UTDWjBQ7vDTWIsJDxNOW+Ekpyns5vMXYK3TworOc0mZ2SkYqMBaJtNehLhOm3KK07ww4ECEMyvORTzOJPgdeLygIm9xmBOlE/gy0FZo6hwS4wb+I0hhDilq8nl7dtYiLZs2lxeikHMhk2DWYLyanm216QpBC7UtA3wtHY1UzguQM375NtTgjGVGR43WINNKhJrPzNmfnpwCIc5pXZta9Xn4BsKmPO4k174sq1KioGq+SYbz1U19HTenCsrBBjFp8txaZPMGJwbgXHFTa4rIsodwsFRrFXabXu1hQW2BA3ltjcE2zH1iH3Gf1+Ltky7R9ukd1kGLtIPEqKaW+QmwajiKWFuJggOYSjW2Fu4udRbGmcghr8niSybLDZKLx70zY2LSV4DK11PAWxXWDvRFZ/yUljy/dGjnug1+zxhE0w8YW6xX67kJ/WFBZwtnHQ0gbKvZ77ssuYb/WAmW8lO4VACzBI/DfXtlE9uEembtDbP2DmatqaIVXOs/ZH4lfHe7b8vCjbZbibsyCPSX3RPg2cqnWZsON8EYbGoNljq3O5Ak5+Y1dJ19w75jQ4s3Cvw0FCuW11tdSA5Wcrc7AnOKFgQPLNV5fZMEhS3fGg2bgoPrNG8sKs73IL6HS/mYmuBe/grh6TeXjcG6pFhWJhgD/13BuBQYbKh23yvFHby4Y+1qX1hKyo2i8b1aG0lm7rvlBZXBrWtu7fbpemMI4gvwiZbwpgb5KovDs5P4eTMqnE+T8mVZjaNZ8Uq9/jOav2rG86efnp3PX/M3qP5nduy0Fz4cgrntN9+Yo1GCLOQ4iBFm+46fOPymN6YPMo5ZbipKJqebqwUOIp6F3BENEEURwk7k3rQctQn3Fj1ECk9g3l/QJT6zA3gnUtuLuJ04OOCcXLseiHPMMTavQ5kkNYe3Cg6CkTmAVcvVGbrf/dZhlbXKu/VcnfZD7rsqUfqTs7pQfsGr2fRse9/l7/uufd91P/uuX6tUE0Ro7AI8HkXIdQl2Yxol164H4pHt8Y1S7fdL9YOZju2rdaG/TqMgcJoU5/rMlluqtX3gE29frvfl+n6W658wJjOdrCgbs9v9rOwuZmW36ez+d0dfmcmWlUB/t2140+Pesufc1Ng+zvu5A7OXUEpUAwVAlnU30wOv+1d3fkXSnI4pqPt53X5et5/X7ed1+3ndrzqvu4MmcITRQxrXgUGRzP+AbNOJVMWTZUv32+qfd092aw0X6z70rlDY5+m57+dTPJppPGv3e7Ur9CBz9l3hviu8j13hL1a/CYlt5FgIM8QTlCTcIS69tnwTwrANneR6+Ubk4st/4/nMb+ImAAA=","output":[{"uuid":"0188430e-582a-0000-2b14-ad7e278afb7b","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/backend\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/backend\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"2mvrdgcqmvepz5u8\", \"$time\": 1684752191.57, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"PHP\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"attr__data-attr\": \"select-framework-PHP\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"div\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"FrameworkPanel\"], \"attr__class\": \"FrameworkPanel\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e458d299-0a26be2f319ee58-412d2c3d-164b08-188430e458e1e11\"}, \"offset\": 1997}","now":"2023-05-22T10:43:13.576236+00:00","sent_at":"2023-05-22T10:43:13.570000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-582a-0001-7181-20c3ce2580fe","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/backend/php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/backend/php\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"pin8vjsmr8mepxwj\", \"$time\": 1684752191.58, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e505ac40-0ebb044514cabf8-412d2c3d-164b08-188430e505b26e1\"}, \"offset\": 1987}","now":"2023-05-22T10:43:13.576236+00:00","sent_at":"2023-05-22T10:43:13.570000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-582a-0002-bea7-d4b125bb0a9a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/backend/php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/backend/php\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"mtqwb993ug1ltj2o\", \"$time\": 1684752192.856, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"Continue\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"panel-footer\"], \"attr__class\": \"panel-footer\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 11, \"nth_of_type\": 5}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"InstructionsPanel\", \"mb-8\"], \"attr__class\": \"InstructionsPanel mb-8\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e505ac40-0ebb044514cabf8-412d2c3d-164b08-188430e505b26e1\"}, \"offset\": 710}","now":"2023-05-22T10:43:13.576236+00:00","sent_at":"2023-05-22T10:43:13.570000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-582a-0003-cd42-5039c46a0c3b","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/verify?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/verify\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"i2zgz88lw17xt1x0\", \"$time\": 1684752192.862, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e555c203-01d0ef0ffe354a-412d2c3d-164b08-188430e555d12ed\"}, \"offset\": 705}","now":"2023-05-22T10:43:13.576236+00:00","sent_at":"2023-05-22T10:43:13.570000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/engage/?ip=1&_=1684752196089&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:16.094175+00:00","body":"ZGF0YT1leUlrYzJWMElqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR0p5YjNkelpYSmZkbVZ5YzJsdmJpSTZNVEV6TENJa2NtVm1aWEp5WlhJaU9pSm9kSFJ3T2k4dmJHOWpZV3hvYjNOME9qZ3dNREF2YzJsbmJuVndJaXdpSkhKbFptVnljbWx1WjE5a2IyMWhhVzRpT2lKc2IyTmhiR2h2YzNRNk9EQXdNQ0lzSW1WdFlXbHNJam9pZUdGMmFXVnlRSEJ2YzNSb2IyY3VZMjl0SW4wc0lpUjBiMnRsYmlJNkluQm9ZMTlxU0dORWRUZHRNMXAyYmtsdWNHdGllRzFLY2t0RllubEtkV3Q1UVZwRGVubExaVXd3YzFSNFFqTnJJaXdpSkdScGMzUnBibU4wWDJsa0lqb2laMUZFU0d4d2VIRkRTR2RWYTJnMWN6Tk1UMVpWVVRWQlkwNTBNV3R0Vm13d2VIWlBZV295YzJZMmJTSXNJaVJrWlhacFkyVmZhV1FpT2lJeE9EZzBNekJsTXpSaU5UTmpZeTB3WkROak5tTTNNRGswTkRFMVpEZ3ROREV5WkRKak0yUXRNVFkwWWpBNExURTRPRFF6TUdVek5HSTJaRE00SWl3aUpIVnpaWEpmYVdRaU9pSm5VVVJJYkhCNGNVTklaMVZyYURWek0weFBWbFZSTlVGalRuUXhhMjFXYkRCNGRrOWhhakp6WmpadEluMCUzRA==","output":[{"uuid":"0188430e-61fe-0000-6f49-83e60118a598","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"$set\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$browser_version\": 113, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"email\": \"xavier@posthog.com\"}, \"$token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"event\": \"$identify\", \"properties\": {}}","now":"2023-05-22T10:43:16.094175+00:00","sent_at":"2023-05-22T10:43:16.089000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/engage/?ip=1&_=1684752196112&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:16.114065+00:00","body":"ZGF0YT1leUlrYzJWMElqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR0p5YjNkelpYSmZkbVZ5YzJsdmJpSTZNVEV6TENJa2NtVm1aWEp5WlhJaU9pSm9kSFJ3T2k4dmJHOWpZV3hvYjNOME9qZ3dNREF2YzJsbmJuVndJaXdpSkhKbFptVnljbWx1WjE5a2IyMWhhVzRpT2lKc2IyTmhiR2h2YzNRNk9EQXdNQ0lzSW1WdFlXbHNJam9pZUdGMmFXVnlRSEJ2YzNSb2IyY3VZMjl0SW4wc0lpUjBiMnRsYmlJNkluQm9ZMTlxU0dORWRUZHRNMXAyYmtsdWNHdGllRzFLY2t0RllubEtkV3Q1UVZwRGVubExaVXd3YzFSNFFqTnJJaXdpSkdScGMzUnBibU4wWDJsa0lqb2laMUZFU0d4d2VIRkRTR2RWYTJnMWN6Tk1UMVpWVVRWQlkwNTBNV3R0Vm13d2VIWlBZV295YzJZMmJTSXNJaVJrWlhacFkyVmZhV1FpT2lJeE9EZzBNekJsTXpSaU5UTmpZeTB3WkROak5tTTNNRGswTkRFMVpEZ3ROREV5WkRKak0yUXRNVFkwWWpBNExURTRPRFF6TUdVek5HSTJaRE00SWl3aUpIVnpaWEpmYVdRaU9pSm5VVVJJYkhCNGNVTklaMVZyYURWek0weFBWbFZSTlVGalRuUXhhMjFXYkRCNGRrOWhhakp6WmpadEluMCUzRA==","output":[{"uuid":"0188430e-6212-0000-ba3f-5cbd5ec58bd2","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"$set\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$browser_version\": 113, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"email\": \"xavier@posthog.com\"}, \"$token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"event\": \"$identify\", \"properties\": {}}","now":"2023-05-22T10:43:16.114065+00:00","sent_at":"2023-05-22T10:43:16.112000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?compression=gzip-js&ip=1&_=1684752196576&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:16.582542+00:00","body":"H4sIAAAAAAAAA+1cbY/bNhL+K4ZR3Kdol3qX9hAUTdK7tM1dWiQpDgkKgRIpS2tJlCnKljfIf7+hLMuW37Kb3TvYLT/ZIofkkDN8HpJD6dPnMZ3TQoxvxt/hWrAIl6LmdPxsXHJWUi5SWo1vPo+/Y/Az/heORm/fjf4D2ZAQzCmvUlZAho6udPsKyfSQs0VFOST+I+U0Zo1MJHSeRjQQy5JCxitaTQUrZUZUcw7NBzXPICMRory5vs5AjSxhlbjxEELXaTGhlYCGrqHBNF5+X2ZYxIznz0McTWlB/hZznNMF49PnZdJWKwtDfcOKZEaJRVKAMGTu1bul/aZrum5uJWe4mNR4IovTQvvwThapIk5pESQ0nSTQqO9am8RFSkQCldgIQeI8pYuScdHLepasvE9eS1umTM7SEJpZ0FA2Ag/bw31lu1eGTE8LUEsEKYHUctb4TuHe8XBmorkjZL5IZV91x7Nc29B9Cwp6z8YkhX4XUVdu8tur11nZzF6+nnyYJnZlvnn7+4ff7B+ifwt9mv+eoWb+Ft8aVezkW7Zsi+oedAFR0wptM4o0RMzIiVzkW5ZuE0+zdIMYkUk03bFC5GkbcYeYnqyslqP6DVqkVUBozgLw0lsawVDGOKsoVDjhrC5bl+2zxqhrVzN1h2rgCUizdexoMXEsx7Y8x0fS4xmf4CK9w9IltksZPvJXpSxCdc33bdc1sI4s25KaFJXARSRd4qD7jr+AVltTK4DBx2FGSQBdB5MGVUqg8Fp/HIl0ToOY4lY4zvAEevPpD8jaTgtKvMwYJrKn0EAJrSVsMnARy2wnJKc4y6VyIEKJFmVpNE0YjDvk0RynWauHtCiew5NUrdemymCCncj/DiY4hQksZ/vhuVulk6Jup+RKFOZcQBg0K3Xcn54tFq1holW1BQm6NR+lVEZzkJPj8nksYDC6GV2VuAD5KMNVJWHr0/gN+EjxohaCFUEQsUJIqIOxxEJwSJCCUO6g1LNxIZIgStIMvBNalU8s7nRbaREI2kj/YnwkS6VFTUeLFAxRi9EKUqC7YP+BkmHbTtvZe1ZwrEeQsfWkaULCNeYSxwbp4J+irrSSp/mB3LjOMm2FPDs5EQwD5ZRsBqwzTN+Fo8M4OqjYMHWo1jBvo9QwvVdpYB17xzrGzpCTdD4cxB75tQyQkBbwrAGhaK37VfsO8hX5067yNWWkD3Qd2295O/MbmllVVomlnLTjHDerQb0ZwYQrm78/UvMXPCUT+ivw4YnJdUho0KzxUOsdqFFbcFzCcuVezffCT9f7FtBOtt1KPFWDkP7T2iVf9mOatNPmlBqj3VKjrszjDBJnVK7yjrUvs5+4odXyCv7KFC1ix1tdSY56uceZQFpRA+eBqtajvpukabA8PeQNu4KjYyUHKsJKcKCi+TUV38Fy4gWWzt39C2DVugRG2aRoWpISAuO3p2MnMNopun7uC37DKH4jdrVrQ87YDmjsDstuFSEjkuk62kpgvSDXG+2q+6QL6l9gTASDXYVcUCdRcPs6elW7uflxXvxUlNOwyX/mv/wYLn+up8sfPr68W/5C36DqffPCbJcqYAS5TtldHbtujDRkmojYxPBMLzbpqdWxBzKmrG6RFoQtdmvzTWQ7GrJc7COMQqSboXWqOmzY1FntfiaweKfDCm3bjgxkakgniMYojqlpW/hYdSBNdAMYGFadLI4rCusX3fdbt+x3kut2znQbWdWgUskW0O7T7yW3Kr/0DWWE9YYvk9AyxR3CM4kpBzaUvtpPqv3kee4n/2JAHumhAX815IUYE4uQ0LYM92iFIE90oPMhlHvtKmwN5T2und6gKpS/XJRHBmmanMzymZ5G1Sw6gvKwaFMwr2BewfyfBObRYMV+/rEfBef3g3Omz1Mj44i4NJ7OuTwq3YVz+8pDroJzBefnCeePjAIdCrC8ZHmZUSF7e79YypFgyelQSoZ5ezb70PgKHCKGmvHoMMvBSMqpKEur78MCL6NW08cd/z342BfCejSDgAs7GCIZ5D5OM8DsQQFLBVVUUEUFVVRQRQVVVFDl/75Jcx378FEcpxHjBJ6rESwFC2BlKKK2bRe9bVs0C4eXYRo3tnM7ncq5emDbpqtTOLVtO9Nt2xqle3QKWAlOCrJdg+sxBGauWEaDDPp8RASmtQQKaZpdEcUjD+eRwVlf1O2FwYeLFJbsImBFyHBrspGgOB9BKUCYyaguCQY5xSwXzSyTpGE5F+7tEi/ruJLhun1m8U1HMYtilnNllhaPvoJdIDjHWQ1qCF4rovgGonBMZ5soWp9r3X4knTGELWyVsIXcqJ4PH6yuYH+/cQNYXnQ+8lz6wb2xv7/LfdFg70fpjCzmIZrkbiVm7TH4PtjbcGNegb0C+3ME+67zAlfTCuZyLaEI3HWD/IMcOGrqciReAybJQEWfCVcTFQs8mAV0d3A1YOByYCxwW8UA58sAnp/EtjCMeYYXReW2vdxnAE/d2lUMcKYMMOhiewOA1UTDRQGoHrUx/10hOHOqSjhV2uikYP+hsG8PL/4q2L8s2J/YlS5EOA3r2ey21GU4aBf2nSuE1CmPgv0LgX1OZzVghdbqqbVvG6TRyj4K/p8e/p3B2c95vsKnMH+A+aZuZZM7YTYxZ3fZoj6M+TaopzBfYf45Yv5fC6cdnQAsE6gvtHyKTDcijnd81oA4+GpoD2Ba9wax3CyVmCmpEgKXQQz2Orc3OBRmDz/SVaG7BGXJ7cItxCRpPzq2j9nq7QwF2WcK2dtTEpfpdZhmmfwe0ty4htycQh+lq/zzx/fwSGremcqBM/zVuwhwrR7B7FPQ/1DoR8MDmnYWgV/Ca9nttwsV5J8p5Lt5nYCHmR7R04aidhj3IV+dyCvMP1fMX1lvjQZrA/bpU7q8vzW7Mi2kfR5LF4M7zXW9+hTB/aro4OEVjXGdidGvvT6r+3tgwg4kVxeBOp/cHA7B3Ltjhaziw/uXW64CoWV58FTIPDCeYqgHMpTlDT8DpRjqQhhqluKIYy+bU1EWUdjSyT5DKYJSBHURBDWwxzGWOm2cfZa6b8kOGSRGVVkt+UT+hUktr9IHWLqVgQz43J6tGcZ7Hd1Y5g2Caalbrm9/BNF+tNa27MwocMMKli+lUiuC629BAUQMcjdAuyWhWO6pWE7twy6S5ZpiUYdOkpnNHKFmOjvIcroOVwoVzSmaO3+aU/swxVCHGWr4wp9iqEthqGUjGub4hhnO5jM+PXR3VzIU1K0YSjHU+TOU2oipjdj/kOZs/8sf/wUlRV8AwWwAAA==","output":[{"uuid":"0188430e-63e7-0000-35f7-77eaf3880b39","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/verify?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/verify\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"pqx96n7zrbq30v6t\", \"$time\": 1684752194.578, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"or continue without verifying\"}, {\"tag_name\": \"button\", \"$el_text\": \"or continue without verifying\", \"classes\": [\"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-primary\", \"LemonButton--full-width\", \"LemonButton--centered\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--tertiary LemonButton--status-primary LemonButton--full-width LemonButton--centered\", \"nth_child\": 5, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"ingestion-listening-for-events\"], \"attr__class\": \"ingestion-listening-for-events\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"text-center\"], \"attr__class\": \"text-center\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e555c203-01d0ef0ffe354a-412d2c3d-164b08-188430e555d12ed\"}, \"offset\": 1993}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0001-3713-fae840c9d31f","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ca1xryhb43tz0aqr\", \"$time\": 1684752194.59, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 1982}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0002-9a6f-fdc96fab0d9d","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"ingestion continue without verifying\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"02dxxmdqmq1icsqc\", \"$time\": 1684752194.591, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 1980}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0003-c4bd-0cee63372ea6","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"o1vi2lr0d7efkvrd\", \"$time\": 1684752195.807, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"Complete\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"panel-footer\"], \"attr__class\": \"panel-footer\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-screen\", \"flex-col\"], \"attr__class\": \"flex h-screen flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar__layout SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 765}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0004-272b-0bdf0f14875d","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"ingestion recordings turned off\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"wxw6rpbifx56jkkv\", \"$time\": 1684752195.811, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"session_recording_opt_in\": false, \"capture_console_log_opt_in\": false, \"capture_performance_opt_in\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 760}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0005-82ae-014d11c490fb","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"completed_snippet_onboarding team setting updated\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=backend&framework=php\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ghxomrt7jyayufsg\", \"$time\": 1684752195.936, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"setting\": \"completed_snippet_onboarding\", \"value\": true, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 636}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0006-a1b6-c15b0e084f14","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"activation sidebar shown\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"9ciqdwvb0gm7stqn\", \"$time\": 1684752195.955, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"active_tasks_count\": 5, \"completed_tasks_count\": 2, \"completion_percent_count\": 29, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 617}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e7-0007-2bd1-96f9150acdd3","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"89hf5t22vlawns70\", \"$time\": 1684752195.989, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"cloud-announcement\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 582}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0000-38d3-622f0b7db0b3","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"g5s1ttbkbuqqjp1f\", \"$time\": 1684752196.006, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$feature_flag\": \"require-email-verification\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e5c1b2884-08baad4ddb54278-412d2c3d-164b08-188430e5c1d1dde\"}, \"offset\": 566}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0001-26c9-ed81e4d2f77a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"314lgzt3xfrozlwu\", \"$time\": 1684752196.054, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 518}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0002-e82f-aba2ca288e21","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"client_request_failure\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ps0zh0lhjw7ntghy\", \"$time\": 1684752196.07, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"pathname\": \"/api/billing-v2/\", \"method\": \"GET\", \"duration\": 65, \"status\": 404, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 502}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0003-e240-08053caca9aa","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"7muhndo38d1ixe0p\", \"$time\": 1684752196.089, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"project\", \"$group_key\": \"0188430e-316e-0000-51a6-fd646548690e\", \"$group_set\": {\"id\": 1, \"uuid\": \"0188430e-316e-0000-51a6-fd646548690e\", \"name\": \"Default Project\", \"ingested_event\": true, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 483}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0004-ced5-5ad043dcd37a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"qiacra8lvetpncbm\", \"$time\": 1684752196.09, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"organization\", \"$group_key\": \"0188430e-2909-0000-4de1-995772a10454\", \"$group_set\": {\"id\": \"0188430e-2909-0000-4de1-995772a10454\", \"name\": \"x\", \"slug\": \"x\", \"created_at\": \"2023-05-22T10:43:01.514795Z\", \"available_features\": [], \"taxonomy_set_events_count\": 0, \"taxonomy_set_properties_count\": 0, \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 482}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0005-b2eb-104477bc52f8","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"xnwub6hl3xv00xkq\", \"$time\": 1684752196.112, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"project\", \"$group_key\": \"0188430e-316e-0000-51a6-fd646548690e\", \"$group_set\": {\"id\": 1, \"uuid\": \"0188430e-316e-0000-51a6-fd646548690e\", \"name\": \"Default Project\", \"ingested_event\": true, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 460}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"0188430e-63e8-0006-74b6-493aaf93af42","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"yxtxo6923bqvqrk0\", \"$time\": 1684752196.113, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$group_type\": \"organization\", \"$group_key\": \"0188430e-2909-0000-4de1-995772a10454\", \"$group_set\": {\"id\": \"0188430e-2909-0000-4de1-995772a10454\", \"name\": \"x\", \"slug\": \"x\", \"created_at\": \"2023-05-22T10:43:01.514795Z\", \"available_features\": [], \"taxonomy_set_events_count\": 0, \"taxonomy_set_properties_count\": 0, \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 459}","now":"2023-05-22T10:43:16.582542+00:00","sent_at":"2023-05-22T10:43:16.576000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?compression=gzip-js&ip=1&_=1684752239786&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:43:59.790498+00:00","body":"H4sIAAAAAAAAA61We2/bNhD/KoaRPyeHessBhqFpNqRttqDIUgwNCoEiKYkxJWoS5UeLfvfd0YptOY+2QwEDhu75ux/vjrz7MhVLUZvp2fSE9kYz2pi+FdNfpk2rG9EaKbrp2ZfpiYa/6Z+UTa5vJv+AGgTpUrSd1DUoXDJzwxlBedbqVSdaEP4hW5HrNQq5WEomUrNpBCguRLcwukEF69sW0qd9q0BRGtOcnZ4qgKFK3ZmzhBByagF2v+k607Tlsi5SpqtGCSP4r6btEewJWkOAsScqGmrKmlaYdgh0AHJfgev6B2JF66KnBTqJ2rm9QZeOtULUaSlkUUKqeRzshSvJTQlBQkJAuJRi1ejW7GyTAIPvxA/WgY9iJTNIsxIZJoGPQ1ZnYTzzUC5rgGVSyUEaqzLM1T13g1AnUq1QbyRW6EZJEIee50ezOIHYXHZG1mzwK95fXKpm/e/ry+J2UYadf3X94fZ9+Ir9ZdxF9UGR9fKa3ntdHlUHR2Zd3QRKIMIPstBnzCHcZxGLyTwI3JAnTuB63GM+d9woyEji7M0j7icYrEdW/wcK2aVcVDqFZrwXDKjMqeoEBCxa3Te2M3eqKRnyOr4bCQfOnzihSyMn51EQhUESzQn2im4LWsvP1GxJ3nl5czLfegVcuM58HsaxR10ShAEiqTtDa4Yt8WSXTr8CqoMJSoF8minBUygdjjTtJAfnB/yUGbkUaS6oNc4VLaCau0+gOpSlDd0oTTlWCgkayFbqYtQigW/nrhVUVQgOTAR3mJJsUWrgHXSiolJZHHiidAlfCG2HplOULV7Qn8AcC5hTHOqnR7STRd3bgd6a4pByDWkR4+OhtIP4sA0sVLsLxME8opUSlR3YM9hSBsgY5rhbFmDOFO06XE530ytokfoNAz/gjxrTpqnVgu1e9aAZpm/qCmywrexhUEfCXCrcSbWukcOtDGf4HFYatM2ETLwAfjvdulI1phwYWq1Ws5U/g2479SxFFvUQWrPedgeYW453GtpK6pSSc4HMDdutNmXKSqlgfoAX/NL5wN6Wp9SINcKHJhwz1VAs/Jiq894YXaepfJ6ykcnLCI5yZtbTnvIe13MYQHHw5TgGbxzabo7lMHum75ymldUT2lo7ANPgNXakKWnnjKscmm6H8tnaJ08CG0vHsMa6PaixfAfph1jlcjkm8RUuELvDbmCxnFNbhO5EOlT26Fi/5fDz4eyOpDEO3mLN2sFpaTK4J74P38Af+k/Qe7L1/clQvwPLOCXc/KOUwbdS7qPsalN0o3vzOPVgMDk2fBGA/ySAHyZpi8Re063WRznhPfHy3GuOgzlMWQmrG1e/3aujON6jOMCB0Qu78JqSpfeX7KKPK//jsn5TN4tsXb1t3/2ebd72i82rj68/b96JK9L9vT737a0BFOOVcfxQieOcOMT3CQ+5l/hJ7ouXHioJ2PgYbiVrrlfH0eY+CSOHBDGdE0oy4vpZ8FI46oUC+xSenwW8o8Q4YOTywIs5xMuCuSB+zHiUPP+OAnN4vWQhPjF0nncCVqoPV8rXT/8B8mE6/L4LAAA=","output":[{"uuid":"0188430f-0caf-0000-dc00-030ac4424349","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"7lh5fljd145o8ilw\", \"$time\": 1684752236.783, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"svg\", \"classes\": [\"LemonIcon\"], \"attr__class\": \"LemonIcon\", \"attr__width\": \"1em\", \"attr__height\": \"1em\", \"attr__fill\": \"none\", \"attr__viewBox\": \"0 0 24 24\", \"attr__xmlns\": \"http://www.w3.org/2000/svg\", \"attr__focusable\": \"false\", \"attr__aria-hidden\": \"true\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"\"}, {\"tag_name\": \"span\", \"classes\": [\"LemonButton__icon\"], \"attr__class\": \"LemonButton__icon\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"button\", \"$el_text\": \"\", \"classes\": [\"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-primary\", \"LemonButton--no-content\", \"LemonButton--has-icon\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--tertiary LemonButton--status-primary LemonButton--no-content LemonButton--has-icon\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"ActivationSideBar__close_button\"], \"attr__class\": \"ActivationSideBar__close_button\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"ActivationSideBar__content\", \"pt-2\", \"px-4\", \"pb-16\"], \"attr__class\": \"ActivationSideBar__content pt-2 px-4 pb-16\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"ActivationSideBar\"], \"attr__class\": \"ActivationSideBar\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\"], \"attr__class\": \"SideBar SideBar__layout\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 3000}","now":"2023-05-22T10:43:59.790498+00:00","sent_at":"2023-05-22T10:43:59.786000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?compression=gzip-js&ip=1&_=1684752248800&ver=1.57.2","method":"POST","content-encoding":"","ip":"127.0.0.1","now":"2023-05-22T10:44:08.814768+00:00","body":"H4sIAAAAAAAAA51W607cOBR+lVHETwLOPUFarUrZFduyiyqWalVURY7tJGYSO7WdubTi3fc4E4bJMEDVX0zO7fvO8blw98NhCyaMc+Yc4d5IgjvTK+YcO52SHVOGM+2c/XCOJPxx/sZkdn0z+w/UIMgXTGkuBSg8dOJFJ8jKCyWXmikQ/skVK+XKCilbcMJys+4YKC6YnhvZWQXplQL4vFcNKGpjurPT0wZoNLXU5ixFCJ0OBPXvUhQSK8pFlRPZdg0zjP5mVG/JHllrCDD1tIoOm1rg1sKOgXZIPmXgecGOuMGi6nFlnZhwb2+siyaKMZHXjFc1QGVJ+CRccmpqCBIhBMIFZ8tOKrO1TUMbfCt+tA4DK254ATBLVlgQ+Nit6kmUnPhWzgXQMjmnIFUJWSA5/1bW82CFqt7qDbcZenEaJpHvh8kJQsmxQ7k2XJDRr/p0cdl0q2/vL6vbeR3p4Or68+2n6B35x3jz9nODVotrfO/rMm53nmxw9VJIAbEgLKKAEBfRgMQkQVkYehFN3dDzqU8C6npxWKDUfTKPaZDaYL2t6i+w4DqnrJU5NOM9I1DKEjeaQcBKyb4bOnOrctCI6wZezFx4f+RGHo7dksZhHIVpnCHbK1JVWPDv2GyKvPXyM5RtvELKPDfLoiTxsYfCKLRMhDZYENsSB7vUeQBWOxOUQ/Fx0TCaQ+rwpLnmFJwf+WNi+ILlJcODcdngCrK5+wqqXVne4XUjMbWZAkAHaLWsJi0SBsPcKYab1pIDE0Zd0nAyryXUHXSsxbwZeNgXxQv4stS2bHSDyfwV/RHMMYM5tUN9eEQ1r0Q/DPTG1A4plQBrOT4fymEQH7fBQHXYBWxnHq1Vw9phYM9gSxkoxjjHusMC7EmDtbbb6c65gh4R570xUuSwHISxGw1qiY1RILCG4HfQ6tgRps5JzRvoTkC1X7IcuW1Y5IatbH9dwUPMWqnYbFwkD8cTXsUQesjvZZ+XeINi58t1O8VbrNb7YmhC0+sdbbt27YS1K9c231PSY3G3nF4sxewQ7FQ4BZ1ZyNkj4KR+0X799ipE+WJagLEoz55qW6yNmGKDXftzq3LN0KGvokdvodv+dHHXuS+2zDOLCSCs7wlg8BbgDeyAc6xAOP7K4dSsZX8AeTSY7RtOCMAJ+gkCrzX4QYcNk82xkfKNpPdDFJLavhy7sIYRtitgOISTOP6zOFADI+fMroyuJvn9Jbnokzb4shB/iW5erNoP6uMfxfpDP1+/+/L++/oju0L639V5MGwPKLFdHfsHK0lK5KIgQDSifhqkZcBeO1gp2AQ23JILKpf70bIARbGLwgRnCKMCeUERvhYO+xGLN/+GVHBP2TRg7NHQTyjEK8KMoSAhNE5fvqdgDlesiOypkWWpGSwYL8nQw9f/AYmT89LGCQAA","output":[{"uuid":"0188430f-2fef-0000-25cb-b19f141897ba","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"r7cv0okqfhk3x0gu\", \"$time\": 1684752247.007, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"http://localhost:8000/signup\", \"$referring_domain\": \"localhost:8000\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Load more events\"}, {\"tag_name\": \"button\", \"$el_text\": \"Load more events\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"my-8\", \"mx-auto\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary my-8 mx-auto\", \"nth_child\": 5, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"events\"], \"attr__class\": \"events\", \"attr__data-attr\": \"events-table\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\"], \"attr__class\": \"SideBar SideBar__layout\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"188430e34b77f0-0330d5d2838f3e8-412d2c3d-164b08-188430e34b82833\", \"$window_id\": \"188430e34b93056-047a90a0b013b48-412d2c3d-164b08-188430e34ba25e6\", \"$pageview_id\": \"188430e61d427d-04b49e037cd68d8-412d2c3d-164b08-188430e61d548b5\"}, \"offset\": 1790}","now":"2023-05-22T10:44:08.814768+00:00","sent_at":"2023-05-22T10:44:08.800000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?ip=1&_=1684771477160&ver=1.57.2","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-05-22T16:04:37.164197+00:00","body":"ZGF0YT1leUpsZG1WdWRDSTZJaVJ2Y0hSZmFXNGlMQ0p3Y205d1pYSjBhV1Z6SWpwN0lpUnZjeUk2SWsxaFl5QlBVeUJZSWl3aUpHOXpYM1psY25OcGIyNGlPaUl4TUM0eE5TNHdJaXdpSkdKeWIzZHpaWElpT2lKR2FYSmxabTk0SWl3aUpHUmxkbWxqWlY5MGVYQmxJam9pUkdWemEzUnZjQ0lzSWlSamRYSnlaVzUwWDNWeWJDSTZJbWgwZEhBNkx5OXNiMk5oYkdodmMzUTZPREF3TUM4aUxDSWthRzl6ZENJNklteHZZMkZzYUc5emREbzRNREF3SWl3aUpIQmhkR2h1WVcxbElqb2lMeUlzSWlSaWNtOTNjMlZ5WDNabGNuTnBiMjRpT2pFeE15d2lKR0p5YjNkelpYSmZiR0Z1WjNWaFoyVWlPaUpsYmkxVlV5SXNJaVJ6WTNKbFpXNWZhR1ZwWjJoMElqbzVOelFzSWlSelkzSmxaVzVmZDJsa2RHZ2lPakUxTURBc0lpUjJhV1YzY0c5eWRGOW9aV2xuYUhRaU9qZzBNeXdpSkhacFpYZHdiM0owWDNkcFpIUm9Jam94TkRNekxDSWtiR2xpSWpvaWQyVmlJaXdpSkd4cFlsOTJaWEp6YVc5dUlqb2lNUzQxTnk0eUlpd2lKR2x1YzJWeWRGOXBaQ0k2SW5CM01XeHBaSEZ6TTNGMWJXaDJjekVpTENJa2RHbHRaU0k2TVRZNE5EYzNNVFEzTnk0eE5pd2laR2x6ZEdsdVkzUmZhV1FpT2lKblVVUkliSEI0Y1VOSVoxVnJhRFZ6TTB4UFZsVlJOVUZqVG5ReGEyMVdiREI0ZGs5aGFqSnpaalp0SWl3aUpHUmxkbWxqWlY5cFpDSTZJakU0T0RRek1HVXpOR0kxTTJOakxUQmtNMk0yWXpjd09UUTBNVFZrT0MwME1USmtNbU16WkMweE5qUmlNRGd0TVRnNE5ETXdaVE0wWWpaa016Z2lMQ0lrZFhObGNsOXBaQ0k2SW1kUlJFaHNjSGh4UTBoblZXdG9OWE16VEU5V1ZWRTFRV05PZERGcmJWWnNNSGgyVDJGcU1uTm1ObTBpTENKcGMxOWtaVzF2WDNCeWIycGxZM1FpT21aaGJITmxMQ0lrWjNKdmRYQnpJanA3SW5CeWIycGxZM1FpT2lJd01UZzRORE13WlMwek1UWmxMVEF3TURBdE5URmhOaTFtWkRZME5qVTBPRFk1TUdVaUxDSnZjbWRoYm1sNllYUnBiMjRpT2lJd01UZzRORE13WlMweU9UQTVMVEF3TURBdE5HUmxNUzA1T1RVM056SmhNVEEwTlRRaUxDSnBibk4wWVc1alpTSTZJbWgwZEhBNkx5OXNiMk5oYkdodmMzUTZPREF3TUNKOUxDSWtZWFYwYjJOaGNIUjFjbVZmWkdsellXSnNaV1JmYzJWeWRtVnlYM05wWkdVaU9tWmhiSE5sTENJa1lXTjBhWFpsWDJabFlYUjFjbVZmWm14aFozTWlPbHRkTENJa1ptVmhkSFZ5WlY5bWJHRm5YM0JoZVd4dllXUnpJanA3ZlN3aWNHOXpkR2h2WjE5MlpYSnphVzl1SWpvaU1TNDBNeTR3SWl3aWNtVmhiRzBpT2lKb2IzTjBaV1F0WTJ4cFkydG9iM1Z6WlNJc0ltVnRZV2xzWDNObGNuWnBZMlZmWVhaaGFXeGhZbXhsSWpwbVlXeHpaU3dpYzJ4aFkydGZjMlZ5ZG1salpWOWhkbUZwYkdGaWJHVWlPbVpoYkhObExDSWtjbVZtWlhKeVpYSWlPaUlrWkdseVpXTjBJaXdpSkhKbFptVnljbWx1WjE5a2IyMWhhVzRpT2lJa1pHbHlaV04wSWl3aWRHOXJaVzRpT2lKd2FHTmZha2hqUkhVM2JUTmFkbTVKYm5CclluaHRTbkpMUldKNVNuVnJlVUZhUTNwNVMyVk1NSE5VZUVJemF5SXNJaVJ6WlhOemFXOXVYMmxrSWpvaU1UZzRORFF6TWpaaU4yUXhZakV4TFRCbE4ySTJNVGcwWVRJek9EVXdPQzAwTVRKa01tTXpaQzB4TmpSaU1EZ3RNVGc0TkRRek1qWmlOMlV4WldRd0lpd2lKSGRwYm1SdmQxOXBaQ0k2SWpFNE9EUTBNelE1Tm1FMFlpMHdaamMzTkdGbE9UUTRPVEk1TmpndE5ERXlaREpqTTJRdE1UWTBZakE0TFRFNE9EUTBNelE1Tm1FMU1qQXpNU0lzSWlSd1lXZGxkbWxsZDE5cFpDSTZJakU0T0RRME16UTVObUUyTVROallpMHdOak0wTmpjME9HRTNPR05pWXkwME1USmtNbU16WkMweE5qUmlNRGd0TVRnNE5EUXpORGsyWVRjeU1UZ3pJbjBzSW5ScGJXVnpkR0Z0Y0NJNklqSXdNak10TURVdE1qSlVNVFk2TURRNk16Y3VNVFl3V2lKOQ==","output":[{"uuid":"01884434-96bc-0000-a64d-d01794a3cbbd","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$opt_in\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"pw1lidqs3qumhvs1\", \"$time\": 1684771477.16, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"timestamp\": \"2023-05-22T16:04:37.160Z\"}","now":"2023-05-22T16:04:37.164197+00:00","sent_at":"2023-05-22T16:04:37.160000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?ip=1&_=1684771477161&ver=1.57.2","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-05-22T16:04:37.165076+00:00","body":"ZGF0YT1leUpsZG1WdWRDSTZJaVJ3WVdkbGRtbGxkeUlzSW5CeWIzQmxjblJwWlhNaU9uc2lKRzl6SWpvaVRXRmpJRTlUSUZnaUxDSWtiM05mZG1WeWMybHZiaUk2SWpFd0xqRTFMakFpTENJa1luSnZkM05sY2lJNklrWnBjbVZtYjNnaUxDSWtaR1YyYVdObFgzUjVjR1VpT2lKRVpYTnJkRzl3SWl3aUpHTjFjbkpsYm5SZmRYSnNJam9pYUhSMGNEb3ZMMnh2WTJGc2FHOXpkRG80TURBd0x5SXNJaVJvYjNOMElqb2liRzlqWVd4b2IzTjBPamd3TURBaUxDSWtjR0YwYUc1aGJXVWlPaUl2SWl3aUpHSnliM2R6WlhKZmRtVnljMmx2YmlJNk1URXpMQ0lrWW5KdmQzTmxjbDlzWVc1bmRXRm5aU0k2SW1WdUxWVlRJaXdpSkhOamNtVmxibDlvWldsbmFIUWlPamszTkN3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRVd01Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPRFF6TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFME16TXNJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpVM0xqSWlMQ0lrYVc1elpYSjBYMmxrSWpvaWJuUjVkWE51Tm5Zek1uWmxhbWN3Y3lJc0lpUjBhVzFsSWpveE5qZzBOemN4TkRjM0xqRTJMQ0prYVhOMGFXNWpkRjlwWkNJNkltZFJSRWhzY0hoeFEwaG5WV3RvTlhNelRFOVdWVkUxUVdOT2RERnJiVlpzTUhoMlQyRnFNbk5tTm0waUxDSWtaR1YyYVdObFgybGtJam9pTVRnNE5ETXdaVE0wWWpVelkyTXRNR1F6WXpaak56QTVORFF4TldRNExUUXhNbVF5WXpOa0xURTJOR0l3T0MweE9EZzBNekJsTXpSaU5tUXpPQ0lzSWlSMWMyVnlYMmxrSWpvaVoxRkVTR3h3ZUhGRFNHZFZhMmcxY3pOTVQxWlZVVFZCWTA1ME1XdHRWbXd3ZUhaUFlXb3ljMlkyYlNJc0ltbHpYMlJsYlc5ZmNISnZhbVZqZENJNlptRnNjMlVzSWlSbmNtOTFjSE1pT25zaWNISnZhbVZqZENJNklqQXhPRGcwTXpCbExUTXhObVV0TURBd01DMDFNV0UyTFdaa05qUTJOVFE0Tmprd1pTSXNJbTl5WjJGdWFYcGhkR2x2YmlJNklqQXhPRGcwTXpCbExUSTVNRGt0TURBd01DMDBaR1V4TFRrNU5UYzNNbUV4TURRMU5DSXNJbWx1YzNSaGJtTmxJam9pYUhSMGNEb3ZMMnh2WTJGc2FHOXpkRG80TURBd0luMHNJaVJoZFhSdlkyRndkSFZ5WlY5a2FYTmhZbXhsWkY5elpYSjJaWEpmYzJsa1pTSTZabUZzYzJVc0lpUmhZM1JwZG1WZlptVmhkSFZ5WlY5bWJHRm5jeUk2VzEwc0lpUm1aV0YwZFhKbFgyWnNZV2RmY0dGNWJHOWhaSE1pT250OUxDSndiM04wYUc5blgzWmxjbk5wYjI0aU9pSXhMalF6TGpBaUxDSnlaV0ZzYlNJNkltaHZjM1JsWkMxamJHbGphMmh2ZFhObElpd2laVzFoYVd4ZmMyVnlkbWxqWlY5aGRtRnBiR0ZpYkdVaU9tWmhiSE5sTENKemJHRmphMTl6WlhKMmFXTmxYMkYyWVdsc1lXSnNaU0k2Wm1Gc2MyVXNJaVJ5WldabGNuSmxjaUk2SWlSa2FYSmxZM1FpTENJa2NtVm1aWEp5YVc1blgyUnZiV0ZwYmlJNklpUmthWEpsWTNRaUxDSjBiMnRsYmlJNkluQm9ZMTlxU0dORWRUZHRNMXAyYmtsdWNHdGllRzFLY2t0RllubEtkV3Q1UVZwRGVubExaVXd3YzFSNFFqTnJJaXdpSkhObGMzTnBiMjVmYVdRaU9pSXhPRGcwTkRNeU5tSTNaREZpTVRFdE1HVTNZall4T0RSaE1qTTROVEE0TFRReE1tUXlZek5rTFRFMk5HSXdPQzB4T0RnME5ETXlObUkzWlRGbFpEQWlMQ0lrZDJsdVpHOTNYMmxrSWpvaU1UZzRORFF6TkRrMllUUmlMVEJtTnpjMFlXVTVORGc1TWprMk9DMDBNVEprTW1NelpDMHhOalJpTURndE1UZzRORFF6TkRrMllUVXlNRE14SWl3aUpIQmhaMlYyYVdWM1gybGtJam9pTVRnNE5EUXpORGsyWVRZeE0yTmlMVEEyTXpRMk56UTRZVGM0WTJKakxUUXhNbVF5WXpOa0xURTJOR0l3T0MweE9EZzBORE0wT1RaaE56SXhPRE1pZlN3aWRHbHRaWE4wWVcxd0lqb2lNakF5TXkwd05TMHlNbFF4Tmpvd05Eb3pOeTR4TmpGYUluMCUzRA==","output":[{"uuid":"01884434-96ba-0000-1404-a179647bd08a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ntyusn6v32vejg0s\", \"$time\": 1684771477.16, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"timestamp\": \"2023-05-22T16:04:37.161Z\"}","now":"2023-05-22T16:04:37.165076+00:00","sent_at":"2023-05-22T16:04:37.161000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/engage/?ip=1&_=1684771477165&ver=1.57.2","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-05-22T16:04:37.167862+00:00","body":"ZGF0YT1leUlrYzJWMElqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR0p5YjNkelpYSmZkbVZ5YzJsdmJpSTZNVEV6TENJa2NtVm1aWEp5WlhJaU9pSWtaR2x5WldOMElpd2lKSEpsWm1WeWNtbHVaMTlrYjIxaGFXNGlPaUlrWkdseVpXTjBJaXdpWlcxaGFXd2lPaUo0WVhacFpYSkFjRzl6ZEdodlp5NWpiMjBpZlN3aUpIUnZhMlZ1SWpvaWNHaGpYMnBJWTBSMU4yMHpXblp1U1c1d2EySjRiVXB5UzBWaWVVcDFhM2xCV2tONmVVdGxUREJ6VkhoQ00yc2lMQ0lrWkdsemRHbHVZM1JmYVdRaU9pSm5VVVJJYkhCNGNVTklaMVZyYURWek0weFBWbFZSTlVGalRuUXhhMjFXYkRCNGRrOWhhakp6WmpadElpd2lKR1JsZG1salpWOXBaQ0k2SWpFNE9EUXpNR1V6TkdJMU0yTmpMVEJrTTJNMll6Y3dPVFEwTVRWa09DMDBNVEprTW1NelpDMHhOalJpTURndE1UZzRORE13WlRNMFlqWmtNemdpTENJa2RYTmxjbDlwWkNJNkltZFJSRWhzY0hoeFEwaG5WV3RvTlhNelRFOVdWVkUxUVdOT2RERnJiVlpzTUhoMlQyRnFNbk5tTm0waWZRJTNEJTNE","output":[{"uuid":"01884434-96bb-0000-669a-d0fa1bef0768","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"$set\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$browser_version\": 113, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"email\": \"xavier@posthog.com\"}, \"$token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"event\": \"$identify\", \"properties\": {}}","now":"2023-05-22T16:04:37.167862+00:00","sent_at":"2023-05-22T16:04:37.165000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1684771480232&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:40.235238+00:00","body":"H4sIAAAAAAAAA+2dW5ObOBaA/0qXK4+howvXfssms5u5bXZ2kqmtTE25hBAGGxAGgcGp/Pc9wm5sJ+5sestT7Sn0kIrR9ejonPM1SIjfP85EKwo1u5s9iwVTTSXmccYWc86yTESz57OykqWoVCrq2d3H2TMJ/81+Zvzm7a83/4FsSJi3oqpTWUAGRrfYuUU6PazkphYVJP49rUQsO50YiTblYq76UkDGa1GvlCx1Bm+qCsSYN1UGGYlS5d2LF5kEKRJZqzsfIfRCl9NXUOA0R2eUTCUFy3WzQ8F99wfZMKZHyRkrFg1b6OKisN7/qqvUvBKimCciXSTQSeDZh8RNGqkEGnEQgsQ2FZtSVmos69u68TH5vrRNdXKWhtDNRoS6E7g41tet490SnZ4WIJaapxGklmLT+m3ieSu15q1f6HyV6rFh17c9D8O/W+zi57MorVVa8H29xS+v32Rlt371ZvF+lTg1/entb+9/cV7yfyq8yn/LUNe+ZUtSx25+NBlDVezDEJCgduhQzi0UUe5yDwW2jZ3It2xMIsJpZGHXDpFvHYq7EfV1Y43W6v8hRVrPI5HLOZjZUnBQZcyyWkCDi0o25WBzY9YM7fu1KHaFBTOPLAcz14oj13Yd23cDJKBNWS1YkW6Z2il5rEUCFOxq2ZHAVhA4nkcYRrZja0mKWrGCa5M4a3+zTyAVaxQkloOfgPJZCE4yh6HDlM7rNILK9/IzrtIWnOnIqWA0v/8BWSeOVrI+kyzSI4UOSugtkYsTE7Hp4FGVYFmuhYMiIrJ4lvJVIkHvkCdylmaDHHpGWQtXWrRRmjpjfPWV/GfgoQI8ULvrswj8FdQ9pqbFYh5J6EGLc5R7PAzIqUWtRbYqUQtlwQ89LCin5ErommXC58s3/HXj5fRDW3xflKuwy3+ofvwu7H9oVv3LD6+2/Y/iJ1S/6/5GV4NH7po8mKhNiRt6EQ4xtpDwQhf7NiPUd8Amz9norrzAIhqCxCYtIrk5bs4OXGaHFoo9z2YisP2ABO5DbenCDkEU7wLOAvxHfNGaiymHBl1qu57tM8/nIf9Kex7BPtWmJeMY9Da7o8j1Pj0/CsuDI4BpFSqNexOQnyIgB57Xl2i5UQQnxYYvHgjIrgnIJiA/YUAeZujel+8naUxfif7bZ2xfZ4hIH2fajOCvjaYZ7Okbm9g7/2sRsyZTN/8a5QHxhdbYfB/iVNXAiPd2N2pA+9dWFrqJ9+9eHZnDXA24KXQeTJDBy2PxQgxergsvpbvxky5s1j0Xy00VGLwYvFw7Xk50/hBjvj4BXzLmW2vuHV+HmDprNA30T/BcprHCtOkQRKiFHIuQdxjd2fQOge+BIwXOByg6auR+vvZTpVgnC5n3Wqgdnuo5l42OlBAHTnIPcfKohGHUn8Koa38ylUiwx2+l1X3hvzKxQsqWKWbRIkLI3Xrrs8QilBhiGWJdiFhcFrWEgJ1B30AkWUWaU6I4M8DdDcUYOg+ld78OPndabdYOdn6BR2F7HVl07/4n6gTYlDCWw9AMHR5FBxIErqHDNdOBNhEtVCPJmglbbbwH6ABtGzoYOkyODjupLDCLvFS14cPl+eAYPlwzH1pnjSLlL6jjLRvqDC7wJR9AOoMHg4fJ4aGQSoRSrgwZLk8G/3Rp3ZDhysgg2n6L5NrZkl4G22wIvWfIYO4cDBqmiAaeySayWFHAggMXuY5ihhEXZ4S5e7hqRsQbXottR9cbsqz6qjnPCMesPRhGTJERoJ91ZrBwcSx4ZtHhqrGw2GZq0ybuWsUxdeX2PBbMJiqDhWliAcYCHg+vZbByMGSDh4viwexYunI8LLu2wA3KaObE3Zo/gAfP4MHgYYJ4EKzKeotxDhJY+zxDictTAhtKXDMlMkXiREYq7TbuKm5aQwlDCUOJkRKxEFEI47FqLuDNBsMHw4dp8SEmjpPyjb2sk0UW2Ho2zvDBN2sPhg9T5EMl1g3kWcOgLeghjVM+viJoWHFRVthmQeKqWeFWbR9Tr6g8D9myXp5lBUyuYYVhxQRZcTguat+zlRZxWqRKWBn4gwHGpYHhe4EBxjUDo28Q8sMO4y2mee46BhgGGAYYXwFG3eQ5q/qBF3BtkGGQMTFkuH7bMrlssNMsENmePynWIMMgY5rIGHusB0ZYLbH2ghpW/NmsuO/IAOJJX6hbrgjE0UhtVU9owc4DwixYGEBMAhCTCuqeb7vUscDzYofhOAgi14sfbs/zHUKcwTYPQf2zRYbrPSx2cpF9bfOaFQ6raNN3DT6/odVGJrKbyD6FyH56Ou04TZ+dTHt+zsZC+6Noa1iE+FqwmdyRrpcgCTkhCRimjuh6wwCcBT+PwdZ2O44NUZ5uc1PTeSlZ+UiRpl+z5CxRXGKIYogyBaIcuzcr0xdhmmV61aEl+oMDuQCFabv7x3fv4DJqqv28Uxu+lAFTqhrQrY3AmQ0qHocKF52gYpzpG/0M7yYWiidm3eFpUbGMCtalXom7xmdh6Z5FhReYc5oMKqaACq2e+c7+KYbwv1+QPtaRefj0aA7Y1JzEdNUUyNrS7VAXLCqOcG6fO8fVv0WeeVvOUGAKFPh89TliilmiKzO5uzeADa61dkNzsOvFUYGdkxfnIlYnoWRVdKNNVt86DGHJwOIpd7du2NIJWhpm/jqo+bnXIXz4yJ1hhWHFBFixj0s/62dKNfRRaGX5g/nfx67vwfDgXsKg4HEoQK5/jALdh4huRq0aCDwlBDYhwbRa1VHHaxkm9CwEAmQoYCgwBQr8jy+W0lvXcwJC9RdLwZrqhFXwyON+ZGVaFPpyJ/XQlBYtl4NM0e6z21CzZnkJ44c17Xz8VCk4mL4+urwXJYSvm/YwDTAfu4bHyDn4gH64xWr1bxHDPYt+DH8qt3uHCLjyre8TLTQMXhd7qaMRtmHhUEFIjEYpIIRUQunNRCD3Pm14ehYL3nOQ+ZAWNzDYbKxpuPhYLmJKyKc//gu3Hodi74cAAA==","output":[{"uuid":"01884434-a2ab-0000-af34-b060321fb9d0","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"pewv8vh77ktqcv8n\", \"$time\": 1684771477.161, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"session-reset-on-load\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 3067}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0001-adb0-d108b4f12665","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"977yp0jwt21hnwcg\", \"$time\": 1684771477.166, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"project\", \"$group_key\": \"0188430e-316e-0000-51a6-fd646548690e\", \"$group_set\": {\"id\": 1, \"uuid\": \"0188430e-316e-0000-51a6-fd646548690e\", \"name\": \"Default Project\", \"ingested_event\": true, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 3062}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0002-1714-12b5f9fc4941","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"p6w8hxbuqycejwr9\", \"$time\": 1684771477.166, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"organization\", \"$group_key\": \"0188430e-2909-0000-4de1-995772a10454\", \"$group_set\": {\"id\": \"0188430e-2909-0000-4de1-995772a10454\", \"name\": \"x\", \"slug\": \"x\", \"created_at\": \"2023-05-22T10:43:01.514795Z\", \"available_features\": [], \"taxonomy_set_events_count\": 0, \"taxonomy_set_properties_count\": 0, \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 3062}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0003-6004-77b16abfc46e","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"b3aji1adgd006z7q\", \"$time\": 1684771477.232, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"posthog-3000\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2996}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0004-6b5c-d8e7d6e97bd8","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"3ud3ntuo2qae4tw7\", \"$time\": 1684771477.233, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"enable-prompts\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2995}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0005-3e04-fa6fc146673a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"v5q0dt8g357ju35s\", \"$time\": 1684771477.24, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"notebooks\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2987}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0006-5a0e-177fb26860c5","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"evyz0oq5z2yo9zl8\", \"$time\": 1684771477.243, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"cloud-announcement\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2985}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0007-dc5e-bb723cae292a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"fwcsezx3qw2jryru\", \"$time\": 1684771477.252, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"hogql\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2976}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0008-5f7e-6efafcb7b2be","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"gzltwvh6qtff36oz\", \"$time\": 1684771477.266, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"hackathon-apm\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2962}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0009-c180-af233b44b000","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"gjxvn1u0l3l5fxqc\", \"$time\": 1684771477.267, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"early-access-feature\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2961}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000a-bd12-c96d4565d278","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"lt2fhodtixw6kfuv\", \"$time\": 1684771477.267, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"feedback-scene\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2961}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000b-02bb-7a705b509049","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"f255icw4jshgl94d\", \"$time\": 1684771477.282, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"require-email-verification\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2946}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000c-e137-db1cc0161182","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"6rvyf37nr7704osj\", \"$time\": 1684771477.349, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"session-recording-infinite-list\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2879}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000d-7bab-cce00f8e396c","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"yu008bx11z13mm65\", \"$time\": 1684771477.349, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"session-recording-summary-listing\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2879}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000e-4ed9-e08a5a7ecb60","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"68vvaoju15ug02zn\", \"$time\": 1684771477.349, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"recordings-list-v2-enabled\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2879}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000f-15e6-c0c8a84bd8d3","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ejk2l0xdtzty23na\", \"$time\": 1684771477.382, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2846}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0010-a367-b217cfce9758","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"q4csan5ar3uyxu1c\", \"$time\": 1684771477.402, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"instance\", \"$group_key\": \"http://localhost:8000\", \"$group_set\": {\"site_url\": \"http://localhost:8000\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2826}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0011-ab12-d83075b735b3","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"client_request_failure\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"fux7i2k80t2uyqah\", \"$time\": 1684771477.622, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"pathname\": \"/api/billing-v2/\", \"method\": \"GET\", \"duration\": 341, \"status\": 404, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2606}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0012-7b5a-642b99f7c343","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"recording list fetched\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"jdnaxi7p1xu8abp6\", \"$time\": 1684771477.793, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"load_time\": 311, \"listing_version\": \"1\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2435}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0013-9ed2-da153edee668","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"lvp6x0x9grc01m4s\", \"$time\": 1684771478.077, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"data-exploration-insights\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2151}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0014-b658-f962d9c6de38","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"dashboard loading time\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ywaj59v3bl8q9scj\", \"$time\": 1684771478.16, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"loadingMilliseconds\": 816, \"dashboardId\": 1, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2068}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0015-fdde-825234a6f71f","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"viewed dashboard\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"wb213rksdxcsobh3\", \"$time\": 1684771478.906, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"created_at\": \"2023-05-22T10:43:03.675923Z\", \"is_shared\": false, \"pinned\": true, \"creation_mode\": \"default\", \"sample_items_count\": 6, \"item_count\": 6, \"created_by_system\": true, \"dashboard_id\": 1, \"lastRefreshed\": \"2023-05-22T16:02:16.882Z\", \"refreshAge\": 142, \"trends_count\": 3, \"retention_count\": 1, \"lifecycle_count\": 1, \"funnels_count\": 1, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 1322}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684771480251&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:40.254254+00:00","body":"H4sIAAAAAAAAA+x9i3LbyJLlr2A9M713IkwKb4Dq7ZlrS7JlW7Lkl2xr7oSDoiCJbb4uScly3+iI+ZaN2P2K/Zr5kj0nq4ACRYLiS7Ye6LaIAlCo56msrKzMrP/4x6PkIukMH60/+udBp94bnHWHjx4/6vW7vaQ/bCaDR+v/MG++HNeHdT4Zfu8lj9bDx4/SB73W+Wmzg1R63cHwrHu61kmG37r9r391mFr9e6tbP2Y8G1E69YvmaX3Y7Hbwznm07oSxH0WOH4Vu7D5+5CLK2XDYW19ba3Ub9dYZUlyPbdteQ3Tv0br9+JEvv4H8hvIbyW+MxHCpqYuDzOSKPHhBwo6DK9KQK1LxgwABJKQCSCu0ax4K4SDBNIhUr5QZGUgR15wq6+ci5dDxwghBvKq5CHjINHA8BPCy5jOAfNUrVXxPldxTRff4IaP5LHW+Rf788/GjYbOdDIb1du/qu8fonGQwQLm+NNHAKGzs+54bHkXHzpHjVOwkOgqd2K+7XhzYccV33GO34R1XnNA/wr2JnzjJsY3K/PO3Zue4+y2fnF8L6/5RxT6JIr+e1Py45tbCorQYOXBt1B/F7n5NBBRnjS+/bzc2z6O2d3jRedHpfT26bL/sv9o6+v7y/Ov3J4cbf3x/lezYg/eXT72v+PK4ORg2O42hKsbpm83tVu/y7xvbpx++ngUDb2fv4MOb4Enj9dD52j5o2ZcXe/Xf3cFJ2H6EJumenAwSQNqNvdqfj38CwvvJoHvebyRz4Nv1/DX08LDZWEP7J5fVxmCg8B7YKMVExMsboFNBXiN+FPC8qDFDuPOCNII4VmDnvYKfgLzV7LDtBd4a1hqZLPcoMKM4LgQm35XAvK/A/P0GcUmyPI7LQaPf7LFtJiAzAt0ukfnQkdk4O+98rQTv/M2P0c5mDqKYbgsgijeLQDTUfMIcEA1Aba9CtDYFouiAEqJTIYpBfEchevh089Phh71nNwpRfDcnREMy1SVES4gCop/fbvjh3sdXNwnRCAudeSFKbqOEaAlRQPTdi41P+59ePL1RiM4/0YflRF9CVEP0w/sPb7d3t4ISoiVEbylE/e0d7+3L17slREuI3lKIHr7Z/eBGnw5vEqL+ZGHoNIj6kFOVEC0hSoi63rOtt89udkXv1+aH6ASJfQnRhwnRNy/e7e8/eZlfLnGOnQxRvFlIdM/AnKL7cYjW7GKI4l0J0fsK0e3tV5+3dl59uEmI1uYXOsXju0slRB8oRKNXT7Y+bx56NwrR+Sf62jgvWkL0gUL0yZNXh1u7uy9vEqKL8KIlREuIaoi+2t45wP5ndJMQDai1NydE/RKiJUQVRD/522/fHjzPT/SgeQUQxZsFIBra8/OitQkQdaZA1Ckhem8h6j3fCg6fH76/SYi6wNucEHWcCeulEqMPE6OH4fudg/1P+SX9yjEaoX7zYpTqUSVGS4wSoy823ODZh928xmhJR0uM3iqMvtuIDqOdd3l9vFVjNAjmp6N+SUZLiGqIbn6IPu+/3r9ZiC6wqB8zpysh+lAh+unNc6iSvN+5SYiG88/0NDstIVpClBB9/eLJk+1X/vaNQnR+25AJ5kslRB8oRD+/fLWx+/L1kxKiJURvKUS3Pu692//4bOtGITq/ymjAQpQQXSFEozsLUX/7/Zb3afNGqegCktFSMFpCNKOiO29c99V+eKMQnZ+KhiUVLSGqIfp+/8XHj/sfb3SiX0SreYKyUwnRhwnR50+eH7ze3subL60corX5IUo3OyVES4gKFd19477dfnuTO6ALKd6Xyk4lRDVEN/bD/cPXBxs3CtH57ZTjcYckJUQfKESffHi6Hzzdz1vYrR6i8y+X4nK5VEJUQ/TZy2fh6/0dt4RoCdHbCtGt5y8O3715e6OK9/MvlyZZ2JUQfZgQ3Xn19KO3v5+zsKOH8ckQ5ZuFIDo/FSXhvQrRYsfhfFdC9L5CdPfVvrvlf3xeQrSE6C2F6N7O3qtnnz/6JURLiN5WiO69DnfebJUTfQnR2wrRzb2dg48bh/s3CVFnAeMle4LsvsTow8Tok93wxd72Rs6W/gYwuoARqJxdUmK0xCjd4+193niyu5dz1VxitMTo7cLok4/P3Rfvgs8lRkuM3laMvt5237zffJ9TJSkxWmL0dmF09+nu852ddzmf96vH6EIOH8o10z3HKDYRZ9UaffZhe3Nv89ONYnR+IzvHKe1DVozR8JZhdA77kP03n5+Ebz7eBYyWdPRhYjQ63Hz1/vX7FzeK0flF+I5TyvBLjKYqT683nrx59fxGZfglHS0xupTO06a7vb/5NKeWt3qMhvOf/O0EE/TySow+TIy+PXzx8p3/MedW3CE6JmMUbxbBqL+A7Mmb4N/Jm4JRr8TovcWov/9q+7Mb5lyQrR6jnLdXIR8tMfowMbr3dPfZJ8/NK4+uHqPznyDiyCERJUZLjNJp86fP7s5HL3fiYonREqO3C6OfNt3Xz6NnefloidESo7cKo+H7l0/397yDm8Soy8C8GJ1gElpi9GFidO8g2Hz/fCfnE7fEaInR24XRCL5JPr99mdfNWz1GF9EpmeD+vsTow8To5v7bnfebbn6fqcRoidFbhdFXLzf237z9uHeTGPUWmOu5zioxWmJU9pmehB/ePn1zo2umEqMlRpfB6MeXTzd2t97fqOzJW2Cud8u5vsRoas+0teluHh7mPUCsHqML7Ne75X59iVGN0YPDD29fBs/y+qMlRkuM3iqMbjx//mnr6YsblT15C+iUuKVOSYnRdM301D18uXWYOwi8xGiJ0duF0Wc7z8Onn9/e7Lp+gf162peUGF0lRoNbhlEsimfdC/34KtzcfJ7z4Hx7MepPwahfYvTeYhSnd7559/nwXYnREqO3FaOH+wdbnz4d3qhOSYnREqNL0dFnwcFbN3h9o/ZMC8jwvQky/BKjDxOj4ed3Hzf9VzfKj5YYLTG6DEZ33u8cRk+e5d0431q70BKjDxOj0eaL3c03L/K2yxBcFWAUbxayr2dgToz6E/jRYApG0QMlRu8pRj+/ef56d/tlXqdk5RhdSO+pxOg9x+gcdqF7u+8PN97n/TivHKP0OTL3XD9BN6+c6x8mRvfe7+5tep/z8tHVYxTfzT3XI1BitMSoyJ6eft4+fP4xL3taPUbnPxLc8Sf4eyox+jAx+vyT9zp8tXmja6YSoyVGl8Hoh81nrzafe3l7ptVjdAH/o5RXlRgtMSpnNrx9snsQvczr5t1a2VOJ0YeJ0a0X77aebO3lzxBbPUYXmOsn+R8tMfowMXr4ae/d1qftpzeK0QX2QikLKDFaYlTs69+9e7/3Jszv168eo9jXnBuj5X59iVGNUdf/cPh073Xe5q7EaInRW4XRaP/Dxv7hft52ucRoidFbhVGcIbYbfdzPn3O3eowuYHMXlHpPJUZTHxCu9+bg4Hnej/PqMbqAfDQo5aMlRlPZ0+bGxscPezeq98SjIefFaFjqlDxUjNq2nWK02eh2Bmsn9QsGKsfJRRWBf7/4zbVddKhfcWspaClOnwxavAHa5ApgyZXAVQEDXXWfpkX4RjHlUUgvqmkBvwPcUBca6aZBArk7PEv6KY5ZnzWnyqbI4dmNxuRUqGchnvmuxPN9xXO912sllWH3vHFW4ZNqr3P680Dti64+Ma1DJaQfFqR79SYyLNKdPmn2B8MKUDpEsU7OW5U0OlAV0rstsQBIXEFCLSxGAt6VSJiOBLAzt4q4FSrSk7gdJ43mcbIGEub90uz95vzy5bfs28gJnV8ukv5vTjWIqq7CTRwVUTZ5AwIUR+D/ACy5krKpgKFscax5TRVAYjVb85sqQMrGg7+EsEmAdO2y3WId+snfzwHMAgIX09mokLirFI61KcK1vCtxfU9wPe6I7NWLlxufXhxmCyW0ehGI+WaBhVKNh4vOu1C6qkcSOVGhfYi8KyF6tyA6x1p+w326t3fw6g5AtHApL+9KiN5XiPru5pv3z3Y3bxSiC4ibrkqbSog+WIjuvfz08vXuZ2MdchMQnd+CiYc4lRAtIUqIvt9598H9HJrNzxKiJURvF0RffN799ObJobENKXnREqK3C6LBk8PtD5tP35S8aAnRWwfR/X7396Qx3O62k179NKnsPXvhHXzeNufWxzVqw00GK978oCl/ElhL8dODAGtuW7OfNLr946RfuaDuPfYB8sL+ml8EVHkDhNVoaAQwyZVgVQED15p4fGBaEqCwnwRZQFvTzslqXujrfcw0OA7eK4L+mm/TDo9AHtMtjVy3GMh8VwJ5KpDd+PYCeQAkn/eQfFLVSVYb3faaxquox03EK98sQFgd255s2XSSDBtnkwnrmH1oBBgV4xHvSjzeKB7Rg+mDs35yUkgSzzBjI9FvzePhGe186azzLGmeniHb2OfxBRN70HdK9YjrehCNt0wPggCkDzrdYzxIX2DINc6arePXeIoU/iN9jkmoU0dvsqvbLeZzftRqNl6wFrgbfB8Mk3Z6x7q5LGCW2bB++nrk8/pw2G8enQ9VMVv1zineoU1Ry4n5j6aRgMhdSaPoQ0BumFwON5TSBj7+W8fS/+mSiqbJhFyGzWGL8F0sm5Rbtf77v/63tY8hsd091RlGf/6nCsgRKoXF1LFlrE4oXjtB/11tSN1HF83kW6/bJx60tgoeyjD8DYqDzUZSkZvHVrPTHDbrrcoA4zb5zRlrfl3Q2iwF5aidvaSNs3pfwPzofHhSiYtydqBHMCVrk3kOb+B3RuNZz5TW5MD6xdp7V4E+VzuxcH/SPD3vg2vjmzQdGVjXVlU40wlVbTU7HOWjVe0nLby6qunGUdP8gzFAjexL/OHJVHI6p+JcUYvm8D6lhjlszlpDXSv10aNmG+hfQ6lyFfXcS4+s8OzVTPVd5cvZ6xjNVMeiobVUHZ3wEuzNInWUL2ev4/RBqchcSoyLhuaUmmLW6A8b50NLV3n++kzUUS6ojTvjOHeL5pUpNWnXB1/TITd7LQb1k3q/Wek1O53kuDKsH1UHF2OqqY1uq9tHev/07JkdeBuFtZuJqrgLUJV2vdM8Uepbs1Rt0ASf/y05Sj+brW9mohhuEcWYNlEpKtbuHjVxQbkqeFBJJ14zdaXzZ1EBZxruiutdoIDNhswRFXk0T7GuG6EjY9QrGqPZsn0xRuTdxtsX+++/7O882dja3tvZ3HqbZuimfIh33eDLDUAvB+UZiinfpItV6d3zAYQj0DqUleoKahFktcgNn0KCmGvvIrwWVEST/Bwg1n6vX9R17BVUJM4qkoPzzNR6+L2VDM4ScFRTKEFOqIrFUHJZbQwGRej1pqM3ZaJzLeoXIpiFK2pQJrw2qRyFmVdbJwBQ9xTyjYH1Dws9ktT79U4jWbc63U7yqyXM7boFWcO//GqpBee65fcuf7WOlGyuXz9ung/WLY/PGuf9Qbe/bvW6UOZO+r9af+YzWD/pNs6ZTfd8iPbP8lAprVvtBEm19cPRL9dB2P5AXh3IrIf9euMrUhnL7AjPT/vd887x1JSv5H41I5DOr81hZdCC8nEfDO55+2hCy1j/o9nmsqDeGZp2caQR0iaTm7QEdu9ypExZuwVs2HzJ+6dHf3GCx5YL6Q5+wn+9tllHGucWl7dV/46ESPOh65/0rXE8INc0crfXhX43XoLgYWHV7SCnpIXxdpH1mYqSGhEgKiQCzGQdazGmVznCeP36ay6F+tGg28KgAQh69UZziKjOr9ZFc9DErCm3Em4hAlDWGZx0++11FUTWyV9QvsdWxcEFtTzqXlYGZ3UIQqQR6n+xH1v6X9UN/1Wa0MVfgD+pmKSTlqTVQjRvgLY6ajYqR8kfzaT/F7saoSUrdtV2JRX+1CIkNZhc5+pZ8/g46bAJ0/rYo/VREYqrg3rkO+do2Ml1UK5FT1oJalBvNU87FTA+bSChgQJc7Vx8P2lknjRbrRQp3mMr/UHmKfiEpFxJqVpvsLuRoPk+Rvuov6zkN4wUg3V8DxZVlcOFRb6V/qAoeTzdOuj06sfHzc4pyfjVAbZ6JNX7/e63kd4wbflHRebKdTQEqITKed3626O/PWKbDIddJFqp5WYXme7WLXzfPM4eZnMSWkd+TK1aycmwIih9bI3e8zYtGp9qYtNXlPC8AwGKTkMe5RPJPZD7NBn9raSjkswnw4kW1e1h2fePjKw6KK1URoFIkUz1I7AYJbVMOOu7PK0OJc92vY8NjDzVFIBindX4Omn08rdy3MS2mh4l3W+/Wr+fQ4p68j3FAdoa/Y8Bgb2QhN2dzhW1q7kMkhYS6vZJwK/JCB+ct5HWdXnlUsfGSQ/Tz+hgnVKNEyRZodwAbYyRU0CqxgowQsOu5jwY9rudUxRAt7p/tQlMVOnn/yCT9dvfHslM/LdH/4kvc12U/eUG4dRkOpjJk75KR5cguNrvxYliWUDRJYnn1aaZsbdma8ErXajhKgNPI1gzK2lxUkApDI8AeoQwpeJ2U3sYRxfHOgtyza3ppm6VXAOQSvaTdlEi9Tza9Jwg4xi2aF0l3hxlG8cSWF8/SkAcOWnl6Nt//5//RxKniqfoyUjjXE2GDzvdIVO5WhydxrDbS8l5bla6Orte5bv63dbA+jekeZE11khp9KOxzpuUwro24eQ6w6SWI61TPm3Vx7/MU9M/uQwavJO1zvqwf57oVVG2zPadLHTt0ipdUhWtto+6x9/HV1QUbMs6kVtsC+yIZKt8fyaRj7/YElpWfLm1M2Sn/caiehUFq1e/aA19U9IUv5Z1bk7ONEXsk9+B+tGinyDDZDBd9HO1oEVwxPiYLPnpd7ErOdOm3oQkFv3uUQNjlbL4J50OeOAGRgXmqfxNpaI4w4WLNjGLL18are4gmS1RSJMLE91J2t3OCyWx1rvnj5ykTdGO3j7Xt1xj4IbUHXecrJ52L/HAtmzL9fEPT2Hn22Gienh9+/at+s2rdvuna66MLymHrGrrWERSFaPeQiVQtn6znrYTRi0o2mw169VR4KtVIxh2nZoVVn1nx4mqQc0KMLVjOmCQT3EfqIB6hpeIITHVU3nkeBKWx3yCfyqsniOCiv6HaR2s7PronQ2R1U+iF4N3B8/zJDtQY3nscSahC7It3KCIBHKPWtxGTO7h993e0/p4aeaGn0rny5dBciogv3Jfkbltddmc1dtH5/1TVKzE+GSMtz3Lic8c76LinlUc74+2XQnOHFvd2ryNLlz1/o+2E1s1gLfiVYOYPzUrC1UcgFj9cFzg/2UQHfoTER16mSpC0b7ahJ37K5hodU+7120/LQ6X0d5PgRJXHTcYBYsrt6PwcDzbkg3nmfAhkdYvRco+KapTq9XW1OuZqjNWmSWR5bhVL7JjEjw3DuNGRS5gg7wIQHHsIEzDXs0NLLtVqYZxSBB50NJEdMeuBfjIqblpmBpXlo14tdBrjcRm2rWKJGhJ4lF2o1OX6LyV6EgvlKRVSCVsScI54F7U+3+pVI6w2DyG+Oo8+dfZ0Fu0c3d9W7lB1Q5rSzSVc11TObmmQt+YpuLNeFMh+nhTuatrqtyuzawEqwq9twBNhfI4fq0hdfSCauzFPsmQ7RNzVc+N4xgVC2y/BQIFP3MOqRWujapfs70af/0qtPzS6GwTxPb13RnpGj9D8/outJbZmhGcMPmVqufHYXaDS0BC6WHdG7sNNA0k9OjH2LVBD20/qLou+jKwg1aElENM+uqKpvVD/AU1JOinsSRw5kPG6KGorBopA6uFAqKSgSTFzkQhWxUHgQBaqirAsrJycsGztBlQTel/3kkmQtmrUYwOnlZgJ0Cnxy7YFBVYTZFrVcdzHRRPBbyRUuOSL7YtqM/KjYKrYqM78iW/Wm4Uxo4wt6mrdLkvP7p3mXS+24OLii4/ehmJol+RKgqELPHATaFU0SmmV112xMkVHUnmSo4pUxUcmUIV1Mdgg2rpzN9dSLliFKywthiebuCCj+Rlcl1lOM9Z2YLR/T1ptbrfZhvfUdFSccr4Dqu1WhBwfLsOGohjBe2tLvlWS+E9gm4p+AU4cteveWwy3+NoDWSUyoiV0JkaNmwsDG2SM76Sga1iqWF9UanWPM9nMzEe3oOyRpGHtoqi0AO43Vh6y+YXTmR7FbcaANJ4D9IBkuBHfogIkWODt0Kr11ybc1IU2iHqEsaRD7LioLQB0gyCCOq76kYuziD/DN+ZVyg6vvZkoDEmI2JZIWEdcaBS1A91ivquoG+/Jt9n7NjcQnw6FzH7BLHoRBCgJTyhY+wk6UM362pQKf6epcRcyBMnMDVTsA81gQL2UwqlI+vrrEQVLO9MTVcklZjCHjjVMPIBHIz8ABjELAyU+fo6F/2cRE4mzFNBALVyzFO8zlitInWw4moxR8wnXAyDpM8+eak6zFOFCzQViGM0Y1WUks7Y46JFx5SOA5sRhJQEAIXBtZOLouD6stIpQjVD1qETB38/OZ5x8BdJEa5DMAYrIBs6AZgFzw/9UWYBzZCNxWwo+hjlNHKQy9IFLxKzXl9wNfTCi6CKzhjh8qa19EUljS8dj9kj7Xiy67rj80vlxaqlJD1jzHVRbWei0ROngnkHOPbQXYgJXEzCgReBCDu1qMZG8WxwjC6a1gkbLiZZEDfbQic7RHYEr3/6BowMCB3mSMStORwEMdTC9OOQsMgihWdIF0sbMOMqn0pctb0AfYBp2LZdZIds4gYmQDJ/BB3MGKshTMxUkPOybXMdCDgGIK/VCNO7foqcTZTZyEg8P/MThtWoFkVssADO3huoCPxpo0RosRgLQPBEbuild0Bd6KImqCAqWuNbeJVXj102chbJRRQHizzEVY8tlVJ6pyIhH6YUqpe+eprmpu/AxQDxZHvUPZCNEYKFJzQhnADvwB15SECi6DvQZnwP8Rf6IYaQFJXy3PROx2GjIxkMDqajnkLLNxeHNMEPuPhMS6ASSu9UpBm7poh9Ke6aCGKMWixYlq4BmgBFQS0WAlI49py+86pgTUDZgE2HVDuo+lyeoow1TGb6Drh1pP+qPtg1EMaA7CWW52wi8J3qBtyN5AMWz4lRVTZgFDvC0Qeh05KCgI+p1RwKWZAKOU4s09kbTiy4gK2Yj+zcyA7TO/SxH4YCnlCvDzwR5sldNibBZYUBOx80i/2A4usbkHAuii3wSCFFCIBTFea/5E89NFAaZkWEx60JvYtg485yqVJypLdwyxogR7BnSIaMbgDIITvMlT4xGeAhFk6BR5Ej72bs5vlnbLAYkJuhquxBl1TaFhwGke2jdbHqRZWw/pSaoEXBcrKSoG4qrPpPeHpQGdUs6Y2MA5IjNCmpnx/WXAhW5Xqmr7zH8oe3tShwsX0od2pw4E7i6jtkH9hc5ajHHEFRLUYLSRciG3S+vquGtohtUNSIHJZjc++hGgY1dB2KOCtJm593JUeONoHk2IlD8G6Y3+3AEWIdglxxMtAP9OyAlbUTRmGYmy1qmC7OdHx9h+5BN1xwHS6xZyz//EwqhF4YgWbckyRzYJAqyihKCbI/kR5nBDgXRVNjTYw5rDJarInuKClOaa/KZpQQk7RcpcMkiRkZFqIzRoVHiPBEGjxCgkcosCbAgvuU/upcZiW/RUaM0ySkNuZ6TAVoFqzyMIsjb6kNJJgYlzHWEeARZKWAyTu4kHkEYyMlpXDowmmeRAj0RlbhiiIp0R4Yi5CcWQwaCLqjOFD0N6lTROYP65MqvBPoz5AX+FP0EVDogDSBj0lv0LfSzuQJSQBjH/sm6kZkP2BIsCqAaFXGuIeSxhAcgLYEYGiAcTWXRJzewDmyeUP0aHYTe0SYj46SsoCsqDB4UafmD2Ru0DcoVxhhK0ZugBd5B7zwVvDCCSXGOlemBE6zxKu+oTQYhIwP4yidlMkP6fdqQgKXxRTUQxkOaQxpQLBWBAuTTfmDLItZsRJOXoxN3vQMVWyGijA2beMQSqjflt9+PO9AL7Y/oKEqFMThJ2DpFL9V4L+9Bd3H0YQrUHKd+LACXazj5KgOfaez7rcZ9QagZsbdzWn7n6JqaYKVCpPR+Y4819YAI1GpydqrcyewYhR+y53ZQoqHiREC7DOwFFgHY2DhL2pUaxQHytjHxg1og3BLXAPpmUfmHYxpbBmlfwP+kKWTp5ATqxsJ6D/QU2z5yKRDGXENpAPLLKbcQq7I/IKlCPAY4lKyusLvgqZggDc4nrFIIXkj7RQZN/8G6Q15t/RKro4UUz1Mb9K3y+wa13Lc3phyUKoTVFN7yGPfFi0IRF10hlHx5UsaU6uN6aGvPBdU+PkEylA5aSYtOgeADl8jOeu2qIQBTSh5Wa1WidM6hKV4VmjsrvbKGbLTUJxpd8WZEkhs7PiL6rqU5oboFi5P6ah2YIlCbEXpw2pLByhwiqGDqNIqhdFeHwbN/e/QGh+qB+4lfTXk2xsKns0Tbf82qCCzoZAJidI7qzTqveF5P5HYUpTKeacPTRgqhua+a4BQEUP2CuhoajaRWkWwOiUNnEADJ1EB9JzoZaGszrbjb4AMchuYAmSwIGBxwdB67/hanjLarktVrAPX3vYOnNoOVbQOHGcjgICuZgljzXhYyx4gOXI7IG+w6rDwGIm9Yw58BF7qAJE2wPrLJ/iWj/HPOXCiHclkFy+RNH5jRoFsysJKHvty79QTvj1w4m0pweFsJG0yl1Mr0jC4fgp/Wj+Glw8TBKfQprVL/kkbXx6PPEkNTagTrUc6veHQ8HnasBijwnQaoZxvZMrFMHvMCFihFuzDgr6SNxGXoklVWWpWxM7jRAw5dqZVy1N95ifUSat3haJqojv6RrVMfQAF/HP4TZqjbR5ir3ORCtGQ6N9R4w5eNhqUkMgSlXTKB/uHpZc9YJAQSf9RmsgHeC1BXuUfVD2QZkOkjUhF6+vFlXigAhaCFpey4Pp4owJ8RiURB1Iv7sSTvaPgEVJLyz/jhga40Cp4TQuL0oFrCcnkvgiEG4wX4BJIPHwdWLIxjl+KRJkc16JIiv8vB+7Jq0PIBEpScgOkZPJGk2NnvCU0NhagJe/A8O0rAx0opfdhlHWFpHDaAZt5jvnn9BSttTSbOZIjrLzRJAmN+sgO5qa2fRgIdehyylIuw6zmYDAz+Zo2F1/J0mrPuBCeVqedZAgGGSzxV8zdaRCMOZfZCRcZaa0urf91Cac3Sf+vOR+K/zb7LP4pncXhfipzfJXNJlByMsFCU5FyGC4+DOETa/LzbCkIoXU2uWceR8KizpgGqhedCwyV3e5xvVXs9GwR/uEdBFRYQVr6CsXs+nc4KVjF0NYJKvcSK0wwdaUzU4rnY34DV1cQWmvAAnT2Eat9UqfjFm6p0r7Le7K64r1l5eVXKwphDD82h2eszhMxiNVrjQlvKliRHycnWCnPSCDHDQC0PCZjSCeUJp8/BJV0SAnBwuhTGvadD1Khw+g77URh5BkFtsqMfvQ5WGHx7jX+lCLb1PFXfvKTiY8iEbojYTfSUSmdNV5nxiAOTVck+lUFhZSL5VvhVJWflhaYgeAlK0OyYUIKnXvNXs+5xvrUZLVx80n9vLVEcks3ucjN8oK11ZdlrJs2Va2tK+QHrq/SnoJ/rixoiBI8fGXBTDrg5B2U3Th6Z01zlFxh0qGt/uR5XKqT7U/hxGZTs0X2qqYVgpREE7yZamtW6z+adHa6OJZY/BmMPU9dyUwknymH1qofKY9i35rwyG1pCrmc3OEm4FTyu4vzu9oTwfhzQxvg/i0LmikBrumyIWZoTt695XyjbbN5QZbSan+v0EwP3hbUTg92Netw01I46uFUoWSy5qAUWGpjU+cK43QTDNVxfXB21K33j3k0y1ReKotZ0pAfKg+FiruPLeIaFKehYyfWkDG1l6BzSLuugFqK0I0JoYsd1ULRUfFrsK/B1gzUfn2onEBxhjYGUCZzoV/DA0aovANrGaikhLaLaF6N6mdU8oE6Dc094GBfRJSesjyAQaUtqjfQDKQiEa3/qHNci6DTZiML6iviJoYaG7WHarHYXtXgTIc6PyhxQJmoHULDkjVwYxvSUirCK4VmKCmjFDUERDUy8F2WF5qDtA6IbCjRcFueVp2270IJmeJdqIUGkL+0VEvodoGtIu5FQwB3MByCYYUoYQbQkAshRxZFIujYwxqUSk4RjTpxGBuEtaicmIDaPisJRbMgoBYkaymGMX4IQz0qBUFxh9v2vhPaEOhCrQntQR0pG40oak4weUKxqSeKJsDBGyg61COpGOtxvwy3sN1AzAhGO6KUDO3ICEY80PhkD0CDDXqwLn8jdGZE0w4HhpWwNLdhEwslS9oCBPwKxxycsdfQvLA79Byo6EI5m1pg1LOH/kTNpkYtssN3UUATrQB6pjAvgKaXF0PZOmTUMISJcuxBNZYqcoIuqQQaHQ3EO6rrQ8eLdhuixhmGNFYI0DBoAICFH/qUZ1Oh3YMCOlTPYdDiQ7PCQVzgFDqKsKCEoiIACD1Zz7ahdEE7JWAPqhYOTDnsmh2JeQ/1x5CSmKciNIAymMDYDyN5imrgMex48RUe12xoBaPuNCjwYISAVFE7JNuizSa0OPAVhkmorlQH41Oan0l58EPDNxYIRVOf0qwfFmMqbZUNsoTaL42PVJAFofKvC8VXKRkKicT0K+iYsTb4FN8zCgtoUl6OGygQQsOHTjbf3rn13/jSKT8zKUf4ZlEEDzxZVe/Bogh+erKgWQTmnTrd30XRRFZniUURKokNC6gVkdHR3uBHuJwCydKEmBNXUFeWXfvynZX7rmSP7uISSzsLGn9uiCp8B2VBI7oIzLrKzwWLlljlgufnLngG9Qto5sLPLUB/7Zoni1eO6R84pgMsdmDfclGrxrCJ+QNW0lxt0Hwxhm0MjQagmIGHFk2I4gusXPhkmcFf4G7JMf6WnEKHS3eHo3phUK+qZEibsRqAXdrd56fgVScLGmHYQmq/q+SnfgoRXRUrpdFTSFg1rZyVpOIgzBkNTUqyuipJEtSEzyqhEEz4g2AQWnb4CaFvF57BgnAJEgrvF5OfGxIK9xFZ0Czl4CEnG6AmCJeOK+WfbiX/UjiW+gldGl83lFQsetpdzaxSjqU5bGKt+CJuQSgGr1ZwGClaohDTUbkTmqBUCgWroowvlWKoBaNln0abrBBkqNQapfGu0iPFgwoeUOMV31AW6/MlPlluUE7Wnne0vwwJFqqc/2S+xppqijjG3LxNh4zUCnbvaQXh6SALGvoCDy5ZsIjheRCk5iRBoqAyrfrptcshHfeLilwSnB9JcCAoD2FNA7E+pMgwvRZLdbgEcH3O5xCow1AbLym9xh93OfAaEm+8r4TQo4cTBKrUw7tARO36iN4EWlVU9AwOGRxbvBEsRWu0ed34czPSCq3e7xiteabGgfVMDxqpnDE+xHaBCRpKC9cLWXDF0qG7RXKSS5x4jYPEO9cLYPJR7ye9gZUIDEpuH8XZrdENDGydsWvl7cD3geNvY2OX/+3A8i97deBu65gHvD3cBaWi24Rt7IB5eLpB5wyMLDaG3EDG+Siw08FjbrTiJnZ2sFEpCWKP8wB7ktGOjV1Z/CdGhP4GDpxya6g/NibpHhjfgLORtwzLpwzDrhC2h9h35HM4dnCx4JH4fCgp7dAYEekfIKMdbMXR9Rry30CQxt9iP4SnMX07qGocLkMSteb5+HNDCWCNfS9I4tbImFZVM6QPdo1Z0EwHMFfPgouIZhbXBt/kWfoqa4wrU4oHvehMLmaiyCrWnSHGQlaN03ne3S3Wbxe+8+HziLoFUW0DDJ0NLRTwfXDgA1JGehXDI5EEySN6GxIEWYPbPBUbGkSQ/ahkoB7hRztwvwO3GZCvu8EGFDWgx4IPfHjHgyIFbLThtkYHVYoSFA+GeCqx6XiLCUMfgjc1ZydfSkwCcERIIqwKjQuc+qWlBjWW5FhsFVa5qLAquP5ClVwnpoqO3OAaTZedTrnpqUcVHnfIUpVehXXCElblT79QFWBiVJ+SGoyUmfOYnGvC76Gyow45gaMrelbOMpGwzkTCOhP1AR1ziUm+Cm1wSqElvXqMBsjiS1inI2GdvvpAZ5yWB0XD/kgWX8KYWukemWsDqQBvoJZCX4NoVT/e4SsmEkMDKuRHMmUiClqZtvtQKUJDMKSSRQhNTIfKKipUZzidqlRCHgjDxIGvKN5Q2dJbP/pPigOvl1lY0jvchTs8MBHyAab7EA6iQrAGnNNV0kQT85NaswTSOyyUiqXLqj5V1dhRaaoabuANSq/qTs8Duk0kqFKUoCqaRFYlzhVsmcnehcHt5OfZZI+xeD8m+x3a+myl04WqWjbZY4ibYDbZw4mgCa7Y4upuTbN8iuP8Yb1Ebul6rVukYaLfuen3HshedoVugMhsk6AdcDbZxiN4Pan6pGtwuwdfLLiBe5Wqu40HB66TEmlQc0bewFW8tMDNgCbrvMKjCl7RUaV4q8SP0CtPctqAwz8q8ZIqy0tcJV+8ETrNBZNEUXNGmp/OGlFVMV1VboSXom/a1Hn8uRnYsHu+F/SNywNrNz9GVfUMNdMGyBI09B3G3hmNe9BLCawFB/AbdR1xS6OV1OyHSpKtMN2H0k5LcE6G7EfxT+9E6U0svSFlw9mz8ugpuuE4sQ9yY9gLQAUb6uPwqws+Fgr1Fnwri8sUnMg05jKFjjnhVU/9L95PcFEB7QZF5STuPummhbr2Pi4+z1/jVrW4UhEtfpgJ4M1ynlNw8sJkguaaUewsr1t8Kwjavhpo1i/WczjD6GVcm7EJg466CeYoumHrnKKNwgdB0Rrds27/eulIGq2kaD+QolGacGVnnRRNbahnpEzTOotOpWCmwZ15ujOH1QjsXsS0hZZODugZd+PpzRM79/RoTNseWpUwktpZ43FgsPlIQzgmiQZGONID5yVx342HXcDMRvyHhvhHoyYXvkK5oBVLpblKS//vPt1bgwekf2YxtpK8aTsFcxT+iWUYLbrEezMMn6TAWJbzzDZYILGwdBJYhdw9RonowhScKj1ku9gT5GYhnY+2eSYaPeTjmAh+IW7tSXVFQ4He3nEdiGEQNxr5oAp7MVpbqZLJURLKsz3dUUc4ioppO9R1UC1D7/E8+Ek8IUs8ZkfvzBAh0XUyDnETH6xwdM3ewG4nD/ULcUFpxESLRmiB2ITxzKiQXlYh1qEHLxuC/bAB87UqOWZWkdZUaHo616/EMJcTv804KAlyA8tBwzNQQbDCtoVxGsJy5RPOZZjKaO5Ez7JqQgK7jqkMLv0R4P9iJYeLCuCC/5eblyYrbcDntqHQ90RpYyOjqlIt4wDBNQ4QXOMAAU7iTfBBa23UcYo1z1mfgcnORy2npR84LbnY8bwgzY1waCJJKX+o2YW5AdRQOGVy0TBEvKo5RjeCo5pjLTDN+CgeUxfDB9TeAIFFNEkTOp6T75YhStqRwPhzMyLhzuNeECWeEG/GllTN+FmAKa0JGoJsHLq4cOjyI7cyn/QMOw8vC2kp4DHiAZPHXuv8FIrm18oflK+UtTrb8H7SRqy8b6d6Cc3bKTbYID/o4UgPhKGBgUMnYc7PI0RgXw8xADb26K2Am30QdcJ4HTHB4+HcSn32iUSXc1sQg0RTomRxsxRVSvKLlLKHOlr2nSRk2Rsm5Sw3kz1Ko1My5cwKbhJTYZSLgTTB9Is0WZWaqmH6zMRJv1EJMXC4Cw44xD5wgCzwOMTWlYrN/UtYmWNDiyny8AQkCSN7SVL2XnGiFGzcZfcTO52MqW48xmOQJyYhcvYRFh41kyKnsgA56+xQZjQIz/lTJaGISPbaIm4Yw92CxKHnb8aTCPIBzkFCT+jUeAq1pKazwlFDphzqRsqXBqXc2Ue6UjpFVeMsO90cWVmwg4wfSMclKWyC6m4SJSLdfelHCi06RYMSnR9QoksiIRSPR9erMpvIukY6KVVdk5FujKwQbClVMjZg2vPyOIVF+o2CjE4xxUqanWBFl0XCUj4dkjJnX+ja6ORUXbO8dENkBeG6TtpNNyA7JGtC6Z2sbmn3qJpL55g2TLsHCvS6YxBC+6UwyiKmOFOpCARN82UI1QVQ8GUpGMqaTx6nVUq/UfVVCaZtkWbGdkpLwjA3bkQVQkqaxk7roFJSFUxzSSufFkEBN20xjl5puTQTeZ3mnn6rxptKNy11mqdUKS2QulGDN00x/SBNVCWmqmuepXHSb5iOarvRtLNc09pxpKnE0tKa0ktVcvWK1bBNU0w/SBNVialqmmdpnPQbJqDbTrCsJo08wjPkZyOB6SqSasZMNoZyoE4HHaMxlI1F+cYMVpWiIvlmXJuBrka+GmuYPHLUIaUYORKCYZYmlVGbNCNx05NSJVUyilF0cbMxmY1RpqUrqnPKWiEb7ZoGpJMHmy8PcQP9PF5UombIpENIxlNucEkZdSgdiul8mY1VM1+aQZ2Ncj3s0cl2hj0SB00m0ogKLCoVQ2AyiiODLEeMpFwSMgMxHZh5KKt6prmZAZ4OeAw81XCqDRXVG5mesokrm8gMqlMk5abA3IQkNwp6uUlUT6ojWFYDzkzH2fSsp+t0GlcjeHR6zyb+EdDoFDWisuzITWSMBcpGrStd2mw6zaZXpqPrmU3EuhGyiTqdwEdGMCdfXXbCXFcrzUHXWU9I2USps1AjVaef3rDSEkxTTT9Kk9ZJqikze2pipV/p1NQkPJqLyT8tD+ufJmnKrpJWinlpDVUJJaBSTKOnierEdJXNUxUr/UqnxEl4mbW89hM0/tys5Qtdz9yxtfxTxB4kVn6RbFwHwU+aCZq1vPHiAVdjP3Qtj9BJ8/QcrhQhfMjKaxb1cIBUOhW5jU5Fht1uC2cNturnHTnxbKrUQUe+pwKHnyqMLTzXfSqRnFeuy1OGqYIKkS5UEfiHIwprOKIQrAU9BgY+zl7FWYFgHvAHv3yw1xO/f3R2x403CHtxaDX8EvKH77mvxQvjyMajj70nOeqU0elCERtvstkW8dxl6oGhjfkz2xGiOIJ+/vNmsYcm+tjis9Dn5igKzhLLD7cwlUhbjlSWLUE5AdH1WzxBh54B8SMf8YfHOscubBSxiYdK8CVP3GGN8I+OXDyKyXVscRYpP+J1kmwadxuxH8gf7iX6SEs2AOVHmkp+eGiw/OCMaN9pMToaihuJMaZTOVbawadp58j+onSA6iGRAOE7U8/J3ac7nrVCcth9hP8p/MgGKk6glgdZJ0mN026TxzN2m3aIMN6dk51VucZZlQtnVbdzBp9DzVjIKQ5aVPQybROjqAKfPFlwedXDn+21xoXnnSxoeBC4Giu9AC7nukbPuPDZp+fnqbO4iTaL/z8N0nJSv8PO/+BieDI9NX6kXONHCud3m6BZMhk/gLAsetCbb2pXbYATTjByZ96Ey+KXQ+jHuheHP20aE0A5ip6V4YLaBosWwuqKR9bHUJGyxUU0rvKArBGZwRBqWnDfDUZHOBxfGFp8BzYYPJlP98rU4nLwId2Ou1BSg7toudKEG2/kcGzwjsiAvDNO0KZGHp03k2+kL4tINOjocjwG5+hRZwzZqhIxU5Ux/hDFhy4FTcVsG654eOIisoD+F5XN4A+d7CKVx3QC0MpDQci48gRI5iFZsBAVFoIqgVQuY5WYGVORK3lc8MVSNR8qfgE+on6eeJOmM3A2EzOkF25f2kpkkUxTtAf5OmQbkE8cIFE0rbph9iY2OoY+r8Hys4I6NZUTMkezolWRPdqUv2xRS5VWae2hSXVl0KC0q6SGHk0KdVWztmAJkInkoxoKioCqFaURoQ2HtsBqR7UxP1S9IDqBFFopLUFyxrqndFeiUOSZ2c0QksXyS9EuteRYBTgSZ/tptEjhcWoxQcasFJjAT0MLRhQLmSBaEBWhlIy/kUdw0kM7LrxFF8u58BS9wQW+qnRM9pta6wSty6KKR3uNHIgloenIrmTfxNCBpE4iWpSpcf0hqSu5PMtLHUC6e6oC54Q/bwXk8p4foOWYniSqgBPjKusnUZZ38CEVGrl4UsmzWCyVPKJpj2AfZaHbcRQY7uWZusY9FO1RWzpfV1f6RydUUQW59VrpB0hDEsjSy9KX/ESHlb2OSGgEKS5GP9sLypUyQJhark5pPaviRD/ACsurITd2PZ22q0aSVRsbQD5A+6N7JU+64m9RH1OdV8AEpSjsEdNDGAqi04r+Q+fT47q4mFcGBSiHrDEFn9qyQN8Y+4L0AWwWaIKQMzYYXLE2MJpX6htRq9JBdwDn+RLgef1pSKvlZvYSy3EdBRaj2pWeBO+Jxag+/8rKMQWqfoargiOyLGi4KuMz0M35DISJxvxroqPTyrczcChXz9M/6vZxpE1lBQcv9gAJnVoLWjc6mJ76NHJk/zEPKbXQ6R2csZ+oM/1/Px9gyfK9cpQMvyWY4WdbnS1+NtnW0w/PrTaipb1hvKa58Jo2uYV/5EFZFXCls60I80e4z7pILPnMH6nuBZsxzAN0HABWgRts4s1APZU1nATUM7wUs/70NWcYsW1lWB7zCf6psHqOCCr6UjRZeysbf25kM8aPlws/XllwEZI0mWAc1zunoEuK3Er4CrmYZ/+rDunEupWN8LhoWC9MRi4vklaru9ao9+gTDa5L/36eDFDw83YvzdW4RnONazQXDuNW1WLf6v0OaYI0mb5ZuM3eJhfNAURlptXgqGjVrYbNeJzv0TjiqdQqD4Mq+ITKgsYMQvvylOOGc8GiDp1ph5Knf9NrZpFEEg6s5k+9XW92KlCpTcnv8vPq0z52UuVU9BmlAzMm9uXLUXZzRZxhXlTs5fNc1WmvOKTIQGMe29TZR3OWU8YLePDXkWVaBLeZ5jRrpO3NqYH3bKqzuV5w4CUHVoNYQ9ANE2c+Ln/pFsjNnmMRH9R20vgOvPRgxSxBN3u6lKqHp53PjD9fRLGhYOSM9mpu4Mip0N2ONHjh6HJ+2im8nnFUg6PPlsK1zDttvD8WllVuW/zwHuF6ZdJ2Tx/NP/48Wwx6cKZU0pk7RGfg+WXy80W44rlnaPcH0JBtc+S8qlnGr3naM44EzQoaPi8XYG8VhapkB9wvL5NAKhVhOSqQ2CwhaJiQyUzfnTkzlG0O3zyTablhib1ClnjqWl7Kgrwb/WZP1O8KeGLPuAfBeZuLdDEzUnKSGbnZVYlcBgloI1ZFU2UuV0ZZHpA44uUC8jOIburtSjtpH8GryKqlKNetGSYcasQyWWlx0k4yo9M4NvHg2GSFqhS3SxFy4tny086dn6bZKIekpSf6DCo4qie9STVHfuIpP3ND5HXyzTKnEykkGFZjBb4UfpTmUDHi0hiVCqDME9oLyZfxpuBpRxMSLOJ/75qS0dzDYAkdoxlHzHG/2zvufpPKTVIoKuXQd3ERA28Hk5+bmQeeALKgkaYYzyYefJhkQfMUbgYX5Coog53GusBZw4KjfFNRFav9vcIO7Xelm8zKqjDHRTikYbdXaTUhQ+YAhDARZA1uYpp/8K61PEecJr98So1uG1QXS32kpkXRq020QhOSWRM98xZe3HzunvctdY5Y67vFQYwjjXOHpEpfwhgm7VY4mpjcrTetqWaY1J9ybuvcrMcBmtLC1mTWhtnOtgcTqyxoaEah345pkNFzfzZcViGwHh307jyD3thSeTA1W6A6DWQF/VosWVczTrlpJG1U+V4JV9Q4774mrYTghUNFMPvHFrTPrG+iNfkDpBE7XTEt++//+r8psswR8vB6boILH3l66+pnRo45fNuD0u19qZ9hxrV1hwQXGT+3sn7GVsODrUYWXEQ+eDvrZ0BpjDdw6sC9qZ8BpdF/94z+u2fsaj1jV+sZz1kePGctOrUVTjVQ0F+cvywZwLyIBIzfAMJgy/julhY2yog4keR2c3035E58aabPaDZ6RrPR09qfElxELnnLmD6jnuJBqbNk+n4IUTaqaJ5RRfOgxnlPJh19pqgEzcgpPEb0ztXPKMbhZEQTXGT83M76GVCaw/68hQ77u531M6DUZz1K8N4sSqBjllXK8H/mNEfPHK7tmcO1PaOp78GyctVMnw8luJLpWwHT91YEfpT7oRI5qwxfH2UmwSIs3xKery/HvuPCqtw2zs83Gk2+UTbzzalo/kJ6eLeL8/PNqUd+4alHJee3WsrsmwOofKOJ5EMT6X7MPL45eAYe0Exw4Q2s21Y/s9/nG4Uiv/C8obtXPwNKo4DjL66Ac+vqZ0BpDqXwCw+luHP1M2oivlET8Y3ajG80HX2t/CkTgHlqrAY8OMqfv1lSFUjY8QzOjrr1/vFcfNG0pE2KSjFy1TNpQWe6M1qSrLorzVRhlBN8o5zgQzlh/u4RRVL6+xhUyHqBbzitY+K9YxqNOT2dn848bpzR1s/KwJn1nxl/5sQJXx/IIUFDgKBc8mM1PHwI+OfP0dRx6QG31e4Nv2+uLr0iXD9+NBh+lzZRClDrlmf3k/avy+eolpAZrehVoKF0JpQ/42J90dvGoVRSOgk3uq2V6XGPriyY380SQmfNY+inUEOjnOBrP7gSXDVj6ay5P62K2PRP62U25XyzKecXeve9dmCkiNNDQ6NSDZekcyxjxVmBwaQeE9q8NzX45bhL27niXrYs2q+etLrftI5gwfBAwXLDV7QZ1+Fc+V9+tdKhHNv/ssRAvkoY4KZkmeSuh5emDj+RPTSOJn2jmOEbxQzfeEPzC72h3VsAhOF973+jzeEbbQ5fuyuVTjfUBuosD6z/Pe/e978Z/0YFxDc6ML7RgfGNw1bfaKb50HlcGBUlq/RDOtn4XvLNjo6/0I7ObWWVjBKGbxRdfCi6ZOBdRCKXW0NkTJGG5QonGbhWC90aiFhK0OB5/L7THbPJ6BuFE994EfOhazN7f11t0gAuEmPvoTWpwb3RIfGNKzC/0BXYTE1KZ5CB+8Ca1ODRaLD4RoPFn0uDZYy7gEdM94G1qNEewfE6JmioAdRnlgApPPzH0QMb90Z3BQPUBA01gPLO4k0Kt6I4uBvroAfUpIHxzuQbzRnfaM74c/mEGGtSOEetBc4Da9IMjwF0JbJgRg4CbDMt3qQh3A3CDesDa9IMj4FRSgmMUkqQU/8x+sq+MQDwFzoWa1RQ/+XLCfbySnF9uQadDlaj5xIYPZdg5XouP3ENGhhVl8CougRQdcmCC+/PluL6UlxPBBltnMBo4wTQxsmCGVMdLOZb+y4D4N6L6wOjwhMYFR5wPiZoqM1COkx3uv/vvbg+MHo/ONzXBM34N05JAqP5FRinagF0vEpx/e1mlYyeUGD0hIKFPLDcVlYJfkfSesFRRhY0iF3Iw80PEdfj1FAX55M/rKWm0bAJjOeTwHg+CebaBRxrUpw85TwwsV1glHsCo9wTGOWeAApOSwhEcHIS3EQ9sCY1eDRqK4FxqBJg23oJsR1O0HXdBzbwjfpHYLyZBEZrKCg8nnamJoU/YyjAPqwmNWoUgdG+CYz2TTCXL5WxLRCcHhbHD2zgGy2WwGixBEaLJYAWyxL7dDwO0HtYG3WBURQJ9KGq0o6GHECMvARKcVxbXIsfWJMaPBodlcDoqARGTwdU0bDCOcmp6Qnte1+k+DllrJz3SXPQEXYI52em6QH2ok43lPqUGWvsCZyoKpeeS3fbWNKZ2YXVG8IKtHeJHuzhkIkVmIFOyqvVHSRftLnK7TlLu8jPbXkc2m1xQ3tXjkMLtBrQ+HND5o1KVmBUsoJCH0Bn3JsrkNGcH815GMCb82bjq/VuWO+nnr0Do84UQJ1pchl4MNli1PnDILHoKzWXsXV6DnqAA+CtVoKTx6z6Ufd8aCWQdH4fntGt9D6cFWx3T61GvWMdd62Tbt/63j236hA14Nq3YJB2fN4YVrMaGFZkIU83tOBS4rSRfdHRvaAV7bxmArt8VsunXe/wpMzuaT8ZDNBQ5qbSaPYbrWT0mSZ7nS5O2GpdeXUGYXOzc9IdfawPMV1tQZEPTUjHJdeu3bs0smt9ewJMwf/+HwnkGD4eLE8rJ7TTGCnEXhf/liBfU3KrDPv1Jn1fCImzAvtxYFtty35c8YO/dSzLqlt+8NjnodIgUHhesyc+rtRsacV+9ytdQnQS+LVGisKVmRfZdMDIJHCVLmS8zeF3VjXXCzo6TQ7r/X79O9o7dnFqste7fGyCv1q5eN2TExxUu25JR6FSMBDlvL8+Hseyq97ASuogC/bgcf695DXx9eRv1En7OIwnTN8h1mNL12nkm3GsTKTRhacWzt2v+oMVditSmreLTe/yQJlrO3zd6p8e/SVwH1s1/LlB8K8jXayhEEOXshaEc0OBTQR3g0MkMRZ5HAbpk9HOzjp3JOHjc9jbCtgIDnR2/uWw2ebJlic40lLFISIeF/yOJpvATU6apv4TgM0Op8LZ9I7DaSb6oeAUA0pOLQRYwolwAkpKIF0LpMlnEgaFbuGmLjpGAEZmbXbuMUg5LqMKGxhVWBydZviwIu3iMU5y9LB0ORLQamMpOnupwKVB3gB28rSblc+IUI3vtWAhN4ETVrFn3dPZ+IFme5z5GPQbhb65yJc1G2vIOMFJQOikE/QRDl5HhpUX7w9e7Ox8elvtdZioOTv2spJJXOrnaIXukXhr0C46cstAuIHLLwN5W+CjIDR6uYHRyw0WOn8YfbmCLeQcOM57GP4NTuzy8LLYfe0YUj6e1Yf/c2B18PDfNVRCoyQbQkl2cv3OyaPNf5Bbq7nQZz/7bK0f6mtk9B1xrBm7qUdvsbseqcPVvvBwtS/qNLNU8gDfaAmX7/qoszqQUm/rE89+tNeS+Vah7ctZPcIUE9L2lAOYi06DG20iPR6Mm6Kw8FjK4mLIek0vMfVIFcJu8YRoVnSOUm5ROABJFY5YUSIAyKDqneYfQpdFTnCUdIDxoXXS77ZTAUJWDzPEjROm0Ah2Qyh435UTzQpJthFDh0bvN4S++v04pmyq/FZNDKs6pWyEhLz72uxZw/rgaynuLcW9Yyx4qF0Rjj03ioCh0VYPjbZ6aLTVQ2NIEhae4lyyEjfKSjRwNDJYiZNmfzD8Yk6+HKEEGxJHTT8SMTsis+QoRryyFbZTOmbMKCg8RfPHsBW7dQgXB0mHexQnqsR0fGcdfbcEEZw9wFtcLb8Z20Y3PDRHE4YwHrjz7ITRbw6NfnMIW4iSnSjZiZKduCF2QjumHH9uiIuxTAiNZUJoLBNCY6QUwkipZCd+NjuRd096DUMxp5fPB8xSXPX5Gho9rrDwrNUfw1RguLcggFWlTc9VpeCiPqHUZlybI01Do1wWFjqLvUOshLFpCI1NQ7jQ4XYlK1GyEiUrMRMroT0Hjj83xMWYw4TGHCY0/n5DYxUXFlrF3VFWYgWn8dw0F4E95MbXL8D0sNv+AvW4jpwsnjvcZ4Ch0IOoG9uDVRyHtDbE9h/w3hqsSWxqFzW+ctdaFO7+CZuKXNFWznsVlWglSxRqeacJh+SXo1a98xVP+kJjOl3stUNHy8LcmJwkwGC5fzI6279nG1uqPanDqNpTDS8zpgo9uv4YlmRCIWW/HOpYbXRsnkvB47yOZVoXYzUTGp+1oTFPCgvNk+4Qo2IMhkJjMBTC2K1kVEpGpWRUbohR0W6yxp8b4mJszUJjaxYaj8mhsUINC52n31FG5a7IPDCHDHGQnxZ61Hsys40qY0iE/CqesUp2YlQdY2IjaWwbw7YQhm0/VSHjEoHjVNECOlU4ETOxUu3Ieosa2N8AfJY+Y4jMsd6hcZMXGru8sPBY7zvERBiTntB4WQ5h5VkyESUTUTIRN8REaCO28eeGuBgH3aExxguNMV5ozK9DY34dGn+q4VyeU2+j/usGzkyGwTZYj7T2hiLDiO42KL/+UHanUm9dIT/TWJ5Rbuc0AZujmB1ZUY+zO4yRn8hVtJJ8/Tjy1XZBo3joA/yoVd0QNAeXKK7AZ48Tql8LXqbCWkV+LfhG8CP9W6vU/mjjeS2uWX4VfrIaVYeEC2QLHhXwmQ0PICoExXFU36vWJGnXda2o6lbwx+uAAfVYQnyc3aURG04VR72yDauRR2LoI04cgDbCFU7QQulqIctYCxu44IAg/Nouq+B6DsLQSakEVcfnc4b9ai2MLbvCotuei2ttICELIf6PGHJXqTVsphU5lapTiysuPGZHFTQAkraD5Wgy7J8mPzeEeDELlvvF8E8mE2kDGQJtbG5CY3MTGvfzoTG/CQvNb+43VYaE+7z3ZQCjGTLJI8f0jzKTEHye96D5JRHz5/mX1Lmkzg+COkf6RIDx5xlvHMEK7IFT52JCkTZRRpQjcwhAZNYMUc68zFjShcY+MTRH34RwXpF5AzFWi0ZUAoQbd3ALLEXed+vi0aJIdhHlTjfIiZfGWkWsi+W/v6WkN8qb0ARXP7A2kBE2l2iYiBUM1GtbzYsEs0Efdj1WlkRu0pqQZxqt0BRp0Og3e8KHT5qkmNza7/WLuo42OwY23r7Yf/9lf+fJxtb23s7m1tusJBkjE+W9/U4oO/9X7ZWrxyK6tG+TemO42z2ut/a7fUgKC7sSNkA3mfoi8Js99UXcaM2e+jwmUcJWwAG36IJXzpsgHvpw9PzDnsoRNvWFmc5jejyWKf03LJLpPCpWY5nS7dACmc7lDX8sU2/BTOeRzIxlSrZokUzncXI6lilt6RfJdB6qMZZpuGCm8xCTsUyjBTOdh8aMZRovmOmNkh5Yvtxk6rnzhjLvhBnvgK+aHXikqLf2xFcGs2glJ7jaKEkX3j/sPxEHfi6wHq23ce9AqhphIokiH0k/fvTP6cJKGtuJYxxr5IZH0bFz5DgVO4mOYOvm110vxolJFd9xj92GdwxXgv4R7k38xEmOSUb/+Vuzc9z9lk8OZ6zVfexGnSDPelLDIXVgqovSYmQcvuGROg7h24PLkt5Z48vv243N86jtHV50XnR6X48u2y/7r7aOvr88//r9yeHGH99fJTv24P3lU4+KTsdN8ETYtlLFOH2zud3qXf59Y/v0w9ezYODt7B18eBM8abweOl/bBy378mKv/rs7OAnb7AfldgSdGSvfAWrtvo6m6tR78EpCbkT7K2mqXs3eQC98WDf8Sajgyge91vkpHDegMkq3a60Dn/7d/te/sqK9+ncAGkX9B3yuwGFLMoDkoMH1mZPrMRgmgQtxCx1NHCcNbCCt/fvFb94vzd5vzi9ffjO97bnRL+DLfnOqARY/SNlDyjZ8yTzyBSyB/IbyG6Wv4jRQSwNwGpOGUDTEdVAe/YApYumEEJJ0HC9GCMnqEJL2bTSIg9RVAKlz4crK9JO/nwOhrDCykvqtYR2Gexfpx04N5cWLGurvswS5asWFAOe7EuDTAB6pFcHiAEd/pA80ZIXwgFHWNCxP/cCNa883V6midjKEVMcd91DSoTzrjPzCQdijP9H1aZooyarSRLFxRhoOaUnrcHwsoUJCyqNhS5xNxdltI6SYqIsJab3XpJ8eLmwHa84ayjfodgZrimjWXBRmIs2UNyBqNc8RiilXEkwVMPRS3TMtCQi1dDEoFLUUCopEncjnWySMJbwmlyfJsEG52yQqCSqM/EkmnRro5SiddLGUL8Iv35X4nY5fWf/cJfyeQw4zWPtrO9GwjWsA30TYyhvC1gYNJWx5FdjaxGQOtgJDwlYCAtsgm+QlJLCVz8hDqNB1uIWEWk/vbog0RnHrQHuoCLd8V+L2nuE2R3exQkK/ZnTXB+gm012+IYB5JYB5FQBLIAdguWdacF6oARwS7AJgCRHAKXxnAa/ngEYLdh1OC6M0Fz4FCmku3pXYnYpdWC/9QN50hOkD2uvceXnBxXaMr2TVrRSMNDc4FoX+53jTwbYm7rCMN0VZtZHS/H7vzWbrqLXR+CBU3M4aAeHZiecfBV6jUbGPvUbYgDQfhyQEx5Pxo6KHx15cLM3giMg3HZ7M0XTLb1GlBYH7iysF8ebpw/nOYr5yjB8O08AJgFcPgZQDm4vOW8uK7YwVG/RnJcXGbmGnQYPs9IzAwsPfssJwah4tDHJbbWFEG077KVfOwpWuc3GhvLFCgRBMK9TYzs4z0dgYsKcC6wykA87XMy+kEVw1TOkC1QQLInhw1m92vk7YyswqF4zljcXCaoYPHZM26Za42Rie98fVALIyhGNlwMy3kl7fgS1jgt7tfwXxSoOVij6HhnOSnIsAqVEdahD9v+bMJIuLG40VF0zAXHj4lHU911CFgFdAWxDvhHlKuJVP/KTdPOq2juF0WA2F4hqO0VTvGpo6VsMJ7akTh6uCP6cIPsqF43VMjOx33FUGvNPp0nnxPEy4kvZSCKHEvuSIVSgnL1YPRF4sIXLiWAZqTlyFkDLc1gsnLtdrOXE/XUUGY1LikhNfCsTKq8ddAnHekyykIFqRak05F87AHBSCmW+4ouSVK0peZUUpgdyKUu4FyCL/ECA76ZJShZYA8viSUoh9AZDZTSWQpwL5bmx3yDECy+1uZEnMu5kBVJQomo4ipcNwV8jh1F1hP/bGd4VdGwuaybvC8oqzvAQ4y0tAZnkVys3y6oGkKIRQiCP3SBRxdEk6V7ErLBsyI2QS1SpWe+B5aCXApwCcthwlmbyGTMI1XYmi6Si6bXsPU5Vnks5p/RRkcoxEOmEwRiJjpjV5M41vQMLQHUIf41grzaiAoY7qXtJKNWZcP90DViHuRZBukvDyOjNl9EkRSRm9kJR8dEMNbjcKN9TwrgT1VFArG6y7Auor6/lG9wzKhoN/b+HYq+FvOA8oXdRz23Xyoh5vZFHPpYgs6hlQi3oJ5Rf18kCthdLtNTXJq+le7w8vuBbiFsvVtRCF4UVroVIl594huWBRr3LPFvVhkUqjvOGinlcu6nmVRb0Ecot6uRcgh6TDijZn0ikJEcg4UVGQ7JHvuBbKIisWKEcs4SiU4Z6xEMp4V0L5RqE8nV9NmUsA/qLegoEkTll7JDvBk7jYYhb06imGbHgXXEU8cloi23NWdndxjrlwtxueMUa8ZUzY7YZx2xz7PdfbJqf7GHCQfGUfg/pvM2d0R6yZV+VRK2u1sU1Yf55N2NXtqGNBdLUg12y8rsBMNMt8bFOWHPTMmc9nWJplOrYbSzX4uTbepnoGi+BYe0rfqvZdRddONGjPajm2gcqdntkBdh9M4CfaVMucPdow8xDG0oL+VlrQQ+useMM7gKPKkhebyovB8eftWlYsuleoTs0crOnVMohIumtYqP/PN0utlmVNLIsMCVE47mltVBW4dpGRSn4Al/H18pRFBt6VwJ4O7CV3fR6CUBxrhhJF01FU7kBfiyL4RilRNBVFcNF/xybZnBRaiv1F3GULalJ5dGD/ol6xjL+poDrdQE+8UZEFk7yhZI9XSvZ4FcmeBHKSPbmXHWkRR3PS9WSXmpOuZ+tJ1wtkl5qWdzHkz9db3iE5veFCId+Vaff/t/ctSG4bSdpXQSjG/u34BTbxIgl5HTNyyzPyjLyeGdvyrscTHWwS3aTF1/LRUmtjI/Yoij2KjrIn2S+rQBKFxKvAaj7a3Fm3yEKxMpEoZGVlfpmF3MfcZZfyIs9TvWiqY2e/X9+eZ8q318IEafmqc6+a6w3V7iu43qR7zITnDVliqZ1ssOeElgd2up3IoTub58E8oYGOJ9ScT0+oLpURUxke5T49vAVp4joOxXo+PRRESBMtcSRqHB7URoG3ggcrhfvgDj0c6pNmQieB4/E69GBVpQWj4+k8O/SO0qGHU7ryHXrAQJwNsGIDDAl15x1r8Y61hXqg51lUPIuODQxYvmMVedDdEWK0w6s30f0GYCKQSJnbULoiwNEUfBTgaIHjE+Bo8SkJjt6UzJKfxFaUagjIraj4hJFjiEkVrJQbYnMgN6Hc94uj73I3obh2nr2Fs1cWEDyi2QursXj2XmO5pB3KnRtPWwdXcmu90SVR640+iFpvIgVP1HoTn5K13kSDmLgiSCEm7gbVLz9haNqp0swV/5ZNXUyBeOpyLD9uNn/qnitvlU1dWVj5iKZuueIVZGhvejO8fQhUn8wvFfOWij7IeSs+kcKlrBKhcsWHspkbONgiSKh1BqqPNhR5qD64cM8zd98zdz5/G10DBz1ZTEdRet6OQIv2GHDOYHMhcHhkrf7jyXTyx+5whKII1md82rqef0F+qmHvojdYoXzDz60fXr3+67/92Ph18Qw5mZ6HemyfYzjaOl6NcQDvCBmojdlkNr54EwH3BeowNazPsNVpNGFpdP+AHV0juFD6J3pejIbXyGTtR+9AAZyIxvbFv1xcjeBjunLwaRK9fYWHQP6mxT9kGaW/RPf/vPiXP1Tg/+sXl5c//fjdT4J/H35LzO3Pn8AM3ojqH0/+8xcppV8gt19gWJObDp99uDB/EeLHl1+AdYNTddjHmR0iW+GXJ7jYw12Ji0gtv7qhyg6iuR8tIWJx4etJfzbFmcXwlFHJD/RoiC5kzKMD7cP/C1/nEY4yjua4L/GreMGTPdd3LK4Qz1+tr/7Xk3/mvpMtqoh8fieL3klZzf+8GSzaDJ5TD0pn0SlXA10fq37xe5wagrwwAHcB5IWeHt4Mo/5Vd/mpDGd6zU/lHX/Z/HTRvYv6X5JT6tPr7mLYkx8hydEKOh7KcX5/tR73y7Ub03jIUyQuCLOntTF7WmucUSAKkIrdANWDLI95otxXbPlQHsQ55mnu/QhwZMxZy5ZoWQLSnGdR0SwiMNZpadmSBK95BPJDBGbvK+8KZQKicMUJwIdwxYlPSVecaEihQtrrIi7yE+Xi4k5oC0k6Mv5YqiQ77VacwOg3WW3b8/5wt/l9ckhj8DWewXIY31+tP8bOOfgk8pxz4pJAFdMHgSqmDxJVLD4lUcWigUaUDg2ayr6ANdFUlp/IyUFZueTjqFJX3FuXFRd3mCq0AVMut9AGmXnnKVw0hamIxMlN4WR5ZpTTuofb4fZiOBbH8BBkb9Cl0083bmfjR4xImJ6Y2eQ2ljNbfCLoXlwyX/xbOrM3sL0WVQJLHSySr5zp2nlmF85sVCo93Zm92eJBUYvdHQFUov7vyzdsTpPiHTnTnS6J6U4fxHQXwREx3cWn5HQXDWK6i4iKVOTr+iA+VczXDg9uYiysKIiHU/JzJzuunSf7g052U5XKqVgG3YkHQGdOpXLqYgS/F8F8ubcxXxYom5OPNWO1aQNTDIgy26JCdW86QkltwoJml92Ov45xOl8eny2GgCUXydEhYMtqt69VUv59MmQpVQg6ALJU4BFURkqQpQxn+RpKGcfPxne8HpeBRpMPUt5tHcyoKNA8Rmvfwo2Iw2bJhZF7ewxGqrJBmNJ8NgZ0YGYOH6ICdi5Zhi+lfbCWVP99urIGcFci+kInI6NxdG/J9Y8LmyFJxY3pkPv63QxIPtR5HwwXVrwE/7/FhhKdKtwbDQX22kJ9hOnbxoY47LNc9xAdeH1etAoXLXls5alYaHGEEi9BBLuMatXj0U7nNqjZ3/zw+ptXr/7t74ipEiRYgl0wEfPALrgkwS4xSkt8iMEuKZSWbKAR5d5ZmGGU4yPNMEr4F2CXZqcdbzzWH8kcG46Jn+zalS4hv4VjiJBhachLfo02unae2YUzm97905nZqb0HpRBdT7vz/ibznBBXD+cmEgEgMa3FJzGt10EhUT5Q7C+o7kg5FoZgaWKHQd3PjiJzUxqzaP8RoXYb2s3wOZ3tNsx/42d/nuZ5okpi3yEqXemfIrXAyTsjtUlurmAcZifg5VqqrHgOnf13iI0I26FSxMbIDnU2woFOYuMgjuTO40D4XhQOKGRvJs9Nkn+3kP+K/Us+H2xvSIaMljVvQdlhgiyn1va8JHGAY+6TltJ+8Jy3NtsWUlTPyFOGzxsvy/QWyz/OiUp8sXvDeW8UqW3x2yUPsUpdGuAwsOHkZqo2x4df5d8a22q2dUqfVb01cAY0IbrGelDm9z2z/Obs3ReWTO9bf5MHGA3fR88sx0VDPu9sv0phVTNaQGGepmk+F2wbm+QCl2vnQ2ZMB/RV0x1h3tF/1ZLZ2ixFsb1zimIBs3b8A7JdvrWQFB40rbHVfGr77i8Ty7K6Fiw9nzIGHfyv+bTjZzbbHcrnlDnyGGnz2QYmM+p1YQw+EWd8bS9skkdbaJviQDrEmUhe+EaJgLbatJ6S8sfPrPnt9Wcwui0Hef2W2/r8C+glNT//mYWJSteQ3xgiR1/pIc0u0eULHL+1We1jUws/Zp2fMgrrls0VcUtPrZhzZeC1GQGS2VaE0hs2NGV23OBYMPmjqLsAmey/Kp1oJG5doG7i/ySRRfY7yucfW7aT809YjI94+tFWoOL0Q2qMhYCL5QZB5vRzgibya8PzJKwzCZnJpk7CYotCexKC9+HI7CysOPNK51pyRm2mkeaMyphI4oGIJyceT5pW5uXs38jaHGL6xNfEo47vSflNxYff4dZySfUCZqWSd12OxS3epJdemL+mC0x0mDlM7iEt/v+2gkPc+h6VRsgBIEfl9m2ysESbFZYoEArsyXy/Ogzps6um0FWDUOJxeR91TohYP1hEfoBuh4Zf/J41Xa1Xii8/aX/1iesS2/jnE+85/m56UYt7ib9Ind5cXf9yc1EUXZKXW03ZRDLqAuS5+dHtkj62X3y6ld+XnwRffRK8kPXCNt/i8+LXXyHF6OoGOLsvbdfpfyrzua9WKODzZQzRd5t0sgvd3ZckUeFhDbF65HhY6ZLwsNIH4WGlD9LDKj4lPayiQXhYBX6UPKxBc30Yhvwkynuuq3tWAuKtERwcS+oLvZJ34tXZv1ry0qL2/DEgOHwHWpvuBCLLQXCILntEcCAXIb1e6WzR94bgECEzlU+dqkaHQXCky471p73FxWoRzWWdMYK8r/Vw/o0zl0BHqz67MU8qDoNOM1Li0mVGyKuoO59YYwACNtYIs/WTj1Xe6h7wHDjLrIgNyph6CDxHyAzUUNcl+8MA3nkLA8SAjng6wUc3V0AXa4Hj7L80yRJvNCP5bRfbgQWlS+NYAwkmWQ4i69fuXVdK2lpMhrMZ9hnQvqsZwB7y/AOkZmNLEm0QHiGM2nxLNDzHwUsWNRx9f1yWaCIOTpqPKrrdzjErsJMR6k9+vOi13KiP5xZdt67dVj8KOl7YbDo3zb7n+r1m9PvFl2Hr0/6XwMOu4R5OPuqWLsGykkHxOCKuhsPjWHicL7FOjQTSQtpo1ICRMpMiY8sszoRsOxSeV5G1KDCWi6w9Fx8rmcL4vyObwqWFd7KhHM7F77HqY9iBihYnpEXOvKVLAqZEHwRMScAyBExJfErClESD2G2IwlG024B3Yz2TxSfKYGuimlucwhY0KyX/BN4a1NGmeiipyS3O/MmZ3OcDUksmN2S1f1BHh9RhDrBhjvjjrXBKwkWYAEPg2eSZLzi/iAJ/l5jk1rXIh4eNgX24fTsf9m2y5a3eYoGKqJMFDI7xIr6KFwFpnqjaSWU6o3Tj1s0Zgy3op7KK72Qxwq7+M/BHFVXWzlJil54nWBdvV4J1wrsfG+tPLb+jcQfsgR3DHQD7qHEL0DTHdwthS+MOoFWP7w6c0K18C+3wKB8CilJUvwV9QFq6REEN/JnJYxdNF0pQHENUDl74jnAkoLKDE4BlcQW1XHO8SlpHLWY4dXJ3sAydQuIz4j7amFdiii7st3McnVdQVT1knqzQlCcr8boguDldLa0tc4T2sFEeJvFGxNWwEbPr0AEB+fgY7HHTLOs4tYpYfqiXniwSm0iCRpzFYfdAJOPufXHzlijqIAKR3WuUZQOjcXiS6D6TH5nC+MKK8UcUzcyXH/ONkRY04xtj8uhO+kB7ZTfDOZjPZNpvpnV0ZdYzFmbpDTY08pCCoQ2DP34yyVaRsU0F7/JZS3vfhCrZwam6cf0KhFg0Hl5PR/0YpAjcIP27xruVpLptXrALJ4999EuxjxZdX9a99Xw2s16sqcW+qQ5yDAr0mtQs9V5M4XN9gcx6eiXplAT0nKNaIc3HCPQoNpZ/x9x7l2ALl3dz1uOF6EX2O5zcsPbMX2MTHWEblctQOjCNFpPhgy1HAr6wXQ3iYGA+Y+mFSWEMlw3xlc9AejlCiyHJ/DXGCP1xOELYpEAG6eVFYQGXd5OBEsoZ2J18PtJqGi1Hd/Izymni/JnUQTMZgPg19J23LvBKr4+lSS6TFKi2gfWBmGL1Ipqw7t1GV4toBHeSqEcqz7XoLmbT2Yp26bknWwgJplcVtDxIWKgId91BvcgCLnB5P7GpDmrJFvFBftuHF4aTXjPQcj54RSRDMsHomBm5B69864TimJPOS6fT8F67L51WI3hN33GCCr4Gjbb45jc6l3QMC3W1PBxW0sJHHNnSeu02Lz3LbTbCkK45Fs7r7+B6R3x4ScNf4qqDM1voAs55kX3Fh9cY4BKfxHCyF42/5ulnyZ3bJPKvcSSMI8YDyfgKWBaXMIy8goafdzgJpeOwtVjMex1z6HJ9Hp8ckK+hOrkzj3hCs7Xd0Vnb808SaiNVwArof3awy6k4OHKIPTpTO8uU9QFwR4luZqs/1VTfiZXk6roGTMcrLKCZW1OA4L7weouEG/J+T6bCi7D5Psajx54uPQT9hPrF36kewnpvG/vQi/elcMKzW9ZZnPdj8FTNANwYPKg6LuDpW1sm+RxSVwWcWGyprvI3cA4zHygItX9sS8dl9kOSEVzeh/3gMvuBKjmd1W0HUeK0YIzYD2Ms2Y43sFt3rYHt3tn427qz3QH+tAYuGls76WCXrcdiHumsx8/7fUvZOXRQwz6t2E05XCmLVXoG86coW/hcU5vaxHvy03A5+B6bqedij5/URuqVWF3lc8sWQaq19rDcXl2JbWCspHM5Y2tikjNcNuSf77hsJSLYy5GtRBKnV7YOISpP+iQnE71kOx7/VkJ61M25GmWQfiU4eAWaI/8BspXLfZiVq3DFwKlrBVzg8p6WTxyJUMhHcaKNKWGwVULLmf14l09EBdOC0TpoN3/5bDdagdVqeIE8L1T+8Rq+jxM+PWxhcFl+8ikZEhvnJo4hxVGnYruN40XXXzoLfG1bHRxUGjf0PBxRCqG1Gh0aLAhwqe3RCo2zQzs9u9HB1rrheXSAqG/5ghT+8RpeU9IJsE/H/6E7zl1df2ktxNcW9v5xA45EbbXEsaoO9vF0kCkOSHUa7c5IHKYqzlAdtO/s9k6WgMcWbfFKGNEWxQqsMDO+g8KWab52zc/PJcXWYjrF0IwIRLRpHPWHK4HAyqbPVtwkfVwuL36xNsE8tqaqQ7EkaY6DR9FZazUjL/Am4uSxFUXwpGMtuhaSdkk4VndTn6PjszVCLAumVypmBVGAZJ5vu6170LouYmG5Tw65O+nF5RDRg8JqOhkWDHwNMyAxqLBhuj1eTnPPtK9un9h97LgBmBT3pxM+AIYkJVRKPtu3/SLwpSoXOivT412yfbZgaIEG9uBg9DnaRsmY0d7BnDxyJUbrVUKuAHBYJD6BR/yNiU/CNStKj+N+FOn9BiefxOtWFB/L5FPFV1zl5TGKTwK2K0qPpSyq0iuOeT8+6VUUG2rWp5Y0Sjs4LqxeJ2C2ZpJJXN4XjwUsMstNZbEYAmZSjPksMrNOZZFsvJ3ekNfD9/nEmemkEq8P/KL38lucYxu/m/a1tW6xbdrcvBBn3BaIhe22A6PO8wSo0NUBFQZsa57kC5d3ltgVzsOS6fG5PLDtebBrkJgzMe4iQTCXA7arT3KAywYYwMVr7CuvURQlnw/mBxCkTUjiR1QeoN03ner0zaQ/7BVOC3ZmhsIILhsQSIXc/A470yLFR7FO5hjlSidsXLz6+fm4H7z6Uz5bXA8rbNWH4m7FQ1kyufSZkm2ZQt5uGSBPBfRIvk5jB2EoXOByceGEoKr7kJ18gRbd4hPkdPOaABLfr0+46LATLpRhcbncK/i///0/G88gnVlRNFzxviRzbZYFJOg0lMUKh62/p3qz/Y8frrEov/n4gVw/FurKWjEc2prOP36QVZOsO3JWIZEILZTB28h/hkzzldVq5lUo8FMkHa0HZCqs7NyQ/XvsqsJ9K7jrhJNn1MXxJeRhkcVNkiY3Na0DwppuOlY7WpEkLh9ekNWCt6VizJUA0/5lVasfwlHJilyj5eyoJDGwVUir8Ha+o9JtWv4Ayf0UWUR5TVRabyCYZ7l3jttDyc2G0wAMl/43QB/RxXJtaiMgD/Wh36HBRoPtvh/jN2I8uoifvB/bjm93gPJxEVKky/6gQ58778fwkDY3F+yWvbmyi7+UlfGWL+++5zFbRvUKcj/eecwMgbKyoxXncQsO9/QcXqzn7mbKJiaqg5KyWj9AOFvrBzvNYm7g6OjBAT3bbPvmBbbO98KMQTkpKpG2sD578fzHBaWO5/mvUKW4oLpWhxWzRoumXVPElaTBTR3dCl7fw37DAQ6ooDVZja+xf5/eWKvJEIeQxhSXgy6ixSiiK6poIQnagoWH1RiG7LqWVoeVzZX7M/MZ3x1WVBctphTZ8r20o6z1BxgIZMGKlzyXIbY2JxnC5eqRfVZmNzUUc1xlbzKQDT68GW5D+1RyNzWsbk3gAFb+CnMhEdhnlRulb8vEBnD9kskSkmToFYifaU6twozlFSwF+fh4ilwmmF7SKpJYUkdQpjjmE2d6Jkkcl4t1Ivnqqr15TNeodOoHHrAcIPFHVqOwC71krFAiWgyeRiKeuEjPkM9enqOSzw3TRdr1E9dvE6uLqAyFy7vflw0XaH4l1g6VQ0xxYAoMkWThFsDY8UzEhdZmWouW5Y2V5lPxLNUsawFiR60VzLJKZgSrGoIWjVvlN5pTgcZyaKFWDU5RdH++EkYHeUqm/ewjkzjTTN9q1RfRezq2MHATrIL799F8WpFVppOTrOJyIafRaDScSWu6Or/bH/Vo1nik1Xp0qoFHxVHnmzbEAJ6hAF6122BaXas8Sn7eqA/Up+PhkI5G6Dliq9hqEiSUoKYhEkIB+myLj87LILhDcimqfF8Cg+oBDgrwqNdoIY8UGBpgUdEYDGwf6Z2XyPH0gBgNLKBb0TEI6QPscq/9GlkmDj7RTtX1CNcqSGAb67kxOfwN3c5do+nK/S0Ywt8m/uJCEwmm+KHo1HQGgKq2A1eMRedPiEu4B5QNjz+jm+Pd2fi5wK0WPjghpWqPg61zSuA7rHE4jHgaEKzjNtqtVz7SYQMft+0GnUt88Vrtht+G7oHAA0Lp+nT/bav5Eum4zdADDLjtE9gXuOMW9cSH0CexukH7FXJ7aFTnNRC8Pvb9SLH1s8OG/E75SluC7i+uyzvuogIjIQFj1+ymOC+MfFEzmqiiBiqjqu+HxXnKiM+iEuat2FIQxNQSlQZwKsfcmtHJEt3JFJewkxA+YxzhQkzRVeFUJpfyeluBycZwaMbSHEwF7UNWFEVhEpcPHrQPeZkUlcXiiPg+gvYhL5iislg/bl4etA95sRSVeP1yOzsG7UNWQwUtRxC0hxos4guXHz5oH7IaJGjZa9A+ZPVHFA5w2VzQHmdG5ddBClkBEkl771H7kBUhURjB5f1E7UNW8yPFR7FSrhu197+7XN64737NZyutiFNs1YcmVYrah6ykBlr2HrUPWf0KhQtcNhO1D1kVCrTUidqHStQ+ZBUllGFxWStqH7JqDanhiu3aA0XtQ1Y7AS27RO2BXmcD6uxvz1H7zaNhhRwUSeLy4QX5sFH7kFVsQMveo50hK8CAlnO0k8TAViHXTHrRI4vah6zehXx59z2P2TKqVcfiEc9jZghopYb+RqL28Cuy9UhHDxZE7X+KojcsQP7TTmF7HO/BHqquYVPIliDCKmfIHcle4vYoqPtm42FjpTPkFs184D5k9THQcsjAfcjqUigM4XLlwH3ICiykhmLOq0qB+5DVR5BM7Ra4D1kpBOnf2nPgPmQVEdCy78B9yGopoGVfgfuQlVxQiOOymcB9yEowqHT8+pkUlQP3IauigJaDBe5DVn4ALfUC9yGrIaAMhcsPHbgPWbI+Wh5n4D5kWf9oOfbAfchy7dFynIF7BC+LWMXlkwjc45A2dhtG7PJz4L5O4D5kGevK48DlxxK4D1mSuVxcHzhwz7LGpelwyMA9zxIXVR2OK3DP88STTOLy4QP3PJtcZbE4Kr6XwD1PK1dZrB87rxC457njKvHdc8frBu5ZRjlajiFwz/LMFb5weQ+Be5Zjjpb9Bu5ZcrnCAS6bC9zPVnOymnM5Ycp718yJmpF7llCuMILLe4rcs8TyFB/FWrlu5P7yJ+/PzqT5XT5bbJelsrV7XY/iyD3LQEfL/iP3LGFd4QKXDUXuWSY7WnbPt8fBzUXD4rJe5J4lmqeGq3H80Nqz/eMEyh0erEkfyJ/pW8Scr++lnQULLQ7h48iDp1ivF+SQ7yGeSl+Wq/mEbDgR7++TF2Sy/PhBOITzY/kszRstutIe3kS9+55wD8gxuV7T2fWew/nbp8MVc3JTLjLBH3c4n2V5o2X/YVCWQo2WcxiUxMCWJq2s7t9OOJ9lecuXd9/zmK2tba1Tax/vPGbWQVnC9m8ynM9yzqXxayCc/ydp6XR7velqQscI14/is4R0tGiaNFncyLG5aaMbln0J9CVMM0Atb2R4XobshdtPWnWxIZcw8MQ3eNtiq86Co2agBvNZKrrctT1AMJ9lqqPloMF8luOuMITL1YP5LLE9NRRzaFUL5rNEdcnUjsF8lnkufV77DuazHHS07D2YzxLU0bK3YD7LTFeI47KhYD7LOU/RqZ9gUT2Yz5LV0XK4YD5LXEdLzWA+SwxXhsLlBw/msyxvtDzSYD7LEkfL0QfzWU44Wo40mM/ypRVWcfk0gvksGVpyfg7mHySY7+AVL3oedP2xhPNxL2yx1S7woh/PB1m+vmrDCMwG9METD5ZreQ32EdEHlzxenuCSrh88pg8meMRcZbI4ZL6PqD6Y4JF1lcmdj4gpiOtjeFZ7JUV+9wTzmpF90E4vrNR0BLF9sJFeK1XOqMPDh/edJstGp6a9BvhBMK26VR6ow34q6oNSWp3HxPce4wfdtBZXWaEO+wnzg1JaVTNWinV13Uj/++AvP734Ot+VBMJp/cw4270eSGGwHxTS2pea9h7uB9G0Glb5oA5mIv4YiWtVkfquHfNvJ0P+GITrxOS41EEr6u80WaZ6esQ65/WvfeDfR5HIzRfeYekVls5gC3pmsbpeRLAVYaXRPVrdG7ikYMZZQziou6PYkMsN9IMzrhXLMkqZMP4e0Qd6tzejZii4ozsj8ySC/ZBlhoZWLB+RQ/6o4/24R742aGWIm4mUgihfCdxzwXIpB75CaWWF/2ai/pALX0T1CkGYms18hXW1UPmPeDZzI6Es6/u3GPt3mixxPbaGDUT/k3ZFvag/2OP2TdlR9xXsG5a6Hm9OatQEgC0nB1cj/1s/G8tKj/dk5oP4oMWXWZGqfrAwPujzFTfJEnWoHMlHZ75IpUYrP2ErK5iPH3KdLjjbLZyPQTJ8S6byRjYTm1iA8wRbB3EIvn1v+wVPhKvGsvQyhaNlfApsCU8/ZBwWu3lZbDlILo8sw52adHi8nvbJ/V7ttWG57NSkQ21enRTXPElS1MEYKa54UqSKNx46pLjeSZEqdv3okOL6JEWq2JejQ4ormxSpYpe5DimufVKkit0yOqS4QkqRKo4j65DimiZFqth7rUGKJeqmSYlDe82QKtYW1MEYqTJtUXI6sw4pri1EvmxlHcjM5XxSXFskSVEHY6S4tkiRKtEWGqS4tkiRKtEWGqS4tkiRKtEWGqS4tkiRKtEWGqS4tkiRKtEW1UmxbNE0qZJ8UR1SXFukSJVoCw1SXFukSJVoCw1SxdqCOhgjVaYtSjI1dUiVaQuRG2mGFNcWZel0bLtxOR1M58vNToOlWMb8atWgQ2nh7YD83S9LDMjcJVvN7ZD8HdeG2oshCbcoh2S5jvE81x6SIvPxkPyd1T6FTwxJILl4SP5ulqWFZw9J7pZ4yIx3sE5hQ4tCevGQGe+abshODNnaDpnxTulm/ooh29shM1Za3XdHDNnZDpmxouq+O2LIcDtkxspZ6+1xtq8Py9SJLcDKRhr5O6opKJZvo5KiDsZIZSyRKqmSJVKDVMYSqZIqWSI1SGW8niqpkiVSg1TGa6uSKlkiNUhlvM4qqZIlUoNUxmuukioxqDVIZbz+KqkSg1qDVIZaUEmVGNQapIq1BXUwRYoly6RJlaTL6JAq0xYlGTM6pLi20MqLKXSnq47Sq6sfMCo+5vtEWVoMNWkuKN9250BPb9YTlh4TS+9AN8h1S1muDLvB7VrJslHiSWjU8b6+NdzW0yeL5b2Ie113e29Q42M16QMugUjcMzqE4zMbaF3F+233u/M3ssfnX1hKz7eD4TL6/IsCUXGFVZa2wkT1yVZUXCclR6MO5YJvJETPFY9W4p5J0c9vr7ufwYCwnDb+uAgeWs20uPvRTXc1WhYIHEGb9C2hqa7AEQEsHI066AgcsW0ucFMA5kMJnGk7NNUXOFN0ymjUQU/gXFVpHRB9jAJn+hJN9QXOFJQyGnXQEzhXUcaOBT+UwJmWRFNtgXOIvjIaddASeAbcXpj2JyxwDttHU32BcwWVHI066Amc22Jl9QSOXuBMS6KpvsC5gkqORh30BM5tKK2aF8cocKYl0VRf4FxBKbB4dNASeAbMXrhbTljgHH6PptoC57B7ZTTqoCdwvmMUrrRTFjg35MqA5EUC5woqORp10BM433OWVSnaReBWRpttz6L5cJpd/UAyyU2zslOuikTIVU5yNOqgJ8IMp5UWFquye8fJgL4mSVEHY6QynFYqKVNOKycD25oiZcrF7WRAW1OkTLm4nQxka4qUKRe3k4FYTZEy5eJ2MuCsKVKmXNwoV1FGypSL28mAvKZImXJxOxlY1hQpUy5uJwOSqpIqOXZJhxS3KrTwqEbdsg6HrKKpjt95Ext1ODRVSu9AN8htirIDl/L9zg4HqMpJ+Bj8zg6HxKKptu3AUa/KaNRBy3bg0Fapak7Y3uUQWjTVFjhHySqjUQctgXMorFxGTljgHHKLpvoC54pOCVaig57AuaoSJsIpC5zrS23oTULgXEElR6MOegLnKqosLejoBc61pDbIKyFwrqCSo1EHLYFzaK407U9Y4BwCjKbaAucoX2U06qAncK6iyg6lPXqBcy2pDVxNCJwrqORo1EFP4FxFaR2cfIwC51pS+3iXhMC5gkqORh30BM5VVFkO8bELnIOe0VRb4BzvrIxGHbQEzsHO0pV2VG5QDp9GU30RcpWTHI066IkwwwmllQBb3duQBadWn74xJ1QWzFolZcwJlQW/VkkZc0JlobJVUsacUFmobIWUMVS2k4XKVkkZc1lnobJVUsZc1lmobJWUMZd1FipbJWXMZZ2FylZJGXNZc1Q2mg7muuPAbTTV8E26m4QVhwO0pfQOdIPcdimDUhb4JjlMW07CR+Gb5MBwNNVe0Dn2WxmNOmgt6By9LVXNCRuhHD2OpvoC59us5GjUQU/gGYrqxLdZHPWNpvoC54pOQX2jg57AM1TVaW+zUJwsfUtoqitwl6O+ldGog47AXY76lubfKQucaUk01Rc4U1DKaNRBT+BcRQnT/pQFzrQkmuoLnCkoZTTqoCdwrqK0TvA7RoEzLYmm2gLnqG9lNOqgJXCO+pZb8hMWOEd9o6m+wLmCUlDf6KAn8Ay/0QMG9Gq5ylyO40ZTfRFylaPguNFBT4QZtbO06tlV3vy6GSXNlfg5OhgjlVE7SyVlyqnkZuCyVVIl5c91SGXUzlJJmXIquRk1zFOkTDmV3AycdYqUKaeSm1FIO0XKlFPJzaiVnSJlyqnkZuCmU6RMuaBxJhfTPmXo54dz77gcPY0mTc36fAb/1UYbcvy0FN5h7o9DrNFU130FE4Df26HAGKbdVzj0jYuqvp3CodnKaNRBa5Hl+GupaU7YMOQ4bzTVFzi3apKjUQc9gWfoqdP2F7ocF46m+gLnii45GnXQEjjHd0sL4YQFznHkaKotcI4DV0ajDnoC5yqqLHfu6AXOtaQ2kjwhcK6gkqNRBz2BcxUlLPtTFjjXkmWnWBYJnCso5ZBCdNATOFdRYtd2wgLnCHI01RY4x48ro1EHLYFz8LbckR+VN4VjwtFUX4Rc5Sg1S9FBT4QZuz4dEWpsxTIqJidJUQdjpDJ8RCopU8AjHHFcRsqYjyijYnKKlDEfUUbFZJVUWcVkDVIZPiKVlDEfUUbF5BQpYz6ijIrJKVLGfEQc/oymg7kAOHwaTTVcHJvsQZcDqKXwDnR/3ATRxj5v1S/HPss5+ChcHByIjabaKx/HQCujUQetlY+joKWmOWFrjQOx0VRf4NxwSY5GHfQEnqGnTnw/wmHbaKovcK7okqNRBz2Bc1VVVjH86AXO9WUZQqBI4FxBJUejDloC53Btaf2dsMA5LBxNtQXOkd/KaNRBT+AZe6sTxy9wGDma6gucK6jkaNRBT+BcRZXV4t//jpujxNFUX4Rc5SRHow56IszYxenMWY2dQUaZbtXBZaxMt5tRpjtFytguLqNMd4qUsV1cRg3uFClju7iM2twpUsYi/RmFt1OkjEX6M6pyp0gZ8/lwtDSaDrZN5GhrNNXYBjubTBWXA66l9A5ygx7HSqOp7j7Yy8BKl9WCP5l9sMeB22iqux55HDOtjEYddNYjLwM1XZbWceQ2lMeB22iqL3Cmw5TRqIOewDMU1WmH+j0O3EZTfYEzRaeMRh20BJ6BmtY61+MIBc6B22iqLXCOmVZGow56AufbMq1zRo5R4ExLoqm+wLmCSo5GHfQEzrdlD3n4Sa1tmceB22iqL0KucpKjUQc9EWZsy3Ssq+qWqpcFwE6mzaCDMVIZ2zKVlKltmZcFwFZJmdqWeVkAbJWUqW2ZlwXAVkmZ2pZ5WQBslZSpbZnHAdhoOthWgmO00VQHFb05f9PjQGspvQPdIDdo3NpZ/R4HWctJ+Cj2ShzzjabaiwQHWSujUQetRYLDrKWqOWHDhiO90VRf4NywSY5GHfQEnqGoTtx050hvNNUXOFd0ydGog57AM1TViZvuHOmNptoC5yBrZTTqoCVwDrOW5t9Rme4cu42m+iLkKic5GnXQE2GG4aSz2GsYThxTrZCiDsZIZWAYVVKmMIweB0KnSZnCMHocI50mZQrD6HHscooUr35dmxRXClqnmJg1NzksGk017Glv+4pxELOU3oFukCsQ7bPWE/fGbRathPKjtqc56BpNtTU3x1Uro1EHLc3NwdNS1ZywtcFB2miqLXCOw1ZGow5aAudga7mMnLDAOagbTfUFzhVdcjTqoCfwDFvn2AAzHsdso6m+CLnKSY5GHfREmGEI6fg7NNZxDsdWSFEHY6QyDCGVlDFDiCOnU6Q4cro2qQx/s0rKmL+Zg53RdDCThIOl0aRdmfTeam+mPYdLS+Ed6P64itBGOm9faY50lnPwUZhcHHaNptralCOeldGog5Y25ZhnqWlO2ALgsGs01RY4Rzwro1EHLYFzzLNcRY7KAuBAajTVFyFXXcnRqIOeCDMcPDpud41VhQOcFVLUwRipDLtGJWXMwcMxzGlSxuwaDm9G08GWLY6ARlONZdnxN5ORI52l9A50g1y7lGGbCtZljneWk/BRrMscYY2m2jqOg6iV0aiDlo7jSGmpao5qmeAYazTVFyE3bZSK1OigJ8KMLZXOjqC6lvM5+lkhRR2MkcrYUqmkTG2pfA5URtOhVJvPgc5oqqG7XWc9Q3yOdZbSO9ANMhWCprq62+cwZTkJj0mB+Bz7jKa6CsTnsGNlNOqgo0B8BjzuaPnqCmfHPOr2lljOFsP3YiEbdCf9UWRlN9uLKF+GDM6scEnX98VlEZNprZVmslhpmRRlAZNpfZdmkpRfvTfmm8lieDtYvh6+LyCfVnBp8sUoxiLyl915/9to2bWup/N+NLevrXWLbf80XA5e4NNwVCQarinL8M6VmXsVjaeTF8O7ITizxvc2VTOdT4UZuIhAsbuczgs441pTwU6jw85iu7qazYfj7vy+gA2ubkUhaxMC2rIx7g4nBTxwbZrkgToY4AEXr6cT63oEE72AF66Ly9DdlXn5cRHNn/eWmDHL+28m/WGvZIYwO1BhhToYEAuI9ubDGa2A+axwqHeKlRKod5cxMphHN7gwWC5nzy4uRtNedzSYLpbPOs1m82Iotc7i4vlfg8HLf/36poAzZlamOSvW0NWEtJzOCljg5qYxcN+WhR6WemiXAl3H4eQKH9ShkI9BUNnC5nByNGnaP6+6iyUcEla/e7/YmC4cPK4MTB1KB7b+97//x9qOyBVcasRiT1jmEr4cLoWefz1crLqj4fvIwiL98cM11u43Hz/0p28n1mD61sJCjtdqYU3nHz9Ed6C/oE3+vTW9i9CyHI6jRsHj5FpRG07+A37bT0iXKzct3C/ufwk1waQhem6vKgvkV6LVSny27WU0Xw5JEErrYtldrhb4J+qOloPUtXF3NFKbJlM4UMSNqu2D7sIe4gqxOR927VH3OhqBm/F0HqGt34XxQPzHTfb2pqg3fj2bzlYz2nHOVwWWIQfDK7KkDoeX5RjD9neXZK4QOIYfTaa2Gglerq5K+OCrgNZZyYu722I2vpET6u2wvxygzYnG+DaIaJXafL0ZjmieTaYTmmd3w+jtV9N3tDWzmpbr4//R+m48mtCg8br39u3bxluvMZ3fXuBQreaF5ONm2lstyPZHx5vuCJuo9eQc9vvRpGhqLr5//acnz+jyRjR8ddI6t3vWxR2nZYMfPhm7Tcsf2E6rZzsNx2rartUI8ce9c9xe00JbI7Tw1XIH6CO6WK5NbbZ7Z1Mf+h0abDTY7vsxfiPGo4v4yfux7fh2Z+DeuQN52R906HPn/diBTDcX7Ja9ubJ9DL3VnO74knyrVUWVsYAa8/nrzGa+upblMvxmZjO3D8qq31eczS3LaaZn8mI9gzcTNzFdHdfS+4Hd0vvBbnM5w+zR0YgDer7ZRs/fo5toPh9Obq3+lDZ21meYWxtT7vPUGhvb8nYPlqwth8if+Nzk0c4pKecuppVhC+mG1L6HkbewloPIGmPvgjjOGC8XHDgqBwvrZjq37qerubXCDhDmIMw/8atZgrPGlrOMfd+O0IZcifOkEjSZUnnL99L4stYfYFLQvQtlkM8SX8uVzBR0KN8CbITJ81LSo5WnzomdynjaH94MI4pCxCPz1UL7mIIA24QVZkT3drodN8NlZSo5b/0yRuPZ8l4YiUUPgutarXyUIkYSDNjDySQqcIPwXBU06aj8/PmJIeweRi0kz3WS6t4syV8ZkFuw4svIVVKKVDG8ofDJj0arxRI+SYpBF7vjeG4Lmgw99+4knnzYMK+n4WQ6x54knx+eAIMmzRdt83bxFBdlMOqw+83ZcLreFrxZPLMFTYYMvCQTt/ZiOJ5B2W5tvRat6xtTz6eYomrbtXxLtFaw7apZIjznBk0aN8tvFdN4ssCaOha2Iz6PoEY+w/aBVnfVbqVv9nwl7BbywEz72VG9DLa5Gi5Lzilhu+gJ2cJSTjAL/t9H82lVZrmqTjJLHQqZjUaj4Uxa5tVZ3v6oR5PHI0XXu6dPVMBgvmlD+OHZk3bVO+HaXvC+s4H/re80Wo5nOUEj9Byx+Ww18Tn0bRctltvAiXrio/MyCO6cTgMg08sgsDyv4baswGu0OpYXWIHbaKIxGNh+s+FcOs2G51J7KDoGIX2Aje+1XzvewMEn2vvi3DLaNhEJbIyRbB9/Bjm3c9dounLHDIbwt4m/uNB0qJMnOjWdgeM32oErxsLGQ17CPaCkVPwZ3RzvzsbP27R1KHx0QkoVHwhf/5IPhDrUfCCQreM22q1Xvt/oBD7uHGWGL/EFqRINvw09BJkH2DRZPomgbTVfupBO6PXsRtsPsYtyGn6LeuJD6JNkUTb3VShHdV677sCHMyFsuH529DLjZjNWYN0CYD8Mojn8wfhvMrXG3SXIYisQu39pJ7AcDBfWf6wivBcbuhlLrr6v997qIVh8S+Rod4FdGKLJaIiewgNtzYa9N1h1p7g0jz3TTyVbdFW4rslxvd2HsKStdhLW1OaopgPE51kimMIjXT+C+DxL7Uoz+bDxeZYLliZ/wPg8zytD01HE53kymcIZddhHfJ5noaFpz/F5nrmm8EAdzMXnb+cRnIL5vHANXXZuyUPF53nqm8IKddhXfJ6nxqVYKUmNqxufb/76/Z/ffP3TsoAzvt9Kcfbg8XmeYYemA8TneW6dwgd1MBWf50l3aKoTn2+r4XmePqeMSx00w/M8hS494g7h+RfDRU84W+Gn/fhh3J3cxx5YeGphBsM8osh8fz6dffwwXS3x58bqfvywiGCdTXq4evPxQxzDLwjU80Q5NGkK+48rOMFGW6lwLVeWhnkO1Oc9Hq6mk7KkDoeX5UMH6nluIZoOENrkWYloOoc2hRz4MiWOkTkH6rmoMlbSQwTqeYIoms6zWciBWwrijJ5zoD4tqAwDyFCg/q8IQtB8sG6EdfHUur6Hnwcha8S76gfpeb4vmjTNnWLOYjoZNpBu2OcHcrVF77pk7cW0rIUI2hM8UxiEMPkSoflB9y7aWId9y0OI/jayiNUFmJxP30QTRPUB79wyvHWY8ZxhuQV8iMA9zwhG02ED9zyjWGGJOmgE7nlGcHq0uoF7nh0sOds1cM8Tc6Vrbe+Be57Ti6b9B+55KjCa9he450m+CnnqYCpwzzOA06T2ErjnicNoOlzgPuDZxWiqGbgPeP6wMhh1ePDAfcAzi9H0SAP3AU9cRtPRB+4DnhyNpiMN3Ac8d1phljqcRuA+4Hnbkvdz4P4wgfuA56grD4Q6PJrAfZCRQq99TmGNwH3Az/iSZsUhA/cBy6xP1i5sC5NAb52kYZ/cjKYwQia39moI3896A5lsnU3nS9gDWLRzF1CWT6+UVcRlA7xRmKQWb+nFPcWbfpSe80aLeC3e0mtxijf9ED7njZR/Ld7SC26Kt+KTharxRstRLd7S62uKN/0YPueNjK5avKVXzBRvxYGvarxRUZBavKUXjxRv+gF/xltYl7W0qlfy2RCf3t3lao3n8PmJXebb7nwi9DJ9cd9RiO70vLEBLyWAJiPeWBhKoQUzwAoaWPcbjkshgwYSzwdOA67+BpabEf7C1Gm2HFxpdrDMN5CdSWacQ39hYt11GsjDQEgBJkQYhsGdLRpGsg+ZBy6ZCvitHBsu2kD2RCSi4Q/wn/wZ7A5Bl/71G2KcpjNCZ98WvG1+Jr69H7cbDn3FetaQhhycOESu2QLDxLT4D60BHl8jbKCwEf6GREMYouhOd9BswYgjQ07+J/uTf5h+IP4Nd3ISB1R/4Z/oSinj8L2MEfV0Wh2EmSlUTDUQnj753SJaLGAaXIkXzel0kKDitq7bfecaYm5G7euW0/G7LqpQNjs2asL03Z7Xx5371/i+7R85UZ+0xu/eDicwK5LDwanR9a/t5k277XejEP5OnNycNxZ1Bt8eaccl+U1pGg16V7++7L1Ytcfez3eTbyazN9fvxn+e/+Xr6/s/r97cP//58v39X6JXzcUP777y3pCCGC6gGnpLycbt3168HM3e/cfly9sf3wyChffqu9c//i143vvXpfNm/HrUfHf3XfdXd3HTGpNgpzc3iwivptOBRYTpL2wmDPO7xaQ7gxt4CQIzBP8pMCun9ObKFWkmxWZbNyzg9+uhCWqGdEL8zJJvxj+ksguojp76xsRVAuON8FC4dPor6dshHQEfcxMAEvGuyiFAeLchOgTWzRlio7oA7ul9hjOtZu+s/2+582j8OY2A28JH+H3X99hHKQF8yp+HrfM8LJ6H2CTvfR5ih1ZxBjjN5ieWje0wTQR7OxHiqRTSAdH7mkrheSqVTCVv/1MpJAeCkalESQPVplLQ3nEqoQDCeSoVTyXYD3ufSnTgjtZUEhOBTyVchqOn2kgdZ9e5hJfuPJcK59IBLC1MgarqJJ5MciZkTCZRg6+aYmqHO04mZBafJ1PRZGoDG7P/ySSy7bU0k5gJWZOJEkYqjQRM/K6T6WwwlU0m/xC2N7rkzIC1GykIvNm75I6Nku2LNn0sgBq0MAEphJo/ZP4SmROTnb17agEiXDZuvt4tYLXSyPnmQQHHYatkXAnJ0Xm75avJ326RRFfn3kt5bBN2pIxFqmIof6WtKpAPc1YVhaoCWaX7VxWE+ixFJBtPKdimDuh7jELM2wqLllIa2tzZXwpOO06LyI8a8DqxaKpR7snqxymtm4ERNSx4184usZJ3DfUFjt8lJmZ6hhuD6usewztb3TVX450Vd3mQd5YD0MrK1VZ8ZxGxLnhnO+d3tuSdDU7A95j3zlLy31G8s5V9oI5TzZ2uvLRa+aImX1qOaBQ1u3d/aQGXyH9pkeB4fmmLX9pDBAx0vbxyqmd5ealo+zG8thr+5hprrbzPg7y3HC6qXW4++70FXqfgvT1jF8reW2zXj9+jnrPaynrwR/HeVnft11lvH6zwffmLy7FdZVXLq724VNw//8VFtt/5xS18cVF6+ASiF7krrigQcRxvbuU4Sr0394EqYZS+ufzUCjQZeXMLfVLIUz2/uYVvLiqwH+DN1Y1MsDdXNxJAOZXnmVA8Ew4QCWi3S6J1HPXZ8Aj3qfxttpIw0DbVRzE8Zg2Aa4UxtRGv+WNqvw9kap/fh4L3oUXbtV3eB7gD1g2z0QoZb3QzqCs4mN5eTKLl2+n8zR/oRmfde2SFgNX/xNHASIAYzhdL1BuYrYiUk3hoLbeDGeOiz5vontLgcN3Dki0S8L0Qvv6nT3wMsX3KeBUw3bJnAF07z4CiGUDHMZjWiFAjcSyczmDaptQEOJ0k+0GFTXgVzg+q6EG5yIM3/aDIEF6/xsJmyXs8DiVcnB9PweNxENwy/Xiwd4AuFSuktCaQeOZSSi3y+JFlFT8zKolND+27mJVm7lYhbOMQhPNjLHiMVFzyn/8H43WuUrrzBAA=","output":[{"uuid":"01884434-a2db-0000-c91c-f13ff43dd8c8","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"01884434-a2da-0000-ee43-723c53b20b8f\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAJiSa2QC/+y9C3PcNrIwOj+F16dOvk2VKQ9fIOk9+c4mchJ7YydZx3EeJ1suWZIlbUaPo5GVOFv56/deoNF4kcAMRHJmOBqUStIMAIJAoxvoF7r/v//3fyb/njyY3Ew+TK4mx/TT40k0IZOH9O+DydHkgNYcYClrdzWZTd5PTiZnkwssZWWXkzltd0r/n0we0Zpj+u03+u168uvkb5OEtnmILQ/oe2a05oD2rXqdyr4uaM0t7f0E3nxGW17IpxNsldDxVZN8UtKfBP6TSUpLUmyXyt5OaR839K2P6agewXsPab8zGCkb8WP61BR+Hsm3ZPj0FL/nje9F4ztpfC8b3ys5alFSt0oSCQG9LGn0lMiZsTpRllnKxJhzOtpClhbWUiLXfEpHlkkoJnIm7ZrKc73q1ko8oqV7Gkakck4E5kLoW0WNeLqmn0u5OgIqBbQXpalsm2ulmbWH5opmrTXMWquYaaNRb8i1dXPh5J/0h7dm8DmntDEHmjqn0Fj+7EOkkLHQ5zWM/5K+45rS0vGKaTOF1XqEEGPwO6Tf2NyO6Lt/p5h0SOvmLdot6IhJT/rV+6gabeoWZU5bJW7qnbZod9qi3GmLags6iqpFtdMWtU4tVDoDmP1qpcw2JbZxXq3fYowvYYzdMF48GzB+Mcb/a4fwPemI73MKtWsKtSsKwa44X8ozMuD8JnD+kLZ5D7tWTFfiO9riyeQH+qbn9L+bCsoBqKAcFRUQC6+2Pioo5JmznArqHlRQBypYSgU/Tz6juP8j/f/95JvJFztGBeUGqYBo0mCggs1SwU+Tl5N92oZQGvhh8tVOUUEpJc/NUAEJVDASKvhu8oxSwY+Tb+nvM3ou7BYVFBulgsARjYUKvp+8or8vJ08nLyafU6wIVBCoYPeoIKf4/5y2ejn5++RrSgmBCgIV7KJ0/A+K+9+DpYfJyLtEBXlny8AQVJBLPW2ggk1TAWvxBeWFXtK/u6Yjyul7N0kFaaCCkVDBP6hM/B2Vjb+dfEp5okXScTUAFVQjs5pNN2o186WCmtZ3pQL+bKCCxVTwlP58NfmJngbP6f/vd4oK6o1qSitv23GgglVTQUlx/1NKAz9NnlCpINsxKtgkR1R7ywWBClZNBZ/Sn68o/n9OZeQXlCfaJSrYtFwQqGAsVPAVaEpfo/9EuVNUUGj+25uggjxQwUio4EewGLykP68nXy7kiOoBqKAeERUwP55soxyRPxUkPaggCVSwlAoyivvMZvwz/f/z5NVOUUEq8XATVJDAWAIdjMVuRij2M66I+RJ9v1N0UEo82AwdEM0KE+hgs3TA/OlSihtfUBp4sfCOQTgPwnlwnz1L9+k7foabNt8t9K++b3RQwD2DTcrI4TQYCxU8oadAOfmJ8kRf099do4LNaoqyQAWj0RT9g0rH3KuOyQi7RAVkozxRETii0VDB11Q2YPYz5k3BNKe7RQXlVtw+DlSw+nuXf6f4vw/W468pNQQqCFSwe1Tw+eSHyTfgW/oD+FnvFhVUG6WCOlDBiG6cvaLYn1EJ4cmOnQWbtRkEi8GYzoLnVD5mq/UVPQ/IjlFBtVEqCGfBWKjgFcX9Z5Qb+gF4ot3iiDZ91yYJVDASKviSckFfTl5TyfgplQ9+3ikqqDdKBVWgghGdBS+AI3oJ/qVPdowKNnvjLHiWjoUK9kEa+JaeAl/TE2F/x6ig2CgVkEAFo7lx9v3kM0oFBfz9bMeooNooFQTpeCxU8MXk7/SXgBfRc9o+UEGggl2kgs+pdPyMckTfUfng5Y7dOJtuyR38QAWrpQIWheWzyQ+0HYtM5L6Dn/TOPaX3MRYqqDZKBf4eFGkPKggZm5ZTwQuwlqX0RMgpLXwZqCBQwQ5SwTf0NPiGUsIXk58oFeSBCgIV7CQVfEMlYwJ+FJ8HjihQwY7eNfsG7t//MNmnEvK3O0UFyYZvHk/vYDcLdLBqi8EL+oZnlBqeUkrIdowOyIbpgAQ6GE306m+oVLAP9PDNgqw2gQ4CHdzv84Dph1KI5V5Qigh0EOhgN+8fP6Wt/jF5RSWFVwu86gIdBDq435aDz+jvl1RWZpGJXuwUHWw+QleQk+9KB8nK7hqwGHVPQW/E8oHvFh2UG6aDcP9yPPcvv6Vc0U9UTiD0/w+BDkZKB+E8WHWGm5/pOfAVlQ2+pr/PdowOqg3TQbCjjcfL9GvQmv6D0sKXO2ZHC+dBoAPlZ/qEtnpKKeAJlZhf7hQdECl1bYYOijv4Wwc6WC0dvKR80bPJ3yffgadpuYAOkgHoIBkVHeQb1ptmd4hfmvWggyzQgUe0rm8h89lPtC1ZEMX3PtJBsmG+6C72g0AHq/Y2ZfaDLyY/0pbpwpsH95EO6g3TQRXoYDSZbn6Es+A53EX7ItBBoIMdzW7A5OSv6UlQUir4IdBBoIOdpAOW/+/vEKXlG9r29U7RQbrR2/mJEfEg0MGm5YPXFDOYjx3zMXoa6CDQwY7ak3ncup8mL+m58NOO0cGm/eyyQAejuZf5LaWA5+BznS60Jwc6CHRwf+ngK3oK7IOv3UvIfrNLdJBtmC9KAl80Insy8zT9nv7/jNLC60AHgQ52kg5+AG0RywT4OeWNftgxOtgsX5QGvmhE95M/B8ngyeRn+pPvGB2QDdNB8C8aCx28ptj/PeiKiskXC+8fBDoIdHCf8358SX9+pKfCZ5QKftoxOig3TAfBz248cvJntNXP9Dz4HE6GQAeBDnbzPtpzeh4QSg3MkrZr+qJ6w3QQ/IvuSgfZyvwqfph8Rd/xhP58uSAHTqCDzdJB3oMO8kAHS+mgAFj9Y/IdPQ9YJpxAB4EOdpEOWIzr11Q2+JH+/LxjfnaBDgIdqPPgC/r7msoGTIv39Y7dT843fD85C3QwEjoglB/6jsoIT2i7r3ZMPgh0EOhAxW15RX9/pu/5FPLGhngV44xXEehgtXRQ0pPg2eQF/fsP+n9R3JZ0ADpIRxa/aLN+dvkd5IOiBx0UgQ6W0sFPFP+/pHLBCyoj/32hn939o4PN+5sGOhhPvIpvKA28opzRPv1b7BQdFBuOZ5fdwe868EWrvp/8CjJCPaEtf1poP7iPdJBumC+aBjoYCR0U4FHxlJ4HX1Ku6Osdo4Nsw3SQBDoYCR18CZHsWP7kr+iZUAY6CHSwk3TwPcX+L4AGvqRtv9kxOqg2TAck0MFo8gS+hJyxr+mb/r7Q7zroTTepNw10sOq8UCxj7OdwP/ObhXnE7yMdZFuT/yDQwWrp4GcqH3wDlPAjlZY/2zE6yDdMB2mgg9HEL/qO/ryitPAPeNNu0UGxYToI/kVjoQMW0+17eip8Rinh64X39AMdBDq4z/FNv6V0wCLasZsIeaCDQAc7aj9gecRf0Pf8QOngxY7RQblhOgj+puOJ48XasIiOr4EmdosOqg3TQbAfjEdv+oRyRfv0NPge7uzvEh1UG/azI8HPbk10MIWfJh2wv5d0FnP6+R0tv5UlMR31Mf2+hyX/TT9/AvkyUjqamP7PQbaurZRSDUApVYNS9LLaUqYoxix1U43ZLrOWihmUsBZNCmK41bbAsXqW6Ym0KKldoyjqEvDqmGKDjZ7EGj+ipXsani2iL5Yd3U8fy/GjG32JZwN9LaavA9oDm/MxpZsbGBc7hWLZZo/WXtBZB0rzp7TcuMOm6KxZHqhsVVTGWrC+b3rcG3pHe7iGEcVICzd0ROzvOzr2GS1tvyXTcvplrTWZeq5HTf92XQ/+7G7f5hK7HuMVDumc2P9HuINlk4+AP/yEvuejyRv433x3CWWs/hZo8hOguwK8OtPWWlewN/Tb7/Q+Kq2sbux3epna78xS935XAWya+51ZKmZQAwyb+51Zqva7qYWrMEvVXvc7xeGZXEeGGf9L8YLjd7fdr9IyEug7oM/+J9a6C72pZ8PtyWXReL+aPIPMNT/S/z87pFm2e6W9KakelTRbw1s2Kc1WnlRQdrw7qZ4NOp1lOp192opZfFl2y68CFYySCvIeVBA0OsupIIcsHf+YvJp8AXEldosKNqvZrAMVjOb+8N8pL/R3iCjxk/Pe5H2lgk3eH0695PBABeugAhZj67vJ97TlT/RNLwIVBCrYyZgqP1Hc/5HyRJ9S2fibIBcEuWAnI0l8ChGo2Q1iltM1yAVBLtgNKviWvvuSYjuzVtxQCrgEqLKxn4Bd9hvI5JdNXkOklS8XUEY2AGVkgUvqRBlBezq8p8I10AQb/RFY4uLJrbwzxu15iyxzzMOqL0XofVSyTN2srS1lijLMUjd11EZMrcxaqixztVZaaKXtqL015MTMLb4I7Rp/yllukauBXtQ9/Ezz2Mi8KCqFOPzdKEo8e/8pao409Z5+5vM5Bo8dfcx7QEHntFWbPnSP3670ofoYw4mRwD7S9b7xO1hdxpV2PTFyT4mCcQJVZ/zmz9rxO3fi9ylg4LtO+/IpciUCBr+BX8UNQEpE3VDx65kv0xnFvVPYQThnkcv8bHebbQ5r6ppt6pztBR3xkWzXflKsHVvtMzrro8nX+MQcW/yP5Smxm1/AuI8NWDK/BrUbvJ+8pd/ZOfZM7gcKh+a01znQ67m1/kyWpUvnfgOc4tdLRnQAq8329bd0bDfaPFnPM1rP/PzE08ewB/7ZCUbLxnWMO+TycXV7fybffzz5nf7dN7y3xDh+od+ixk8b+kVH6DMMvwG/yk3PsylPRHTm79GTM6XfvkVqfwqnRRsCJR3dP1ulbi8zf+i331V1hPY57NsHnrjepF3mX35Md7QrONUVz3NoHbna+T5Bb/QzOM1jrYY9z7iIM8CDA/AWnOPuegxcqx9tteFerwTuPjvsEJBncz0Ar8pjbUzvwacyhnO4G1SSBV5Y/nBxwca9AxdefUdUftdvMcxpyUf0l8WDidHz9xwok9e/g7OTcYlcEhLP2MaWrQgf8o74MAO8/9UTHxg/MpPPLvNFV2fn2eQPrSfOw03pLPn/B5IL6c7vrNZjviuWFytabbKB1W6uqvneB8Ah8hPrEULUvf4ZHefv8He1a9+8jaTeu8pVL1e06tWWrzrD299Rxlvnqqv3rnLV64FONDdvm3Y+9ftgwBygfw1yNmsbtXBiPavof5Ow2wqmK+RJ0s5SYZ+VY5Q5B1vdelZsTsvfAb94Brdt2MiZ/u0ITt4DOuI92ubW63Ya63UGeCfG+B+UK/uCtmQayf0eq7waDizdCAd2DlqAM9CD6fcxhlvfOchETEv5G/37tvXGVdBhsaIVIhuRVnX+85yO9y3AgsufDKIxtogtOgi7RNvUAHSFc7kiOFcbhfMZ4LaQw2Kt3TpgOhQXsJgTyDpzAjZb0aY0Xd/R8peTZxT2ryZv6N/nk09pyed0Db6hn5/QTy+tc0+tGq5ssPPbfY5nC06PYeGuv7Vp1VI7xnvQzFzDactPBmHNGuO6FY51ywfmnN1UQ9a2ek05yb47PZr8C7jcg8Ybxrh6lWP1yo3IJDdguz0G6eQYtZPdeR+7nw8b3xHAktlk5zCebudCNsi5EDl+2quV9zgfBGS74bmYxaM7QawrVPZof+9w92M7I7fYz2FsEdLcMUgkjGtlWn/23AX0dTz5K/2sbADKGv2fUKPbRDlMr+hIWM1bw8PlGnbcM9iL+chFu0PQCc+hNathu/gZzuIaWvzpnMFjWnoJPYjZXAL0OdW056GPidWdg9zFRnXeaLnonY+RR/0D53WBHm038O2QUisfy/KZvcX2J/CO90BJ3ce8eO7LZsT57F+Bu2Z2nRnGGLhGbT5731tPnIkm/w/ovITt6QBmbcOXRMOEJpZlVlziVm5W44ZTG98KibEumF/TkreTv4A++iHqkXL5iUw+HgBb3ZgT4NsFvjPwTuIjEhLLAeL/NW3rsz9MJRT0ni/h9xb7+TeOZI62T/YmPqdjGAMru7XQmd5LM94K75XBcC5n8hgtrGJ8MXhbXAI8/+oYA9MYzUEPxM8dvhNcwSqcwXnBV5uV38LbzlC6VrWqfIY93CDmzQGC1xTXHhulM/TL+gvCj61jDCcD+/axxJjfYS85BWy5pBioMOEAnn0ofVbE3z3QQXysYWGK/wv8r1ZMjacJkxn94b1lsPaH4LnCOBYG02NKgWewKn+BNiXiZAzfpnj687GITzVtxUc1v9M678GJfgSUcyGxsLk+04Xro/fQZXU+Xkg5b2nbCwcF2XH0HeAJX4MDGOkJ9HCGvj98TzhECCyjXP5+3zPzHcBm1tpTOC/U/PSxdefLrTTfHtMerJKgbj5C2/srxB/9/8f3bE+x7euxpoHW4ZHK6MPNTx8v3J/CrrNs17mCOR3BWp5IbnzZCbYNexLjda5htdy0YcPLP2DfETLoY8SIv0q/EzHnx+Aj9gB+BZ7cgK/DOT5VO2QXJe2xdvz9bO7tlm05ieOO+mRbqxlI4jcAA7GXsvVfVC9qm1ATbU3O5trgCd/Dehwjx6jGoVq5RmJvETnX0HyvGo8+StdohEzLV/cKrar/tnCrCcJWrYy+E+lcpv7pYy+uVoy4TXcuvppo8zwHiHDfexevqXZQbg1ke6zv2Ss+x9DuGu9a6WcJWwvW8l8wpxuwSX1o7Qccrzn9H+NedQMSoaDuplxRL53LHM4xPqJLONOHmNEhUv85jqvvvOxj53cLrlD6WXSydluNdzjKWPp/CJ+8qztzVcsh4ObDls15Dvv3JeA8h4CJ6/lSLLD1quj5f6Q+6xPYl5VMzL79E99pp6L2/2Tg0VygTM7gpo/HhEGxlN67jJRbKoQPr+A8l2HNsLQ1JA4upkJzd42N86Gwcjdt6DR3KH0fdu/Qbo5pDjbRK/TsubGufWKs/l36OqVP2rG7WIArdgwQvOQ1rNBdR3Lg3NtMOUGdx0fy/uKBgT8ubePyETDN2FtY80uYhZAF7fzbe7Agl7Re8HI6DOMG11F0Wh/R8gK+H3vCyBzHDbylyaPbJaZlEuwyrdc19MrO2P+L47y14JcbNmar5fTnO4bHjYjDwrphG5udd+321hlYC5e/08Wh/qnZi+aT7zR7j5BqWHTXtkXJbnXPwepuKx/O3mWzb3W1w78F29MHb+uW8PXXrZbinty6bzvZfRLyFflL5Ru0lisrot0+LjyM2b3c9d9a72YDzjtbyrffZycH+7cNJtUK/KUW3Q7cXb+pwrGDF4P4TfnAvuuezU++u3tNsfPzssdOPdSoNvt29l7OM6ibGJ+CZzbXdR/iCcdlRldNTH90veiDLZrVG/pzCCfCHN61ipFzv/auI38Oo2W09azhqd+8t89u0bC2D5w395sthFVH1ApZRtQLWfMzsC6IVlPgGblfO/8r2vN8ABfa6M2T9zf4Yf7iGf17CRzpI9BMirNXh5SyYh+gjVbFdjgA7lf3RLqG27lNTBQ8A+dcV7O+V7Cmp54rrPagF3AzKIK7R4xXfg73wxg3UcOeyLUcXI4SpaItry+MEr0df7LAO2ftpxOtVQKrIcpLrQ/9r16ut09lG9H7H0784nbOa6S9fe0ex934JiabsJySi+WSosFZ+D5VOc7C0lFOOkd14NET9FxBd90dXoG0/Rng/xj3XX18bK9lNHuinSmL62NNPt6G2Z1C728By09wXcOJEk6UdZ0o5xQK/DbwKezSt3Df6hRswxndl88pTFh0zlM4K/TaqawtQcLVn2c1rE9m/+JnQAzQZjul+FSDRNAsi0E6zuH2l/pUaKdXMZrTgoBe6u5PZY6YKl1vdR4MthvNwA/zcpCbf5vfx5btLO29q4KIdswDZ9kelhotFu9RCdywiowIBcPuUqq3x/Szugvh33MCMQRr+kl/fhUreLIxKbf7HpmABo/dTpnCrpag9q6CiGXMa0l947o91jbGCIlT4Pia5RlEciwAR2bQA4EeElnPYxny3lkvLNoef1MCzzbLReS0CHblPchkzHab2YK+xbhreEKMMNJGXlpqzLGr3qNW73x8RBu1XqaPONJG7N7hb2Gv+gvweW/RDn6Evnpst/14ZXt913uow2Edg/ge+IDXG8K5pDfOJQ6cq5w4V90B5xJvnEu3AufqteMcOxMqhHSFsSb2IMJWDlinVjkDbKxghXLJuU3B10WtWQoZJitczwLqZ8jt8TzoieQD+fdDWCPmN5UB98g/5zCKHE5Ss/fa6Dtv1J1K3lK87RDxNsVYugJzS8zym8O3DKwJxFLDvxWS+83Qq6ACuuE4xW9qcGqt4BvnZadgX9sDqFZy1AWFSIljJqgf0b8f4twJ/i8ALhmMqNmXKjmFz1O4jcihKlZNcCVitTgE+UoW2qhiA5Iz+M5LCoyzqpcIuIqVi7W1iy3YwFczadWpmSj5Yg9gzum7K4QTKE+wfYp+A6pkTFCuYQwZzDhvlWQLYJ1JSrRBe6qdBG14c4jr0ObU4YL5MnhzyExhF48a3w8NyhafTNoVo3ZRewHSpgn/KZ5WU5DOYjnWWI42xxZpa1eKG2Nsfjfhni+Aer0A5nUL4jXusAnOpoC4t6t5360Grwohdve1JVhaAIbo3+6yrsmaVvbuJ/sH8MmdgR/qqk73srP1uM/pTuCuRA2wEad7CntgjjttBrFcssY3FyY293b3zq7W4xbtDin0kUkczOEs5SdvoZ206gRWZafGWSTwjp/pgrMTT8WWXs2z/Ba+1UBzucQ10V+M7xFPZIh1JcRoz3BfTyETt5jzVL4jgZXOQC+2B3brFPnoEnmSGK0jJZw+vIcSYMQ1cJwSasB9IYGV4J1JcF0IvDtHXiZB2BY4zgJ2lKpVo74lFGNc7SLnUymedxXu8FOjbSJ7rBvlZo9zY4xmy9j5XNKBqtl9gQ8rpOekIz131WcNJZdsUrooEIMyjbcTdKeoNrXsAbnx+bTF5StOLG3xYpWkWpMbyxv7GefHzJ7jhrQyLNfLtdurwc9sAxqMBNahhN0tQo1GDrsg20+J/JYCvPTvq+NyfXkgP6mNfcs1qU18X9065mtfRzF7Lh8Jez6XElYh+d22eL/h1+cW8Y1zruUK16voYCUqO1uD+lBrjfRHpK8H32GLAWQrXUowv41XztHxpU3dd+M+riEezeq4D7LR3b3CmJh7sEMkwKHneIIT4KndmpHKcRa3T+Ic+QcCz+jfxrce5cbXQz9tCYVDAVLEFCA6ND3cgpXe7D/W+rftFoV1tyhGuJZVJzt/ORJ54G5yzzq5iinSN/cGSVH+L8Cawzn6BG6w1RK/MtjTmd41RVxPQApOUX7nfOYUbgOw3SGRp0QJWEcaNTnqHVIpPae4UyXy7Kkwpp/ZmsgdqN0TAbsDt48Q0Lb8gdKPmE8MlDWF2XLqyZAupuBLk+LeWYJ9OUEtTW7I2TVqTAhmi9NLhZZgClySvuMWyGvvAY9RaRoF3jZ29rIqDqnagCaMwFlVwmqUEvcKOLkYPmWokyVSn8Sxr0Jraoo7FwEomXUFRklJcbX42vK1ruWzBE8x1TqVON7uKcVeErSKRuiXpVpHxpiadXpPEXqj8DER48ncaNucm1nHdVMpSuxVa+6HUhtF0HrM47QksPNz3jLDt5i9mHUJUlYpff72kG8opNWY20aadWY/ggaI5j8hxqO3zeWs2v3k0i+tkFbkJgz0EcXOmeUrpKdk7fRUokdMDRxZm55SxKJK4p+yrCiYCmo06zLoucC1mOJem0iJoIBnhRU6RbrO0HNQr8vRo1BR5h5ICwWuEgEeRO2NROIX1wzrNVxvVRnapQK8Hvl6F3L3rAAbIqmvZvg30yCSISbW8LSQhQi0FJDhUliM1vhK20lygEKOs0thtyatuhQ1pwTmJPYk0rDDZJo3pqprn60ZnmyF3BNytPPo0DdrcunDIfTzBYxG+HrwfWsPs+amkvtP0JPFLBcrojTetcZJlpjLWsBLh6XgA2ZYK9ZgD0d9KEeTylXl+x+fHZd+c7mDFtgyRw42k56tom51dL5+HQHX03BfwVzSOZfjC8nxqz20AHzMEau5NT1BHFZ2EI6lHENTyXclyAnq5TplKlsH54x0PGrWqDNE8FocfwUnmcNJn6L/svp+2vgu6rkF7VR+K+HNt406/bS5bfRr1uUIxam0h+mtIynZ1SDZmfSZIY+qfKpF3R5YiUpjj0rg/KmlJ1ei9V/ATGI8w7OV8nzr10YLmwLHoAgt0RXAKEf5mEugicbBE+TVhARitjBllARPJgL8OHHIMDVKMaeN/s26DPG3Qqxr9r26lck3oLmYwmoQB/cguPGywYvnTk48vwMfXjq58HwBD25y4LmT/869ue/SyXvnCznv2pPvTp1cd30HntvNcfvz225u281rm5x27eSz67Vx2cVGPIKnqHMoJJ1M0V56qEFErVqCXmYCkwTGEc0GM8Xz+1aTiA61E0Nxv7XGyyqOK0XNTNmq2TO8LBM8d4UWsEJ+M0V+P29wfTO5lolc3wI4tAxlVf1tEY4jlWck3z8T5M64rqhZk+EM9b0lRWrn483xLpNeo/y0uPamRi6d9au4ggxhWqEPyCHyKDl6lhKg+tzg8sWKZPgmoedKrTUVrI2SigicE7omyCznmt4E6uaafGLWRPhMiTeuVA3fedRzsVGrdh4hCVVoM1fSSCF3PCIlYlUjPL2JhsFcdm7K6kJ3ZT6vS2Kl9Lms5Z7TlIiafeSatzrR9pxUSiqmBqM9i1XuOKSTJbHupL0mjltqxRribrRvqp1gVMgx3ul9DznseIy0ucz3zbMSHMI+P74x/0bH+I625JGYl80gxmjR/i1jjL13BNkyDjB63Cms4cXK7gZegc1jiFvOKpKurTSGqHV8ZOb83e3NfCTuXkUc6CvoU9xqt0UVDzfFw03x9d6C5PIv9+U/RW1MjTfJBL+XQsTJPdBUCB49l6dyKTma1Cjj+g9dTlMyGucg+G225v+5/BS12katmqhVw/8fSjlFl86ER3qN/FAqNUlszDOcK5/5rYRFga1r6feaa/faa+SgCN6sEJxDrfFwheSIE60kx/k2a6JGi0j+Fy2jVstmTfPZsdywrxdoOP2iidlih9WNG/i+Y+lqVVGRk4c8mdh9/XbPzdiHJh/yEDmHAxhpLMfhy8nEgA/HsLqiNx5tlEXIOgVsENFpRHQ59fQe/Kjd+gD9lEXbrrEQ60YcBlU+tZZXjviBlSOOT+WI40M6Y8SmI+Rc9447usr5ifgikRZNOzZiaZv5b+JG9hsVwVuPxnwFI+BRbT9gXoYbo0UKsQ+WUwmPdXwGtKDn35wDn/Yes2VcWHq6grkdQgzlG5jBsda/gk8MlHiN0Z7ihW88RH7wRuN4xslfNzP2NPPtiFULfG3ga1fN1/rzNYKG9Rh8HO7J5CnwtvvI4Yq78sI/PkENrYh5xz5/J59WbUVvL+Avj433GtbjKX3iNXx/LiPzvQYN7D5851qxSLMciP64tf01jm4qeWAR849bWdTIvpNzEK24FvA19rSP1hf1llp7v4j4l8DoSjpaNZMX+CQfNf9cyV64B12Enhb8Nul3RptS67kCiAsY/LwyrrPuxB9WnaMBD6W3+AzOixPYW22lXCtzLjNwudqc43uPFrRp5p0ScfxN7kLM+wyySR4PcEb58f1TC9/Prd02To5n6rXzkN3jbIfzJpw3dl85sUurmHbxiKRufvP4z07PpQ4qIhuRlE6B47xaypc3OfzFz+lYdwC79yVI4Vdrwryws4SdxX0DSETzTLVonSIiZ4y3JYQnk7LLC54xR63tFOl5LksjrVT/GzdaxI3S2ChVcUMj6e2ay9jQU7gXoet/K/idGyURlkbSzs91tKJGLxHtRCyhBH0RRdwLoXUVvro1fssBgsqeze3pMXLLMcQtSMGLVXDSqfTbi7WbEwV+K7T+UvT1i7SoEBH6DOkW50Ra2vmoxO+YzgnS8ZwoAlcV9r6d46qqjtRSOqWWfCN81Xeomf3WyA/HM2NcYzbI5VyWkBe5Zvo9SpAn9Ge2Mvxa1byvwC7PMexYZqgVOl63HPwt5s27gLoDkMrndD5z1HxHgCfzlXKT/bQPi2fO8sgdjXItn0N2TW4x4FaBX1Hb0SzlFhPhdXGs2b7a68lsDv9F/x7gfn89+RtqSm7APnZC92qeXfX/rlgD8qNVA5I4oo8noH1z1STOmmk4x8M5vmPneNIpoguPoes6x2uH3qRwWJ6nGzn3mffBLZ6BLwCSB7AO3ez2/H71pvgXNjpuI48a33kmBJZB9RJ9K8bJiZgjnoMl9WiFOWSGHvFh49xaxZjfaxb97cqq14aXyP7Es3+vlnv4Fjief2HucxsnkTr5hbSzXwzD4O1cLd1KpjTGP8BeeSrX8lMtk3zU4ZkYPUOOICsL95tYFXftm1Wm6XFm05j7wssFE+4ZzXY2LiUxD55FbUVG3vfgIWN6/ix67gBhfduyoJrthOd8rPGbi9pzO0EMktGyeYq2wntePbVcolaStPCcilE+E5R8CrLHMfBnJxqv2CfTj+hz7F79OqS596SA63ZIp4ukzk3IkqnDXzIBv2hXDRmBl4SJB+vgQ/qOXnkp8r3/AHagMYx2fLSjvD9d/qPbCjU/an1iYEjkwctlTsk0c+qLMif/lzn1RZnDyyaBG8xj2xdWez6sZtyLuEcuJV6ht/51D7k9c9wOTeDGs2uFN3NztCucBPdl8rOrwAS7j8mu8NxMvxoDVh5J2XZ5e/PMvgvv3daOzkCXeqxpFOfA19/gjc4r6/65eS+gbd33guY/aP63U/OfNzgb/+emzvOydtZUzhq3dJU7a7INn75PoB+hoWZ24Q/g6SR6vobVM28u8jvwB7A39eFX8s6R8oJGMmgkh+eOuHcHv4F4em80jUfQ3ylEAGEQOQLo9tcvtvsNHFXgqAJH5ZvRpQD+I8GIlzymtZkrupJx/XhEXpGhs5BRfxOMfEswh14NnyIZV7jGnHSJjEHKI72KzA+ZzBy0h9EACcakIxDfT8RuK2TUrhq8HFLsLYP4CwTvhouYGTnmBxLZ+HLNazozsgzxjNVTLW5aLXP71cAhVDKiP++1xOigU5yFiATMayqMAVrJGCWVlumyhmemMlJcIqGuIJlhDqxExrKeoi+5yKGi5x/IMANehu/icd9EfOMCZk5kRJVUZujh8aBLGfduitkfY8y+kmKcf+GtTzAOLPcUmxk4YeILz6ScYLaESsK5gngJlcyMrGI2Fxi5lOD9g1KL8lxhXGwVC7CUmbtrGcNQ5I9TWcGngAe1jIGZQezdWt5l5WuZazH2CObQjeSba8xFkuI9BAJvE3ECU8QPEYlwipio4gHmGMOPxz/PtPiINcauniJGZDK2eSbvzPLaFONGFxhZL5MRkEv8nAK8VDR+QQO1jN1bwMz2MC53Kp+Zyr4qvG1QAQZWMrKyyKRTyHflALlTSWsV4v4hRmfk8d6nGD94T+bBnGJsSp4JppAxoTPAET6iQmbGLDDydoG4KKJUEsRD0SsB3Cgx3uFUi4ueaXuXWgmO6aUWOzmVGWdyjAEuKCCTWExkLqICMSbDEesxGnN5H0PkH8kwv0eMEQ9TeD7CTNK83wzzS/OYiiqOL98BCe5xU5jbHsa6T+W+l8uIlTXiyxSgITLpiQiWOVKjyJfOy+YYWzLR6J7A80TLuCVaH2rviuXbKqSNUsaH5pm11FhjbbQzmVk7lZlkCZ4mxPgu4kuKtiKjqYJPghmNYgNCHGr6WzM5lqQxbn02fJaFkYVQLyUyY7HIj5QhpguY5RhHtP1UYawNf2slYVVqELSNeUxajqqjlsOll8iD1XcUVl9f+6FbjtNXtXDa+QqnvisPdr4tsvMVTutv4bT+5sHOt0N2Pn9N1qbsfGcoq4v4WXp0uisYB9dZLtZf3dXnzrffu1gRl9stv9XeGzneGzRnQXMWNGfbb4ssOnLphZNLL5xePYXTq6dwWhzzBTVZsMYFa1ywxq3UGjeHO8jHyGvN5XkzjE2u3WfgKwJfEfiK5TmvuRWOZ3i7hc8VZpL7A+r3pB1K5IGuMLPcqbRdqZaRzGDIWhGjzVg4FdLgBPyfc2kfSGd/nqBPXL8+8Znj/DFXtHSuNXHWFEGXuDW6ROL0dCROT8dNRdgfvy7xIGgQR6FBNPe2bpz6mbWPIbnzR8Dj/Ra49MClBy7d02+Oe9qfQq4mxVnfwndRyuOrElmaYksyIu677Mh9l07uu3Ta/kqnRbh0WoSJs6bobCvepJ4w6MXurhfrdm5eww0RHqlpiFNT749/Oxy1NBfOzXBuju3cnIJG6pb+ztDbkumquHfuqRZ/O8HsN1OMqa1H177FVir/uB5lO4Izl7dQvUQyv7yIz30r36ZieMfyPeJXxSIncnziSf6WMZ3kdceTvHKeymXQow2oR4t656n206i9tJx8+rpWTg6tcnJolZMPK50auLKzBi5waLvAob2D0YlY0O8AVicDWSDNvt8YvQduLXBrgVtbzq2leJOC59/bw/sSGVglM3l3hmnhT/G2Uy51IRk+O5P3fsRdjBifjrWnY7wTkaOuhOej2cN7ajxLTQ2cXylz15T4VgK5tnPk/Erkz6bwDK8fD4/WzBXs/xxxnstF4NG2kEf7wjifIvq9efrpq1w7ua/KaTernHx95fS9q7bS9y5wbOvi2BhGX8HszmTG8GH4NVfPgVe7v7waz30WS04t8Gvd+bUXwJXV8ifCe6QZXfcEs0Pnk6cYjUD8PMccze2nWIbpp40+X8vanyFvc4pxGCLIRV3gzXXedh9vGyeyZ5V3utLKRG7AHLk9vYZpI57jbXE1Qn5H/TXeAWfZpKcYB4D/qMzRLLt1DLUpnKBT4EMjvIE+xZHvoe5MPSvK60afBPNL78vb5qp9gnd560b/oqUa03OZyZqP/zXO6Dne+M0wzx+f/z6WEtRkqlyKGcIyQg2lvho/j4bjrTtmQ6mdPErdmUcJHO8mOd7PF/IO5gpXzrUvnTXEWZM6a5IR5JdZd+6OJ8jxtSGfOvOcJ5j1PkgGQTJwSQa3A8sDt0EK2LAUoHPjNhlArw/a2s1y/xXGI8uRIxVc7T5qVqcY06rAb6XkQAUvWcHdT71U6H8z2osq5ZwpkdKC6JvHkksws7YaTYURkxinXssYWxHebWHRkPYxMtIU5ZQpxtHJ8Okcs8HtQZwcs1QfoyoVfLRoq/rOMbu2GHEu467lGJntuROWXOopkDMnLUjzbyVwKU1YJxjhpwltvVyfi16uQ9x8hw5zc2Q61PncCpQWdLgnGF2ptkA+kXeITNjr5eaICwOfMglZ8x36CoiRiSiCag3ccBaypxhvId/Po6UljRrxLbXMRJWbM1Hl5kz0N+QYfSrD582yfSmR1ZbWmZQJzf5VuTkeVW6OX3+DOeMmfDjUasd8VXmO40mlTJtpeJfImlLGilQ4zCTT5/IpMZIKY/kR+SYlUZcYE6uEnVHBtUKMEGX6aFMZq7ICfIgavfLoYUIS18dCYHT6yEuM4FbB2NRsK4xHmMveFTwzS7kaH4MzkTEe9Tc8R/rIcJfdx1h/XCehjzprzC9rwSBrQUrvy4Sr/lZ9NZ4b49TXcB+fSVrrnhu6jKxVqo9RlepQUz3rMLZDbCxajBROjz87PWfXYvAzM2gxtk+L8Vzm//vcIvGYK1w517501hBnTeqsSYJkHiTzhXHXbwD6jO54XsdzidXDxF/no7G9IUjywfcqSPN3leYV78S5sqeS23wt5ayn2Oq1bCV4T4Kc81NZ81o+/xRbvIYWTamAyxmi5338nhiySVPyEN9fa09F2pP6J8UrZtqc9mGl92R0eMH3qyf5dzXfVMoB+kj0XnSZqTk/c9a5MWfeuw7vHJ8ZCy+aNG7e+T/n4iCSzhxE4EU3yYsKS05E8dl9tpsrTZw4UDhrcmfN1MmTBmtR4EkXRUBlM5tDD8PYi5o9Bj4z8JmBz/Tx8Y8gq4J5VzIy7lTOG3cnI3lDUr8vad7SNG9PTukbeH6UCKOm7ml5OWq8YZDK/D2HmFshRm2lyPKTYP9E6188EWNOE3FvVGTbEfk4Ish1IUoiLBG/sVETGyWxLMkbc4oxu0mJ76xk3pgcv7EbDfxGqogIcYi3Wve0XDc8Ww9/Jh7RrdIU4Plnp+emTq6hDtzmFnKb3xonbDT5iP5+CW9m8cjtOtDUyTumjvghKfy4MKdy1pSB3wz8ppPfZM+ewj45lH9Ss8fAbwZ+M/Cby/hN4f+yODbH3BKJo81hRq24H+LWqMrfl2C+sAzzkk0xr2Qps5rF8olYiwFSIz/HY4tkyFeKHH8ir5noSb/Pmsp7sVPJQ6oy5nVwK7MdnqL3y612F5Zzh3uY+S7V8s8lMo9eilwzn2+OngKxnPGqYJvL3J2ZNk4Os1zmeCzkmFMJlQozqYn/Kr+oyOdZyFyZpcyyp+4MR/i2EvMl5tq67WHmvT3MWUjk6ueo685kJtIaoc9vHXPo89x8JfauxqDi0JQ4nlTy7SpWTI73lvn3uZZdUNx2Fi32MBdpIm9SK5iVWnZOYpSliGP6uBMZv0bHmQS91mKEhMATvb9Y9qcyEiboacP9OQ7Rc0TkZVS0kaGXTYJv4N9KmZVRZbWMcR4ip2gqS+aYv7PAsaoWvP4QM7XuSd2/WMVa5hJlI8+QLmNYpz3IQCHuwUTo1RehfWCO+MlLYiyNJd4W+Fk9qbcRMmaq5TMliI+ZIe1xq8dclkRYIn5joyY2SmJZko1K9qs7yn6Vk7cPEYW2Ufbbt/LP+spmTqktc8QUSiFfpwtPiLMmxBQKkp5b0juADGeXOOazAS0Mrp6D5BckvyD5LZP8Urz/fSt59hL2BsVhi08iziKXgW5Rv6/4MF5PvOJEph5xImdoAeBvqjwiQsbSG1zYK4SOX40zu3PdWLi+rFOMIp6d23XOhxhF28j1fbrwLNVXOHdycpkjPhHPD++qyZ012Q7e2P6UPnHlhLyL786DX3jglBf64MxoyQlmAxnGB4ftSv+CyOg3wDELvA088n2O4pSHKE6DeX5zDWMhPV72pV43gx9RzmMBlaAXJeiBwvnqHPOlRfI2b4H6XXHrd4o3+lSfXP86xRIh8+yBZzCRvUVydPsaV616affbHqM+JvV539HS7K39vkiD1xR8wttjbs/NNnsOG3NMNni2IW4bmV5OZF+ZBarNdzRHq49NX8NmO1s/iWOkOn79DLED+D1PHiugwFlkMrZV2ehb3M3O0XZAtDHyugJHSRqjVHfiK7yRK+JdCYlPvKNZk8n+RCm3EIie22/ilqraOkYhgRY4Z3N2KVqq+BjbMBGebJX2TUQfIBLiejvVX2Z5Qy1H0xxbLecg+jRnVcvVbMJDr1Hwa5YqeLffZK6UOUZ9jduzM7GjDRciYUfwLoYaFbHseCoim0l9zTfpe4s5RtteYs5vaoHJtAG9qQXOtp7NNTJHpa+ubUYmZrQhIXBKh5nAwCbNq9ZpizrM96RO6sha1GHOTu0rJlxUuYKfWabgnFn3F7U2qXN/Sa37i8KINkRIA3ZNDKzljt2Efa2dNM11a1KPvuZ1Y8VNbGlSz9RCMbyM419zN2r3WC7gAkiDCzBn0d5DTQjou6+ABZGr24S1at1cJbLgjNNH2MSL5swEPjVhIsorDeeaMG323VwHfUz6Cjbn0lz5JhT0HbeJY+LsVTjXnIl6ujn35nuJY7xNWDfnqVapCSG9Rj95m2NsvqE5UuJYXVu7Zj/N94jx6Hi3aNztuTbXLrFQRhO2NtgTKyxUTWWcts0xNt/QHKk+Mn01be2a/TTfQyTMdLxT+7Iuabj28Pae3z4TEoukYTtn2ueQfadunnT7jbL2uajeYztZ9THqXL7tvLad6JWxR+vwm0IvNt6hyWPYuZDCInvYeJvmjIgVEqQBM2KFbvucbJ+jhUXuaM6pjQvtsz1rwMuUPAT2uXZx267v2l/0kdpOmeYppM4n+8ml4GiWNU/FpnzZPldt8qXtpG6f5eZpH2HE330Hl9DkJpo91g4qsHEwbR6HWCHQPuP0vc92IjZPTNeunLZixBVOidd2wmcIL4VxOh7qvJ5bempLXG2JzLZXN/ckuxRol5BUjb7r2SVRU1J178vEoOK2dNyWnisLrejwU2fwIum9LfG7dxrS2mnUHtWeHbHChEi4ESts29JpW3otLJyUTSI2MaEtUZMWzFxnsJB8TbgnWjw9tVrNOZjrbEpIbYnSnIV+pprjb9ZkrTO4Odbmm5qjzpx6jnZbW1+VhXZMGApJeNFcbPNvwqewysJ2uOujJtaxkhYMiRXezd6bIzVHZq6yra3eV/NdpAU7MqIYF3lHH4Tc6YOQBx+ErfRB+AzfMoeYa4ss4YXTByF3+iDkTh+E3OmDkO+kDwIvfQcWRLYXKI8Q+1q4vBKKrfRK6Ldy9h3gB7DFn8p1/RSs5mcLPRIWPcM8A97T/5xOz8Bye7SiXSV4adzdS8On7RzXta9vB+uZndFvAR4z2LUvAAOG8PMwew/eHcEDenc8O3zXdlgOep1+2omMaJLI+4EiSon4fyrvyc7wTmGB+ME/3craArUw/P+hlj+2lvdkU+N7jh7dM9QciVuXIi6MyiFbyW+iH3VrN8e7ezHKd+pOaGrc2lS3M0v09y5lrMEUPcP5pz9WJu0UnSOi9In8U8rbiIkRN5HDLtbWQkA/0tah7bcfaWXiDqyIxMMhOMOaAm88ik/qTeLTrVy3FDMIl3Jt1JOR/MRr+N9TueIza98C64j2SbytljpHcXM3wcg8/NOt7DGVs+H3W9UnhW/qU4a58dSnUzmemew90t7IdRL876lGATNJg4qmYoOq4hZ1KU8p/j7bet6FLs294NYYXYEa51h+Uvera9DFqhZtGouddBivgQ5JQ4L0p9+q43OlU0oiQVsxoLZi1THhFX8dTV41uGMbnrliHxGn3ExGGKN11Zz0Ksa9SIJmc+JvPQR58q77jL5ahXMdcyfNFxvRK3WFk5BUTZl+FZhg0xrsjt6ByWkxYOWR1PUtb2/u1l1uk5yhXMVm9w5kw2ttn9X3uNiqYeijsbD3qMt9B0Cpl/SXRZ9ry37NJ2YgVx5rEpm5awedRtBphNsqd5UiS5QeI7Ah8t8YeP6xWBLLjjw9cdqMiNNmRJz8PHHaJYsF/EAW7m+G+5sL7m/qty3ZXnID82T8wWruc7bfEU7JcEqGU3K5rrXGqIkqX02FkfZy1NelqDHlsfVy7RtrO8folRnWE/yuWsykn2UsvTAPUasbo8ZN6dhyTbfO3zfDCCcZ+ssd4hvE0wQjAor/BfaTG9+nMjbeHs50ito8kZMykxaDHDWLh1p0xT2pD57K+JoqAmatzVZYAERExkKOOMMR5cb/XPZfy++nqFWc4k+OfQhP3VhGHxRxFXNcyVQbQdQYAcHYl4WmG98DHb+ah5pFKseUyBmI1U0lHPWZibGo76UcbYbzEXPMcCQzjFljrm6C9anEJjHDBOeUa3gVaXgVI+6KyKfiaSLxoJSYy0fKsVavmUm9cbvvBEdRyDVUK2iOTZ8Tn/ke5ixN8W+BeCo+CxyNDNjqkT45lporwzF0T2YVmGImVXWnVq1qGy8EDFLN3pTLNwhsbOKiwsQI5yjW38TjmYUWVJTSVPPZFpFMa23U5g6iqJJDStgBBDVncgRT6RspRpghrU3lfhdL/DP3FgX51Nj5crnfqJ2J2w8SHGUs7TyHGg4mMqpooUUYLSEeZyznIEakamf4tlxCKEPrj7pnMkUfYWGdEFkrEhnlVkA1kXuXuefUGJH1UKNKQTcVRqElsjd9bIlchVxSO9HGGsvMZtyP81Tap2aSjmJZq3Zy9bx4w1QbdaaNVN9xKvyea3PPpK1mhnNNpL2uOXoBLQEr1SrV7HKVBpcK8aGU66TGbu73GfY4xXkWEhcLGad5ru2qOVriCnnKzFpviBsjaI+vPX41PxUFWtB6LWPzHmrQzdBOmGtxcxPtTOVjs69Tcz33ZMRljg0JWvBmhn03QxucjklRY7UOtTdUMh51rs0zk+suovOWhkVVcTwmjdhoiJ8KKko0pz9O+YWMypthT3quFw4PZWdW+6eZ/cWssWWCabY4NPLJ2HPEzJdkibFFqdPfo+LLmaUp4GptlESWVu1Y3e1MPWPSxXTLol06dSRlyKK9pblkdM1C5NRemGtdObGgdNYQZ03qrLHHK04wp9H6LXRvaY9Mt3SK2p8IKXcO1BqDHeQMoXxNa99C+RF8i1eGg/3mdIX7kznWGcaRMkuvoN9zqcFT2GvW8FzX15NfaRuxn1yAfotD7R1oF36nn/4F+gZu2foANkyGf7/Rv8crtGKuTifrmy/0c7pbfU933Aj2riNNs6JTQOWkjaqzF8J9tOE2se8A99DhbLlz6J2d2au1+AY9btDjBj3u8th8Nd5RZnLQc02zF8n7p8KbtjDaKvso0aSoqaaLSWUfpeXpRGuVGJJYqfWh/9XL9fapbEOkl/RY5IOqwa35P1c4z6vcWZM5azbD5d2FIzqCnfUE+T2dm1fli7mh1d/aPEDfGl5ig3PVmZfYNCf1O13nY6CVS8gFyCLd3gC1MI849vd/AVfnuCrv6fivrFConbdWK6cHQtU5i876MPQ3wL0LyeMoHDVrNomlL+n3W9hnuF+jC1Nr4Ie3E1N5rI0ENGiH9N1Hjhm69snauU/Wzrw+pQNzE8z9a6+pRnKr+hJom9+a/NDDB7jekD8vkxjOgNPn8bfjtWik+o35M9gzmVR0CDvAOX1uvkUjZXjz1lKz3LPH9lRMafbBSO+x36C8fI679gP5Pqab0rn+31e8e/9o2ckykExc+9U68o6vluuwz9iurcnAqu2CRdUZFsNKx9ECimIS4xXg2QFI/9dBlr5HsvRUWrMSkEafY15dbvWqMbaXkLSFr8FUyrHt9glGF6vhCbN/xl88xxZ6aWppO55ISBlY+v7s9NxmotTc7XyM7tg+hhNU3URS1NHtnE1GoPtcrcbftO6Zdjf7aVI6TxMymjNDSZPn2MeRpn9XtTPtveHMCLdNVr9b1x1368pJdUXg1ALVBU5t9Jxa0pC2/J+bbgGn1pXDSu85h/WUlp7j7ndi9afIQPtq390Th3Y1A3x3ye6b0v7rvGMMN8bMmY/Ry4iPLdb0UzF6uG3aH8h/Zqt4+ynIPsNB9cHI5BiXpSHrYWno69Gi4MahcgjPXmmRVLvZGjL0TLTPdlO7hZir7rc1n+xahJA54OglxGXt51+2/AR27dAx2INu0Xs0xnvN5yA9H8M5fQzepNvmBzaMXctvB3qmQTBqwc1Gka4zP3We+WlnO3OITzzuKARND+ZuO8CqIggz2fUWdWg8G7SQyOfgXcqk62ZNbIVVn2gDzTc/wjeHXcm9K30NEIoasLPvR5Vz1ylDZLk1R5aLBugjBj7yDE6lo17x6jLQM7qwo3bWkBCvbsu40dWfQ5uKVjf06XYEvPQV/c90BRedYtCFWxXhVkWwL2y/VS/rdG85g3gYfzrO28xZkzprEudJXDhrEqdeiGxYLyS85vvqvrKN8yFPDC6MaQc+gE+86PkaMFTfC9vW0u6z34x+7wbOvRhiDfJbDuIMP0DZ/ho1UKzPP2TdbJR2guZsxjjGQ9DmXaH/kxqrefNim0Yeywxmqxv5KaWQTdsIf6Kl72FnuAbdLK+dASctuAvGn0Yt/tQmyefOsyPvrD+8P1E77TrzIfVSQR/lxvTXiM0R3lq34y9x4m/hrMmdNclGzj9TE9M++x6MOg+hnWNJ18Kx5E5NZL4hXoZbYi8BY4WUNW5eQNzEVFgYwwqSkePdd5Nf4d7ssTwDruFs5FYhdgKe4m3S37S4yvfbg+g5LVVZXH+h707hTg+x7p6F8/QvnJJjsaE9Mqz7kOtOnKtbOGvysO5bv+61c3UrZ00Z1n3L1504d3Pi1OmRDXkSh3Ufct0L5+rmzposrPvWr3vlXN3SWePmCaZOGat21kydlod64zJ2d3mTbGj0QaccdMqr9vziOuQ53lOIoBceF/bCoT0unadI91jKQXu8KIeSuSJBd9xHd1w6z8LSeRaWTo6q3JC/ddAddz3LSyf3Um4ofljQHQcZY1tkjMp59ldOrXK18TspYd37r3vpXF3irCnCum/5utdOib5ynqPVhs7RsO5DrrtrN6+dWuU62AjvwboT5+oWzppgI9z+da+dq1s5a9w8QeKsmTqlr8RZM91a3XGO0WSD7jjoju+T7vil5oUsfJL5irnzr+UYic5OJXnQHA+uOb4GvRmPQv/I8BwPWuQ+WuTcGW0xd0Y/zTH6qb0mC1rkLdIi5xhxz76WddAiBy1ykDac0gajHhcf4IpzmGOcwyBlbve6V87VLZ01JKz7lq+76w557ow6mGP0xbDu273uuXPdM2dNsBZu/7qXztUlzppgLdz2dc+cu7krBljujCqXO+NX56046br0lTprEqdv8mbslc3o1zyP5ZyWvwWoX0O8qdXrE/vNwjZmPVL2tmkV7rYLpCvLs7etO4BLssucvEDm5AWyDfECKqo7j9Z7DvpZoTvl2qYTiD20uuwEIR714njU9qh7Qa/tpuR9iDfIc3tHln3bTs+1kzYrZ03prCHOmmRnI3DleG9q/bO34cD4zubPwdLK3vdkK8Z719PjIeITG+HMsPOquJMik9c19PfXUc5bt9m3uaYryJU4Rf5JSE9t7X+uZVWJIUOTgqcqPwRqHHvWlcU2t1juMPeHH04mjyieirLAGet7vOu8zJ3nZb5luu8EotCG1W+vZOHUWBROnUnu5L3yDenK3NmuzNPN3L/1U/AYMEedf8lI89SbZ5n+LW58m8kzukkb7HT/HU6pS4gefg1wuQRK0aMr3+2E4xB0cw0qmvVjiIpcTP6T8gtRi5tgXkL/OQpOYjnnk4A/03+OlO/psluaPFDQJ7t2zdK5axJnTeGsyZ01m/EtCLvMmHYZFgsl7DG7tscQp8WIOHk24uTZCifPVmzIyhT2mDHtMczWEfaY3dtjSudOQpw1hbMmd9Zkzt0ndcqS+Qj2paAtDNrCsGMoqiyd9Fo6uZJyy2J3Bm2he/UL5xrnzprMeSrUI/KiaGsAzR18OyTFarIHkU+YXxvnSpvcakpbFIHX2znarZyyYemUDUunbahcSxzu5RhfAMaXEOUpYHzAeBPjXWdS5TyTKid/U63FE8gH4xk+5xD3JA0YHzDewNHKib2ls4Y4a4pRYDzzwduD+E4B3wO+6xhaO3fr2imN1k4+qFqLPOKzw9cUn3lsw8DTBIw3sZc48bpw1uTOmmwUGJ8DPqfgSUoCxgeMlzhaYJwyO/bWzprKWVOOBOMZTzMFqk0CxgeMNzDevl8XGI/JXpM6a8YhtzLfhD2QWUtpMQ8YHzCe42jlxN7SWUOcNW4rhduq7faC34wP3qLbReyG3ju8fR1uGoWbRsF3IPgOjH+Xd0VXK5zR1Yqti64WfAfcq18517h01hBnTRpuGgX/3HDTKEhG93rXTJ0nY+rUBaROXYAr3mGB8Q7DLYBw0yjsMbu3x9TOXaFy1pTOGuKsKcIeE24ahZtGO7jHZE4+JnPyMZmTj8mcfIwr0meBcZvtNUm4aRS0hUFbOLIdo3bSeOWsKYO28F6sfu7c+3Pn3p879/5sQ/Gq7u9NoxT8V1LQ6mbBmh9o16DD0kmhxFlTOGvyUfivMOmVY3zwyg0Yb+Jo4TyTCueZlDv5m3wtEVl8tIEZ3CbN4PwMGB8wXsfrwonxubMmc9akI/HKZT6KCVhQAlcTMN7E0dqJvZWzpnTWkJFgPAF85tHxA8YHjFc4Spz7NXFK58TJCZG1WJh9btdlcPOigpuvAeMDxus4WjqxlzhrCmdNPpL70xlIrilYVALGB4xXOFo69+vSKbkSJydERiK5FqCdTOBGdxUwPmC8gdeFE+NzZ03mrHHbKdx2bbcf/NR5p8mdc8EdQdOVWdLliZ/i/fL1W0o+hTxqrIcD+M8xn2e9+gxuPUUebeKGN84YKXv5LNr51yK4CRaDrfAKrEQ5fHoL9iKy1fNkXlQsZ9mbRl7A3cpdyO8NMDj1y1y46LkL+iluYZa7/SnMimUyPFzhmqwuh+Gqx31LS/uP+pkc5cOGz59om0BbUa/zOvYW72COM1l7Af0fy3qWTZDlqPyMlv8uWzFPoynMOse/ov3vtO8ZZLNU/Bab7RX99oj+/AY/e/SXyRnMp+kE/Bqm8POoAal3tMUh4O4BhZXOxbEbrTPYCxRUr4Eimrv6Y1yXa9rPqm7AXsGannqu8JF87gV4mkWgV2Mn8HOwGO6B/pCd5Jz35Dc6RKloy+sLo0Rvx5/kfZSWpxOtVQKrIcpLrQ/9r16ut09lG9H7H078Yqt5DfdyL5BHm3XKOzmn58PryZdYI1a3zQs1Y/T5P+eSdSon31U6ZZ2ys6xzqt15vrs33HvaUuWxXm1m1n/Qt7Fd9Ff67TuYzTXU2iCYO2Hb9Qbn1cYli+9hN4pA3mIcqB0ezJPnPXI2EZw/EawNq2Wn/gF4er6HlsfoPf4B9hYusUSTb4EHupk8pf/Z90M4EVku3EvwDbqEt3+AXiKoO5Lfr4ETvIY5vweea8+xQsS5Qpvxyz9v+a8uihew6L7luGMgtL1wXbMc4zwOYHQx4tgJ7POsPnLWsDmd0W+HMO9F7UxO9gLw/By4gMVPneLNC0Y/74BG3K0Z7BhnwTQhN1sFYT6/C8QM3zsijO9iMqLtnohZ+w73vBjylP9B2z5Gzo+3GCPf7Idzfpwuv1Mr/o+Bi+w63xj6PIBRq9z1iiPlGqSH8JdlEI/gWwxed7/ASRMBDfGMLA/hr4AR/2XtWeRM/9YxtNfxls3gVxjtDDD7GM65KzlOpde0P2WTjKYtrjSm7fmNgTNYKbXqbhoy3yOyjx8AV3tA2/J1rkBnXIJmj1HIQ2vpX+GOhq2/SzrGd8BRcGpUlMhX7wLoUOhKHnv1E4F0kMF+fAz1x1A2h9Etm5f/03d5j1on/hSL1NR8jvf1EHgrfbXc7/nryuSJurP30bqou/me8dK2GNf66N5O7cng+8JjsLyc0JX9C/irsWdq/J+CR8/HC2jf3EsqjGhdg81s9XuJwLAr1DJ+8NxdfPaQZptF+0Gb3t0jPgLZ5sDYD8V+w/cD15M3oA/lshW3gV2gLlr0I3aWh3f8vGi0x0DdH1rjNP+rfW/Ve1oW9rR7tqdNV7qnVYgXCdgA+X5E7rCnTcM+FvaxFexjeSddb935Tndfe5R7/xO6xtXqbAur9q92av9qpzdC7dSKd8+UcNUZrmLWMe2Ptz+Cs+EGY4iuEqZc78j9arh+94T+tcO5ckKzdEKzGqnPBbOingIWr0Y7wva3E2+vL8Y72C2QM7ApMr3hKWrSH4NNR9gfb3CGh/QbhwE7Feb0G6fSd0ijl+BDwmccT55NXtG95tnkOf35cfKS7nlXsBe77f5Md/k79ND0xDqAGV3CufF28i/gF240a/wB7PNuG3COOiq3DTi/gxarjYXEGWe/cMbZLzCr0CYsBzejjQRh36fe07dzruVQajVUy9/h+VXuYD8Ar3Mz+T+gWbnAtv9t2cOIM8o8wSjzXdb7vaaZHN6at/jdjEPf1LuH8gLVfVt+AI7tVO7Vn2pcYdThGebrI/xJ36E0c7RzHlhz2JMvAQ6r88ESp4KpJV3ugzXHddO9sZr0+QDW7hati2/gL4P+G1iBc4j1c23xrZmBhfpY8+h4pvXDJWfRV9Toa7s8wg4be+P4KF7ZxM+Bk0hXZpfqPsK4h7XM77Tyx7/2GZU4vHwIelevV2pRNlbTum2e/UqOiWBuMeq9fl8DrD+XHincSzCCXUT3LuEefYwimZX4QDtrhJ/KW+iTnx43YFlmXPV5y7PFvl6Zc71SJx9SO2uqDcn9m/NDXcW4F/ELbE4qKtt1D5nDdQ+BODMCEMwIs8kYX3eDkzi3g3d7F+92XYZa5tl+BbeAjqRX3So94ZfzUOze1hkdUwTzmdNvwX8++M8H//ngP79Z/3mCsf7v/lzi5LpcuV2IM7cLceZ2Ic7MbgQzu3U5+4MOKOiAdkMHdAg7yQHqgN6BzwUb6RtY1bl29i3nYfa1vnSJUPUatXoNGqGgERpeS9EFE20nn+s0Sjvfe7/PuqEXdETc+3cONeI21DtjDY5g5gegB/oAWiGxVkIS41oin/WpnetTOWsKZ00edEFbowvKnHpbV04Hgjmwgi4o6IKCLijogoIuKOiCgi7objqdrKFr8X+ucJ7WubMmc9akzhPeXZMEXVDQBQVdUCddkLj/8BZi511r91T66oNsPQedUNAJbUYn5MZz/SzJnSdT3uNu8P3VC3G+ZIZe7DrsTR3PXHoTHdxhLQrnWuTOmsRZMw06oK3RAeVO/WzutAlmne8iBR1Q0AEFHVDQAQUdUNAB7bIOqOgUT5M4s94RZ9Y74sx6RzDrnf2Ed9eUQQe09Tqgg6D5GVzzw6M7sEilb/B0uIEbGW8wAukFSGfqDLyGdW6eR3M8ka7wFge/Gb0H7z2n5Td4q5mfKjO4Ua36j7VRiCgWKlLqf+At7BtZ9x6yIeujja2j5XFXT+BpMeI3cPYxvudX2e7a4L/Yqc0jh1xgRFEuXR/D7W++34YbbUFXtRqNyStJCZGB4ZEVw82ztHaeslXQTXWEtIrjwaMonuNe4NJg8dauiMu2dSNO7og4uaPCqf/oni8y6LHWr8ciTn0ycdoziw1F4Q56rKDHCnqsoMcKeqygx9puPRZp6In8nyudpzVx1hTOmtx5wrtrsqDHCr5MQaO1JLYRHxl7h+nNdADR1XwjG6leXF4jor+gCwq6oNVEN/LHwfZpUTrPnrLzjaf7HeHodyw7akUk4vEQD6X2pxll+QBOWJ6Z4jc8Z8TazB2rUzpXhzhrMmdNGvQ+W6P3KZ262tJpwyw7+7EHvU/Q+wS9T9D7BL1P0Pvsst6n6pQjgmBm2n9aa1JnTeKsmTpP+KlTV+SOY+mOf5iEmOtbEXN9H+zOV7Cn3aDeyYY3pROjSIi2HjRwDQ1cDKfizUo0ccv0bycAvZuG9k35Vfhq4EQ/Lt2H3mPgMQOPGXjMxTzmOdyGZ5zgHpwaKR095xNT4DEi+Y3pIWKAWQURGPXPEWSl5E/E2ucIsrkVcDvL/FxD1rw/QGvKM1pW0D6HNiy2wCH9lEi+kvOUCbZKIDds0SrjeUY4ZmTQqxp1Cj8R9Mdmyf+L73NZoreOW63bdc0eD2EkOb5tivMuYU6c/82xnwr85/nIE4hGNUPY1QBf9fkQv+XguRfj/FO5CinEQIoRBilAt4Bec9lelOcAF8YjsNHFEupTiKXAv9cUHqoswjLxGzfq2DOH0BcfVwlj2QOuvwLN7R7EXS+hlGEAH/UU1nU80kDd4Or9n3Px/JvLJRdsPffB1nMXbseGfaUTL4mzpnDW5M6aqVMOqbfQOyHIAquTBfjNDSZfv4HPc7AL8L3pGmd3LTXXc0/9MvfEfg86ZrPXyNFrkA2CbBBkgyAbBNkgyAbLefwS81d2ec6u9S8xa2qQDYJscHfZoAvHY8NA4sTNwlmTO2syp3xQO2sqp+TgrnFlHS+debJd/lOJM8vIpu5UvYIIVzxr+Dv61q4eRSVmO7PNzO3p54d9vwCWmT+/WPn9cmFWvcLrbRGUH8kbliIDNrfU8bjxM4D5MUpI15iXMHKMqB5g/raekx47+iHgyJVhCbmLxClG/IjC5wDyspt9rnY32p+8pNz/txR739C/zyef0pLPJ08n39DPT+inlw54pY69JFmgN/NfI/Gr46h77TYTD/4lxCFk+PwC1oPJDd/C3s29fLvTf9LZo3e8Myru3YzIvZvROnJs6hqld8D5HxjRQM6QB3uIPfMsI7Gz7ZUxd95vHxhUI4FBskEY1COBQboxGKSd+YGhYZBtEAbJSGCQbxAG6UhgUGwQBtlIYEA2CIN8JDAoNwiDYiQwqDYIg/vG9aUbyqy+yhnZ9U+uSOKZQ7/0pyy7gPtRPNrdNxTT3qEtVM1vBjfNRclUwu0StJG87E/Z4w1Yc4/Rynsu2yTo116iDFjC3c50AU9KGnSgRsQ8kN/TVTvTbFVmRL9HYI+7mfwGUP918jeN52Ra6g9IUUdar1PZ1zWMn3s2HGrWssQ5F57PT+gAUocdbwaWOQbpUxztY/Cy51a8I9Ag8dtojyb/TfHvE9rjR6Ar+YS+56PJG/jfhmMGmP4R6pc+AZtGgbYdMfZMjn0KNhlemjdWtWh8J43vpbWXylpaW0sTCedmedJ4VyLh2GyZaXrSVCvPtXIWY1WUF45yIqlnKrEtkXM0Syu5ptxaK1aW4cr/gpVlrmnJEjl7hQOP0Gr2QGKJmF8F7cU6iSdriU+5BjP76lcdKTBbSIGZkwJN6jD3Ba7/au9j7l30fxqnYwFePT577xxGrGzc/5aRQnnMOxU14wjGeyBLlFV+j85zDm9b9JnhAvfQ+FODlznmbNRj/qcWRZTZ2W6tpw2/oamXd9vZqwUS1vbt7GmvnZ3drj+DqLMsxqKwFMxhR3iEN3rmsMKsrLlj18CZ9duv9T4qbX9JGnu1XqZ2arPUvU+b7TJraW7s3bl1j9b3dCLLS/BEaO7S3KLU3qXfAZYcav4ld9uT+ZlDWrtyAnZDn305RStTF/pJF1qodpV+3qMdjVHJ3wCWbWqpwO+zH7XofShqmcr9vbaU6dQy1fB6EbXomJtZS3VqKRwcTWHhaBK0iTeppVk+HL1wb7Q2F5MCRvjQS4KxQbrQS7Iwrkg4b/Tz5kCLgGE/b3KJgd3PG9VHZSmrLWU6Bemliygot1JQrfHtOgURjVoLR7miIBv1rIJyMhhDm24S7dRffM7wLPPdzpnUw+9g9fz/Yt7030hrwq/umabfqOSbz4xS3b+tydP69XeBM1G1F+h5/lB6/R41uPjtzRHRPQJMv3h6yzI2+Ox3Ogf9CGizggyIU1rOsve+Bb/bQ/BJndL3ss/MQ5V73eTAVxa0vAJ7DNtfjsArNqP/Y6Ab1scU/VLbvRPos+ql16sNudKOnbzdOrBzXL6cNohlmtZxEcSyNdFzP43xO9AH/A6+ZDcAuznEPzpGWDGvsX9h7H/ugRfDbRPGlxxDq4jCj/vBR7h/XqPd4LeGdz/LEXAAmUlSuLfF38yt8t1XI/FcjWQLVoP7TV/AjsNzsapoajHAPhkQcqkn5NKthJyKT8Ij1h7AWG8AfrY4i30gmXlCMhsIkn4+f19oNyPnkloLoFjORc0hAuMJ/fbAOq+8N22Z+LPJk2EOJ/0Z5DXy90i3r3fhCZdiK85MLq3xGyXHwFedgaT2HmbVB07EE05kC3aY55hj6xh3jGvIi2MrZRznNVAYj5Z9pOXbYqesfjPud/BOPoPn/+bIDtZnBUrPFSjXujP96Nhvqt4nkrnLbvI8EqdPU0pRp88c+K0zyOI9A+/95inWZ+VrT1jWa115P3xvzydHrru7VShotX21dBew1jea/W+d2jrTVl47LOhTR/kiW77ZMnOUK70d1yy39XZmOZF22LSlt9PLhtTb5VZ9d+Fpt1+d3m5XKYnfjD8An6Y/GnTzt8bd3EcAg1uUt+2UVQxAWYVFD15Y9OCFVQ9eeOrBC4sevGkb0ikqsWrCzfIxUZSvJjzrQVHZKDThm/SEEZ4j4/FzaY9onV4sRbDCD+KfmINvpK9/YgrRCfr6J+q9VNbS2lqq8zRm+SKexmyZOXZUfQdONRtlYZSXW+KfmHruyjnGpe4qMWRhVw67srEr5wtwYte8xo/hRsgBxORju/Ly3TiB/Hx+u3Glja+7N5Xqo9LK8sY+XEEMjeYubJa692CzXWYt1fff3Op7aJYrX5DM4guSrWXvzbV9Vu29GWTj9vOqIpr+965eVWTBnaBdo7TF+hqmMzvFOz5zSoczgDPbTz+BGF9Tq+amGkBzU1k0N0piqq2lpuZGL1+sudFbZo7y3MHNmFxONUI5s9I8R5fJmUkPOTMJNNVLc6PDwq65Ib1vOOl9VJay2lKma26I16lltssMn8TccW5NHeeWTRdaW2gq03jFIalKtwjpVFVqa7GYqvIe/vL5yu8xtTluIuN9HgBGKqsVzyrTTbLozsfPof0lpf1j4N8PwNp+ALRzAFG7Hkv841xfRaH0O7yP6wf2QObKoFRZS1cjr+SjlVfcljqiec+78x75+5vyiHLrsLoPGbn6n1bLoJ8HWL4mD7AQLXu1mXPGmLnajpepJ16mwbMWIZF5QizbAn+hrpFZ7ZDJPSGTrwUy64kGa4dE4QmJYq1+Lf0ykZdw47PvbmFSxjj3irtF7rdjQOkJqXJN+2rIFrBN2QLuHrM89/RLzNfEUYf8BSF/Qchf0Jem6x7+pAVoz4J/Rh9d6jmcVW9lXAXTcpE6LBfFABFJilZEkjFZLnTLQ+EoV/4ZmeW2uFk6pJ7VZhFMNX3kcutF1cN6UQV/jOCP0diFg+dkwAkTJ8p75TmwypseKlvmG7wjq6Jy8nPbPJOZV+pHxlMCup8YpYd4X1y/y6NO8LJ3lCS9j8pSVlvKdAupXrrIQqq3yxw+BLnmlaN7YhZGefvkzuA8J5aoYpX0EBg2qhgfedufR9lOl53eRecoSeLZsVhJs5FbSQmuI7PFL7KTrsamWBjRe4eyKZoWuDFbFAvPCB/Fjkbs2X47IudpDiklmqfVG+0cmy+NEDTHM9h+i/URzPkGuIUz1GjqZ26sjUJEVz+B+OoMU/4Do2HrkdevIDKLGm1sHe0NwPZEi6X9gM7rLaw7i33wQNstTN3wJWDPBUZ94bExjuHk4XL7dQ+tfeFppS/WZKUfvzW08LQfF1sRn2VIa2jhaScu1mQn3pw1tPC0CxeD2YX9rKGv5M4WGTtWZN2xzBkVvfcJkybuqx208LQYF2uKrhLsoPfdDlp4Wt6LNVnegx002EGDHbQvTVc97KBkYdb2oIHfRQ08wcznQQNvauBVpOAD8F3mNW9AG/jBcZuoGkBXXrV05eZt09pxB3XqKF8cjcCeLcks13XmiTUagVlOGtyA0panK8knUEstpqkn97Vykx56cnKvsgkkg9DOWzwjhSbslvb3yJpfbNqbXsxeKmtpbS0184tNvWxMzZaZw99Dp5jUQTGphWIqzVJVWsqGoxoevbxNNak31VQ9braGnE+2E0fNTmgl34GkfB/vrZox+XR6yaz3Vs1youW+qC2nTLISm2wB77BFWPC/t1r2uLdaDkQzTF5h0eDfYjSCC8B6xsP60MsMdYJKBuL2Bi71PDTsNceSN30A7/mClp7BiK4h5vdfvCglBf+4R9IyxXRxj2Ad32N06J8pBF5NnlMJ6tvJj5Pv6fr9C6w9Kd5hzuhnttYfyxEKTcYb4K+PQMvBo/vt0dFc0F9mEfoV8rPGEgrHGIP7Lygbst832OpvKOkz6fHRgv7tfT6C8/ItRhs8AqmKz+JRo2VJS/6L/r6BcmYDegPeIf+Fu99vFA5nACthG5pT+Ou5fL4CDvaf8MzfBlqBzydPqPS7P/mBQv8b+letQI4W8MdAKWwF/rkAwxim/Hvyi4FVv0AdL9Wtk7w8R6v4LxpV8Jpf8AYi9ydg0t2bRqSVXwAffgEpna+XepLHVH0D+j4eGVq1PoKdhOOyeuJzaMVOpDPQBkTYR6T1saf1IiRL3oPQtP2JtdcQifo9UNE1rpl6l8lj6X0211k9IyD9WevZP2FV/uwoPy7KCx50CrupU8gCh+fp1XeGq8otItyzns3tCKNuxWh7uZFnyRnwhcewnzEIfmR4/WXg9XepZej+BErmEMmbPfWJ1C9+RPHvAN5+aJSeQUxzthb8DOM7JvMAedMa7ycWu8d98h7MHXwqcfCpxOr3Xxg5THXptdb6GdJ/MNc4Zp1bre+J/2A4E7btTCAL+IRwB6tLPCtWw2Fxhp5rH9aoLTDjttUOj+upo3yxXjq1Rv53+3KX1jjlZrmK0sjXs7Dsx82aIXfkCnCnHU8uh+jR26RBuG88GYMc573OgcdplrZ111yD1Vd3rfdSWUtra6l5tzH11F2bLTOHbi3XOIip9W5jbr0hoUdwLC1lw0ZlJNaojKlnnOlpZ85GPBuoyD9/NM+d8QE1Vicg85zD7K8BSuIW0il8PnFk2UhAz1MMYC/KLSeZXlpbS017Ue6l/W62zBy3jHSaSxw0l1hvJSWWqI3JSmjOfhuJaJlPFtFcBntBN5oTzwaa66JNOEfKEzoE4T3JNAH/PZiknwAPUw1AnZUlTrFeWltLTeqsPPlMs2XmsM6aJ2LuOBHrDfo/2C25iSdlltr9qbtSZjkKf7LuWeFVfGc9Kzy/UdYlK3y+przb/W53HCNvy+548LWeY2aAPv7ttae3cb1VebZV9uZDsJLy/Nnilttd8nCbtefg/9od3sTz1iMJtx573npcdq/Q59RunqV91j3xXPdwM09AIvWEWLrWW1evkROLLNhhm0XWm95NrNjMHTqVafcc27NMunwVD6GfK0133X3N80GglQ8CrVPgLbvCS+Wg7g6NwhMa643C+hPwgRFI/rfgEcT6jWBs4rScwc6tSy2+9EI857zeu5af07orvCVyDHc9TsFfJ2pIdP8HypozjSjWfIBbmjOw2oq76BHGzL6Em2x2eJQ9bo6UEC0lyODiRqDuB8Xp9Rjlbb6K75CrZnOKcaYxxaxXdNd/Rs+855MfJy/R1+zE4RudD+IbnVt9o3Orb3Tu8I3OvX2jc4s0nTt1XZlD15Vbsv5UIK2XFo1Xu6bSItCfa/C9ezbDVLu5rFtscm11lvlJkx5+0iE2lp/eS0TneQvjvgZ5vRmjUnjv33drDnFQG3FQG7FqlomF0lItnvmwntWVhcoy7W2bsuhsylelhFvB6/IM2aPzFN4h7s8si+wibxY25nrLxrxen6DhxpyNeszd/IoWRSrb9VxAXGt4A9Ek+uiyFj93LvNhuFvpWkwujd0ltld3CbbylOaqoPVCSNSeENsGffwVRJRLQV4W+qMZaGP7wKj01KGXa9Kh941MpUPnd9AW2LRtfeCVeMIrWatOJUI7C9+7bkCHdEL/PrDOIe29k5hUc18jVpWeGudyK/J7HWAURy4/nqAUO4csT/YadtadAYc/A02du5154vJIjPwsXfzUKejrYpDU3wHWulvziM8HAM9+e17uuar5Fq8qh+kFRsF8iLRmcsN6rDLhh8vi9P4VNNIqSlmz7h3uQTHw0H/QlsI7mbfoszaF59oUW8HluFdH7Od9YEV6w4r3svloen47kF/8O67lEv9XFYeq9IwtV44ytlzXFYhb71FR6l6AZM74xALgf44aoRi0WL+AfBfBDh+hXiuX0dcS/GXtmX7Kv3UM7fUdTkQsF+Oy1cV4+/cY9K1XmmZX3FO2P2WL7Uhk20uAziF6+yq8bMZOi5e2bO/U+jgeg4XwhK7sX6RWOZHx41O4Zf5X5AkXxW/nOjsVr51gxLsao7i7+9BvV6pe/ooU09aucKy6Qhmev3l5zw895tBs035GrRmrMyHvHnFTH/R4if6nrQty932DOnkeHegdYNyh8aZjmCefzV0+L5rPMdC8WHV1F878r89k3uEs998/q977p64jDrvn/dw9k5XuniJmlfAgTiGes//umcBq8YihddhJw066kZ20HmQnrbdqJ+VrcwZRmMa+l65mvxxqP3Ttbe1dbLV7m99OpihL0aqiumXz8n/6Lu/R87yoXct8Tu0G5mq537O6HaPy1vqTtWqxM6u+uvLWuU8X7m/JlufOqDz19tWavbj/ATmAeO6M7zCbz41jJbPe1hNuv1gH1uW9vEXLexVTOluh59ocT/gztNxc472Va9CMM86O5XVc3upNi9P8ZPKfdAaf0b/M00qsAf/2n3ROn+Lndl+iTUpxhn/mUa/bzzbf2X5SZfPTn2Z+93orwdMfYByT9ptOaI0oLSdPJh81ZIEzgBCbcwFzLqCNnr+mWafHymvXcuw8hgiAPBrEJ2CBTmjNR0Z88Dd0djwP2CeNWFwpRN66xbZiVRKrB2ItOdA+Hoiql8paWltLTQ9EvXyxB6LeMnNEXVEeiAXkXWh7IJrleq5sW6bs1cSTsN2g9Y3Kkhtn0129D6tReB92vUGbAx40zw8eraDLDVq9v127QVt5Wv+qNVn/7vsN2srTglitKSPX7t6gXZaZ8wj4K8YxvYdRXRu5N0UktCbX1AczSk/MKIO3IUKi8oRYtVbp7DloFa4B08/xzp1d3ql77wUmPtzn27W1pwZlMbRqTWOxzbdra08NTb1mr8hX4MV0jb5r1637teZuyf2arhfcSH1gnXvqOffpWuf+go6D6zHnMtvAh9Zt4xuAEOMzDkBO1WmEPXkBn69Q88o1sCyvM7+XK3pkMOP5FObIy+w5YJX10qnUO3IbcN7Iy3kCeoFbnNm1xhfopSwfAeuZ7XgE5EO2IoT+Mp09w6gCNDk1aGUSiu1TWprBnTaWGfAYtCyfAJQ/AlmeR3uy3c1NBol0lbQiXTXvCbZvCPrcDWzfCmxHd7TFri6N6Kqk1bJslfjK4G2Zux2ZuoRZ+kWsmvaIWLUoA+KuZuy6y53aBO7Vcj6ej/p0aWQ4ot3j7E4vqpfKWlpbS8277Hr54rvsesvMkeku12in0u7LFo5yYvSfQaSxJnUVcB+1XMkt28x6z7bU8iUto7xcs2HclfLyBTaBTd2zrbRd+O53Ka/Rf//EsGtzO6/7/maF1vku/OozIx7IPtJlBDQqckZw/o5rt2M4Pdm7Y6lniUCWnmPWd9bfO7yxMG88y+mbRyZ/i54Hpyh1L2vpsr6bd1DFm/Wc2Lxmhvr3vyA8P16QYVxAXODFpVwNtS+5VyP3vLUcVkOtxkPQ1VZrWJsqrE2HteFRVFa/OmVYnQ6rU4P9c9VrQ8LadKKcGuy2q1ydEvINhdW5++rwfEyrXp11xl7xyRO03sgqvpmLQi4l/1xKiyzeudWCnht+Aa5nVQS1M6M062h9z0dnfe+ulc89NdPbcM+5rYuJDbv5b4CnV+iX1MeWUXhCbRt8EOynIL+9cAk9RFbIihu2MWZcPF7gS968Kc6j9uW974LXnt4J9Zq8E/qtxPZxLUITFcu5qpiS+lxi0K9eN24xLMaRXMOQSMvgpm4CHIA16RK0zjet+zhi3o+N0uX80l/BUqXHPJgOgKelJ55uh6/Ecjw6AI+YmQcuma1j1I93h7SPj0U+wvPsTLud9A7tIDfy3hrz44lRA96kPVdrlT1qplmZukO29oJsNtjdgmE8wWxedyo2yTH4Ib+FXeSoERtqZnxvxpbpn++gfao+AptEtxWqDIufe4V4u/X6GTCvuE+B84omT1qzfmCdS9KbMzU5sM2d68qz7AlmejuS3pkf4MbhAykXXTbu4B3jzIXPfx/sSHtDtNKiVI/Db3aON+WOgWIZLNsesW/R5nwMs+0DwcyTvrKt8j22wVDdq7RJUuatkD4QzXtDtNIiqo8Vnn0gVHhCaBtkzW8bkQm+AFjc4N2fPlAivaHEexkTHrl9/k/hJnkfeJWe8Cp3PKZwH//+OYz6ADD8dICows1IwT5t58ht6PGFl8nu4lZfjPEDbowIgbqcIlpey6y0bwBiM/SSutSeO4C1PJBju4Kd4L3mEyJuo/fB68oTr7c9AnLfyKUVZOPsC6tKy+l5v29vVOCP1h9eyUD+/GPGrMRLdqvWdouxX9xJBalnjR3UFnvlGKy0D1saTXsLEZ9F1F5A/75RLFPg1FNNfvyd9j1DntymjxDe4r9BfJZLiHH1CCz5ImeTDql3oL14D7c23xpyaduTVd/dz8CWdqzZArvu7H7RSiq8SeuDcekII23qUYmYZ8UexA7iEXSegu8li/7ympY+Ba/RPcrFvpb1JX5ntQX9XGp1Ofzfx++J7JX9j6CWYGkEzxLoZwpPRPBpD/R+kdZDCr6zFT5faSVP5ej38dkEnlVPpNiD6leVvMYR7GOZGp3elxh/E04/G7BjT4jZv4Y9lz2vxsdnaT7Doaye4qPRn+EtfnZS7yFYwgXe7YMup4vOxh/rM0+sX6++bx/1L5faTmeO21fqr8P5EM6HQSil8KSUYtTnwznd3fnNCLZfi192w/+PEe1KvroYsvUaq3hA6aL0xNBydFBzaxKakUhNjUKsxR90R3nk90eixneeQUL59LTrz3Gf4TboZSMRbxH9mfXzBX4L5v2WIfwF+H0bP2yogn5uoPgbq9DPXaCHlMA+u0bNTT+Ln48tFuQ3vSzIiadOKFkbZzZ2rVDqqRVaBLFKy1J6f7VCqadWKA1aocD1D8KJpp5aoXTkWqFz1H0weZ9Z/ghdDwKfWOTIGD8T+JzSz0SWptiSjEg+SD21FumatRafAn5GS2xeFVCWn3yzDT7zIj+k7kHcZ48vPNd2G7wV7KfhDwCpU4rt3LL6qeYLE3V4xuQv+8CeeMKe3EvYvwHbs7B2m/JGd5iWvWFaaZEqxn0fqIJ7GX7zDXJnd7lTjzrXV+Y8gpNXcGJHa/UfMd/cjGO02L9k0T2tA2N3NKPo9KHl2hO36533Dck8pdp0oVSbjV6qHUoLkHnLtovhlewAZvnJZOu6RRK0APddC5B5ypvZmrz2+2gBSvBJKCAK3x5ERjuEW2Esu1xhfMrgUw4aAd4yRo+GwijLZWY67o8xpWUleGdUsiwDD5PKUlPRsfLaEr5FlhaH0KIEj5IpjrySIytgNiW2iKVeYw/86Njs9mDfjHDMGc5tCvida7Pi3zJoNTXmU6C3Cf/hvRP41KxhOZlULUGvGLPFIUCZQO0UoZOgt0qNfispjDiBWVWQfyuTpeLTKa27BZiORzeTeeo2srXpaPudZH04zf75tyugMT94ji9Xefc5E885k63AIXVb9BzknDP61HkvnCh7w4f3MqT2M7LqOjNPfcCysVYDx6s/AIk6gtsD4g7CkWMGtecMyrVqm1O4/XkGfXMcjAB7L62zyD3lsWwwnfkmJIzl2j1xr+66k6a12YfQn6gbud2pOveWAHf9RtWNzEW2Gr0Z9wu6wohOPCLC8vamJuAuWrTV6rsYhnLfjyPwM7pY632q3FNaz9cmrY9Xs5F7ypn5muTMoNm475qN3FNaW1e0m/vs1Zx7x3xbnM2nDHHGdjbOmBk1d/g4YzyecV8s1aMiByzdPSzV426vBkfLQXA07KS7vJPqkftXg6XVIFhaBSzdWSzVc3+sBkfrQXC0Dji6kzi6GqwsPHXVxZboqsca65ZBMOkNad7LfYdzPying0A53QFs7gPlbBAoZ6M7y14DzPpAJh8EMmOIkSnO9RcQkfWgcb4zr/2o1YbZXIRV7wmWnYEetg9MC0+Ybtc9FXt84XQt8YULT7+PYmEMzmIUuqcmDjILkXlbvA+cSk84lVuAezZInQOFXvSCUdUbRryXcUKIP/0WbeVvgYvvx6PUnvCqtgCnvod96Vr6btzC7v8BztUjsNT2362It5SyKJojGeiGw/AYxiFwCL1cabm5usMrGQReyVpzP/TJzmBK8Syr9/PJzxQnz+m4C/r5y17QTAeBZjpS7BO5BLvDx08aIFsRad8GIeFvxDmyeS9Y5b1hxXsZAlKnkNd9dd7FxJNzJ4Nx7nf1Sc3Aj+EI8qfNrd6cxJNTJgs5ZeLNKft6AP9C4ZJCho/U4Q9MPHnXZSMvN6ALuQEuQpd+XoP/x3s4AZiMfoxRxI4AEuwUOKC/b1EL8qtRKrzxIjg7eHwyPSdGBPyJ/sQxZLu8gLMkop+F9yPPgmm2vQFJ43iy12tfqDzXqlorpbzCURw5qaP2HHcePGk7etIOnZlgKB9a5U83Ay3jsebxdQ4U5WN/ES1jx9qt3m+29JQuyMJcS+VA0sXu+oKvC3u744mfVFVuyX27VfpXl54yUxlujgf/6kF8fktPKbQc/c3xFFY2h/skCdxh5nfFE7whzfj9PbwbHUHUOJYp4RCe4u32MFq/+D3FflQvrDSW7URsOtGPeF8s3yN+/6Djm6JPlxifeJK/hbWIoZ5lveJR7VK8Jy6eZs9WsryC8gRxuv0Eu8cdW54Zi7d56SnTl1t932+oc6HwhFURzoVwLgxCncQT48jIzwWC926WnQfz1v7f3u3tO3sCbVf5hhgjcazqDeM5E3x1b+vhf0+1neCumrcn6MPxQdOb3YL+7T3a/piO7C+03adgDfy4h9dlbLy/+ylTedL8erVp3SFpzs5X55asdXbfoXZ1DpRxCvO6gPgbb9HD5ZLSJZvrBZ3P/wLlmDO/AWn9RpZGdBSXQLkR5GtgmSOOUQP7Qert96xQqry1OtORWaW6Yn3lqZ+otkQ/cUP3c123GLVKuH5H6OQVJ9IdgmlvCPJeVh/tpfKUd5eNNVuRZe0c+juDk9gV76XylJyqwU5Jv1kUaCl6j/uSO9ZL5e2buA3xxZunMpNHrjADj9Do9qMv4rne2xD12g6dGKB40TNCTeXJP1Zr8ju86rGL83GZWdH7QKbqDRneyzB8dbriM70eZLb1KHaXGfhOzvGWk4qi098Htfbk9urRcXuLsqmpHSaylHFb2DVazfrALvGE3XrliQfWsaa9x8p7GeMqx+izftLrlK09OcN6K2IbuaB0gvHuWal+07OteSWa7sOmd80hC5uPVpVgHNtkcK3qarRQtSd/Xa+JOz3x1kE178bq2uL2/VimF/3YW3Mu6mOAm67pEl5fl6ClXt2qFJ6rUoxqVbrTZmxYDlzQ5+v0B/CHlyuEPukN/cq4pd8H9scAC+b1b1pMVrUO9vcdantdpnG0h3B3SpRn2s00e/trrX25whUsPVdw3BadF3CS7IGeNoM9jMcvZ/G6lRWe6SdFXHMRh72G9inGJi+10mTyFO7s3UK22T3ouaT8FY/cnmE8cx4rPYPeKyjnUdx4JPcCS07B6j6FXvbB8rQHqy3a11qPBXzjJYl872vMgpdgmfBBSOE5YakUs+DeCxnUmuV8dinY//fQ51n5OxAccwJ1/DN/Ygr2pRTfWBrlpxgHvoR5q3GlxrgygG6CceqrRjnvjc3xFqxR7O16hPg+lKqwalVUVPWmIt7LZmmowLVPYS3J5DlEG9pDnVuE68ru5uxjTQY+7XsQlYrzcxlmJRBZAnKJJyWUPIU8p3uQqSDDPAclxGKJJJ3mkCuB9xmjD04usTqF24/PMTOkGGtCV4pnF8jRj6aGtnmHO+j+6+6rUajX7Gl+Cic/9+u/RuvRJWiSGTZwKPD4wKZ3PucRufXoDEq4bYmfRk0I1EClfhBYv6/9B4iLwyM1nMj5CmuaiJ4eIRd8givM538F/jm/oq7gEp+6btx2eGjATzyr7k2IuxJ7DtglnlFM6xAvpocUX8Mp1xfSvJcQL2YRlLNBoJyFeDELoZwPAuX8nsWLqYFbHQIyRYgXo0HDR8Ln7UK8GF+Ylr1hynu5z/FiapAR/eBU7Wi8mBqkqL4w4r1sQ7yYE5jbcS+IJV4SiwmTXY4YU4PGqS/EeC+7EDGmBp3EEPBKtzhiTD75hsL6hvaX0pn9qxc0s0Ggmd3LiDE1aF594JNshdfcKiPG1GAV6Asr3sv4I8bUYCPwmy3ZUMSYeknEmBoy//adA+9lvRFjarAZDTHyKkSMWXHEmBry0/itVT2iiDE8p47fuIsQMSZEjBlhxJgarIF9cZj3EjD4/kaMqeGk9cGTdOczctbgU+EHq5CRM0QG6O8JUQMM/DAuDxFjQsSYQTGv8MS8JJwLnhqBdE3WvHAu3P9zofTEuDJEjAkRYwbDOl/dWzb6iDE/gMXzV49IJz+MJGYMg3/tSfXr1af1gaU+v8xb75be86gxvyFE9xxw8tXtJPcibkwNdx58ZpxtiZZi3XFjarjN0heCvJdVx41hb8kHGWu+sbgxNdw/8ptDNsq4MTXcQfHzECh2Pm5MDbet/Na73Om4MQwClSekqp2KG8NmXPeGDO9l/HFjari32X+2+Sgypq0ubkwNN3B94JSPjtvbdNyYGm5J+sEu3XDcmBruePYdK+/lvsaNqeF2rB+M8hA35h7HjWErXHhiQhHixqwpbkwNd8T9VoWEuDErgH7ZG/q8lxA3ZhNxYxjsK88VLEPcmBA3JsSNcVBR3ZuKauM+c4gbM/64MSwa2tRz3af3NG5M4akryDbgcT/2uDGFdzSTKsSN6SXFF94RTRZp/oodiGjSD8r5IFDOQ9yYhVAuBoFyce/ixhSeUUyXQYaEuDEaNPwk/GIrrIzjiBtTeMrcxUJpoRjoRul448YUnlJVsRURPlYTN4Z4SiDFwqgKZBQ2zeVxY67Ai+Sqp2cS8ZRZyFbkxVhH5BjiadVcBLHayOt4vyPHEE/L6jJ4ZVscOWaf8iLZ5O+U1i5oi296QTMfBJr5PY0cQzwtk2Qr/OZWGzmGeNoLycJb6mQgGWHVkWOIJ/dOBuPe7+rZmi2NHEM8uWWykFsm3tzycJFjiCf/umzk9Sgix3wP/AS/X3CD2gweyY/pOHlMjbegc1X6T65ztceQuQDfkIeoO5nLm0WHGONA1NxAzYXU37aj1RxJzy0GWb1G3YjoE1Wm9OSxyzVr+Z+DR/gxWJYPDb8dc/S+3C4JsWVCbJlRxpYpveWPRX4m5UDyR4gtM9bYMqWn3FWuLV7KeGMIlJ5SVbkVnq0hhsD4YwiUnnJquSY5NcSW2ZXYMqWn1F+GmGMUBqUnrMpwLoRzYRDqrDwxrgqxZUJsmcGwzlc7l48+tsyXhhaO+SEdwo37C2h7oe0Km4okU3nq0Ko169D8IWfOxlentt5YC08xLjfXhX6QcWJU7Bc9RozyDNY1saa+1a6fVXXc49XUwEboC3e6NK5M5a3XSe9JXJnKU0NRbYmGYv1xZSpPvcUiCPJeVh9XpvKUeJeNtdhgXJnKU3aqBjsnh44rU3n7MJIQVwYy/Pqtd7XjcWUqTw6yWpN/4njiytTe3N4ir8R6IK/EVceVqT25wWWzTe55XJnak9urR8ftbT6uTO3JN9ZrljgeWMea9x4r7+X+xpWpPTnDek22kBBXZlNxZWpP/roeWQST+x1XpvaUGeo1WUF2K65M7SmD1As9J+uBbkiFuDJdVrD2XMEqxJUJcWVCXBkLFSXIgfSlI9FPiC2zPbFl+Jolnmuf3MvoMhwGvjqDaYgvY4Geb9yTMkSY6SHPc1jnvWEt+glRZhZDuhgI0kWINLME0mQgSJN7Fm0mQT5yGOiUIeKMAQ8f6V+0DFFn/OFaDwBX0dN9jj2TAO899YRWshVxQlYRf4bPPhkATqKnbYhC8xYspMc9oZZ6Qy0JUWgkLLIBoCZ62oVYNHyu+WBQy7c4Is0flN//ivIrTyaf9/JB5JAoBoNpcS/j0vC5EW8okR2PTcOhUA4AL9HT+CPU8JFW3nOuNhSlplwYo4aPrR5gFqKn9UaqSaBuOtD404EsDX3j1XwHNw8izKSr34nQ70DoNx4i5O/YXYe30IZrp7nuVaw/09C+Qw9Jrr2N4KzhURdmDQ3uXq/9IPXmrdM12wBeAtyOcd76vcrmDPz53DLEnwnxZ0YYf4bjcTYAHoueAh7f3yg0fI1z73073/GIAxwKhTe8ihB1IEQdGMS/IvWWTtM1SachIs1uRKThOFV6Y18WzgjwLPSFVxXOiHBGDESltTfW1SE6TYhOMxjmZXfQyBWjj1Dj1hdtJiINh7Cvdi0bqXYtu4N2bb3RGX7AuC8zkP+vW/NZFJlmzzlbfx1Mdi/ixfBZ595Y+v+3d7XNkdvIWT9laqtcZVd2xkO8I/kSn+8SO2VXrmLn8iG5D2tJK20srRRJ3rX860M0CA7nhcOm2IMBRiiXvBKHA/bT6AYb3Y1uUWrG9HJREnAxjHToyjH+OYqMYnW0+jH++RqNRCZZQ8bThs9KzKEX3vY7JvDI54T5+JbX9XPwvj/DjnqaFlq0JMSxpp86qwsVH3/eGrPfxtl+M87XaHo5rwXafhSR8hefIIPuonPW5zBvTIG260Sk3Do/44fFzAgwh5FywczJMPNsMAsyzCIbzJIMs8wGsyLDrLLBrMkw62wwGzLMJhvMlgyzzQSzHGGN7ccsifLrYmCuyDBX2WBmZJhZNpixdpiM5JfDxkemYBYEmMNIuWCWZJhlNpgVGWaVDWZNhllng9mQYTbZYLZkmG0mmBXaDhvCrMjssMNjrsgwV9lgZmSYWTaYORlmng1mQYZZZINZkmGW2WDG2mF0faxxMUR39RpygZ56IocKbU+pyPHPn6AGzmUv3QZN93GyS1zuVx/tFk07PxLtVQ/tGm2j0PVFGks766W9QtNeHYl23ks7Q9O+PBLtopd2/LvfHol22Us7/h1ujkS76qUd/y7WR6Jd99KO922oI9FuemnH+yjkkWi3vbTjfQ3iaO+mZS/1eK8Bi+TtvTiw9WlGeA3YXovbkHkNDo+5IsNcZYOZkWFm2WDmZJh5NpgFGWaRDWZJhllmg1mRYVbZYNZkmHU2mA0ZZpMNZkuG2WaC2ZLZYTYbO8yS2WE2GzvMou2wWL0ip5552ndKwJ2Q/7mh3l99M4l3HM27uJ7PH4ES31flTQ/tAk17VeZ9gyMSzbu4HpFl72wrNMXLLE9Ubc61n+O3jWQ5ym7W6nG9g54wV1BL9DeonOwrWPmz5u6uT8CjL+GE5P6zQ3M4dfRQj9cd46uzf4KuNX1jfga5c6O6O6fJo0bPblzP6Be98mgIKA4j0WrQYo8WYe3AWH3I09ciV5/E3fdl65uqIHL1FrCF0/7LQW25gOrU74B3TxM1pkL2egt3Hl9jKmSHsiGKw0hxNKZC9hTb3A0WjUlRYzha/ngiGiMIKA4jxdMYidYYXTQmaY1RaPlTiWiMJqA4jBRPYwxaY1TRmKQ1xqLlzyahMdUIO9Lu1ZgK7SWdrjH4PlImUr2zojEv0xhsbyt/ZxoawwkoDiPF0xiB1hhRNCZpjZFo+ZOJaIwioDiMFE9jNFpjeNGYpDXGoOXPJKIxloDiMFIsjcH3hTKRIrFFY16mMQztq43dS+qLXtljBBSHkeJpDEdrTFU0JmmNEWj5E4lojCSgOIwUT2MUWmNOJw9ghrzPyfU9VGP9AJimSbRGy4dORKINAcVhpHgSjc/NrE7ijEw1ol+BHHhb80xyM6sRHQSGMVfZYGZkmFk2mDkZZp4NZkGGWWSDWZJhltlgVmSYVTaYNRlmnQ1mQ4bZZIPZkmG2mWAWZHaYyMYOE2jflijnBrZ4x9C8Y0c6L2J79lkCHbeN1eMip3kXaN6JBM6LVOg+AFW07jHlvAjVeZEK3fGgWutocEzflEB708RAZqKImJko0B41XrLfk45PCHREWSSSmSjRmYliIDNRRsxMlGjrkpfs96Q1RqJtXZlIZqJEW7hyIDNRRsxMlGjbkpfs98Q1RqLlTyaiMYqA4jBSPI3RaI0p2e9pa4xBy59JRGMsAcVhpFgao9C2JC/Z70lrjELb1yqRzESFtiPVgI9dRcxMVGhbkpfs98Q1RqDlTySiMZKA4jBSPI1RaI0p2e9pa4xGy59ORGMMAcVhpHgaY9EaU7LfU9YYjbavdSLViDTajtQD1Yh0xGpEGm1L8pL9PlGiOVo+eCISLQgoDiPFk2h8FuXyRLLfx/RSGVp7VDaYNRlmnQ1mQ4bZZIPZkmHOJYtyTMeX/ZhNNlmUYzq+DGGussHMyDCzbDBzMsw8G8yCDLPIBrMkwyyzwYz12MWqu5dPRrBB+8hMZB9ZyARnPX0YK3QHmCpa9bic5t2ieWeTyAS3aN9UqXyWXya4RfvxbCLxYIv24dmBeLCNGA+2aH9YqYaWtq/eov2ENpF4sEXHg+1APNhGjAfbEdZliQenrTEaLX86EY0xBBSHkeJpDN62LPHgdDWGobvTsES60zB0dxo20J2GRexOw9DdaaqTqoZ2mhrD0fLHE9EYQUBxGCmexkh0lLycC09bYxRa/lQiGqMJKA4jxdMYg9aYcgY8bY2xaPmzSWhMNcKOtHs1Jl53GobuTrOey1I0Jj2NwXanYYl0p2Ho7jRsoDsNi9idhqG706xnvJW81pdItETLh0xEohUBxWGkeBKNzXcUkfYZh86nYOg+K8uBii1sratJ6pgtGWabCWZ8n5chzCyTfEeG7oiCwVxlg5mRYWbZYOZkmHk2mAUZZpENZkmGWWaDWZFhVtlgxvrcWKS9eC65fgzdT4Wt9UuJsVf4pv6Gz/F800O5RVNeMnvXecfRPjceOXbbt9fi6NgtK/WwMsvwZOgeMoywrsY0jwVH++D4gA+OR/TBcXQ8l5U86aS91hzt4+OJ+Pg42sfHB3x8PKKPj4+wLUtOdNoaY9DyZxLRGEtAcRgplsYItG3JSk500hoj0Da3SOLcDUP3rtlPcRgpnsZwtMaUnOi0NUag5U8kojGSgOIwUjyNUWiNKTnRaWuMRsufTkRjDAHFYaR4GmPRGRYlJzpljZFo+1omcu5Gou1IOXDuRkY8dyNH5AWakq82SaI5Wj54IhItCCgOI8WTaHw83JxIvpocEQ83AzOlssGsyTDrbDAbMswmG8yWDHMueYlqRF7ifswqm7xENSIvcQhzlQ1mRoaZZYOZk2Hm2WDGWlIqUkQ6nywehfbdqcgx4JC7ZXssUYX236lIUdWcZl2jeaeTyN1SaI+ZKhHB7HK3FNqzqBI5cazRHjQ1kCOtI5441mgvmipRwqQ9uBrtWdSJZDtqtIdOD2Q76ojZjnqEbVmihGlrjETLn0xEYxQBxWGkeBqj0X7tEiVMW2MMWv5MIhpjCSgOI8XSGDMiGlsqJ6WsMQZtX5tEsh0N2o40A9mOJmK2oxkR7S2Vk9LWGIGWP5GIxkgCisNI8TRGoTWmVE6aJtEaLR86EYk2BBSHkeJJND5yrU8kE8WOiFzvnymbTeTajohcD2GussHMyDCzbDBzMsw8G8yCDLPIBrMkwyyzwazIMKtsMGPtIFsq6mzxzqB5d5yKOlVP10SG7s6yLstl3iuok4Oz73j0fii7rXA+oh+KLTV1MsvL4Oi+Mf7O4+9lObo/yn6Kw0hx9rJ8RI8UW2rqJOzP5Oi+Mf7ONDRGEVAcRoqnMXjrstTUSVtjDFr+TCIaYwkoDiPF0hh8jxRbMmiT1hhs3xh/ZwoaU6HtyGpvlJmvdSM5vMZwtMaUbNm0NUag5U8kojGSgOIwUjyNUWiNWZYo8ySJ1mj50IlItCGgOIwUT6LxUWZ1ElFmPqpvi9o7U7n0beGj+rYMYa6ywczIMLNsMHMyzDwbzIIMs8gGsyTDLLPBjPXklW4W27zTaN7pI/Uw4T02CkNbVrH6WuQ07xbNO5tExJWjY8Sl/0J+EVeO9u3xRHx7HO3b4wO+PR7Rt8fRceLSkyFt3x5H+/Z4Ir49jvbt8QHfHo/o2+MjrMsSP0pbYzRa/nQiGmMIKA4jxdMYvG1Z4kcpa4xA29wiiXrZHN13ZT/FYaRYGiPQtiQr8aOJEs3R8sETkWhBQHEYKZ5E4/2Q8kTiR2LEaRc5MFMqG8yaDLPOBrMhw2yywWzJMNtMMMsR8eD9mGU28WCJtp9kJNs9H5+6RFtxMvKpkRBL4b3va4m2i2QkCzSneRdo3okkYikS7YkTpVZddrEUifZY0tW8mbbHkWjPnBzIWsB3XZm+x5Fo75wo9euS9nNJtMdSJlGHm6N7ueynOIwUS2PUCO9cqV+XssYotK2rEjkhrdAWrho4Ia0inpBWI7x/pX7dNImWaPmQiUi0IqA4jBRPovEeQ3EinmE1wmMoBmbKZIPZkmHOxWOoR3gM92PW2XgM9YgTJEOYq2wwY60fHSknKB9vmUbbYTpyBPbHmpLn+jfd8+bSaFtMF9/4Fu8kmncyCR+pRltXunTAyc5HqtEeR51ILqBGexv1QC6gjpgLqNGeNVUiDUl7fAza42gSyQU0aG+jGcgFNBFzAc0Iz5opHp9JEs3R8sETkWhBQHEYKZ5E43MB+Yl4fMyIXMChmVLZYNZkmHU2mA0ZZpMNZotehWzZB69xxKKtKBvZigreD1dX600P7Xh7ypR53+AdQ887S8L/YUdYRyVWn5v/w6ItSZvI6WGL9iDagTiDjXh62KK9iKbE7ydKtEbLh05Eog0BxWGkeBKNj+uyk9jNCXR/kv2Yw0i5YK7IMFfZYMZZaYKw78dpWLgC3YFErHUYibmzcdUG3vTQLtC0l8juJu8kmncpRHYFuvOHIOw29RptMoHuVuLvPL5NJtBdOfZTHEaKY5MJZGcOEy1HeOrq5Wh3u+2nZh/9WH/zj87e+RpGv4C/ZqPuntefXk6SaVw/ETNQX9WPc+q8nsrpiojT1SuQ6mmcZkScZgm8D7+vn+64dVXz4ensb8C3adzhRNzhCXDnW6D+oraKL2H8Wf29O7ji3vlzsBQ273E2wX+B5/H67M/NNcfDqVKHtbqrLCq6/lDfcQvv7j/DOB8ans7qq881B1k78gN4eLsry2WD3j3TzcY0vkoCvoaR0pNYZ6vew/dv4ZPnidxSaG7l4KHdxa1b0NePE/mkCfgURkqTT/7bv4AOz+BN62M+0/hm0HzLIY72n7BaPZx9A7aJX+fc078Ha+RDfZVmDbMEXAsjpSltng/nMMp9uyefwjVsB5thrlF1sHmH5Ng10Pm+/d413H1f//V1/Z+Lhjq5uqmv30Fs9h8hZ9P99zWsbCt777G+8s3ZX+s31/XZdzU9f4FRp/C0IuNplagkOn29n8glrOc8jyrJu7h03vhlvPX2OJFfnIBfYSQKfl3XGnPY6ApD2/sscs7BDzD7T83TZ5Cb8Qx4duOQBDjCSJQ4Zmf/U3OHQXSKwZXd9Csy+tURPCtPYHF0909uj/9Y3+neEG637/wn3jvi+OHeEu/qn18an8qva1cvaso/g713Db/NGq+Lfxs/1n87W6b7jct6XfHcemzzc57hvk9gGXXvfYJdyuXZYuJqodEzFteP/nNDx8UebTFo2uNkf3uZemqsLZzEdUffPcI+f8CfOvfPeq7PIT/MyY+TmSBT/fc+Arqn+upj89clyL/z0ez/3i3cd7P3LsfhefvG87O17/5rwD4H+3+drw+AZg7c+aX+/k3Ln1vQrMv23gvA4+4Ns7F557xn7sIzPBX3YM38BhZNyBx4qP++nKiDlkCOw0hFjnfL8W1D9UUSMvxyaeHonRjPpJ7mbr45u5iGXxWaX7EyMD7VV6dz6/uNFfEzIHerdLi3gnvD59f1X2Evu/uO98Db1Tr6EcZfraPOI3NZP+dP9fXfO/HjGfwwsHBZJ8v/93rsG7B1du++P8N/i/qH1/+/g/OpX4OV6fff65x6D3t1p80hVhVGfQ86+9ihtLtyO65cwIxPXbUfz36qLcJ/bT4Jo+yWOoaWuji71nuQtGuk3F103pAMZlfU352DB/Yc/l3UP27e57AfWNRvHv8bq+fMRevO4Vv+PvfprPnU/Vw346xGcVfn7X1zGGXejhOeN2+fE37+qOlbNpwM9IVv+qe4O+bw+by2F69hZAb3rr7tvmva6wauV41cb39jXo863/GdPi1yMvvQzv63bQ7+IaUP7wOw5R2B7ngmCDuelXdEeUdItNTJxN8RqsnYHno3PG69C7ZX/t2rfAX3HvIJc0BxuCek9H7A++ni2MTXnRVhrI/uPyC+cwk8dBGbK/Cz3rWx6dnZl80Kue2H/QrhI1iPAs0hXvQAZ/S6tEx5+2j0OhDXB0fD2U20eK9d3HPKPzV+2kfQnmvw9d42UUF/dvK2eavOmphiP2fcGO9Bi2c1JXeg4TN4L7nvPHZ8uqtn3fdwctHLSXwkO7UKSC/XFoH2f4hM/B9P9buh68WcbV3xPqQgMSvLZgoXKwIuhpFoI01veihmZBSzA8X4bmG8D/CGv+ycKdtEwtFI4nYkl02s6rdmjXLSfteLAp9jmUO/7823vNvz3MPzV57kqTon0fMuM+DYbg7NgZMfYaWaxi2F5paKtDt6+Qrv6XK242Wjd1O5owm4E0aiseDZwd/8hgyzSWLFcZL5G+iOz88O1TwosmoF2jrMoxPXO6C8uy7PdlzzsbmHJoo3hX8SbWfKyPuUNz30VgT0hpFSnO95k5d/NfE9LNGWpIx2Fn+Kt7ePU1fwXv4AV286fs9tH7DqeF92eYBF53z7fv+ugp1r934q/+6hfGESbY3LSHbsFdoT9gBz/9h4Gm7XfNf+k5vGKvuyiZJ9hfbjh8/nwLuuvy1kqd2Bz/yQMyPQMyOSmpmX6+h8LZbRNwN+rv4AK/LuoDMgCWYgjEQxA5fAEXeqYT2Sc6jZ2P28887Kxzt27zmcGAvXeedU3u77Hzr364POo0LPY9qRph/h3bKAfn8caJYQaXG5O6tMAQUVQRZQUUpA9MXfM4PfGOSyrq5WZ98B7k9womrRVJf/Fq7NYB4XkI88gzfAAqpjzyAmNwMLYQFP81euITNgCaN8CxGxBcx4uN92RpTwl79Stc/9G/x7DWfLF508CQbfC1HUgMJnWHD4dP26R8cgR2HR5G2vcjJUQ3MFn/nf/TeWEPdizRP12vVrkJEFdNJgHbrYGl0cuOvpWgKvutf9aA7jJ4iSLaHexR97sgTx2rqSqsNpkibQpDDScTVJNhLAYEbV2Q+wF1xAVW3RyAaDXrTfNp/4HmsL8Bl4O8/rg2wipzPwrXpp0XDlO4ggLqCOCIfI6gK+a5vYagWjqXbMeZMtJFrZZsCpH+AbK1qrer4Y5N6IJuPHwr3iBSfyx8w+3gNhImfLX4M14M8n+H/dyDPYMT01nPBxqvVTBiFK9QR3uCv/B/j9u2k3F/CehfhnBp4hVudrWFy1mEOM7aKxh2eNhXzVzLTnwT1kE/3aeBbumm89bJzceLvGw/Dd1RmQcO6jL26H67Cr99Zm1CMqM77WKi+4nsB6oAamH6dUednPaUbE6VOs8oLrD4zhTqnyss4PrH9AlSovo/gqCfgaRjr1Ki8KvbdWr7rKi0LvnNRA9E5lUuXlCvBdTuabQfOtVHlZ8cIScC2M9FqqvGh0zHOIazrrKi/Ls/89++ns3+q93F9qu+VpIk8rMp6ebpUXjY7J6lLlBbjACfgVRsqjyotG2/v6aFVe9GCNF422rvWAda2PUuNFo+3dYfrTqPHyZ/B7nncy8v05gG61lVt4wvNGDr8/E+CjD96ruFnR5QK0/X7t2h1QtX7lPfgYu9fcU7wn9iNkDc7gru4d69VkFhPXE42e1bg+3X+Bjlgfm/oeu2k3aNpLHZhSBybVOjAavWfRA3VgdKkDc/J1YAx6r2ZKHRjgQoXmV6kDU87402QMGPS+1pQ6MKUODLn04b0EpQ6MgE4bWG0tdWDKO4JKSyVa6kodmFIHhlLy8L689OvA/LU5GxZWHpfbuPIdvYVMlecmU8/X+fCncI9d/8WgfW8msu9tGkc3UeK9dMvoubM+89Xd9a716K6jndV3rOrDhIreK7+w9+HurvXi9v2fYMRNj/EFUOlrv1zBHYHXjw1/HWd/BYpnnYriu/i+6OU8PhZ+OnViLNo/YkudmD1crAi4GEaKUSfGonfcwxQft06MRe/ebMJ1YuyILM1SJ8ZzTKLnvdSJsWgb1r7COjEWbVvagaxLm02dGIu2NIcxn36dGIu2Dm2pE7PFPwneJwz//J3HrhMj4RzvdHrDSKdbJ0ZCHhSWU6VOzGnXiXFzzNHSUOrExKsT4/gt0DNT6sQcZgYkwQyEkUqdmOPUiXHcV+h5LHViSp2YUiemX5M0gSaFkUqdmJzqxEiQZuye+lTrxEiYSywXSp2Ybf5VKM+C2GNR6TWfTaw9+ofO6vEechi8z+ojrMS/1b/5/LztyHvf3ffAu6fGVxO8Gy/f2VcoH4gYsFWrI/g/cLyVR+UtI+EtS5S34qi85SS85Ynylh+Vt4KEtyJR3rKj8laS8FYmytvqqLxVJLxVifJ2eVTeahLe6iR5a4/KWYPi7L7uyGYtQpVSlr3bTT00ecKrmP5nwPixs9cIn7D635uSm59cbr6Eznk4P06sHnsvz82v2hNY3nszA9+naP6uwE/yqfW1sObk1KI53bRodpM3ze+zxqOp4DyW992YxvuyAM2tWi9r1f7uvZOfwH/qO7FdN1b1AvoZWshp+gQnqcIdN2vjBG8Pa71A/rldun0Wv1wbc9Y8RYAvaNHkM4Snef/UCm/42925osf5Nm+akQV4cAPftp+2+sxRo+Gv8Knf/y7WPK0+Uy6gW4KnrMvp1b+LJvPLa+sCPFqi9WS7n4DjvOM7sy2//OjzZrTtf7vjh/ML4Qmrv21CJxkkYHJ6+vf2CU/g8b1sMs5u27P6FfjlfR2LUD3C97ldaflun1J4S66067HJgTxv711u+J+2UfXr639vvLklzBlGyx+B4puNNSXEdj90MoYugN53nWphYe33+ebLpi5Rd81bp4gnRZHp1FkfT9GuN6mvE3Z+9iWsVLKWg9/r6/8A2vcAb9mvWnr+3sbnvAXwaaccv4P3Vvf6S2VUJSejptP5g5r/FVgPX9RX5000LMzGfOdsbEuHhZXutUiHTU46bCe6laJ0rHow0EuHGzsd6eDglU1NOuTBtHNTOlazgZUOP4I5GH2uykhK8sGTk4+qifnHkJDufOAlRHT8vfQriNtvpSMhvuNsahLS7S596DVkNR9jJEQdTEJ8xdiUJMQmaZ8uX8z/bU+chNyue/Ca9e9JVh11p+ySMNm6Es7sOnq+2uM/xFJtIlG9BFregpfENH9R0K+jc50eg4o+Bxay/CjoXz/Fedj1uLv6Ydfjbo+VuDJCx2PdObVGw+EKdo9i7blx3xw6wd2J6dQ8oalSlX49y131KWN73WyjAbQ2TX/sxkSrbXO42mfnG1mOL4/LMvTpQBa55sBPbc2Vi40OUrtxsEnxAP2qfa0rHcT70lgr/WW1jOeFjr1adue5rJYMfSaUkZ0JPcxqKSauluYVxx5eslpq5C7pNa6W8oA+3+ogMbt962WsnhN5rJcSvV7KpNdLNWm99LX7X2s0rquFY6JxDGn9vr4187Cxytg25vpcl1WToc8Ss8jnKMeummbiqlm96hj1eDuzgsj+sqyaR4jfx7c112e7rJsMnbvPyHL3D7Fu8om5zTbJdTNe5sbL7M1u36yycsbMaznOymnKytlZcbBRIJ50FIhPjAL52tXprZwqmZUzZp5AqF2a1nxooC3WmRh3xiyci+n/3cWbHvfmkGiSDLWYNMc9DUVHM0+a5rj6K/fk+ahe/b2H6tdXTaXrcAr2Dp7p3gburPFHeBt8hrysX8/+uVOR4R66sftT7RedUZedU8cfoI/BU1OT/h7mJ3y/6kWkwKMR5pe147luLc9tdacwDm+tg27dcQ6n8H10/21TQ3e5h4d6rUvwOP6H705fP0VPBlvVnCToO6cuAd142i14BGlijCvrbJf0dd+zL6Gzaqp2TaezauX3bm0FWH8P/d6iWlWjem6vreaii0u1dwZk/17rwHs4d//UviFfZjVZqPko4Nv/D2Wm4z9SDQkA\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": true, \"events_summary\": [{\"timestamp\": 1684771476282, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476788, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476788, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476790, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476790, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476790, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476790, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476790, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476795, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476795, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476800, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476800, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476896, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477161, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477166, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477172, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477175, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477175, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477208, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477225, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477229, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477229, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477231, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477238, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477238, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477239, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477247, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477247, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477248, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477252, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477252, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477262, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477281, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477286, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477300, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477327, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477328, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477343, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477370, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477402, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477402, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477410, \"type\": 4, \"data\": {\"href\": \"http://localhost:8000/home\", \"width\": 1433, \"height\": 843}}, {\"timestamp\": 1684771477428, \"type\": 2, \"data\": {}}, {\"timestamp\": 1684771477481, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477482, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477483, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477488, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477508, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477543, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477565, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477569, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477576, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477600, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477603, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477624, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771477631, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477664, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477701, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477773, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477796, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478421, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478426, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478429, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478432, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478433, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478447, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478449, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478472, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478476, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478478, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478480, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478481, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478491, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478494, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478501, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478552, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478779, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771479028, \"type\": 3, \"data\": {\"source\": 4, \"width\": 1433, \"height\": 540}}, {\"timestamp\": 1684771479110, \"type\": 3, \"data\": {\"source\": 2, \"type\": 6}}, {\"timestamp\": 1684771479754, \"type\": 3, \"data\": {\"source\": 1}}]}, \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\"}, \"offset\": 2839}","now":"2023-05-22T16:04:40.254254+00:00","sent_at":"2023-05-22T16:04:40.251000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1684771483262&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:43.268842+00:00","body":"H4sIAAAAAAAAA+1abW/kthH+KwvB982yRb3LQFGc7669Jtce0ssFRYJAoEhqJS9XVChq1+vD/ffOSLuytF6ndZAPTqwvXnFeOMPhzDyk5J++WGIjKmNdWWe0NYrR2rRaWOdWrVUttClFY119sc4U/Fj/pGzx8dPiP8AGQroRuilVBQziXJDgwkF6ptW2ERqIfyu1yNUtErnYlEykZlcLYLwVzcqoGhms1RrMp62WwCiMqa8uLyW4IQvVmKvYcZzLQq3RoTOkgNCUi4yamqKiIHRlDcJ7N+59JMQbkSWtli1dooqo7M+fUKVhWogqLUS5LMBQEvn3xG3JTQGTBI4DxE0ptrXSZpAN/An5IO17aFOWGZjZigyNwGAct4sgunCRXlbglklLDtTMZwXdZZTdxVGWkA3yTYnrI2HsRxHxY/ci8t1zi5eNKSu211t+9/a9rG9/efN++XlVBI334eMPn78LXrN/GbJa/yCd281HeuM2ebgebUqnSuLY9xzh+VngMWY73GMhi5zE90nAY9snLneZx20S+pkT2/fiIfdinKzFqP4GL8om5WKtUki3G8EglDmVjYAJl1q1dZd7A8ty9nZtj4TCht137IDQ0M556IeBH4eJg5uv9JJW5R01fZAHLTdxkl7L54LYSRJEkUuJ4wc+elI1hlYMU+JkHlpfwatRjaQQfJpJwVNYOmxp2pQclA/+U2bKjUhzQTvhXNIlrOann4E1pqU13UlFOa4UDNRgrVDLSYr4XldZWlC5RudARHCbyZKtCgVxB55Y01J2fuCO0g2M0LXBm0ZStvoV/hlTVaOkSCXY1oIpzctqmYrqxAKNblGjEQ16OJLun+5rbqpmbbo8h5YgoOSxP5xxaBCwrwMVJ+EKloLrHnG7DnVoHt26u9YhRsWNUlKsQQ6D/MUyENl9S6DdFDI14hZz6N+ilnQHNCZpA4sAcesDZGB13RoDc52PR7ZtsAdSjQoTOuSKaRv4gV2BWj/i5q2Udt8FjjgFbewSom1BIlBjdHpYV3Yw31M774A8Ul6cdGxKnbo15d07NaUPLh1sc2qojY9gH0La2qURa1sfAtcLFbBn2HB7MvwwhJFzqzJFyopSQiuAXcGRyvdrJF/PJzsjy6l8eCTvH8m38mnz8xJb52ifP0EmXlOMrqoMujtswiHcDyR+J4ONhCf9K/b2Ar+LOSAO88LeqPbxhS6OBScOAABOHPBOOvBkj3tPOrDQSv0Pm8dTZIrfJ6EpoOa7TEIYnswD6Hg0D8TAqJXA5lIXLL15z9620dr7cVP9o6pX2e36G/3tu2z3Tbvavf7xzd3uW/HBab6/vfa6dnPodwNc+p4bZhEnGSG2I6IsJLFPXS8OAB9P4WUvL4jg3aFlW1ZcbcfT+UlI/cx28ijyqUj8OHGT8LG5UDhwHY/0B6AlYLk4mi2K/dALbADxPKAkTxIeRvnj80Vx4LpBB3MqzxuBxxoSY/SHE+LBzrM6Hk4a0F/zUkJnbP7yKrp+5boPQYq3ujsXAPOV97qXQrv7sesOokhx38DfldgN3LF6x9xQ2e6VQ6cnYWCoUXpQWhp8jN727PvI7QWC61fBntdF+gQdDxOA0A8Z0KrhHKHVejBmu4T35v7vM/NxC/9DH55b4fHc0zkLb0gRxHiye3h4ThzoVPPheT48/+kPzy8J8Sh3EpI5tuOJzAeoc8OAs0d9Q3HqOok7RjwvdMaIN+zPAo0Jvmja9bq/DcwAOAPgswTAW57LOFhlmVC5yRTeBk4AIAlmAJwB8AUAYN+4U6yAdA1xCoIQyuphP+yYUIdaQ62lW6or5DLVIhAAHSwo3Y9R595lfML2Ae9rV53kDLlPg1w/nC+ZM8b+oTB2F25leasjwzS/icjNaYzFRjNj7Iyxf3qMfVmIFxCPeJCmYSZITojLYtfPH0e8wCUiDMeI5yaT16qQJwg/WvzSisakOWz9c/sEP8PfDH8j+GNuRU1cVU4VVHJZd8XyEP6S+Yo5w99LgL9xndO6vNznQ3NJLh86gW1AQlU0oLgWEEtMyb+/+x6Gh8a37wX9t3z4IupAnc8Q+zSIDSeXSqNoYxbdFX7G1RlXnyuuulGsq5s7EuZNqJekW88JXE1mXJ1x9QXg6hos9aX6AQK1GJBzgVckwa8W7ypeq7Iyi0qZRQ6vZvnFfBt9OlS6pz95YqwXuTCsEHxGzRk1nytqlkGQe3QZbqO7LNuR7v99j1DTuyBuNKPmjJovADUxPN3nTsh/D+oM2ziKjmI0g+RTQZJ4/tef/ws4HAtKLTUAAA==","output":[{"uuid":"01884434-ae85-0000-0790-3f208b1ec925","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"b4chaybacz87b91v\", \"$time\": 1684771482.742, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"a\", \"$el_text\": \"Replay\", \"classes\": [\"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-stealth\", \"LemonButton--full-width\", \"LemonButton--has-icon\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--tertiary LemonButton--status-stealth LemonButton--full-width LemonButton--has-icon\", \"attr__data-attr\": \"menu-item-replay\", \"attr__href\": \"/replay/recent\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"nth_child\": 6, \"nth_of_type\": 4}, {\"tag_name\": \"ul\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider\"], \"attr__class\": \"SideBar__slider\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\"], \"attr__class\": \"SideBar SideBar__layout\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 4, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 518}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0001-5a90-cd53a406401d","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ue3df3rfc6j1h584\", \"$time\": 1684771482.901, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad091b0-03eb47f8265dc08-412d2c3d-164b08-1884434ad0a2092\"}, \"offset\": 360}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0002-988a-cc85b5fb42f1","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"recording viewed summary\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"xdfl85kbbeoftbor\", \"$time\": 1684771482.915, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"viewed_time_ms\": 5563, \"recording_duration_ms\": 0, \"rrweb_warning_count\": 0, \"error_count_during_recording_playback\": 0, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad091b0-03eb47f8265dc08-412d2c3d-164b08-1884434ad0a2092\"}, \"offset\": 346}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0003-e5d8-07b69a479391","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"y6wlixr7tcrdj71j\", \"$time\": 1684771482.963, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad513139-06be1f112c824f-412d2c3d-164b08-1884434ad521e66\"}, \"offset\": 298}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0004-c4f5-64ddb410d79f","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"client_request_failure\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"c2nat8nn0n5nlgp1\", \"$time\": 1684771482.995, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"pathname\": \"/api/projects/1/session_recording_playlists\", \"method\": \"GET\", \"duration\": 113, \"status\": 404, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad513139-06be1f112c824f-412d2c3d-164b08-1884434ad521e66\"}, \"offset\": 266}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0005-2691-56f4c4e7d4bb","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"toast error\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"278rnjz16fs6rg1t\", \"$time\": 1684771482.999, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"message\": \"Load playlists failed: Endpoint not found.\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad513139-06be1f112c824f-412d2c3d-164b08-1884434ad521e66\"}, \"offset\": 262}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0006-1698-9970569297af","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"recording list fetched\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"i55f3ag6w7zbby1a\", \"$time\": 1684771483.127, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"load_time\": 130, \"listing_version\": \"1\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad513139-06be1f112c824f-412d2c3d-164b08-1884434ad521e66\"}, \"offset\": 134}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684771483260&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:43.265573+00:00","body":"H4sIAAAAAAAAA+19+3LbRrL3q6BU8X7JKYPCHaSyORtbdmJvlLU3vuR8sVMqkIRERCTBBUhdnErVeZbvj+/BzpOcX88AJMgBIIIXWcr2riOSg0FPT09PT0//5vLh94PwMhxPD44OvkjHwSQdxNODxweTJJ6EyTQK04Oj3xdPTvvBNKCU6c0kPDiyHx/kCWk8S3pIMvFunEbTKB7jVVC/Rlq7/fjg5uDIsq3HB1EfCR6yTaNR+OrsLA1Rtu645h+PZWbflZmdPLPvrWS2O16W2XJ8mbmDPCKzZaxmdtwss02EkNm28RJl9r3VzFa7/cevf8jEdBqMJsRr2/F902kbltvGoy/SME1Ru1OicICqOY5teV2/b3ZNUzdCv+uZbSew7LZrtHXHtPpWz+7rpud08XuRPzTDvgFJf3EVjfvxVZGc0/ECp6sbZ77vBGHHaXdQvypalNm1DNsErWl8EY5BZjLonf72ovds5o/sXy7HL8eTi+716O/JD8+7N3+fXdw8+eX4080P4YmRvr1+al/gzX6UTqNxbyrZOP/nsxfDyfW/jl+cv7sYuKl98ur9u3+6T3r/mJoXo/dD4/ryVfCblZ55owOIJM7EZ3U6Nsl67wplZzpi21mzqy0Jrcl1pJPpiIOXKLNNSrOc2RJ8k/aRIiGz6UJlhULRW0uZ/YWmSrU2XRRernymU61OvtthdapVJ6dj3Yk6WZbUEGFpREO6qxrieG5lS5qWB5XglqxuSbPTMe7GMNiO7OtmZuJtGkWWu2R1O7ZpXOF2rGlH2xWD6f57ZBsZqUd20J6yHWs8BrLVlJleKrfZtrQkyOxDH4SC5Hk9xbuwc4fBzTwROxs4VPtutZ08rxw3LBqIRN72KsOwMlle8n4or5/nVTwc05/nzfj1M0GU5AXdKpW27A4IsUpXq3R7DxpNJkc8y/WmqnW8NvwObp3q1nE9+Ej7ah30wnmPKvY0ahNurnvXXHlnIse+qn1gjbl96toHM9h9tU9xMFurO/k2T4Dqm8vc/XR63lzF6MpSc5E7Fo2nYfJWNmtl8zlsDeubz9jSGqJl8oTJcHYeibrE6XQQnx+Ow+lVnFx8S/WcBDfDOACnvx8YyJKEWWM/PjAXDeZ7VhttbCHDYDqdHB0eDuNeMByA3hH8ROcQDTyNeodvZHv+FPbipB+Nz1P9O8d6/fR751XrtxQkbfDlUPzFEV3eFX898dfPH7XFz474a4Il+gAr9IHy6QNU6EPSMEHEg0bhmyRkghJ9gM5B2kuiCcnMBL0DfFqShI2fmOQ5IF/QSdf/o1Jf8Yz1tUZfnfaWrvDO9RUe2q362hvMxhf66x//75vXb394WtDRNtip0FF6tJmOYqbZUEeh1Ss66tXoKJ6xjtbpqL+lTYV1micE50RiEpyHl1F4tayYgyQ8K1U9wzAOk3AyDG7w0QMbfzuLhhgt028e+U8fWVbedngoDehpf5ZAU+MxHj6yn8hcxEz227LmWSnFOsbfi/Bm/rT4unh4GQxn2cueIZOo5sE0TuYvnU/pq/9MPl6IJsvgPn3kZs+EKEvSg56IyKgPIL7w9CyJR/PCdMvsy+IOKjW7Y3CMtFazbRnP2q31TZKrsHvYQzvGw3DV9g5R1hCZwiSJExJAEpCH+AHlwf/7OUjGUEntyzWs7/Nnx8c/v3v1M6wvovaGfWS3vwJBQbg5ATT+keUSgSvw8KQbz6bvJqR1r8bvxqN4Bt+0/13UDZOX42fP3zemj8hcx8JfKiDtDcL+bBjm9AXZ5hS9jmUdOR2iCF2YBNPe4InoPs1J2UbHy2o/CMb9YfgG3To8xvfzsLkoTcO3EUwlapNglob9V900TC7DxNqAFubXiLUWK9mcitnxfe/INInKOO6Hp6OY5J8etibjyejwIgz0qyCanpHeGC2rZZ4i6Vu7Zbbcw6X8hZyHw6h7CDsQXqOEQ0r8Lk5eiy5gHXZD5Ah/QitfC0ke/pX+/+0abP/ivT15//q/3hHbHdfKdLqC6XTWlU4A2UztS7tltIw61pfyL1dgc46LgiZD6RzZXjXPxCV4w0AFU/+t6bVMp2UonBZ46/2W7o4/ywLcshf+soHLA1enwzieWODpPOq1svQPGFvdXxuziyULMGsWsUv28w1G/vX6Y1GNLL/tw/T4RCVIb8a9/9ht1YcYiMBXslT5LO3nJJhgFHmSJMHNJu0FlNvJTMk91aeO/zD1CYP7eGN9MsxMn1iTWJNmyfBtLD2PwxEaCf7bBwzUA9/5dQPd8s0O4LGaQS+BZ0buEknDrhvtZEYhkO4sGvahVySPArsOZDIOr05ytfsgXzk1W/JLJiMKSaYtqgL9kl5Rv7nI7LbRAVTL3Ya7jeg2wvs/ybRqgwmL6WHGIkixOrE6wTANo95FYz1y26ZhIMJFNPDG8Bj/dYPexQZzNcfBJFIwE40vEdf4fhYk/bCfU3wWXm5A0zfxp5Jmc4KugUmVmDSXEnwyxldMMr+LknT6fLNQgmu4WBMmYhHXYQ8DybNs5roBKa8DRKeEVJi+HL8C45uw10YzyVDJKk3U/qdwGAZp81m/41kmlsjeQvZtPDmhANAm5BEHEFyjoz8PKNbRm41mQ9jQ/gbUTBcBFKKWzMbPRTDy5fjpRm3keDaGdaFRROuaAlrgSYYCtiTtI8JjCZnKsMzm4vMRS3BEP+qKNukLzmQEKv3CbB5S8W0HQR6hmiUUm2tlGy0iyeVxHkEPQZWT8Dzo3RTk+eYGg9FoExm0M2MZTEFgMn0b5zoq6Da3nZbp4I/CdHNCpg8bLHRoNkau7jA8hTL9HE0Hr5MoTqLpzQYRx7aRSXSF1AbN3WlDdrL7oaa9JJxm0csNdcctNHaR2gZ6Y63qDdpUkNy0KVyMODANvxbC1h8OPh5koekj7TgY/5+pBi8IhmikBdpP5ApoRDDUpFOnwZ8LxtosjxtrvXg0icdgp6W9HUSphn+BNo71ePJY686mWoR/434EVxAiCLRROIqTGw0W8wLp2g2ASg1+F8Z34SuCSKydRdePtV4w7sEaYOzSlmOBsBaaiPUMkngcz1JtGqQXKHesPUpbj9KPBBd+PAg0RGefn52F4L+H0saziXY2A6YAGlmWjx/HGv6HFzOI/jUwqDD5Me4HsELBFLUnOGoB3Lem6TWW/Hy1eHE1h3jtdRL/hmJfxKOQoDDxll98qx/BW6jK6K6Z0e2sm7FIcSWDeOnJZCIymks5n6LVTsiPW8njFPIIJ+IpFKEfoE0p31KKfMOsegFu2BJl6ys0zK/VgJfLy2Nu2y3EgBcDXgx4MeDFgNe9Dmgw4HU/9enBBsgY8GJNYsCLAS/uNgx4sTo9eHViwIsBLwa8GPBiwIsBLwa8GPDaOeDFuBXjVoxbMW7FuBXjVoxbMW7FuBXHuRi3Yk16sJrEuBXjVtxtGLdidWLcinErxq0Yt2LcinErxq0Yt/pT4VbPgnTQjTFYZlED4DDzJIHGOO1KKGglYxEPmj+qBKNMs5j/3wO24rO/b4Gttjz7m2Erhq0YtmLYimErhq04zPWA9IlhK9Ykhq0YtuJuw7AVq9PDVyeGrRi2YtiKYSuGrRi2YtiKYaudw1bQnhEQmDnMtDhisBRrQuDy3w1rwkV5jDXVYU3i+nLGmhhrYqyJsSbGmu5tLIGxpvupTw82NsVYE2sSY02MNXG3YayJ1enBqxNjTYw1MdbEWBNjTYw1MdbEWNMesKYHDAN5DAPVw0AQEMNADAMxDMQwEMNA93iazzDQ/dSnBxs2YhiINYlhIIaBuNswDMTq9ODViWEghoEYBmIYiGEghoEYBmIYaA8n5U3D76KhmHsph9+ZGFDUHUarefw18pRdD7Wax7g9j1l2KdRqniLMxAcBMirHqByjcozKMSrHqByjcozKcRSPUTnWpIeqSYzKMSrH3YZROVYnRuUYlWNUjlE5RuUYlWNUjlG5PxUqBxiI8IAbicyhpVSYywEcw7Abw24MuzHsxrAbw24MuzHsxrAbw24cpmPYjTWJYTeG3bjbfPZuw7AbqxPDbgy7MezGsBvDbgy7MezGsNs9ht1OgnT6U3iWhOngbXgNwRbhKJh3CErOqjLorAjBoWHg4da/YBaxtvkLJxDF+OlsOs2z+UXACbLDg/KMGDZVsEvJ1S5ibW+T6Pw8TDztSyiQGFi6NxosEBABMHz25ds4Hk6jifdV4Z2yx0vvZ6nFYmSKYCj7LuG8IsfZA6eC7aJ0C09Lc1uI7N8uCsuyy2mS6XgT9cMcXKhtRvjOJZBkMVsZPuqtsZXSWwOLddfAYhnTfeiYLq5xY0y3DtPFRYCM6TKmy5guY7qM6d7jmB1juvdTnx5sDJgxXdYkxnQZ0+Vuw5guq9ODVyfGdBnTZUyXMV3GdBnTZUyXMd09HHDKAB0DdAzQMUDHAB0DdAzQMUDHAB0DdBzQY4CONel+axIDdAzQcbdhgI7ViQE6BugYoGOAjgE6BugYoGOA7k8F0L0cp9H5YPpjOIUTBBAm+32cg1Hu0kbAZShIzVtEeHIkSsm1tMGysLuxkPEl5JuMgyE2OhbhsZfo36nEiIwi+vRTmEafqHvBvSlsjPw+icQrxRo8S4Lzc8p6DHfy1tzztKWcQjXo0UmARpyu1nmtnCtPVt5KoVdpdAnPdylXkbXqTFUyc2vPoy3kK7bP8vMyFNRf4wZKRlMZTWU0ldFURlMZTWU0ldFURlM5+voA9InRVNYkRlMZTeVuw2gqq9PDVydGUxlNZTSV0VRGUxlNZTSV0dSdo6lyypSeROPwe3iBsGYlACiAjzVgUpjfQq7s6fvok0R3VnNjYGGYlmFahmn/xDBtm2HaepjWYZiWYVqGaRmmZZj2XofhGKa9n/r0YMO6DNOyJjFMyzAtdxuGaVmdHrw6MUzLMC3DtAzTMkzLMC3DtAzT7hymZTST0UxGMxnNZDST0UxGMxnNZDTzHkSrGM1kNJOjn4xmsiYxmsloJncbRjNZnf4M6sRoJqOZjGYymsloJqOZjGYymrlzNPMniHQspkOkngKHmScdx+NpgN2oSQablcBMVXmL+JGah3e2MhbMWDBjwTvBgjuMBddjwTZjwYwFMxbMWDBjwfc61sdY8P3UpwcbO2YsmDWJsWDGgrnbMBbM6vTg1YmxYMaCGQtmLJixYMaCGQtmLHhfBxC/gAp9Iqh2+DRgqJahWoZqGardCVTrGQzV1kO1FkO1DNUyVMtQLUO19zoUx1Dt/dSnBxvaZaiWNYmhWoZqudswVMvq9ODViaFahmoZqmWolqFahmoZqmWodudQLaNlf360zGS0rB4tg4C2QcvcBVo2Dc6JBOnIZRReLYNkgyQ8w1PVdBmGcZiEk2Fwgw90iOnfzqIhIhXpN4/8p48sK287PIyTPszVaX+WCKOBh4/sJzIXMZP9tqx5VkqxjvEXc5H50+Lr4uFlMJxlL3uGTKKaB5h0zV86n9JX/5l8vBBNlsF9+sjNnglRlqRn0yL1AZnX07MkHs0L0y2zL4s7qNFs7Ehlza7R7Db2NG+j2fZCs1MMVwTpGqhVeD3Fax8wlsLlSiKMdUTmw+8HxKxpoT8U038/6A2DFF8OTjACjp/OplOMpIXvug5dn0ZkC5dSafCdpfgIg+F0sPzsbDYc6ldRfzV9EKR6BJBaTU2jfigekUJlrNrtO2AVHsEk6FNfVNLBDR1pUV6FAp8+APld8glUfqQ8o955GTYT84JH31dkmU5vhtAYKFMSYN0P9X09tzxg3Ugfa0YKCgsSnW1JuB1nexLQ+u1IwB4oNJYaTJyRcoJhATP91QQ0BIJxWDBX9SBrqIJ6uI6/WlzS02fTaKjD6qPQb4PxVO9jzdn5WDQbMTt/2Vf6q7A5oIApS6ob9KHLcYkmWUkUCL1N4iHVZ5FtEozDYZ6xSF9hroQ+DX4UjkzXKGKRt1iKojslpWBYDcXrFBRao6Tl/IXS2uZqadWk4PWDSDyZTSg8tcwUPVuiq3SilReWhJxVYBh0w+Ew7HepqUubrVjCba2xKuPbCyltj7ZiskrLKW2RmrJWWwRjUIJ+cpkNQJMgQX1foiSrg4pSkbaJ1WIVT8QyqcUTsCzs0HKqA5Miki2v6kHFG5h04oKJiifV71QUgydu5RNxT0PpE3FqR+mTldoXngiHuKSiHeFPlElgpZ7wj3IZu9RGGP9KG2gMV4J+jDHOyOD8wvmw4GkE5/8IRvgOf+hSRnhKTOpThO37vWQ26qanp935D/Ki4Lzo9BJyLR7oFjlJvQHQw3+gxMyVkVW3baHDCzYpqQGfCFaMVxmtLsxRC4NE6woDL+R+0QlY0qn7SfRwhCsyf586bEbdXaGOJtmOtPY///3/tNeYsryIzxfFeCvFiLbfScPqaCvTq2ksX5XftkpVVJlsZkFWNJPA3NYRrpJx+UbmWojpdZ6nmvG2yjgs/06EVsnO6WlCq9UxSA1nI3hulcx1VOYwuO6EudFUt4zqoh1DKdrBhGMnRVNo+EbHNDLF5LyGBVNlAYP9Tlg4G4bXGv2hNtAGwrtGlBCBN516GpzB32aY9p3d5D9HulvDqaVyClbW5jRQ+JSZDvAbTn5BwysnGSmUTAaxamYZhHMsIh8pQh8TmJBBfN5CAPWwH/ewegMrSfXzGWZoKUVAcktWXXXVSpPHvZWVLqvt6Wk2Q6thRbXhNCo2MrQn8OrHGvmCWkBLlLWCDOblrFrz20q5VSuFP3aG4BQCDef6LNLNTj5kFlMncYIdPMhaI4PVIaDAGiYXdQ0hZlalNhh+H2QQkh2eIDgWjvuEedw2ralkUR0qaNLUqJlaKE7EFn5H4DilkeAIwdkhDqLUu4jmXXwNRGAYJ5Q4CAHJfK2d4V30BtTxCMBAMgqGX2si/yAkU3ykGV9rVIYeDFGHI012+yxNTDYJjaB3x+HXGtZao/xgmOfWjRaCLeEoy4+aQVwCzIgBGIyiT0Bnz6NuNAQ69LWGucYnPU6vdcnUKI6nA5H5HKs+UpBFCX/Mq/if2n+gmku8msXn6eV5tRjm2Y6O5AqOYlZZlwWl/IsM2xQyrhL7AP9frL74Fbl6syQlUU9i7C8gkc2z6SnWquUFP16Qp+RqloMxbBXN51HPVNQbM3CDsJWzaAwTnTWf4F4D8ib2M1BgF5/HUQKcBSx8C7U8S6DZoFB8pP3+cYz1GI9QfKFRk5jApS9tz+iH51/h9Y/jHVAgg5G+kQGLaTIL5x1g1eVY8r8zL7emo16Sx1djMF9S6y154imgEhFBpt0XFEKieGo4ouFAaNT8J+Z0ZF9ItPhFcfOn8TXFVTRDsxz8Q+r1aIjuPQ+eX11dta7sVpycH1oUPZfsncW9WUoBEzJswTAVloUmlIOo3xeBWJJIqZFI37z/fkVgqhvkNHGDaLGkatnw4EcTC/oNw9S8k3bL7RhGR/NbjnliYg2Sj2TTmqdjJZLbOcnzm+0TJMiv1jzV+2UhQvQK4vaYzNCa1XRVl6voQ2eO6bb+zhVWmGWeDqzU3M3phtOrMBxr58FEdzR4+061BXdVx2yZUZfctM0YpXGEggwwA/KLPo0pWpMF/g66YiQir41sTnLe/dIy/cda/uerr2vYVr20FbbJZ9uMbYKZ9Gk0RUgkia+EpFdFW8OY6kO5TXyoWxmTbmTNmOyqrlORA8rQlIPqslbdJ0ralccoqosCc6S+psqrntIyG5Shlo0BoUq3KUNN8aoX5G45Ya4uS53jurfMcRWPK5vLIgRBY/bcF8ZRvQrpJt7wfIJzF9OfKuCkSm7Y3LpaOa/JHHiHcxusHKhjhTLsiJVbRKLaUK/JTHct5+VheimAwBXRNLHilV7KyOy0HBuORqvT7rUMR2/ZVsvw9ZaHpdH46LRTvWXY+O3hUyQMscfS1M2W5/ZaZkdvmW7LwnsOki16b6hbut1yvB5eoMd40aL/XORBPvo0PM3AE5Rk4sNHAXip5cATwksuqDigD6i95dtUEKigWLDWRi4UKzmiQmXB+A9ZHGugg2n8z0EOy6UiHKzxBhkHlbQEAS0j4JnEiEmFux1RhiiCmNCJCYOqC8JUJSqMqIhPn8jaKIdKtEEEL2lZdeHaYUkDxEQFonaQBclKE7JCugPhisceycAn4YIoRCt/UPGL3GgYE9ksR1QwoyZLQuEQK6SK4iFT+ksS1SS3VCliFmWLykCgLQdUjTbESdRFVeeyIA5QiChHCmqQS1EIUUOJVP9MxvSibAXIUIhZluei8fKWypoSTEGGopnh87bFX4hPg/hINSA+en+YaYtg3pLKR0VJZfo0QuN0sOii5RNBSBAVcalK+OvbpJwoDYTETzSxSVVFQ7Q7LV9Wuo3S6KtFSmsRq/SFVJ0E22l5bUgbTUlt03ZarkfZJDVk8wR10xNUwSA+bXeAJ9BzUn/6KZRcPKcXIDmiJ4hKxWnj0xGl44tv4sUOROfPyRNbxJVIskgDSPfBSxtyAMMmdQeTKJIK2shooEyXpIhP34EWUR/AFmL6aQ/zF0BDEJjTm9MX5VE5otWRCUIQ7KL3k7xcqi/1VFAr1CmvZ8sS9R7gTwelUdPbbi4kIkvVIoLU7G00ryjTprqj6rJhoFRkeLIWWbQQukIbBVP7ofGh3y70w/J7UDpsTjcc9L9MP/GG35n/cFL6qeFLntCjN6BvJAx6ojup+Kbjmw7NMjR0WnAMilAnegcNM/9qpS1MzugL+kmWpFlaq4M/pDf4gy+ftpqVeaprLMbaJs4Tvp1F5zMAzHOqJU5wk3F0nSlUvqZFH8S0Oa9mUC9xhQvMUIbGzAB7p0e01HoZo5lzNw4ua1hS3WNvVyBlkQOsEKa1ddma3tL03Bmo5lV1r4u8Uobd8EpT9xo2VFfc2xVYtMRG1qD5jLwQBRNfacfIl8bkum4q7quetb8rdGnOa4SV592gRu991akuckEZtucCf2o4UN3pFQ42D0msyYHqta5wUB+NXJeDQjwHa0fydVOVXKlG198WZFlYpNzLTxGVp61EuZ9fYqSI2e6UZiR5yJvmGerqoY0WZlXWXh0caGXYjhYj+Kq1F228N9nmM6uthLvhqrRKEavjCy2OayTiHM0viFYdCUQHvt+i3XIpXqWI1fGIVgY2FrHYhkor0rElRyCU2aq2kuBMk4UbDSNPSyOgWCu4GAEvozTD9o406Sx8rYkwNSF1y02gi6ihGmLIEwMczEfLE48OSLO6CGvUtcet6xmzl8NrtBw24c41prLR2uqQSAsstwt5ZSocjUTsZ76skFaCDIfRJI2oBxUELeDHHCycZ6nmWR1E29vGpBZBJc/R8K/d8ei/ipiRQMlF4GypSusEsZZmJFtEn9rqON7eSfTpRyx517AOMcB8E/8MzTRM08Ln4rcuEj6NrDYicLvLt27NSyZou1r2tjqVWngx+dyK0KlqzVQH8nYTqKN+PBDWeF3zktuqZfC/YuggyqpF2cky4UpRqV5JUVRyFfMDE9WGK7crRaT6LCsiIg9mhyIif2FZQnX+BhFY8vIpIff0KyTUePV8pWxKIK6drhVcWhsoQPIaZkpAsQIzlOHumOmoflKnyVT7MyJ0KZb5kEouLQQOLgGxq6uAM4y7Rg6qZ1OUA2W413JoCl7i4Fmlvo28ot2Bl7iutI4VynAn4GVH9RRok9puRJKECHzRrr1sNRt11BpWVM+g02gRxJ8XR+2onkCjbQx1q72wcOuF6bw3vRem8d5s//KjrXnv2y8s8733wv7lR4Aj9guz/d40X3jvTXu75VwddbgWet5k/vud3Hw+n/R21GFOmK3P0anVQa7TdHr/BsYcx2hoha0cctOmoQ5aYszc2XBOg2b13iPPKJkGF8qnDPdgsCjd41wzWNTtxF4eZekUAPLcoXy560ZJCO+fh1hNSvGnwnR1EbSo7OZSaMqQhKS9aG/tOOAZyni0xAdluKNe5BnKkLTKSv3Cu52JRBmPkMTjkZCDMh4haTfjUQdrFbAMAkNOy35vvaA1xu/pN5B//MT6BvELawyO8WlSVs0Gdu7hq+a2vPeWcWxrltHqdOiZiQWwyIHs4ssLIn+Mp8D+xQMA7TKv+PIeBI7xTZCTuYh+ztMvkjvLoOLfY4WzKeihyOwJWBaPQEY+QcJWA6ZnKAOm1P5G+3eCdEr162MjxWI0UUZN2cNZuT1DGcWRtJNFa1gsgpge/V93t1qB4WHXuMJjE6fn/g3GhQNQqgwyNn7XVZoy1FZaRthrtfZEZKnmQB2tzVtGa6U7PsuP5pgTVYdeUZG7dgGwS7+OD8pwVy4AtuUrrDRFe/9TM7VRNAYjC0mrw5aoExs9Dwc+KKJpsrjoToxe/b70bCP4zvelD8OzfFu6Bh3vhfpNTXDRM9Xhw9zVPK2Oy9pAuWepA4a1q9VFyxvHr+TG8e65fjWgrYniganJTUrYFIhjAulaNrlnqY8jEYHBVrOtmnxrVzFs2tSVt2fNvNdSbX6RBcqwdxbUEWKFhc0XJK3Lgjo4rLDQeDdUYxbUQWGFhfrd3LtgQR1AVlhojsg1ZUE11CssNEe8mrKgzh1WWNh8cee6LKhGdoWFzYeCNVmwSyzqEgs22df9slBvHSnD3lm4zTrae7eO6qFIqyzs3TqqRyWtsrB366icp6SwsHfrqJy1pLCwd+uonrm0ysLeraN6etIqC3u3juoZSass7N06qmclrbAgTk7aLwu3WUdxctJmLKQDHGVyoRs4CyeDOMu2ty8vSpjACyaMxdSEYyxrMI7FDgZADflFyrdCBurJSkjakU+8w/M/K9lXzXWRfXk66Y7OsPPU849WC9vcMJcvO0HQ4bpWK1U7vcLRLWcS1W71R9Ha5AZMyFlXDRuqrXaaRPBr2RBn+HSxwwp3E2DPUI/udhGHAF2n2oi4o/tKLugEJ0RK5jPEOQhpLkm1pg6qsW98apK6GcBTzqLJ2uSuY6vNz3IuCa1iwR710X7Dw53FEd/iRXBY0wLqQFMUFWVgUUlRqSfZrIjKrR8QG5k99TQaJDXsGaamA+qa9wr1pBjJ8V1HqdWDYZDEMVwhB3Wwa3TwSA0qawMjBUrqneDUBMfAN3kmlNHCPQfYRo3znrL07EyoLD/OhMIJAfifPBMqp7IdJqoeliPNzJ1rojqEivNxWBM99SQfJP3pTifz1EOEpD+5M0/8zVVEd7gVvus6rj3rXdTFzNXzh5C0vf8iVjwNiRM9lazQ4FDCLk65zilkS+ll/mqO1UOFkLR7OZ6eyhvtahhRx82lI4WQYTeMpEM4vjWurXqmkCx6JzaucE8EOUHqPEb46tmya/T/8wFO9BxM5y78BNs4qtfxeuqhP0hiuyjkoI7Q3m5GaI/WK+GuqBM68Qb/o6OR6KeHxUz4NGiVEn3ZzuCpR3TIIMruOuoFFlKSe5/jhDgbCYiieWjX6FvJVHYpzrPBUR3rMVUzzVcP60DSnk7q99TTNpDUdJWYPND1f/77/89dfvX0DCnKvdRBPf4CSdvXQT3PQkYeP6vGqkdcLDFFGe5eY9VTL5C0t9ZWDeFth1ms1dqqfWp02ECzOqh2h27C2roOquXY2QUHm2usamGKTFGGz6Cxqn267TSFzVtbPVsBSdu3tnq4gBTlfuqg2p3bTgVYqw6q5djZfSEba6y6BX6JKcpw9xqrbn5H0t5aW7VPYsP4tq2t2ichyv3UQbU7dAnZ1nVQLcfWV/NsrbHqruQlpijD3WusukUYSftqbXV/LpK2b211r60U5X7qoNqd2zbUrlUH1XKI1RSfV2NVC1NkijJ8Bo1V7VOju9yatbZqnzo7sE/qRlIpyn3UwVc3liJp6zogGqsqx7YHvWyrsX7JdssiU5ThzjXWV/deImlvra3YJyTtoLUV+yRFuZ86KHYHSTuoQ8nMa7fh/E00tmTmVWCKMnwGjVXsE5L21drqXjckbd/a6m4yKcr91EGxO0jaQR1KZl67uiVlY40t2UdWZIoy3L3GqjvKkLS31lbtk7kD+6Tu0ZKi3E8dFLuDpB3UoWTmtdPFi5tobMnmqCJTlOHuNVbd+oSkfbW2uskJSdu3trpxSYpyP3VQPRuxKWnbOpTMvD435kWXfdcxRRk+g8aq9klsRtpPa6v2SWw72ra1Vc9GiHIvdVC3ECFp+zqUbAu6jez+NbZko1CRKcpw9xqrbh1C0t5aW7VPje9TL2tt1bNpNKI2q4Pq2dwWhlqrDiXbdj435uWXbeQpYl7I8Bk0VrVPjdDBRq2tbuJB0vatrW7MkaLcTx1Uz+a23TJr1aFk0+Hnxrz8kt0uRaYow91rrLrhBUl7a23VPt22rWWt1lY9m0YLPZvVQfVsnB14NiV7NRp5Z3vR2JJdEUWmKMPda6y6ewJJ+2ptdV8FkrZvbXWbhBTlfuqgeja3bTxYqw4lW5o/N+bll6z6LzJFGT6Dxqr2qdF6+2atrdqnxlfnlrW26tk0207RpA7qinMkbV+HkvXj4lCSz6qxJWvJi0xRhrvXWHWROJL21tqqfbpt2fVara16Ns0u/m1UB9WzuW0x7lp1KJl5fXbMq2Q9c5EpyvAZNFa1T43uEmzU2upqaCRt39rqamgpyv3UQfVsbluMu1YdSmZenx3zKlnPXGSKMty9xqqroZG0t9ZW7dMOVkP76mpoKcr91EH1bG5bjLtWHUpmXp8d8ypZz1xkijLcvcaqq6GRtK/WVldDI2n71lZXQ0tR7qcOqmdz22LcteqgXsWxdEiJe8sZLXSgTcXeQ5DQR0jta6hKL4km4izc6gqqN18scYIMtZwM7GpWomnNXlBfXeeMpIai/UeMA4KyA0a0+R2V8xJUc9N4gfbrYUinq0jaWlAoTlwPOR2EGp0MqqEfiu84y3RR/qpdKsa0IOfmE1mii3NxYhyJhMNdZpFutpFH3KJQTJ3EyTSgLYjV0qfF0L/i6TQahTjHY4SbFGidOTwYB0cOwEF8fPBFKk9DPRWlmu2249iW1/X7Ztc0dSP0uzgx1gksu+3iHnnHtPpWz+7r2GvVxe9F/tAM+7SV+YsrXPIVXxXJOR0vcLq6cYZBJwg7TrtDZVfQosyuZdh0ido0vhB7QyeD3ulvL3rPZv7I/uVy/HI8uehej/6e/PC8e/P32cXNk1+OP938EJ4Y6dvrpzbdHYfDRCCm3lSycf7PZy+Gk+t/Hb84f3cxcFP75NX7d/90n/T+MTUvRu+HxvXlq+A3Kz3zRiTL+OwsDaEWFowQWja8lEryRToOJukgpqafJHSN9zSSDTZ/ckqNtKRneUIaz5IekmDDSPWyViq2+wfZ8GKsXNaH7No4UF1sOe7n519jM276WDNwaK1QQ0kCHvV2JMT1RduSQPU3JAHRJBj9LnNtDvp98a1SleE5sirXqTKcsjtXZXnNeYUKCBuKNsce+MUmdPq1SfNTR+Xmr2l+TLs/Q/MTHlnuNVQewa1VPtHFMWdQj8bagVGQtaNWO6zttAMuTp4wGc7OI6pLklyF3UOcMZHi9JJvqZYTtCOcZNn5L+UtzUkibnuik+xAh1pznuvDwUecK5dGn8JX3TRMLnGixjCOJ1ovHk0wJYLri1NABhrNioY4KjFBwjjGuZFRT4wnaevjwcGvGI2qlMJkpahXCghoxyYDo8EkluO+NBM4S4RivbiV1aKYShYilE32KmNEd7FRAZwU88I5Kc/rYGXHSl7wUZ7XhtNQYzR4SKnXDwBYOzca0I7pID4/HOPk1zi5WDUbhrgNM9OmxwfmosEAGVCIw1qcQzOMe8FwAHpHbTqGJphEh+DrN8zv0kPzMG/X+UTvdH5N8t+G0SiafmMbf5FV/QZf6JaFb3CLMK6IGMV92JiwfxpQVW3UA3cKPj5wxIDoir+e+Ovnj9r5l07+xURFsm+oA32A8SxBUoTymiCJE4wgJhNks28gTTt3TRAXn6B9cBbSsVD4Afqi9odmiwRngSjUCEziAb3toNiFhrdFmKhc++kZa3+t9m85NdzZkHkQj78LouEsCbUvVdW3bOeQDvCMeoe9wQxnLf/ivT15//q/3rV+S48su2PbR2bnK5CjEAXp9mwYpoetyXgyOrwIcXG2OJU41b60Wzjs7RRJ3+KMuJZ7uJS/kPNwGHUPxTXfKAGciET/8K+HpzR4n5r4Ng6vTtAG4RhPPtA5x/H4h/Dm18O/frsG/8+fHR///O7Vz4J/BwvIjmyP+IfURxFCOHhzSPfB/8fnr9Am1TGPLKu6OYhr8AqrhVK+xUl8ptMyFM4LHPd+SwXXOXdezrc1T/sZB2dDX58kSXDz1w3awHQNB5GS3TPdDc/iJPwp7M+u3+BO0BCsb8KfZXb8TEd2y59s6CWRxudRr5Wlf7gIb9zmOkAXmR3ZQgfoQvL+6nQonw2lJ1TYX9fq82+O/V/8kzc/iAI878h0iHxwhkPqfgSqMLXXorLMpmNBVY3dS1VY1NRZFapM3lSmjuEdeT4xO6L6CtE1r7NlOQ7Fe+d0NiBhOxb42b3YZmkomjLsi8pt0o9xLNmRI1q0G8ChOMYpj+chDGVzQp3OkRhTVpjahJTnZuICrffBcLYZQ5ZzZAuOst6ULqILt5BTohH6d471+un3zisijeOgj1zBHmL/GBd+xoT0RRxfNGcSLLa9I3uhoy9BD/0TV+/hduRjTHlxBOMGGmc6ruNnIuyGcDN+hlvdnIpr+dBbYZXmVL4wm9NpW0Y+wmHMgX0fvRtH01dnm3GF5ZHgSqgszRZOYC/e3Iyb6xmtFYWGCEIZW0SHmHo1/imONxA7tjuZ+FvR0UXv1vtAlr7Munhdh5/nPkRnX/xq9ckjjCcj6AWZgLPhLB0Q38eZC/TPWTgLXyJY0twadNo0aIp2mo2RCwd3niazMen36ySKk2h6s0Hjt42stVZIbaBJnbbbBkUiVl3vDah28sG3nOoGFNsYM0T3S3HkMDXouwldeP5q/F3URUirsRRxDg9GcKFZmMngKNPe4InwOJqTso2Oh5kMkZJH+go/Sxr+Dcys4duZIzgJYK77edjO2oCW5aF5BWd5JZtTgePnQ1Q1o61+FURTdHftS5zq2jLrZgNZzuXZACV+FyevxQTO2sJjLc7KOq51ZNf403o6684h/3UmMkv5lyuwGx+bpvmwn3vwsXc2B2h3HuYcgOabb3Ce9AaTe6yzsDJDEaQwZBUzYp5T/jvNKRGaGm+sT/DdKl0a1qR/M02aJcO3sfQ8DkdoJEyOPtDB6r7z6wa6hRNNjtpivK8Y9LD8kc7UF9Kw60Y7mVEIpDvDmijolZggL9il4MJSyE6+cmq25JdMRm8Rp01bVAX6Jb2ifnOR2W2jg6gLdxvuNqLbCO//JNOq5p6xLUIiHCNmdZLj+fh4GPWax07ctmkYR44YzQkyyWe5G8zVHAeTSMFMNL4EKvf9LABc2s8pPgsvN6DpI0wkotSlNJsTdA1MqkQIsJTgkzG+YpL5XZSk0+eEsG1ShHtk21REeB32MJA8y2auG5DyOkeuiJSskArTl+NXBEZvQBPxRdkFFZqo/U9Y+UwroJuSdXAZbBa2rCH7Np6cULBqE/LzME+cPA8o1tGbjWa4LRbLfZpTM10EULII1HMBKbwcP92ojRyc3H9kCY0iWtcEx4InGQrYkjQODYNLtAjLbC4+BDYxO5/H9MO+4ExGoNJNArg4awdBngVKsEyxuVa20SKSXB7nEfQQVDkJz4PeTUGeb24wGCFi2lwG7cxYYiliOJpM38a5jgq6zW0nACX8UZhuTghnZR45Eiq571FW1LSXAJHYSnfcQmMXqW2gN9aq3qBNBclNmwJQDnnpyysQf/8o11t8xAqMj1i7TDeh4ruDs34/ioUc+PGRhpQAN5HBO/gXAKrpxwM87ME3EA+xJvH0jPZuiWSAOlisIR48H/cncQRekUMTObBiEVlowSwy0L6VP/AT6xpnvTDBggLx1nyZksybLzYQz4jrOT788eCP6uWPNpZE8QKf+gU+W66J3WjFtE8H9ZevmH4bY/UZbkU/PZ3SN23lt459SaNQ1+Udc6sPdaHF82TcMj7GtXTTsJBR3KWno7Z0CXk3xtWDIz0Rl8FVrrgubnrCsc872ou5yjvdGxxEmIUotZo/WWW4cjuUujGy0TUMlZu15IWOVsfBvUN+4T7Hz9holTJQ92vedjvErTLILqhExzjH8sy0G9ACteXr/ZYu/RsGXbGWrbheWyM7Re+pssvp6iBcqHwxeS6cflWGClEvE8kX12W7VA4k0eI2pSMNyzSNUfq1tnhGNllckx0eaRL6+lqLJ0EPg+6RZn5d0xbqJtdiW1CG2ra4k4u/sxssb734e50rvmkLI/FK+0ZlFx7GaahnnFbLSd2fuyKn5ucCZDobDFH36h6LTtW/qWFMPRugs+VJNdVlqUcDFMuiDDuzudRequFZTj49/RTD3AjjUyMhdf9vs3tEHuI1pqgAWRsaGAQf+C27pLQva90S6quXoiBpF7ecjkzc3uwOrEtroFufRobexndPfMfN4rrb092Wa2mG3sFPXHzqtOU300jxw8enlv3uIaMNGdJF0KZOObOvyCp+4gv+yRRQFzdI9+gJkW/rdstt46Odyi+a+E+jH/iQj2Xap23uW/XVe2BkX9mFt6Jc+7ti5kSL66JXVXYSbFBYZRBJDRjc1TiQYpnbuN98IKipmeJxLdWMMmx+A/N5El/VFK04Okja4FAHXNScTam0M8zdwv6RVjJ1o8iiLEMZ02Udd3+vfjba1khAGTaR1FAC34dTbRAO6SiKjGjJkLftUTLl9RNOQ3XllOEQSX/2gWUte9dWb+BB0k7uxzY7mtdycD2233I7movrscng4yul4rcrv8g0PEQOkVOmiiTTFt9FMqXgn/wu05FBZt/K5Lfpvp+qvX0IfmBuz8GPmuAHTrm9X6cFzHuh5Wx0RIBt4JIfbvO6NsdB7fdrPyds0ub7OdO/qVs88zn8N4/8p48si9jGxyP7Cf7Oc1GKdYy/wGbnT/M35w8R8J1lL3uGTCIZBQCh5y+dT+mr/+wvC/l988h9+sh99he5nSX/laHB+U8KiZ/SuUPf6JbZ/wteTVGN2Szqf/MXuTfVMv6CdbVUu29IorRlFJdYVW1CFY9oE6r4QptQxRexCVV+K2xClQmCIh3TLzahIqaVbUKV30CaTq6iTaji87ZNqNj0Ijeh0nGhy5tQO2IGXbUF2+UuW9dlTVwVc/dm2qM2r/Gb3pIbQ44zIrarCYiwXUdZfEhJ1qkfXConfKycrocxImNDnE9Q8QzoRtUzeUJAxTNhAyueCWFXPBMnFFQ8gxZXPsOxRZXPxA7jimcCnKh4ViMXXJVT+Qx39lQ/q5GLVMKKZzVywY1Q1c9q5ILbsKqfKXIh05PphEt6VQJdSAI7iQZMdFsTZ/OlODRufkafHgynWlccIqDPv/SDFOh5zfxGCVogqfnBeYjbgT4OsluMjHLGOp/Q0cWQlW6TSdrENrjOBpt//Pq/0C7uGuSxAgA=","output":[{"uuid":"01884434-ae8b-0001-daec-3c60a40202fa","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"01884434-ae8b-0000-78d7-f52731c49734\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAJuSa2QC/+19CXMbt7IufwpLdXJeciqUOPuMktwTx85WV7n2s32c+07iclEStcTUcklKXlL56+89oLEMZiXIGY9G1FcqSeQMlkaj0Wg0+gP+3//9bfDnYGewHHwYXA+m7NP+YDjwBl+yvzuD48GEvZnIpzzdYnA1uBnMB0c6rSPTXrM3i8E5S3/OPl2yz+K9KP+9Th2zH5Hjg3zmsvpc+eyc1alShrpsXuYFq/Hp4IT9LNinpUw1GviDgKX7i1Jma4rYm3xNfmlNEatrdU3eIGHpijXxUqNCTYku06zJHYytauKtKtbkGZR+0M88XXtaE09nU5NL/fHX4LWsLU29oJ6/YP2a9kdMLY3YZ599HrPcAeVWdHYlRV5J33I+hBvzQchAWd8mJX3r69rTmjxDCupr4vJeJq9JoW8d4m9RitL662qKKsdFXFLPeGNp5fRsKkG8H5M7kCCX/UQFLkSlYzaw6lef5Q025APndJjpre5GEpfbvHw7JRrFM3RnvTRsyoM4p1+7k4VYl/7BGI1+KQ+azUlBydhLa19/jPM5yS2pKdJ9ZfaqVzrL2s19XumMFJTMsV6JdrTVJS5JRVk9YaGesFRnxVY9JMZ/sZ7U5kjriUrrsbMZHJLwsnrcknr8BvWI9mwy8lySoqiVkZdqCLOEMnnYhM6Qxsqno9Op5X9933m9bFFQ0aJNafU/Ia3jzrkfkeR/uha5jVqUzmHnbO5aslLng5c5jm3War9G5sLKVl8PZqzVp0SNeKpm2OXgjP0/HeyxN1wrvWPf5oO3g29ZnTu6HRNWz4y9mWhu8FLHuqw50W9yVuR0KtoSEedizT1Xl3TGaFiyGvcZRXtU5xGrdUZUcmr3pdbz2VvBMc6/I/btBdGwkPbCc/btiNpyTK0+ZW9Ggx/IEno2+G7wI/v0dLA7+IMsC0GFp/noG6sUPyfjQe57mPselZYS51Ilue+O5mb6xCk8cQtPvMITv/AkMChKee4UKHc07emTWPfMgvFzznh5TXOXep/o9zu6L/NUejpVpHk6rpXygOa0zUZIUDMf3r8R4rQ4Qo5YmhvWirdsJDwb/DL4P2zMPGN66T/ZeKgeCbHmWpORkJbSn5EQ3+FIcC1HQthgJIQ1IyGoHAn826luy99Itk9ZvltW95RJ/yqZPyNZP1lLXsf0s0c5+UjkJYtvfKzwGfSfrMRz9kbMpYvBN4PPWOu+Y385lxcZ3f9G5jS1/xv2iY+9iV5RipyfsT55lClL8Sn7nn8ulqrSuIPH8vNbluZDSd6q2tOct8SVm1zNfKVkproi2kRJnJJiTafsjXoaDZ5kcl+zHKqEJfXnIldCQHwIcvl474t+sEs/YXxKV+42OYQcTlk/nRCNFyUtG9FK+DjTup0NR0fC+Oq0NE/M2c87lveQdKxo8xVLN7WaI2aSuzNd3pRK5L27o1s2J55OtRdkh/pSWJe/sndz9knI5HDweUtzxPeMv4/Zz6+DfzFb6Vc5R+yTF3XMcu+z33jwhaYypbsLGnjdAX0KDBreSV48Yr3BZ9sly3etpespe/Mv9ntB74RlfsyswnOWmlP/M3v2hNX5qoMWCH8H9wGJz2kbFpR/SjqDS0e+BSm9XVAZEo2cSp99SqnkOnBBUr2kkh4ZY74Lujwmg9zrmJeAM1bSJaOOc+6F1JJTVoJ4zmeyLqTTIQ+5R5/NvuX8uqH56pjlOKRPcxr/c5ayG8qEP2Cf7KHq/nQ76cWEtHJInx2DmktW/jHNBxf0SYyEBSt3l1FzyX4v2Gc+205Y+XzUc1/xidY+Y5bOZb8OK0Gk+pZRyb/vslbv1ZRfXiZv+zlp+HOSrungvWzFnk75A6V8ZswQLnt7SNaQkMLnclS/NyRzb/C1/v22JX7/m3H05eCA6bFng/9mPFf8Tljb3YLeXo/XC/b+MGMDq3me853zeEy/m/K9uvy6PrgLLldJtU9eGj5L7tPO5macVtwUnBM2rbCrviWtvEu2zC7Zzau4Ws65I6K3jxx0SU/xXaX7xMGs1RtKbr0h7nCr25WcOiVO7ObS/ybXDsHgdQf8FTEawoZzDf4q+/SFXHm1OVtWaSWX9rUjaQlFBjUT9v4Dy300+Eeve35Gc+dScmxe0/vZdL+SXX8tV2SPyH7mq4SuRpmIHvALVgr01Dp6itsw0FPd6inhxbi8Az3F/QZ5PQXdBN0E3QTdJPjLPb0zpkGuMj6RPcYVMZ6En+43udY+o13d1x3pr4hkYp98Z5uuAOfSs6Y8UEo+vI3XfGaJqZQcsmfc538s9ZaSkXL++lJWLmmX4qCg9X7L1PKGqDOfZGVJ7dbzWlU/qHemL+m4E0nzaN5J6HOIeQfzDuYdzDsl8066T3CQ01nd7Mg4FEktZABaCloKWgpaqrhy5xbUjMp624FeCmiflceX7FMMQEqJqGXG8oj/h8Svtx3tw/mEO8nrSt73t6wG3lM/slImFG3Cx0qeyicUN9ANpRH5PPinTSjtgsaAMG37FGG1CY2PaMw91ruffI99TpR832lUA29HQO3wMpEV7yn2SKxVnuR2aruhK6T1B6fPli6u3XhExVPJ9a44GMtxFa9FqZCA5xSJM6Wdhy4iBXyK83Sox6PG9L6kWeBARxR1RX9ciGER8/z3NLpEbAhvywXNehMZd9MNdQ6NJyczx88p5/dGVBuX0+86HFGcMk+u6JMSyt7rmC/BKzOq4G7pjmQ0kJuRVzPq5i6kMJKRDX5mjjo0RsyxwTczomox+BuTji5iXSLKK2KCxmtT2YXujOVoydKXjwxKKRQRL7ynT0lWP1TI6Qv2RqyeLjqThrhgeU6o1ilFhi5JSp9UtqwLu9Sl1YH4ZMPtLmhyCDOZj/W7ofUDL+2QRvgbqad+pbinM9brc1plzOn7h46iJ2OiNCur9XR1M9IT8q4X50TRs0e0Slzm4jq71UJB5Sivoq4b7eNaaJ8nOTq7Hh+BXAsJe+J1TXQ3j9L+nf1m47P5m8c0W/4voll4kIS9xLUj9yo8136EoaaSW6Omh28o/XoT+ntTiGoeUiz6BaGYLiWHdtnTl6yd59Sj4i+vj/smRjRr89YcktU5pDEzlN6Ic+lNnOo8F6RJOdV8vAsb9K1MP2TPBNZpKL1kM12C8EYISq7YX47reE/1HlFbjqTNINZnw9p4Q2FzDI04oTNaq11S7TzFkt69le3llH1G/hT+93cDq/M7zQ9DGZv7PeG0p7IHjmTrLon/nOIbqktRki/nd/bDaxI/otYsOvGZxLjwnv+FvEETaSdNqMbnBvKlDMm4S7bfe5JILodlNa4qI63tGfHsD9nen0hqphr3k9YVVdbFS73dqMSg5RIDmrfaLbGKxvoS0poe0QhIS3RqyvxOjrcD7cWrL8evKCf1WnwntcIxaaEPRnnVacw6nLVrcFfQ7FKJO6Q9N0XvBK3hoIHeAXoH6B2gd4DeAXoH6B2gd4DeAXoH6B2gdxADhBgg6Cmgd6CboJugm4DeAXoH6B2gdzDvYN7BvAP0DrQUtBS0FNA7QO8AvQP0DtA7QO8AvQP0DtA7QO8AvQP0DtA7QO8AvQP0DtA724HeAZYGWBpgaYClAZYGWBpgaYClAZYGWBpgaYClAZYGWBpgaRCRAz2FiBxgaRAtCN0E3QTdBCwNsDTA0mDewbyDeQdYGmBpoKWgpaClgKUBlgZYGmBpgKUBlgZYGmBpgKUBlgZYGmBpgKUBlgZYGmBpHgqW5gnRcUZR6WIVmo1cEFiHYqoU9+CTLlkX8VFfYhX2o5hrfdSLQ7MbkDQPAUnjA0kDJA2QNEDSAEkDJA2QNEDSAEkDJA2QNEDSIB4HegrxOEDSAEkD3QTdBN0EJA2QNEDSYN7BvIN5B0gaIGmgpaCloKWApAGSBkgaIGmApAGSBkgaIGmApAGSBkgaIGmApAGSBkgaIGmApMkhPc5pB5Dzq4hU+YV8QhNpLa2LWfEqUTbArGwXZiUAZgWYFWBWgFkBZgWYFWBWgFkBZgWYFWBWgFlB5Av0FCJfgFkBZgW6CboJugmYFWBWgFnBvIN5B/MOMCvArEBLQUtBSwGzAswKMCvArACzAswKMCvArACzAswKMCvArACzAswKMCvArACzAqRHT5AeIZAeQHoA6QGkB5AeQHoA6QGkB5AeQHoA6QGkB+JFoKcQLwKkB5Ae0E3QTdBNQHoA6QGkB+YdzDuYd4D0ANIDWgpaCloKSA8gPYD0ANIDSA8gPYD0ANIDSA8gPYD0ANIDSA8gPYD0ANIDSA8D6/BEcu8H2tFK984ExqF4W4mJd3Ct7x5ZVU7UUjlhS+WMWynHobmqjXKqkCrFfNm4k9Wl+5V9WIfXqS/Rt6Z2kxtvnMryceMNcFDAQQEHBRwUcFDAQQEHBRwUcFDAQQEHBRwUcFCIpkI0FfQUcFDQTdBN0E3AQQEHBRwUcFCYdzDvYN4BDgpaCloKWgpaCjgo4KCAgwIOCjgo4KCAgwIOCjgo4KCAgwIOCjgo4KCAgwIOattvvBF9zTWRiYdaWCN1fIl8AJIJSCYgmYBkApIJSCYgmYBkApIJSCYgmYBkApIJSCbEGQDJhHgo6CnEQwHJBCQTdBN0E3QTkExAMgHJhHkH8w7mHSCZgGSCloKWgpYCkglIJiCZgGQCkglIJiCZgGQCkglIJiCZgGQCkglIJiCZgGQCkqkM63BAdCypZ09IdjlS5SXZ+cta5Iuw8IVcmftdWURRFcJJjJzLxjU4lZilYg0HUjK4ZXpD4zFfWlSJXjk0cqxTYmCNqVpdVlyJWXpJMnhKO39zpgF4qUfS66dG3SGNB2EZvTM4fMLSviQ/3Yw4fE0apLwe29zV9WfTVrXGTJNyKPvcRF6NrUry1+R2leyW57Uv25XIgDakwqVZYx06lQ3ygv3n3o88PmLz0ehV4hDtS7PFBoYt3SYXtoR5DFrCPAKDCQwmMJh3j8GMgMEEBhMYTGAwgcEEBhMYTGAwgcEEBhMYTGAwEckJPYVITmAwgcGEboJugm4CBhMYTGAwMe9g3sG8AwwmMJjQUtBS0FLAYAKDCQwmMJjAYAKDCQwmMJjAYAKDCQwmMJjAYAKDCQwmMJjAYAIdBXQU0FFARwEdBXQU0FFARwEdBXQU0FFARwEdBXQU0FFARwEdhRgrxFhBTwEdBd0E3QTdBHQU0FFARwEdhXkH8w7mHaCjoKWgpaClgI4COgroKKCjgI4COgroKKCjgI4COgroKKCjgI4COgroKKCjgI669+ion4lifrsZl9BfSHYnBsoh+/5xAfUS1NzkVYf4sCnXscS7rC6r+ua58jvXykv8WcowHzNcCr6oRQb9LOf0RQYjMq7EtTwnGT4ffNSzmVtz09uPJHVpLW4l9mhOqJ1TXepj6dVsXnYxXXWZqeZQuQ5IM4l9zlX93F6Z9Xnq61pInbUguqYrynJr+3m9ktaVs6DyrrjVCLzy8gJrbFua3xbnF9GKDbhD4A6BOwTuELhD4A6BOwTuELhD4A6BOwTuEFFBwB0CdwjcIaIXEb0IPQXcIXQTdBN0E3CHwB0CdwjcIeYdzDuYd4A7hJaCloKWAu4QuEPgDoE7BO4QuEPgDoE7BO4QuEPgDoE7BO4QuEPgDoE7BO6wL7jDRxlPzAE9nRL6ifPnbA10n09xce0gBYV9/IUFUvIVIfXcNcoW6xogJYGUBFISSEkgJYGUBFLyviIlYyAlgZQEUhJISSAlgZQEUhJISSAlgZQEUhJIScRbQk8h3hJISSAloZugm6CbgJQEUhJIScw7mHcw7wApCaQktBS0FLQUkJJASgIpCaQkkJJASgIpCaQkkJJASgIpCaQkkJJASgIpCaQkkJLA3QF3B9wdcHfA3QF3B9wdcHfA3QF3B9wdcHfA3QF3B9wdcHfA3SEqBrg7RO9BTyF6D7g74O6gm6CboJuAuwPuDrg7zDuYdzDvAHcH3B20FLQUtBRwd8DdAXcH3B1wd8DdAXcH3B1wd8DdAXcH3B1wd8DdAXcH3B1wd5UoqKXsBbXDpLRpinUopnpMf5cUYXop96lMJJEdUmXdcoMaLNeqcnDXIu5aBOYTmE9gPoH5BOYTmM9+YT4TYD6B+QTmE5hPYD6B+QTmE5hPYD6B+QTmE5hPRI5CTyFyFJhPYD6hm6CboJuA+QTmE5hPzDuYdzDvAPMJzCe0FLQUtBQwn8B8AvMJzCcwn8B8AvMJzCcwn8B8AvMJzCcwn8B8AvMJzCcwnwbW4VHGE/OT1EkfNWJyRlYcEJNATAIxCcQkEJNATAIxCcRkvxCTIZubgJgEYhKISSAmgZgEYhKISSAmgZgEYhKISSAmEXcJPYW4SyAmgZiEboJugm4CYhKISSAmMe9g3sG8A8QkEJPQUtBS0FJATAIxCcQkEJNATAIxCcQkEJNATAIxCcQkEJNATAIxCcQkEJNATAI3BNwQcEPbhhtyKnFDQSVuaEl9oxA+f9N9dcvq5vvVq1BCZzTfnegSbOa5Mf3sUc5rWnd/kN+O5HzwT9K+M7kDvhh8w/RixLj6Ga3xFsSThdRyb2TOKxohYi57Q37suaHXRc7P2Az7KFOW4lP2Pf9cLFWlcdk8KT4Lv3Uxb1Xtac5b4spNrmaO/TJTXclZeCL3Ioo1nbI36mnENIaZ+5qkUZSwpP5c5EoIiA9BLt/UWMfbpM96821yKFvhDevnOWmJYstGtA45zrRup8Ho8CpHh1c5OhbSWkgRbmNd+5R8IQttU73OrObmtGdyIy0TleZP8v0ea0pdPTKrc/1JOwIzmgsWepwdSCuHe1luKK+wusqfj9jPUsuB0nrVaZVtx62OkVwhT2g8ntXmOyE/2oxQKrydq9KfUatGpNVX0a/SLqjkqZErlYo8f3l8yjbxV9jD16SDjzWuclX6I3n+hVqN2PdHNWcjjUXtM2cFFvXCIp/SYbdkp3w6Ca/iaEQxvDYcXZD2mmmN9KfG4YpzDJQmHhVmIdUDY5buS9JlC0lPFUVJryjid8n6vaPI6xVFfPbzLWmqG5svtU/tQNp8Yp9sVQoxlkT8yWSjHNmxWK2BAprjbdrJ5+8RPT+nkSusNtXmb6kPlsT5qTxt5tIYsYtaGiLLGTzNpegRXkuuG8b62yhjBaf+4TnpvlSTzwnHv6gt8ZraNaUWZ0uta03UamuUda+ilBYttqms7LqWJa22TKxPpgYdase+vRbW1VHdUr5XYtPSzei7kDFRI/LfXZPnKY1qOLbKW0993Eo/VUt/vndmlG9Kszr3Tx4ausF+pNa1KGq9RTbjatN2rTeuYks7cJ3WrTO2Nm3lqrH1Wu/biNnrtrCOuyYqRP//rFvnUuxefjx4FOmV8nDdvO7KvGEhX7Qyj6+tKTMX30HYNOdmdYq9i7hB3ib1hg3yBg3y+g3yeg3yug3yOhtKRsJ+NpWp2GrchKXjJjBGslgzrzuGL6WfJ317KVeA2ahc06o3fUuu9hdxv+p/kadqqjWU8MDvbGitfyfjeo9pD4pHMXE994b9HJa82cl5uUa6PlVeWS7uhdvRPrcj2qPjyKf/ku0u+r+yMuORrK7qQZWyC54v5Fxjw/Um7fat2+231G4v55t8nPH7qPY/N+yWofxmerkXJfacoDOwaFHYs7YMB7+zel2KwnVph0rsQ/xEu0lVLQ0tWmpqnT5riZEcx3ymazaSI2uJ7p/2rNZ62d2kUcm+z6LEKk5j9fP8fpEpr0wmnxXKatIrsXWvxPdAWtfn3hvaAVRnBIvV7Ixmr8uGnE2sOZvcA85eEHe4JmzCFZ/WMTZc8fVeWZ+5oqIuP5CeFLIndsCbccmx5pJzD7h0Qmvl9xRjJD6pcTZkvEl3RURslogeGumZWXiO/6A9Ge6DPSFuZ99esCdBQ5671jx3O+H5xJLf2TJ3ZJql9P5XzTPr76MtpDY1I18220lLo+vLoz8WMv7jWlpbZ2Rt7crIvz1WP48LEWcmLOQuwCmdqnAssYfzUru0iXR41tLRv3WIff/z2fAoZys34ZpvzbVuVzEHcrfoknSHOkl/ok/XHVZIUHkrbVY27bWwPU1telVPZHTWUrZ2RONpRFGVxbV/Vepr4pi4K0OV3EyCQiveVnE2aGmspbuv660WhA/5SiMFVYnXMnptSto0jfJvvtO4Oacj67EadTpWd2W70yifP2X06UKvq/ZlFOWM/nK//KGMHnw7+ErGjM+oH1TKM+oPbm98RXbJleRt2tf7MoJ8TvPWjNKl5fP8atUgIru+orhs0ZIR5RA9tE/1p9ZKNl26i69i5VW9vB6e9lbHr4hYyHzZIzqZVkSDcZ2eLX8uZWwqz1UXkfpXMuL8gm4qEVjLU5Lmcyqb8+ArKptrx48kwwuy20xOXRA+f0mypko+lSdGLCS1og1/lfTif7Dff8jerOarU5l/wThzupE0FEvbZz/mOQ5VpZr9UkZT/slopdSuouw3ueeUnp7wWpZ1RJEeCy3VXPeeZ6TsrxJ6FvKkrnyLvyylXqXehMsTwoZcGLEooj8XRn+LGI+xxjGc0N9ziQQxR1/K+6FEQ6U3A6i4YvH9MWHcjwzZ+1Zq0hOSzQuJ9qjOxVvL47PF+QqfydaXj9Q5jQARj/o5+f7G1E+ngy9k7bykftCQ2i8LOp/EjChaElenFXNDbOnRrJ4Z8j76ZvPxreF93dzy/VmPzeq9kIVEIKRR1OmdDCqCT5XsUMnp6ibVZOUpxC5yaikpCVfvVRQ9xwO8NyK5hvTLz60Qf1X696zsmbQEyuLp39HPLvvlp7vwtpyyp66OqM/y9YTG8g15NA4NSeFvJhTpMS3ZUT8jrhyTdKj0SrbWtU+4lL4a/GgloYm19dKNt02dgmhrMap8v9CI35V94tCeyAEbf7sULTim8/L4zuYu+X4OaDd3V+JbHRkVXkwvTu3hTw4K5fO98wOZwnzqlqQNB/+ulF4xHymuP9YW16fr9cDam+it1ExxT/1k7+SJXFnf2FQiBPNesUNCj/Jbe6ak104p74g0hdjH8RutEgJrz+QqjgeG5/LuOK5WVCrKZ1h4MiKtf23cYpOdN3fIZzDX0ajZlQbXsIdsTnZphH5Jrcl+4jN0sx5xW+sRtwc9ohBkI5LqpYyy4hbOO2NcrJL8Zhy19fUFHfn62uKo6RleNOSR3wKPVEnd8qhJqwPrVgf3wgucyoZofx6N3kxGwha4pUpqg1tnxr11beilZtyJrLnTt3iIJq2OrVsdd+plzEYOiGgg5enYqWhLYt2Wbnz/ZTt/27Z3uC5ablNJDa3XFmFHkQr3YccvtF4f1HFNldRvrrUjZa41v9yOpKxd3xq8Zf3zloXWK5ywoxXOpt6yC9qf3qWoC+Gz2qVzyo7Ia+UzDu/SaXm7dG8p/xbKk6vFN552QZ/4zb/ifSi/pylm8s5Nh/bDd+k0+CN6klAazqldkq0R0eLK/ThV34wi50YkR3xH+0jWoHKLGl39P5Dl+JnvY/Z+SCikXdnSMVEkvH+iBaKmXfJycn6ImgJJiy/pH7M28e8RtVm0KDFaK7gWy7JEa00euZqytMUjWav6fka+JuU75H1xpFujWuHLs8RjnVvkTSkY5igIiVrBEUe3PKB6VTvSVriaJke3QPWuq/lotkzRkn6PNLWebI9qoycpETUNc73ryPeulibVQke2yTfkamjI1UjKrpDcNHeo5SDSkisoFVJrvlGtLyvbkVQEug/THszSZrZJtHxX+old+TeQcqo+KxkdZniremqkpTTbM0JCd0lDK/kU0jks9GpRLhQPXEmHr8dgKlFnBVlMJXEo26j6PyvHs5KxIOQwlWazfYEcefkxlR2VglNCDtPR7GkKxtqrryj05Fgba3030vKX1S0p592M5vO1vkk100eKnHQkleJTpCkUMih6JNC9JD7z3lnosZRSlL6dydp8zSFOY0z0R5mejmXb1FNXa1pXc9XRuiurcxJqSSxlW4xKNW5iakFAfetoGVW0OboXfD3aQ4NWwcGhvAMqII3mUOqZHkcj/TbV5Gl+VcPYoNozKDU1Tiy/+0bbPckVR9aYSKmLSqhX3FK8SlO5WgeMdV2+7AmVTvRTSntW33uyxLFsZ6BlMZAl+FIXuUYvpG89GkXZGkY5Cor0FelP2zfUsqDGeiJlWsz9vqbcNeQr0P3rZmgr76d8f+5S61JpEJ8S2bbEkJS8JA1zvXVk1BDLvhc9P9Q9r/pd9Lo5YkJN4bAwRsrGkJgVYtliNf4+ypjpXYpfHMn2RdTzrswv+CHmv6z+HMq6kpI3XBbU26F8kk9xpOsQ+s3XuieR33zSLmYt6pdTPqa9yCPdO4JGN1PPSEpF/qlLsppkngxLUg3pd5coEt9G+pt6/7FHu6GhtU8+bA01YOdbE09PaH13I89wKG+BvX/dvQc7L8W9xKynZUSR7TMdG9rEjxC2wDlVUtecEyeHTI0VsFhDr8LKFfl7yT7dNuRkZD2Konspg4JHo8zpkNmzh+3T5z0YTfget8B3VVKf+a4iOppxK7HmVnKvpTSvBYrxF+UxkenTmY6MHLPWvG8ccRFZ7wxE9wLDWOT9uTwv/pBa1oxXTgu8UiX1kVfiUzMeua3xyN1aHnmt8cjrMY+GlW+K5xpuzkvfmpf9w6yV24rFnZWFRLWom6aKuyv2VqXqgUP2Kd3RyuMy0h0n2xPtuj21cXN5CazlJbgXZ9FE1qulqIcRfk1HQHE/sg9joPuzPjcfD5G19HSLlHxWycN8C2LrFriQ/07k/+5PhN18PCTW0pTcyXhIb+xV90UMNS/KR0i8RvRXvFWxfdUr8fT83aqV+C3pnSyKeJ/OeEl9RF8R+ljhJRTCd9V4GmUid+2iZ4qpJxQNlp47LHIobXko44aajLZ2zjfOU8A5cS1vBavSZJuP39h6jR53dNrQejF7+fmAx4qe1p71m57eNKPevyapXawcFSmWO4+TLpbUpD9c6/7oX0xgeRReSPt+4m9MJx+q/5vEz6Xna6QxmPU92n70YfX+Xj9i/GJrj0nc8xi/X+R9MUN5hvBE7uMHOtbIoRgjh/Zkx/pN9v3ISMF3kF2a4/m3Ppb3KeXCfpc43II91qHFLqxCeDbR2YH1aAvuyYoqXTl8OuusaEPWnbmy/npNtcHWBuvXPQKbS2PYgjRm72eANN6tNHZ/Z8fm0he1Jn3RvZO+saXsNfFCKWqGK1Pkd43Wl7tubr/ZXNbs8aT366Tc6pNw0zM9mnEuaYFzqqSHxLnE2jeYdBT/8dBxvwt5RtzM4paSBXkzp9Yn9Y9yvGkiN04LcqNKgty0KTddocoTa99e0plvr/+o8sTal5WsiG5OWjuBsM+o8sTax5N0FOHTlF9zsjEnxh3L2ZNH1ezfjGuBNdcCYPGBxW/FH5tY+2q6ui+q2cmV4jzJnwjt94pwcD/R81f0/N8slUc7Qa8oFUeAvaI0Hr0TqDmP8sSUx2GfQ/rk9ersycTay5G0tr9iF/XwA7VTrHmqIoAS63Vz0ss937uyQxJrriUdn141kfPihFA8VfeNZW96H1uvZON7gSJJfQBqpd/k9smQdu2a80eVhBVb1YptQVRPSGudfYIVm03ahbw9yFznrfIlHEtkkdrHWUocZ9HLqlIKTBI/R/BNJj6xKnKgPGJpU6skK5F2K1GR8n7PAM3XVCGdhdCcX6qkhzFjhoSlb4tr/oOQssCaX1iDYg3axlompDNFbKWu/2vQRJ7bI04b+kmeseIxXrj0TdyE8Eq/j+R3/lacD5S+E+fkPJbfHV3qkFaxvjwJSnwL6NsrkojH9MSl2xMSOjMkLcGlM0timT82nvykqX8s8zryvBHHSJktN33ySlLwWD5LqTPLUvTn+fTvDO94DtX6V/IWCMegT7Qym0dwOc0lqDHziBT9WcWHdDaOreR3fUPhhOxi1cPH8latqpVcbN0OD/MG5o2WRk9iLXVJz88RjWQc7pBO9BO//Hysjz3SVo6lvybrb0DkQV/8EWU+hs3XCo6ld6peGlRJbUiDibBpqvcPjLKa8MjWy+G05uWwm9+fkPaYGDdtlNPvWdPvwEtDu07N+aVKeiheGsfa6+B0fMLDf1C9Q7qn9lLyq2qkhNZt8GEBwwJuySKLrKUuggXcAr9j69vu4hU3PvYhajl761DZCTb5Mz245p/Rvc1Lfdsgv69W3Ih9TfH+3ML80DDOOSS/ja1s34fd4U153RwTE5Kus+Oley/ODCy7qVTI4pBpeb4feiNXRYesbn7m5Zm+1TvN4dD79DQIcZP1jT7nYJi7W/OYqDiTZyA06Q3Hujece3HDo7jhNT/+3YZcclvgkippe7nktcYlb4u55LfGJX+LuRS0xqVgi7kUtsalcIu5FLXGpWiLuRS3xqV4i7mUtMalZGu55K1h49dzyTPWANvHJac1LjlbzCW3NS65W8wlrzUueVvMJb81LvlbzKWgNS4FW8ylsDUuhVvMpag1LkVbzKW4NS7FW8ylpDUuba/t7bdme/tbbHv7rdnefi9sb74XMNe3FY1p/yB/TkC6U/EH7SnzNyfESx5dtBy8Y3+ntHtWfW7QtdyrUKhCh3Y51I6G2UecG+n9Xgo/Nqe2nxb2s9vAcfnWKwX/XtyZmMY9vNT8OmDfJ3qPaFWKESH1roneyUY52rprJ6RbWpv3jiqp63ikJu32W2u3f89OFxPxSu9b0NZBazwMehHLdi25wjXqB8kpcze4GbdCa26F9yJS8kRGxI1oF3xG++E3UkvxsxUnFAk11BF0I/Z3QbKneCvuQOSnyYoIh5GMHSvuwheR806NrDfrp8i6n6I7uE1q9f1RnLLYug3BA4/DFxKzpCi9u43CF2f5qvn/uDYt743iSeLrnBH3pXF+cFqn4mOzEZS0IH2qJEgfpG8d6QusV9qrpC9oaaXdjUUbWK+d2ztp2G7e4jP1SGJ2dypod61pHwNFQtHBtvwCmhZYgnZi2wPrNXPQEYKlySkMnjyhQJxSEDIZdOhEAJ+wwPxZRN8cejOmMwNc+YZj8Yrp1ckNCb3Lls9xAQdUonrmGOXkaenTCQTBGqv8BLqZ7iWyHSUhdDN0c0ujNLKWuugenNK6K/vEkbo5lpp1TNo2q5s9OvslkqmFTs2nz+rmbPlCN4sU5lO3JG3fdLO9n8e/V3srL0i7LIljw4rnI1qDnpFX7G1jzE1g7bUIOprn1vdamCc6zjTPuAfA5NrYyoNhcprPjWXU5O8iMutp0heh9Ro+vBdotGqunslbc2cN9/FC69V/WHt3hyipzxxbEOLxuLGfP7T2OYT35OSKpTy9dUH8OpdjX3n+bHYI052a7B0lwqrhltxMW3TFfZ1reQ/c+4b94ln3C3wbsJ/bsaRCa99G2HvfRqhPVHTJz3AgT5j05Y+wbdO3oTyLUXwf69MT1ZM+WbzhGt4I555ZBm/ledtq1yiPNT+TMXfvaPd7j43BZno2bIGTqqT7zUm3IScja90R9c6SaNLu2Lrdccdnk4rYOREl8TurwSW9FlbsP4XWa7+wh7E5m/dgZL3OilpbZ32aHoys1z/+vVsxdjsvRGvE69avJKNeIPvubl6IrFcyUWcrmW60im/dbr/nWiWwbom7VT0YWrc77HkP2tpm3r04I+wu54W4BU6qkh72vJBYj65ki7RKbG1txj23NmNrazNqbeXTjx50rXvQ7XkPetbaLMa8UCsTfgucVCU95HkhtrY2487u1upGq4TW7Q57rlUi65b4W9WDsXW74573YGKtzSLMCzUykVhbe3WcVCU95HkhsbY2+3gXdpN2u9bt7re1mVhbm0kP8S1NetC3brff8x4MrLVZiHmhVibCFjipSnrY80JkPbqirdIqsXW7455rlcS6Jduz7xxRtJhNu0XK/vagwBzYarMA80KtTLgtcFKV9HDnhYhu0LUdXd5WaRXfut1+z7VKYN0Sd6t6MLRud9jzHrTfd/YxL9TKRNwCJ1VJD3teSKxHV7JFWsWxtjadnlubtjcqZ2V9G3rQte5Bt+c9aL/v7GFeqJUJvwVOqpIe8rzgWFubzhbtO0fWNzKLlP3WKpF1S/yt6sHYut1xz3vQft/ZxbxQIxPuGvvObu284D7ofefI+hbaqLNbaLvRKq61ten23Np0rT2B7hbtO0fWt7lGmdta+9mD9vvOwDvXy0TYAidVSQ97XoisR1e0VVoltm533HOtkli3ZJv2nT1rT6DXc0+gt8a+M/DO9TLhtsBJVdJDnhc8a2vT26p9Z8/a2vR6bm161p5Ab6v2nT1rT6DXc0/gOjd7Au9cLxNxC5xUJT3seSGxHl3btO/sW1ubfs+tTd/aE+ht1b6zb+0J9HvuCfTXuIkeeOd6mfBb4KQq6SHPC761telv1b6zb21t+j23Nn1rT6C/VfvOvrUn0O+5J9Bf4/Z24J3rZCJY44b3qHZeCB74vnNgbW0GW7XvHFhbm0HPrc3A2hMYbNW+c2DtCQx67gm0v/XLBd55hUyELXBSlfSw54XIenRFW6VVYut2xz3XKol1S7Zp3zm09gSGPfcE2t8z5ALvvEIm3BY4qUp6yPNCaG1thlu17xxaW5thz63N0NoTGG7VvnNo7QkMe+4JDNfYdwbeuV4m4hY4qUp62PNCYj26tmnfObK2NqOeW5uRtScw3Kp958jaExj13BMYrbHvDLxzvUz4LXBSlfSQ54XI2tqMtmrfObK2NqOeW5uRtScw2qp958jaExj13BMYrbHvDLxznUzEa+w71+Od4we+7xxbW5vxVu07x9bWZtxzazO29gTGW7XvHFt7AuOeewJt70AJaiNuxXuvpXu4N9VkijMjVp5Iz3WW6M8jKueaPT2Xd8I36f2wFZ5FmftVmvDsjOa0Tbl2zn5nG9z2nm1JZD0iok5HBG/BkGg5orvS1dgYsjaJ2e5IykrVCImtW9atFf+MqJ9QO7KtGbKn5W0+oTn9gm59P6N8M7oPnucRc376fMpSLyu5YmNP+rUe0yATb9uVBZS2Y4e1b0b6cin5M2Lpz8m2iXVpx/R+wp5Wpb4mHvM0M11yk7GkbnJ5rcvgNV5QziW1/Dp3s1EkT5qKCRslPNpf1kqWalXKlwXZenNGr0o7zklhke5qTv+W47W5QqrvnwVRPMvJwpxm9QXpKqHBR6zsG3qe6nTB+zFL9yXRv6B+MOU0S1HSK4oC41aO/lDk3SFFSs7mco1yWzp6uL1znHm+2bgRHr9+jRuHYqucjfsg1eKKy9eMJqXb3lFNXOPn398F/8Vqon/89yw1V9G2ekEpFlLSnxfm4wXN4RPqNTULr59nxH5ET05lz3Xbb2K+Ku+3sLLfrhn9N6w954YOmLOfdyzv4WCPWn1Jvctl+FtWn5Laa9l6sdLJyvotWUGqvCmVyLm3o1vGdU8qKyk3ykr9jf3/nf0+J16cDz6y/08ZdQsq+Zb+DinPFcs/JJovqGVTvf54R9rvjH1SnpQZWStT6p1jsg2viOMn7PfI0IyLwS7VvkM9s2nfOK2MKUdz6Sqjz7Pj5r3udWWvfdD2W+rbz+8aZ9v1lPHhhDistNaIotZDox319fgb1+NL7LRdPd7G9Xhyrtl8xHktjTjRn1w+r9ibPbLTl0xm+Zh5azXqxsboneYkR+R0KtoSScymki3XWM9yPXvNvu2xH17nEVnXZ5LaffIn8Z899vyatWiP/eUj/Q+5AlqwJ9xDucho0zcla6I3xBdTny4G/6TPFyTn39A5NH9nedJ+/EY+mdN4nrPvI6n7l6y8C7LzxXjmI/wN9YTihqf7KTRu2/Fzc1eQ+x7mvkelpcSlT5PSp47uufxzJ1eXo3smnzLbFlc/943naZS9o1uVf65al95W6ugWms9iY+XG++GILBf1NinID5eBXUOGXd0S3zi53NM5U5p8zZ2yERhnfNfrjV6V977OlztUzw/s6TlRNCdvwedWY9alE3n2JJ+WNN/tUS/yufEtG0X/Zhx4OTgYvGJWzn8P/sV67w+Wep9yJnSO0T719BeaQuU5UOPuhtrFx/8uo+aS/V6wz2/JUzLSXOAtXRDdHks3pt83MtW39IxLTkAasar88jL3SHfwHjmX8/172Yq9XMqIPfma/b7RFsQb0llfSz38jvFBaKQpfed5fqPeULPvf7KnH9gswvN821IPfD94MnjMfn5l3H/K/qY94MtzDfbpVJ4vDPkSPqVz6YkStc4YDzitbwf/QB+xPuqqfxz67G44QhS3BWfFfCla8y1ps13SY7s0867iejmXj4jelNt57oUFjrsl6X4lvXRNmnM+eER6i2u5rzsaCw7ZpD59ju8Jrw/JF3BF9Twny+SGpXpBu7382Z7kXjccdEmPRwVt0mcOmuO6WlKvaJY+Yrmy6X+jmj4wSl93IqEhWWj7dHZcyt8FK+mWen+Vp+FZwS4+0G37usU5/wWjP2Izf8TKf8H0ZdoGbifuUx9+YXj3T0gPzAe/yOiEJauhPWrqOOrT333aU70fEissvEvqPX+lvJqpu5ZWn9b4+4RsSHl7ofs4lb0u+tql+dQn6YsyGj6lqBs6PJI6waP7IXM3ZAemo5PrmrT3upqfHak9/MxYPaSSeBmP2e+E9NxUWpld0MR9N/uZ9Us9t7qiiyPC8jImKHtFdd10yiWXRp6X4VN2tlqUesab0bfa8z5i614+Gp8Nvhv8yD491XSHdDbKPv1NaRY74sfSYhV+4J9I977thJuCkzGNA69Cr/4sKRRz6gW1dULfHktv9hXZ3N1oPIfOHvPJLsxKI7dehffjV+md7IKagDy9bmFNVaTmbyx1FxTFZH0UV3liJSTs+wuW65LkjXufu+RXJKOe8jaS8igfSMvjBbMsLjvRb+r8HqFRxpUcUxQpbj0lHXBFezPdUOmSxS4+rzvTp3M632EXET6f52b3TWf+Ytl7csYve7dLpQi/4hVRJ3SHsg5OaDZZMJ4ojj/O+Yv+N802/PdnuZvWhc2QUExTVFgv3dBsw0s7JI5wLt7QuBf6/BntB1/RX+7L7UYH8P2P/Birp6sb/ZRQRGksqUxp26TXu6E2KVkjr0NtN1TGci3iZFbyRxSrp8bqvxgFx3Lufko++nOap+adyGRInAwL+uuYvAfX2u5+ZPhEuqDLo9uVQxotgUGXsP6PiXOpD8xcFXRj7Y5pJznvR+T8Evb3cSHiwO2IMpd2r4QVVN2fbie9mEiUXH4MrLez8I52rZZyzud0810Fl+bBTXcWsmXW7SyolD9QymfGfp57R97Zqt22hCymfYp82pTXC/b+sCTyvq0dnery6/qgTz5wn3bTlGUaYhdhw12EmHwq2EXochdB7ZK+oNiAy45iAAR6xC3YGDzyRthp/8BuJ3Y7sdsJPSX5e0Vj5vIO9NR4Y18OdBN0E3TT9usmHjs5YxrkKuMT2WNcEeNJ7Dr9JtfaZxTd9boj/RWRTOyTN2/TFaA4PWKpPVBKPryN13xmiamUHBIOc6Zj9tJ96DL++iuj98xa3hB15pOsLL2U0bu8VtUP6p3pSzruRNI8mncSGTuDeQfzDuYdzDvFeSfdJzjI6awuPMueEW+COGVoKWgpaKmylTu3oGZUVhexKwHhssa0r+5nVu0Kz5LfA+5mH86nU1LyupL3/S3FkU0HP7JSJhKLeFyg8glFY3RDaSTjkvyNKO2CxoBwivnYQnsaH9GYe6x3P/ke+5wo+V5j57pqR0Dt8Ix2cF3AoxbFWuVJbqe2G7pCWn8EmciUerq4duNRiE81nrYbSmM5ruK1KBUS8FyetqROX/r8k+8VhhJ7kI3j3IzelzQLHOg4ra7oL4sL4iPmexpdIjaEt+WCZr2JPkOhC+ocGk9OZo4XEVXfG6gILqffdTiifLpDRazokxLK3muEruCVGVVwt3RHNGMJb1J51M1dSGEkIxv8zBx1aIyYY4NvZkTVorMI34jyeoX4Pjsqu9CdsRwtWfrykUEphSLi5YBiprmsfqiQUx5tJ1ZPF51JQ1ywPMUpP1OKQlySlD6pbFkXdqlAR4lPNtzugiaH7rLYp/uMESvbbqys6NkjWiUuc3Gd3WqhoHKUV1HXjfZxLbTPkxydXY+PQK6FhD2x6pSpP+mkp/T8jd/pnXiq6ufxoOK5Tz6dL+mbOj9EvPldr2YmtHY+lt6G/5EIriWlEDmPpKchzSlOonpDVpk4rSpNLTBC4tSPNMf3lIpjhM4lby8lYiItY9coRZ2gJkpQp6L+Jd/OpbfniDghTk5I6yqe02OWmz9xIc2nuP2sNP9fG5+v5ckziPp3Zh0/2dLd8My6l8SthTyR7AOThjfkeZvI8+nq34/k+bGcgyM6d+OczfJnVjlHhvwXU/P3E0I1XUhtU17iQsr9lMq7lLt2PPchSeaSUCoj4oigrMmZedVn30YU2d3/mxRW9cqRPHt4QiN8Wto3q/Os4v/mZ+Xa3iGQdHSHwKZnEc/0PQwjWuv5hISP6BwsxR2M1Way4lrLSv/uQJnLk73Sk/vm5MGfS9TyIbUmLYuXw2MmzogLx9TO9PQ0bh0sjDP60vQzst7NU8Gqzqcc6rlyvqaM5mkfSeqHa6UuStnx2iWsNwLqKCmenZY/j3cnQ23VecjiFDhhWVywMr6i09yL+ZQ1NNL24ZTymsihr+hU92uyjsTKRkj4Vw3HkdfCOFIltTGOxHhZyvOkbUZTtvTyEuruyvnOSD+seD4i+ZrS2fDnxKMPtWlNO39Esib6fHW+C7lLUZeKczidmdVKqDr9GbV9RGPe5Ep6Hr7icHqThGkBzGglNaUxYvK2idz5rcmdf+f6e0Lf5sbJoOvbZodE/YeGXA2sudq3G+abtDpsodWqpP5b8WoU29hqdal5uR8piuAiY7M1k8DIui+ijm6RumVPm9+o9nNOdxbP23cobbo7MjUs3bIUJ8TbrG12adhxt3Tm8Dumza/06dX8jOYh/bp0e5tL5/OK9O9Z2TPjHO/82cvv6GeX/fL4FW7dnNK9b+r05SynTsiPdkNnzB1mtF2dxVlmoS7JQzs1zqFVdlq6Xko5pdJk53TTIltPOheDF4NXgx/lG0VLuezG1rLbzY3JKlrcTnrT9ecFIavFjQcc1X3Lfs9oLfqRvRuzT7F8HhrPd+lEK35C+xH93aVdiiGlT+TbIcU2cY+Z+cyhmz7Em0h+H+beH8kSPSm/jsQMO3RWuygz+1SUOjJqdIiWfBpB+1i2OCbqRR5FfUzntO+Sh34kfxeZJ0Pj/1C/Geo35hOV7mPlSD6idYCSncdkLXxq2U2sZTe8B56sE9I479nnc7l/tyBL1JylVtmwqc4YGfPn5rNbLD3Eq7kcGyerf1ou93e1tJBnrl1SD3369VKzfnVa6FdVUtd3X5b11ylJ/ruGXHGtuXI3d5sOS/aRhmSfnJP+OKb0dvtaOxU88Kx50L97be1HMl8LHOV43URyfGuudXun6o+08zkkC3Rm3E+ap99+He3f815PfUJN+ju07u8Qq74Htur7NDZvTHuytlIX9Xq99gvhUoaEHuG3PhyQbtmlOxv5zY1DOsFLrY6CTFrxPsg8MdOJnIHUV8XcjpHKIRlRzyOjDPOv+dxM7+o0qvT+rJC4DMQb38QqokPcrbtRskybcd1yN/dGemRDbdNtaO6d3obG7z6zuTMtv2v5zeAz1oLv2F8e9a36QHz7jPXPI/m5WJZK47IRLT4L9Fkxb77OYs5bfVa5mZvvppqpriQacyIRhcWaTtkb9TQaPBn8XfLzWu/mcQ7xNgfU5oDSmLcn5N9lEXb5typ+8o2+n/sb8rA57M3fZa0L2Rs3dNrBMUvx98wtdS7dSXcr06pecUrunYtoxDS9d84sJS59mpQ+Ne+dyz6vu3cum9Jsi6tvcTbvnRORM8V757LPQ30bnFe4d8581t69c+I2h/y9c6FeEdTfO5dkduvWvzUy6OFsFBqc3tzmfqltzwPj/q7hyhQicuM9jaH1Uo/0mL61vHe3zt4U6K6iDRKSTbTaXq3LP26UP3sj6ib544b5o4b5w4b5g4b5/Yb5vYb53Yb5nYb5m8mf21D+3Iby5zaUP7eh/LkN5c9tKH9uQ/lzLeUvnRfz+iswNKt9/HWWpj7vW13TXuFQz6dih8L8fiHLOaZZZ0Zz1aFx//Co5Mkx1SKwgs38dIm1xyTp1C/LWzCUUSuiXXw9MSxd1Zh++3L/rSP36TZbhTpST74e/H/5UeeJVioFAA==\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684771480258, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771480759, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771481261, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771481896, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771482397, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771482683, \"type\": 3, \"data\": {\"source\": 2, \"type\": 5}}, {\"timestamp\": 1684771482683, \"type\": 3, \"data\": {\"source\": 2, \"type\": 1}}, {\"timestamp\": 1684771482684, \"type\": 3, \"data\": {\"source\": 2, \"type\": 5}}, {\"timestamp\": 1684771482739, \"type\": 3, \"data\": {\"source\": 2, \"type\": 0}}, {\"timestamp\": 1684771482743, \"type\": 3, \"data\": {\"source\": 2, \"type\": 2}}, {\"timestamp\": 1684771482757, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771482767, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771482881, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771482901, \"type\": 5, \"data\": {\"tag\": \"$pageview\", \"payload\": {\"href\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\"}}}, {\"timestamp\": 1684771482953, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482953, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482954, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482955, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482956, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482956, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482956, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482957, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482957, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482957, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482958, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482958, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482959, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482960, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482961, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482963, \"type\": 5, \"data\": {\"tag\": \"$pageview\", \"payload\": {\"href\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\"}}}, {\"timestamp\": 1684771482968, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771482974, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771482983, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771482990, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771482991, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482993, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771482995, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771483000, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771483002, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771483013, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771483131, \"type\": 3, \"data\": {\"source\": 0}}]}, \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\"}, \"offset\": 2993}","now":"2023-05-22T16:04:43.265573+00:00","sent_at":"2023-05-22T16:04:43.260000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684771486268&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:46.270510+00:00","body":"H4sIAAAAAAAAA+1VXW+bMBT9K5OVR5BsbMDkrWsnpR9aVKntQ6sqMuAkFDAMOyks4r/vOmRq0ynZHqY+5Qn73nPONZwj87RBci2VQWM00krUelkZ5KC6qWrZmExqNN68dWapMMJWTFdLNKYO+l3Q1apJoIQdZGRrgPb07CBhTJPFK2Nl7L6RZbXebjaoFg3MvUzRmATMc1A2rGhveWm6ZfSglpVSG1HWtstZGBLGKaU+tEZaap1VamapiHDOGPWCOExJTIiLZRgHhDPhUe5j7jLipV5CUxeGxLB/w0siUwwvPXrNVFq9vpdjUSBY7OJ5GDIhI8YjLwoOaVmw72FKQMtUuVQgUy+T2cskuViFJX1cq0tV53FbXjXX3+LuapV3Z4/nP7treYP1XfuV5sBMM20ylZjhGIvbi0lRtz/OJ4v7fOlrejN9uL/1z5LvhuTlQ4Hb9VS8eHoelAg+STWfawlmehHFvfN53m4G+8IIaO8bQDZdAdwNEiorhQG73LoQnQueGqijZqVUphao7+15dyoQh32VpBAaFuiuEvB15t1sZuzqy4e9a5aylK5bZIvln01XNk3VwKS9JP5D3EJCT3E7GjffZ/89bgS4lc5sZIaItWgcAK6DeSE8h6wEg2fT3Ulwf9DFyD+5eNxFanP+CS5y/y8uusyHC36L9ThcBhbMQWn3j4DKPhpv0wdohnfSdsQOHX5AU8jqoZAw5vNTSI6FhHDM++dfwg/U6TcIAAA=","output":[{"uuid":"01884434-ba3e-0001-712d-9832c78876cc","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"01884434-ba3e-0000-51fd-ebb6a55b6144\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAJ6Sa2QC/8VU22qDQBA9nxJ8rhA1RtM/6FNf+hZKsbk0QqKipkSCv972zLpuYy6thEAYVndn5syZmb18f02xh4USFTIsOHvEAB4e+LUwR0RLpLXiVyDFFjlmxneofUtqdvwWWj/Fq7ZIjJKYGO/Eit+pT07thrE/O1ZhzIgXa0LkEzNqbA7GGMHV6PhI76E+YJ9TDhlrk3FMTuGTGjdk+o0RMkpAcfgPGU/E18h79mt/VG2AiWG/jGsykZzXJo+9QiSqB5HqRcqVzarWXFecNX0pDUJ2aUsfwST44LpW0vbkOC+3V14zxVcYnYUXZhIp9hhLZvJGKY1u8I/d5lip07Tg3Gb0mLmueiFtYnJKytFU9/f5vO0Zk5l3kzPmaN+MloIZtLvbPUc7vRqb2JXWuMzFO3O3Avp263pmH5eMu6CmPd/1lfVPeMPuUX9I3tvVb7Mav/NW7EzU0NyJ6kDnXHjF3F5cQ45TLtH7Z7j8C1xBDy55AUdX7u5IdSVU6B+N+AMddAYAAA==\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684771483335, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771483713, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771483953, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771484458, \"type\": 3, \"data\": {\"source\": 1}}]}, \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\"}, \"offset\": 2930}","now":"2023-05-22T16:04:46.270510+00:00","sent_at":"2023-05-22T16:04:46.268000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/s/?compression=gzip-js&ip=1&_=1684771486823&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:46.831635+00:00","body":"H4sIAAAAAAAAA+2US0/jMBSF/4vVZTLyK85jx8BIHUCDkIBFEaqc2DQmTeKJ3TaZqv99bngIWHRTqbvuknPvOb7Kd53HLdJr3XiUoYlrpHVl61GAbNda3XmjHcq2n5W5kl6Oih+sRhkL0Ifg2lVXgESD95oIkFEoI7HYgWRq7bysLQgi4XFMeCJihqE0cdo50zbzsRuRJOGcUZHHiuSEhFjHuSAJl5QlEU5CTqiiBVMhETyH989+TbTCMPlkYxrVbr7G8VRInof4OY651ClPUpqKfVljc0QxI5Dl20o3EGPLYv4yLS5Wcc1m6+Z3Y6u8ry+7q1/5cLmqhrPZ+b/hSl9jd9f/ZBU4lXHeNIV/G2NxezFd2v7v+XRxX5WRY9c3D/e30Vnxx5Oqfljifn0jX6h7FjX6/rUQxZSFOAopvSMiwzzj4kfM6Az6jk1uPzd64nYYN3YEbhgO1r0H2+NTgKT3nclXfoyBu/12BVOwfS2A2Q9L8G6RbEwtPXAM7VIOIYzvQUdWrpxWaLfbQWan63b9GjgeoNTr0/7tSE/bcdh2pEfYDh6gjVG+BEScQb3UZlFCesLZXoLR6b98GMEIA8Gn/+KghNVTBwAA","output":[{"uuid":"01884434-bc70-0001-6e8d-1718c6247aff","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"01884434-bc70-0000-a5ba-6c77cd70a82b\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAJ6Sa2QC/71Syw6CMBCcTzGcNUFBRH/FcMBHhETE0PoghF9Xh6VUjfFCjGlStrOzs9Mt99sSFRxolDhiy2iBATwMuTvYIGYmNmjDU8hxQoG15U4M910hMGhKjRYZY0a0tuwUGdlK9DNWdqwAIXxyZ4x9xgEjD66p/JfXvj4nP/Hp2u5bXLkrgy8RmUyjoVmTYsVaLQ47TvUx97nt/r2uddJ43lsflVQcZAaxzCLnacRb7XkuGbVz0bbCYS6msiLSOKhlda4Lohk1zm9+n3facL3ifV9h/pNX8A33ItPUSGxXnzqdUkJ+ih2/2uRDyffzPpU/PcIDAvlBHZgDAAA=\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684771486730, \"type\": 3, \"data\": {\"source\": 2, \"type\": 6}}, {\"timestamp\": 1684771486732, \"type\": 3, \"data\": {\"source\": 2, \"type\": 6}}, {\"timestamp\": 1684771486739, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771486750, \"type\": 3, \"data\": {\"source\": 4, \"width\": 1433, \"height\": 843}}]}, \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\"}, \"timestamp\": \"2023-05-22T16:04:46.732Z\"}","now":"2023-05-22T16:04:46.831635+00:00","sent_at":"2023-05-22T16:04:46.823000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1684771486824&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:46.833134+00:00","body":"H4sIAAAAAAAAA51U2W7cNhT9lUCw3yKbpKhtgKLw0sJN0hpB4qBwEAgUeaWRhyOqFKXxOMi/91IzmQWd9CEvWs65Ow/v568BjNC6YBacdaIGDWKE4HXQWdOBdQ30wexrcGbwFfwp5Kv7D6/+RhqBYgTbN6ZFgpILGl8Qj5fWrHqwCP7eWKjMswcVjI2Ewq07QOIW+oUznSfkYC0mLwarkZg7180uL7WRQs9N72YZIeTSQqfFGl8SLX+tGu0w7y/n6fU5Yz30voQCSWNV09aFGqxwCCF5Hl1trHze7T9jO1OPsBt8LmC9Yw/dJ3IUetg6J2QD+cEIZ+zOqXb+M73d0PvJbQ3i6/N4y02jPoEL6ZOeIJRwUFTWLHfJQkbVJp0foB8TTu54ZJ7ohJu3YunnfTzBg0PanyCl0QGsRVsPqAX0hTZ8+OBdemkB2mIOTT3HjHnK9+CqUW6OQWJCEBwbWHXGup1txn3wHfzdmkce1k2JaVZQ+iT4c6iqizi9YB5vWizLFY1CVKRYXTTq1Yq6l7KtPO8a3yhNMp6mlGfJRcbY60A1vWtaufWr39/e6e75n5u7+mExj/vo3f2nh/fxlfzL0cXykybP4714Yn2VLA8kO7nSDFsgEPEyjqQMiYpkIlOSc05jlYWcMsVkpEKa8JJk4d48UVHmgw1+qj9RRdMXCpamQEk9gcRRVkL3gAFra4Zuupk7KiDbvGFEEwhRBiSMqUjCSiU8iXEqOfEX29hatM3LpPJDL5aTfOPFFdAwz+M0ZYISHnNfSds70UoviZO3NPiGVYnBIdi5wUKBwxelBlVg63ikRd8odP5ev9f7iMIGMRlXWtTYzecvSB1iRSfW2gjlO8UEHWabm/pIIjya9o4FoZe+ODQBFUrdyMXc4NyRg6Vo9FSHP1Ex4p8vbVdNr4Vc/A9/JvFqGg2Fxtz7TQPtiQadHbzHf/fS5mt/547dgnHSOS5MwIXot+eZwvWJ57pDp+VmsBXf9551ZgEe6eayeLqTt0O6jB7H9o+2W5TPyzf27W/l+s2wWF893rys38I70n98vo4W05XeFrnTOI9YUqaKlpSGBNIyoRkXLMpiFPUpkW/sgYKaVs6qaZVZHYbjeSJ4GZIqTbmAnGc5y5MfxfLGMSMR3ayvGi8gHEcTKqYRjVCmSQm0opTJjPHqR/HQnFFIEq9NvyFQwcsOozHCopDEIWMfaTIjfManhfEYfPvyL6II7K8OBwAA","output":[{"uuid":"01884434-bc71-0000-a710-92ebd1e26e1f","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageleave\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"a7ang3vlww1tzbnf\", \"$time\": 1684771486.822, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad513139-06be1f112c824f-412d2c3d-164b08-1884434ad521e66\"}, \"timestamp\": \"2023-05-22T16:04:46.822Z\"}","now":"2023-05-22T16:04:46.833134+00:00","sent_at":"2023-05-22T16:04:46.824000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} From 663f87a2135ba682c5edf2ba08279d2beabd161c Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 23 May 2023 14:34:48 +0100 Subject: [PATCH 015/247] Add timesource trait (#13) --- Cargo.lock | 11 +++++++++++ Cargo.toml | 2 +- src/capture.rs | 2 ++ src/lib.rs | 1 + src/main.rs | 6 +++++- src/router.rs | 6 ++++-- src/time.rs | 16 ++++++++++++++++ tests/django_compat.rs | 19 ++++++++++++++++++- 8 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 src/time.rs diff --git a/Cargo.lock b/Cargo.lock index 2d2ffd6a7267f..f6922b7ae4e7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1139,8 +1139,10 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ + "itoa", "serde", "time-core", + "time-macros", ] [[package]] @@ -1149,6 +1151,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +[[package]] +name = "time-macros" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] + [[package]] name = "tinyvec" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index 23cfe2d24d257..da6382923702d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ serde = { version = "1.0.160", features = ["derive"] } serde_json = "1.0.96" governor = "0.5.1" tower_governor = "0.0.4" -time = "0.3.20" +time = { version = "0.3.20", features = ["formatting"]} tower-http = { version = "0.4.0", features = ["trace"] } bytes = "1" anyhow = "1.0" diff --git a/src/capture.rs b/src/capture.rs index e8edaa005b4c9..9bba1e76fce5b 100644 --- a/src/capture.rs +++ b/src/capture.rs @@ -153,6 +153,7 @@ mod tests { async fn all_events_have_same_token() { let state = State { sink: Arc::new(sink::PrintSink {}), + timesource: Arc::new(crate::time::SystemTime {}), }; let events = vec![ @@ -176,6 +177,7 @@ mod tests { async fn all_events_have_different_token() { let state = State { sink: Arc::new(sink::PrintSink {}), + timesource: Arc::new(crate::time::SystemTime {}), }; let events = vec![ diff --git a/src/lib.rs b/src/lib.rs index d8cf49f4b620a..6b9e84dead103 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ pub mod event; pub mod router; +pub mod time; mod api; mod capture; diff --git a/src/main.rs b/src/main.rs index 85412e19adc35..9bce0ab50559b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,13 @@ use std::net::SocketAddr; +use crate::time::SystemTime; + mod api; mod capture; mod event; mod router; mod sink; +mod time; mod token; #[tokio::main] @@ -12,7 +15,8 @@ async fn main() { // initialize tracing tracing_subscriber::fmt::init(); - let app = router::router(); + let st = SystemTime {}; + let app = router::router(st); // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` diff --git a/src/router.rs b/src/router.rs index 9af45c76454c3..0c7e7978694ab 100644 --- a/src/router.rs +++ b/src/router.rs @@ -3,16 +3,18 @@ use std::sync::Arc; use axum::{routing::post, Router}; use tower_http::trace::TraceLayer; -use crate::{capture, sink}; +use crate::{capture, sink, time::TimeSource}; #[derive(Clone)] pub struct State { pub sink: Arc, + pub timesource: Arc, } -pub fn router() -> Router { +pub fn router(timesource: TZ) -> Router { let state = State { sink: Arc::new(sink::PrintSink {}), + timesource: Arc::new(timesource), }; Router::new() diff --git a/src/time.rs b/src/time.rs new file mode 100644 index 0000000000000..e510f5f692fcc --- /dev/null +++ b/src/time.rs @@ -0,0 +1,16 @@ +pub trait TimeSource { + // Return an ISO timestamp + fn current_time(&self) -> String; +} + +#[derive(Clone)] +pub struct SystemTime {} + +impl TimeSource for SystemTime { + fn current_time(&self) -> String { + let time = time::OffsetDateTime::now_utc(); + + time.format(&time::format_description::well_known::Iso8601::DEFAULT) + .expect("failed to iso8601 format timestamp") + } +} diff --git a/tests/django_compat.rs b/tests/django_compat.rs index 24d526375f7bf..3cfee2263dffb 100644 --- a/tests/django_compat.rs +++ b/tests/django_compat.rs @@ -4,9 +4,11 @@ use base64::engine::general_purpose; use base64::Engine; use capture::event::ProcessedEvent; use capture::router::router; +use capture::time::TimeSource; use serde::Deserialize; use std::fs::File; use std::io::{BufRead, BufReader}; +use time::OffsetDateTime; #[derive(Debug, Deserialize)] struct RequestDump { @@ -22,6 +24,17 @@ struct RequestDump { static REQUESTS_DUMP_FILE_NAME: &str = "tests/requests_dump.jsonl"; +#[derive(Clone)] +pub struct FixedTime { + pub time: time::OffsetDateTime, +} + +impl TimeSource for FixedTime { + fn current_time(&self) -> String { + self.time.to_string() + } +} + #[tokio::test] async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let file = File::open(REQUESTS_DUMP_FILE_NAME)?; @@ -42,7 +55,11 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { case.method ); - let app = router(); + let timesource = FixedTime { + time: OffsetDateTime::now_utc(), + }; + let app = router(timesource); + let client = TestClient::new(app); let mut req = client.post(&case.path).body(raw_body); if !case.content_encoding.is_empty() { From 9da6bfe9af8c63906a5683ed774c2d3f3a138878 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 23 May 2023 14:53:56 +0100 Subject: [PATCH 016/247] Add MemorySink and check we emit the right amount of messages (#14) * add memorySink * don't bother with time parsing --- src/api.rs | 4 ++-- src/lib.rs | 7 +++---- src/main.rs | 3 +-- src/router.rs | 10 +++++++-- tests/django_compat.rs | 47 +++++++++++++++++++++++++++++++++++------- 5 files changed, 53 insertions(+), 18 deletions(-) diff --git a/src/api.rs b/src/api.rs index b3a18e696c105..09e79ae7b961b 100644 --- a/src/api.rs +++ b/src/api.rs @@ -12,12 +12,12 @@ pub struct CaptureRequest { pub properties: HashMap, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, PartialEq, Eq, Deserialize, Serialize)] pub enum CaptureResponseCode { Ok = 1, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, PartialEq, Eq, Deserialize, Serialize)] pub struct CaptureResponse { pub status: CaptureResponseCode, } diff --git a/src/lib.rs b/src/lib.rs index 6b9e84dead103..641385f93d553 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,7 @@ +pub mod api; +mod capture; pub mod event; pub mod router; +pub mod sink; pub mod time; - -mod api; -mod capture; -mod sink; mod token; diff --git a/src/main.rs b/src/main.rs index 9bce0ab50559b..c550c218fec93 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,8 +15,7 @@ async fn main() { // initialize tracing tracing_subscriber::fmt::init(); - let st = SystemTime {}; - let app = router::router(st); + let app = router::router(SystemTime {}, sink::PrintSink {}); // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` diff --git a/src/router.rs b/src/router.rs index 0c7e7978694ab..047ceec932607 100644 --- a/src/router.rs +++ b/src/router.rs @@ -11,9 +11,15 @@ pub struct State { pub timesource: Arc, } -pub fn router(timesource: TZ) -> Router { +pub fn router< + TZ: TimeSource + Send + Sync + 'static, + S: sink::EventSink + Send + Sync + 'static, +>( + timesource: TZ, + sink: S, +) -> Router { let state = State { - sink: Arc::new(sink::PrintSink {}), + sink: Arc::new(sink), timesource: Arc::new(timesource), }; diff --git a/tests/django_compat.rs b/tests/django_compat.rs index 3cfee2263dffb..e305412a0d43f 100644 --- a/tests/django_compat.rs +++ b/tests/django_compat.rs @@ -1,14 +1,17 @@ +use async_trait::async_trait; use axum::http::StatusCode; use axum_test_helper::TestClient; use base64::engine::general_purpose; use base64::Engine; +use capture::api::{CaptureResponse, CaptureResponseCode}; use capture::event::ProcessedEvent; use capture::router::router; +use capture::sink::EventSink; use capture::time::TimeSource; use serde::Deserialize; use std::fs::File; use std::io::{BufRead, BufReader}; -use time::OffsetDateTime; +use std::sync::{Arc, Mutex}; #[derive(Debug, Deserialize)] struct RequestDump { @@ -26,7 +29,7 @@ static REQUESTS_DUMP_FILE_NAME: &str = "tests/requests_dump.jsonl"; #[derive(Clone)] pub struct FixedTime { - pub time: time::OffsetDateTime, + pub time: String, } impl TimeSource for FixedTime { @@ -35,6 +38,30 @@ impl TimeSource for FixedTime { } } +#[derive(Clone, Default)] +struct MemorySink { + events: Arc>>, +} + +impl MemorySink { + fn len(&self) -> usize { + self.events.lock().unwrap().len() + } +} + +#[async_trait] +impl EventSink for MemorySink { + async fn send(&self, event: ProcessedEvent) -> anyhow::Result<()> { + self.events.lock().unwrap().push(event); + Ok(()) + } + + async fn send_batch(&self, events: &[ProcessedEvent]) -> anyhow::Result<()> { + self.events.lock().unwrap().extend_from_slice(&events); + Ok(()) + } +} + #[tokio::test] async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let file = File::open(REQUESTS_DUMP_FILE_NAME)?; @@ -42,7 +69,6 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { for line in reader.lines() { let case: RequestDump = serde_json::from_str(&line?)?; - if !case.path.starts_with("/e/") { println!("Skipping {} test case", &case.path); continue; @@ -55,10 +81,9 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { case.method ); - let timesource = FixedTime { - time: OffsetDateTime::now_utc(), - }; - let app = router(timesource); + let sink = MemorySink::default(); + let timesource = FixedTime { time: case.now }; + let app = router(timesource, sink.clone()); let client = TestClient::new(app); let mut req = client.post(&case.path).body(raw_body); @@ -72,8 +97,14 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { req = req.header("X-Forwarded-For", case.ip); } let res = req.send().await; - assert_eq!(res.status(), StatusCode::OK, "{}", res.text().await); + assert_eq!( + Some(CaptureResponse { + status: CaptureResponseCode::Ok + }), + res.json().await + ); + assert_eq!(sink.len(), case.output.len()) } Ok(()) } From de20a3c07ae11163230fcef134d33b9e191fef00 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Thu, 25 May 2023 16:18:15 +0100 Subject: [PATCH 017/247] Add liveness/readiness (#17) --- src/router.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/router.rs b/src/router.rs index 047ceec932607..199cd2095e0a0 100644 --- a/src/router.rs +++ b/src/router.rs @@ -1,6 +1,9 @@ use std::sync::Arc; -use axum::{routing::post, Router}; +use axum::{ + routing::{get, post}, + Router, +}; use tower_http::trace::TraceLayer; use crate::{capture, sink, time::TimeSource}; @@ -11,6 +14,10 @@ pub struct State { pub timesource: Arc, } +async fn index() -> &'static str { + "capture" +} + pub fn router< TZ: TimeSource + Send + Sync + 'static, S: sink::EventSink + Send + Sync + 'static, @@ -25,6 +32,7 @@ pub fn router< Router::new() // TODO: use NormalizePathLayer::trim_trailing_slash + .route("/", get(index)) .route("/capture", post(capture::event)) .route("/capture/", post(capture::event)) .route("/batch", post(capture::event)) From 351be7f7ee6236facb73e008a1dbd47a18f94d1c Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Fri, 26 May 2023 11:04:38 +0100 Subject: [PATCH 018/247] Write Kafka sink, using librdkafka (#16) * Add rdkafka dep (requires cmake install) * Send batch no reference * I think this batching thing works * fmt --- Cargo.lock | 127 +++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/capture.rs | 2 +- src/event.rs | 6 ++ src/main.rs | 16 +++++- src/sink.rs | 81 +++++++++++++++++++++++++- tests/django_compat.rs | 2 +- 7 files changed, 228 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f6922b7ae4e7d..d6275655449b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,6 +144,7 @@ dependencies = [ "flate2", "governor", "mockall", + "rdkafka", "serde", "serde_json", "serde_urlencoded", @@ -156,12 +157,27 @@ dependencies = [ "uuid", ] +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -554,6 +570,18 @@ version = "0.2.141" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +[[package]] +name = "libz-sys" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "lock_api" version = "0.4.9" @@ -711,6 +739,27 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "once_cell" version = "1.17.1" @@ -784,6 +833,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -820,6 +875,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.56" @@ -893,6 +958,36 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rdkafka" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8acd8f5c5482fdf89e8878227bafa442d8c4409f6287391c85549ca83626c27" +dependencies = [ + "futures", + "libc", + "log", + "rdkafka-sys", + "serde", + "serde_derive", + "serde_json", + "slab", + "tokio", +] + +[[package]] +name = "rdkafka-sys" +version = "3.0.0+1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca35e95c88e08cdc643b25744e38ccee7c93c7e90d1ac6850fe74cbaa40803c3" +dependencies = [ + "cmake", + "libc", + "libz-sys", + "num_enum", + "pkg-config", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -1219,6 +1314,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml_datetime" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" + +[[package]] +name = "toml_edit" +version = "0.19.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d964908cec0d030b812013af25a0e57fddfadb1e066ecc6681d86253129d4f" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -1408,6 +1520,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -1613,6 +1731,15 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "winnow" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" diff --git a/Cargo.toml b/Cargo.toml index da6382923702d..340110740e3fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ base64 = "0.21.1" uuid = { version = "1.3.3", features = ["serde", "v4"] } async-trait = "0.1.68" serde_urlencoded = "0.7.1" +rdkafka = { version = "0.25", features = ["cmake-build"] } [dev-dependencies] axum-test-helper = "0.2.0" diff --git a/src/capture.rs b/src/capture.rs index 9bba1e76fce5b..541ef315cd895 100644 --- a/src/capture.rs +++ b/src/capture.rs @@ -125,7 +125,7 @@ pub async fn process_events( return Err(String::from("Failed to send event to sink")); } } else { - let sent = sink.send_batch(&events).await; + let sent = sink.send_batch(events).await; if let Err(e) = sent { tracing::error!("Failed to send batch events to sink: {:?}", e); diff --git a/src/event.rs b/src/event.rs index c5f09db61cd4f..8cac0ecf8a1ab 100644 --- a/src/event.rs +++ b/src/event.rs @@ -82,6 +82,12 @@ pub struct ProcessedEvent { pub token: String, } +impl ProcessedEvent { + pub fn key(&self) -> String { + format!("{}:{}", self.token, self.distinct_id) + } +} + #[cfg(test)] mod tests { use super::Compression; diff --git a/src/main.rs b/src/main.rs index c550c218fec93..77d1e8d1a016f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +use std::env; use std::net::SocketAddr; use crate::time::SystemTime; @@ -12,11 +13,22 @@ mod token; #[tokio::main] async fn main() { + let use_print_sink = env::var("PRINT_SINK").is_ok(); + + let app = if use_print_sink { + router::router(SystemTime {}, sink::PrintSink {}) + } else { + let brokers = env::var("KAFKA_BROKERS").expect("Expected KAFKA_BROKERS"); + let topic = env::var("KAFKA_TOPIC").expect("Expected KAFKA_TOPIC"); + + let sink = sink::KafkaSink::new(topic, brokers).unwrap(); + + router::router(SystemTime {}, sink) + }; + // initialize tracing tracing_subscriber::fmt::init(); - let app = router::router(SystemTime {}, sink::PrintSink {}); - // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); diff --git a/src/sink.rs b/src/sink.rs index 44592dcf7b409..20ab3fc70a429 100644 --- a/src/sink.rs +++ b/src/sink.rs @@ -1,12 +1,16 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; use async_trait::async_trait; +use tokio::task::JoinSet; + +use rdkafka::config::ClientConfig; +use rdkafka::producer::future_producer::{FutureProducer, FutureRecord}; use crate::event::ProcessedEvent; #[async_trait] pub trait EventSink { async fn send(&self, event: ProcessedEvent) -> Result<()>; - async fn send_batch(&self, events: &[ProcessedEvent]) -> Result<()>; + async fn send_batch(&self, events: Vec) -> Result<()>; } pub struct PrintSink {} @@ -18,7 +22,7 @@ impl EventSink for PrintSink { Ok(()) } - async fn send_batch(&self, events: &[ProcessedEvent]) -> Result<()> { + async fn send_batch(&self, events: Vec) -> Result<()> { let span = tracing::span!(tracing::Level::INFO, "batch of events"); let _enter = span.enter(); @@ -29,3 +33,74 @@ impl EventSink for PrintSink { Ok(()) } } + +#[derive(Clone)] +pub struct KafkaSink { + producer: FutureProducer, + topic: String, +} + +impl KafkaSink { + pub fn new(topic: String, brokers: String) -> Result { + let producer: FutureProducer = ClientConfig::new() + .set("bootstrap.servers", &brokers) + .create()?; + + Ok(KafkaSink { producer, topic }) + } +} + +impl KafkaSink { + async fn kafka_send( + producer: FutureProducer, + topic: String, + event: ProcessedEvent, + ) -> Result<()> { + let payload = serde_json::to_string(&event)?; + + let key = event.key(); + + match producer.send_result(FutureRecord { + topic: topic.as_str(), + payload: Some(&payload), + partition: None, + key: Some(&key), + timestamp: None, + headers: None, + }) { + Ok(_) => {} + Err(e) => { + tracing::error!("failed to produce event: {}", e.0); + + // TODO: Improve error handling + return Err(anyhow!("failed to produce event {}", e.0)); + } + } + + Ok(()) + } +} + +#[async_trait] +impl EventSink for KafkaSink { + async fn send(&self, event: ProcessedEvent) -> Result<()> { + Self::kafka_send(self.producer.clone(), self.topic.clone(), event).await + } + + async fn send_batch(&self, events: Vec) -> Result<()> { + let mut set = JoinSet::new(); + + for event in events { + let producer = self.producer.clone(); + let topic = self.topic.clone(); + + set.spawn(Self::kafka_send(producer, topic, event)); + } + + while let Some(res) = set.join_next().await { + println!("{:?}", res); + } + + Ok(()) + } +} diff --git a/tests/django_compat.rs b/tests/django_compat.rs index e305412a0d43f..c8ac9a4ccd252 100644 --- a/tests/django_compat.rs +++ b/tests/django_compat.rs @@ -56,7 +56,7 @@ impl EventSink for MemorySink { Ok(()) } - async fn send_batch(&self, events: &[ProcessedEvent]) -> anyhow::Result<()> { + async fn send_batch(&self, events: Vec) -> anyhow::Result<()> { self.events.lock().unwrap().extend_from_slice(&events); Ok(()) } From 938b3a193d09170d05b0bda159c5a621ac1aecce Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Wed, 6 Sep 2023 11:46:32 +0100 Subject: [PATCH 019/247] Fix docker build (#18) * update and fix dockerfile * Update dependencies --- Cargo.lock | 534 +++++++++++++++++++++++++++++++---------------------- Cargo.toml | 2 +- Dockerfile | 4 +- 3 files changed, 320 insertions(+), 220 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d6275655449b9..d8e3ef016a0a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -10,28 +19,28 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "1.0.1" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.31", ] [[package]] @@ -42,13 +51,13 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.15" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b32c5ea3aabaf4deb5f5ced2d688ec0844c881c9e6c696a8b769a05fc691e62" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", - "bitflags", + "bitflags 1.3.2", "bytes", "futures-util", "http", @@ -107,11 +116,26 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" -version = "0.21.1" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1e31e207a6b8fb791a38ea3105e6cb541f55e4d029902d3039a4ad07cc4105" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "bitflags" @@ -119,11 +143,17 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + [[package]] name = "bumpalo" -version = "3.12.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "bytes" @@ -159,9 +189,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -189,26 +222,32 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] [[package]] name = "dashmap" -version = "5.4.0" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown", + "hashbrown 0.14.0", "lock_api", "once_cell", "parking_lot_core", ] +[[package]] +name = "deranged" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" + [[package]] name = "difflib" version = "0.4.0" @@ -223,24 +262,30 @@ checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "miniz_oxide", @@ -263,9 +308,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -342,7 +387,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.31", ] [[package]] @@ -383,15 +428,21 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + [[package]] name = "governor" version = "0.5.1" @@ -412,9 +463,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -422,7 +473,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -435,14 +486,17 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "http" @@ -468,9 +522,9 @@ dependencies = [ [[package]] name = "http-range-header" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "httparse" @@ -480,15 +534,15 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -501,7 +555,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -510,9 +564,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -525,14 +579,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", ] [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "itertools" @@ -545,15 +609,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -566,15 +630,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libz-sys" -version = "1.1.9" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" dependencies = [ "cc", "libc", @@ -584,9 +648,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -594,12 +658,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "mach" @@ -612,15 +673,15 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "mime" @@ -649,12 +710,11 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", ] @@ -722,18 +782,18 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ "hermit-abi", "libc", @@ -760,11 +820,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "overload" @@ -784,48 +853,48 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-targets", ] [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -887,9 +956,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -912,9 +981,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -955,16 +1024,17 @@ version = "10.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] name = "rdkafka" -version = "0.25.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8acd8f5c5482fdf89e8878227bafa442d8c4409f6287391c85549ca83626c27" +checksum = "053adfa02fab06e86c01d586cc68aa47ee0ff4489a59469081dc12cbcde578bf" dependencies = [ - "futures", + "futures-channel", + "futures-util", "libc", "log", "rdkafka-sys", @@ -977,9 +1047,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "3.0.0+1.6.0" +version = "4.6.0+2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca35e95c88e08cdc643b25744e38ccee7c93c7e90d1ac6850fe74cbaa40803c3" +checksum = "ad63c279fca41a27c231c450a2d2ad18288032e9cbb159ad16c9d96eba35aaaf" dependencies = [ "cmake", "libc", @@ -990,18 +1060,30 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.8.1" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" dependencies = [ "aho-corasick", "memchr", @@ -1010,15 +1092,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ "base64", "bytes", @@ -1051,49 +1133,55 @@ dependencies = [ "winreg", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.31", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" dependencies = [ "itoa", "ryu", @@ -1102,10 +1190,11 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" dependencies = [ + "itoa", "serde", ] @@ -1141,18 +1230,18 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "socket2" @@ -1164,6 +1253,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "syn" version = "1.0.109" @@ -1177,9 +1276,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.15" +version = "2.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" dependencies = [ "proc-macro2", "quote", @@ -1200,22 +1299,22 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.31", ] [[package]] @@ -1230,10 +1329,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.20" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" dependencies = [ + "deranged", "itoa", "serde", "time-core", @@ -1242,15 +1342,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" dependencies = [ "time-core", ] @@ -1272,11 +1372,11 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", "mio", @@ -1284,20 +1384,20 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.3", "tokio-macros", "windows-sys", ] [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.31", ] [[package]] @@ -1316,17 +1416,17 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" [[package]] name = "toml_edit" -version = "0.19.9" +version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d964908cec0d030b812013af25a0e57fddfadb1e066ecc6681d86253129d4f" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ - "indexmap", + "indexmap 2.0.0", "toml_datetime", "winnow", ] @@ -1349,11 +1449,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.0" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags", + "bitflags 2.4.0", "bytes", "futures-core", "futures-util", @@ -1413,20 +1513,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -1445,9 +1545,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "nu-ansi-term", "sharded-slab", @@ -1465,9 +1565,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] @@ -1480,9 +1580,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -1495,9 +1595,9 @@ dependencies = [ [[package]] name = "url" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -1506,9 +1606,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.3" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ "getrandom", "serde", @@ -1534,11 +1634,10 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -1556,9 +1655,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1566,24 +1665,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if", "js-sys", @@ -1593,9 +1692,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1603,28 +1702,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.31", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-streams" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" dependencies = [ "futures-util", "js-sys", @@ -1635,9 +1734,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -1667,18 +1766,18 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.45.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -1691,60 +1790,61 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.4.6" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" dependencies = [ "memchr", ] [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if", + "windows-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 340110740e3fc..7514974a7e765 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ base64 = "0.21.1" uuid = { version = "1.3.3", features = ["serde", "v4"] } async-trait = "0.1.68" serde_urlencoded = "0.7.1" -rdkafka = { version = "0.25", features = ["cmake-build"] } +rdkafka = { version = "0.34", features = ["cmake-build"] } [dev-dependencies] axum-test-helper = "0.2.0" diff --git a/Dockerfile b/Dockerfile index f9e1ce8920971..d7f528619e1a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM lukemathwalker/cargo-chef:latest-rust-1.68.0 AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.72.0 AS chef WORKDIR app FROM chef AS planner @@ -8,7 +8,7 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder # Ensure working C compile setup (not installed by default in arm64 images) -RUN apt update && apt install build-essential -y +RUN apt update && apt install build-essential cmake -y COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json From fca44ad6c9e245656e0d7053e6789673879d9038 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Fri, 8 Sep 2023 14:19:07 +0100 Subject: [PATCH 020/247] Correct debian version (#19) The other version has glibc errors. I should investigate, but don't have time rn. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d7f528619e1a1..c44e53be54fc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM lukemathwalker/cargo-chef:latest-rust-1.72.0 AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.72.0-buster AS chef WORKDIR app FROM chef AS planner From d3f5de66fbf27e68963bd071e0b48fa7cc8912c6 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Fri, 8 Sep 2023 16:05:11 +0100 Subject: [PATCH 021/247] Only build ARM images, as we only run ARM (#20) --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f6b8ebb7bb44a..76d3ef09d2d1b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -52,7 +52,7 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64, linux/arm64 + platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max build-args: RUST_BACKTRACE=1 From a9cc8a2d6724e1be13f0d8f65a172b444d48cf40 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 11 Sep 2023 12:48:57 +0100 Subject: [PATCH 022/247] Allow setting bind address (#21) --- src/main.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 77d1e8d1a016f..3e6acde32af15 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,4 @@ use std::env; -use std::net::SocketAddr; use crate::time::SystemTime; @@ -14,6 +13,7 @@ mod token; #[tokio::main] async fn main() { let use_print_sink = env::var("PRINT_SINK").is_ok(); + let address = env::var("ADDRESS").unwrap_or(String::from("127.0.0.1:3000")); let app = if use_print_sink { router::router(SystemTime {}, sink::PrintSink {}) @@ -31,11 +31,10 @@ async fn main() { // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` - let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); - tracing::debug!("listening on {}", addr); + tracing::info!("listening on {}", address); - axum::Server::bind(&addr) + axum::Server::bind(&address.parse().unwrap()) .serve(app.into_make_service()) .await .unwrap(); From eea90a5c196555504a8a22bf01fb222ebf4c3e83 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 11 Sep 2023 12:51:17 +0100 Subject: [PATCH 023/247] Use a big ARM builder for docker (#22) --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 76d3ef09d2d1b..98160d19edce4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,7 +11,7 @@ permissions: jobs: build: name: build and publish capture image - runs-on: ubuntu-latest + runs-on: buildjet-8vcpu-ubuntu-2204-arm steps: - name: Check Out Repo From 565d096cf7b1629f6be3d0bc05faccae347babc0 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 13 Sep 2023 16:08:51 +0200 Subject: [PATCH 024/247] Capture several fields to the ProcessedEvent output (#15) --- Cargo.lock | 28 ++++++- Cargo.toml | 7 +- src/api.rs | 3 +- src/capture.rs | 167 ++++++++++++++++++++++++----------------- src/event.rs | 70 +++++++++++++---- src/lib.rs | 1 + src/main.rs | 4 +- src/utils.rs | 38 ++++++++++ tests/django_compat.rs | 12 ++- 9 files changed, 239 insertions(+), 91 deletions(-) create mode 100644 src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index d8e3ef016a0a8..91d5f1148bb3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,6 +32,16 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-trait" version = "0.1.73" @@ -81,6 +91,17 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-client-ip" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8e81eacc93f36480825da5f46a33b5fb2246ed024eacc9e8933425b80c5807" +dependencies = [ + "axum", + "forwarded-header-value", + "serde", +] + [[package]] name = "axum-core" version = "0.3.4" @@ -166,14 +187,17 @@ name = "capture" version = "0.1.0" dependencies = [ "anyhow", + "assert-json-diff", "async-trait", "axum", + "axum-client-ip", "axum-test-helper", "base64", "bytes", "flate2", "governor", "mockall", + "rand", "rdkafka", "serde", "serde_json", @@ -247,6 +271,9 @@ name = "deranged" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +dependencies = [ + "serde", +] [[package]] name = "difflib" @@ -1610,7 +1637,6 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ - "getrandom", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index 7514974a7e765..ffc4b7cf97321 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] axum = "0.6.15" +axum-client-ip = "0.4.1" tokio = { version = "1.0", features = ["full"] } tracing = "0.1" tracing-subscriber = "0.3" @@ -14,17 +15,19 @@ serde = { version = "1.0.160", features = ["derive"] } serde_json = "1.0.96" governor = "0.5.1" tower_governor = "0.0.4" -time = { version = "0.3.20", features = ["formatting"]} +time = { version = "0.3.20", features = ["formatting", "macros", "serde"] } tower-http = { version = "0.4.0", features = ["trace"] } bytes = "1" anyhow = "1.0" flate2 = "1.0" base64 = "0.21.1" -uuid = { version = "1.3.3", features = ["serde", "v4"] } +uuid = { version = "1.3.3", features = ["serde"] } async-trait = "0.1.68" serde_urlencoded = "0.7.1" +rand = "0.8.5" rdkafka = { version = "0.34", features = ["cmake-build"] } [dev-dependencies] +assert-json-diff = "2.0.2" axum-test-helper = "0.2.0" mockall = "0.11.2" diff --git a/src/api.rs b/src/api.rs index 09e79ae7b961b..51f8321a85836 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,7 +1,6 @@ -use std::collections::HashMap; - use serde::{Deserialize, Serialize}; use serde_json::Value; +use std::collections::HashMap; #[derive(Debug, Deserialize, Serialize)] pub struct CaptureRequest { diff --git a/src/capture.rs b/src/capture.rs index 541ef315cd895..b7653857b817f 100644 --- a/src/capture.rs +++ b/src/capture.rs @@ -2,27 +2,29 @@ use std::collections::HashSet; use std::ops::Deref; use std::sync::Arc; -use anyhow::Result; +use anyhow::{anyhow, Result}; use bytes::Bytes; use axum::{http::StatusCode, Json}; // TODO: stream this instead use axum::extract::{Query, State}; use axum::http::HeaderMap; +use axum_client_ip::InsecureClientIp; use base64::Engine; -use uuid::Uuid; - -use crate::api::CaptureResponseCode; -use crate::event::ProcessedEvent; +use time::OffsetDateTime; +use crate::event::ProcessingContext; +use crate::token::validate_token; use crate::{ - api::CaptureResponse, - event::{Event, EventFormData, EventQuery}, - router, sink, token, + api::{CaptureResponse, CaptureResponseCode}, + event::{EventFormData, EventQuery, ProcessedEvent, RawEvent}, + router, sink, + utils::uuid_v7, }; pub async fn event( state: State, + InsecureClientIp(ip): InsecureClientIp, meta: Query, headers: HeaderMap, body: Bytes, @@ -38,9 +40,9 @@ pub async fn event( let payload = base64::engine::general_purpose::STANDARD .decode(input.data) .unwrap(); - Event::from_bytes(&meta, payload.into()) + RawEvent::from_bytes(&meta, payload.into()) } - _ => Event::from_bytes(&meta, body), + _ => RawEvent::from_bytes(&meta, body), }; let events = match events { @@ -59,8 +61,30 @@ pub async fn event( if events.is_empty() { return Err((StatusCode::BAD_REQUEST, String::from("No events in batch"))); } + let token = match extract_and_verify_token(&events) { + Ok(token) => token, + Err(msg) => return Err((StatusCode::UNAUTHORIZED, msg)), + }; + + let sent_at = meta.sent_at.and_then(|value| { + let value_nanos: i128 = i128::from(value) * 1_000_000; // Assuming the value is in milliseconds, latest posthog-js releases + if let Ok(sent_at) = OffsetDateTime::from_unix_timestamp_nanos(value_nanos) { + if sent_at.year() > 2020 { + // Could be lower if the input is in seconds + return Some(sent_at); + } + } + None + }); + let context = ProcessingContext { + lib_version: meta.lib_version.clone(), + sent_at, + token, + now: state.timesource.current_time(), + client_ip: ip.to_string(), + }; - let processed = process_events(state.sink.clone(), &events).await; + let processed = process_events(state.sink.clone(), &events, &context).await; if let Err(msg) = processed { return Err((StatusCode::BAD_REQUEST, msg)); @@ -71,47 +95,61 @@ pub async fn event( })) } -pub fn process_single_event(_event: &Event) -> Result { - // TODO: Put actual data in here and transform it properly +pub fn process_single_event( + event: &RawEvent, + context: &ProcessingContext, +) -> Result { + let distinct_id = match &event.distinct_id { + Some(id) => id, + None => match event.properties.get("distinct_id").map(|v| v.as_str()) { + Some(Some(id)) => id, + _ => return Err(anyhow!("missing distinct_id")), + }, + }; + Ok(ProcessedEvent { - uuid: Uuid::new_v4(), - distinct_id: Uuid::new_v4().simple().to_string(), - ip: String::new(), + uuid: event.uuid.unwrap_or_else(uuid_v7), + distinct_id: distinct_id.to_string(), + ip: context.client_ip.clone(), site_url: String::new(), data: String::from("hallo I am some data 😊"), - now: String::new(), - sent_at: String::new(), - token: String::from("tokentokentoken"), + now: context.now.clone(), + sent_at: context.sent_at, + token: context.token.clone(), }) } +pub fn extract_and_verify_token(events: &[RawEvent]) -> Result { + let distinct_tokens: HashSet> = HashSet::from_iter( + events + .iter() + .map(RawEvent::extract_token) + .filter(Option::is_some), + ); + + return match distinct_tokens.len() { + 0 => Err(String::from("no token found in request")), + 1 => match distinct_tokens.iter().last() { + Some(Some(token)) => { + validate_token(token).map_err(|err| String::from(err.reason()))?; + Ok(token.clone()) + } + _ => Err(String::from("no token found in request")), + }, + _ => Err(String::from("number of distinct tokens in batch > 1")), + }; +} + pub async fn process_events( sink: Arc, - events: &[Event], + events: &[RawEvent], + context: &ProcessingContext, ) -> Result<(), String> { - let mut distinct_tokens = HashSet::new(); - - // 1. Tokens are all valid - for event in events { - let token = event.token.clone().unwrap_or_else(|| { - event - .properties - .get("token") - .map_or(String::new(), |t| String::from(t.as_str().unwrap())) - }); - - if let Err(invalid) = token::validate_token(token.as_str()) { - return Err(invalid.reason().to_string()); - } - - distinct_tokens.insert(token); - } - - if distinct_tokens.len() > 1 { - return Err(String::from("Number of distinct tokens in batch > 1")); - } - - let events: Vec = match events.iter().map(process_single_event).collect() { + let events: Vec = match events + .iter() + .map(|e| process_single_event(e, context)) + .collect() + { Err(_) => return Err(String::from("Failed to process all events")), Ok(events) => events, }; @@ -139,61 +177,54 @@ pub async fn process_events( #[cfg(test)] mod tests { - use crate::sink; - use std::collections::HashMap; - use std::sync::Arc; - + use crate::capture::extract_and_verify_token; + use crate::event::RawEvent; use serde_json::json; - - use super::process_events; - use crate::event::Event; - use crate::router::State; + use std::collections::HashMap; #[tokio::test] async fn all_events_have_same_token() { - let state = State { - sink: Arc::new(sink::PrintSink {}), - timesource: Arc::new(crate::time::SystemTime {}), - }; - let events = vec![ - Event { + RawEvent { token: Some(String::from("hello")), + distinct_id: Some("testing".to_string()), + uuid: None, event: String::new(), properties: HashMap::new(), }, - Event { + RawEvent { token: None, + distinct_id: Some("testing".to_string()), + uuid: None, event: String::new(), properties: HashMap::from([(String::from("token"), json!("hello"))]), }, ]; - let processed = process_events(state.sink, &events).await; - assert_eq!(processed.is_ok(), true); + let processed = extract_and_verify_token(&events); + assert_eq!(processed.is_ok(), true, "{:?}", processed); } #[tokio::test] async fn all_events_have_different_token() { - let state = State { - sink: Arc::new(sink::PrintSink {}), - timesource: Arc::new(crate::time::SystemTime {}), - }; - let events = vec![ - Event { + RawEvent { token: Some(String::from("hello")), + distinct_id: Some("testing".to_string()), + uuid: None, event: String::new(), properties: HashMap::new(), }, - Event { + RawEvent { token: None, + distinct_id: Some("testing".to_string()), + uuid: None, event: String::new(), properties: HashMap::from([(String::from("token"), json!("goodbye"))]), }, ]; - let processed = process_events(state.sink, &events).await; + let processed = extract_and_verify_token(&events); assert_eq!(processed.is_err(), true); } } diff --git a/src/event.rs b/src/event.rs index 8cac0ecf8a1ab..32c1b5216ca68 100644 --- a/src/event.rs +++ b/src/event.rs @@ -7,6 +7,7 @@ use serde_json::Value; use anyhow::{anyhow, Result}; use bytes::{Buf, Bytes}; use flate2::read::GzDecoder; +use time::OffsetDateTime; use uuid::Uuid; #[derive(Deserialize, Default)] @@ -16,16 +17,24 @@ pub enum Compression { GzipJs, } -#[allow(dead_code)] // until they are used #[derive(Deserialize, Default)] pub struct EventQuery { - compression: Option, + pub compression: Option, #[serde(alias = "ver")] - version: Option, + pub lib_version: Option, #[serde(alias = "_")] - sent_at: Option, + pub sent_at: Option, + + #[serde(skip_serializing)] + pub token: Option, // Filled by handler + + #[serde(skip_serializing)] + pub now: Option, // Filled by handler from timesource + + #[serde(skip_serializing)] + pub client_ip: Option, // Filled by handler } #[derive(Debug, Deserialize)] @@ -34,19 +43,20 @@ pub struct EventFormData { } #[derive(Default, Debug, Deserialize, Serialize)] -pub struct Event { +pub struct RawEvent { #[serde(alias = "$token", alias = "api_key")] pub token: Option, - + pub distinct_id: Option, + pub uuid: Option, pub event: String, pub properties: HashMap, } -impl Event { +impl RawEvent { /// We post up _at least one_ event, so when decompressiong and deserializing there /// could be more than one. Hence this function has to return a Vec. /// TODO: Use an axum extractor for this - pub fn from_bytes(query: &EventQuery, bytes: Bytes) -> Result> { + pub fn from_bytes(query: &EventQuery, bytes: Bytes) -> Result> { tracing::debug!(len = bytes.len(), "decoding new event"); let payload = match query.compression { @@ -60,17 +70,43 @@ impl Event { }; tracing::debug!(json = payload, "decoded event data"); - if let Ok(events) = serde_json::from_str::>(&payload) { + if let Ok(events) = serde_json::from_str::>(&payload) { return Ok(events); } - if let Ok(events) = serde_json::from_str::(&payload) { + if let Ok(events) = serde_json::from_str::(&payload) { return Ok(vec![events]); } Err(anyhow!("unknown input shape")) } + + pub fn extract_token(&self) -> Option { + match &self.token { + Some(value) => Some(value.clone()), + None => self + .properties + .get("token") + .and_then(Value::as_str) + .map(String::from), + } + } +} + +pub struct ProcessingContext { + pub lib_version: Option, + pub sent_at: Option, + pub token: String, + pub now: String, + pub client_ip: String, } -#[derive(Clone, Default, Debug, Deserialize, Serialize)] +time::serde::format_description!( + django_iso, + OffsetDateTime, + "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:6][offset_hour \ + sign:mandatory]:[offset_minute]" +); + +#[derive(Clone, Default, Debug, Serialize)] pub struct ProcessedEvent { pub uuid: Uuid, pub distinct_id: String, @@ -78,7 +114,8 @@ pub struct ProcessedEvent { pub site_url: String, pub data: String, pub now: String, - pub sent_at: String, + #[serde(with = "django_iso::option")] + pub sent_at: Option, pub token: String, } @@ -94,7 +131,7 @@ mod tests { use base64::Engine as _; use bytes::Bytes; - use super::{Event, EventQuery}; + use super::{EventQuery, RawEvent}; #[test] fn decode_bytes() { @@ -104,11 +141,14 @@ mod tests { .unwrap(); let bytes = Bytes::from(decoded_horrible_blob); - let events = Event::from_bytes( + let events = RawEvent::from_bytes( &EventQuery { compression: Some(Compression::GzipJs), - version: None, + lib_version: None, sent_at: None, + token: None, + now: None, + client_ip: None, }, bytes, ); diff --git a/src/lib.rs b/src/lib.rs index 641385f93d553..96e98755dee98 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,3 +5,4 @@ pub mod router; pub mod sink; pub mod time; mod token; +mod utils; diff --git a/src/main.rs b/src/main.rs index 3e6acde32af15..b7e549fca2da4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use std::env; +use std::net::SocketAddr; use crate::time::SystemTime; @@ -9,6 +10,7 @@ mod router; mod sink; mod time; mod token; +mod utils; #[tokio::main] async fn main() { @@ -35,7 +37,7 @@ async fn main() { tracing::info!("listening on {}", address); axum::Server::bind(&address.parse().unwrap()) - .serve(app.into_make_service()) + .serve(app.into_make_service_with_connect_info::()) .await .unwrap(); } diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000000000..c5a95fbe2481f --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,38 @@ +use rand::RngCore; +use uuid::Uuid; + +pub fn random_bytes() -> [u8; N] { + let mut ret = [0u8; N]; + rand::thread_rng().fill_bytes(&mut ret); + ret +} + +// basically just ripped from the uuid crate. they have it as unstable, but we can use it fine. +const fn encode_unix_timestamp_millis(millis: u64, random_bytes: &[u8; 10]) -> Uuid { + let millis_high = ((millis >> 16) & 0xFFFF_FFFF) as u32; + let millis_low = (millis & 0xFFFF) as u16; + + let random_and_version = + (random_bytes[0] as u16 | ((random_bytes[1] as u16) << 8) & 0x0FFF) | (0x7 << 12); + + let mut d4 = [0; 8]; + + d4[0] = (random_bytes[2] & 0x3F) | 0x80; + d4[1] = random_bytes[3]; + d4[2] = random_bytes[4]; + d4[3] = random_bytes[5]; + d4[4] = random_bytes[6]; + d4[5] = random_bytes[7]; + d4[6] = random_bytes[8]; + d4[7] = random_bytes[9]; + + Uuid::from_fields(millis_high, millis_low, random_and_version, &d4) +} + +pub fn uuid_v7() -> Uuid { + let bytes = random_bytes(); + let now = time::OffsetDateTime::now_utc(); + let now_millis: u64 = now.unix_timestamp() as u64 * 1_000 + now.millisecond() as u64; + + encode_unix_timestamp_millis(now_millis, &bytes) +} diff --git a/tests/django_compat.rs b/tests/django_compat.rs index c8ac9a4ccd252..29e344aa7bc62 100644 --- a/tests/django_compat.rs +++ b/tests/django_compat.rs @@ -1,3 +1,4 @@ +use assert_json_diff::assert_json_eq; use async_trait::async_trait; use axum::http::StatusCode; use axum_test_helper::TestClient; @@ -9,6 +10,7 @@ use capture::router::router; use capture::sink::EventSink; use capture::time::TimeSource; use serde::Deserialize; +use serde_json::{json, Value}; use std::fs::File; use std::io::{BufRead, BufReader}; use std::sync::{Arc, Mutex}; @@ -22,7 +24,7 @@ struct RequestDump { ip: String, now: String, body: String, - output: Vec, + output: Vec, } static REQUESTS_DUMP_FILE_NAME: &str = "tests/requests_dump.jsonl"; @@ -47,6 +49,10 @@ impl MemorySink { fn len(&self) -> usize { self.events.lock().unwrap().len() } + + fn events(&self) -> Vec { + self.events.lock().unwrap().clone() + } } #[async_trait] @@ -63,6 +69,7 @@ impl EventSink for MemorySink { } #[tokio::test] +#[ignore] async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let file = File::open(REQUESTS_DUMP_FILE_NAME)?; let reader = BufReader::new(file); @@ -104,7 +111,8 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { }), res.json().await ); - assert_eq!(sink.len(), case.output.len()) + assert_eq!(sink.len(), case.output.len()); + assert_json_eq!(json!(case.output), json!(sink.events())) } Ok(()) } From 8eb6a2ac8911c2f5f65066436b002e948d41e5fa Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Thu, 14 Sep 2023 13:51:41 +0100 Subject: [PATCH 025/247] Initial workspace setup (#25) * Initial workspace setup I'd like to tidy a few things (dependencies etc), but get this setup asap to avoid awkward rebases going forwards. * Format and delete --- Cargo.lock | 11 +++++++ Cargo.toml | 18 ++++------- bin/send_event.sh | 2 -- capture-server/Cargo.toml | 11 +++++++ {src => capture-server/src}/main.rs | 17 +++------- capture/Cargo.toml | 33 ++++++++++++++++++++ {src => capture/src}/api.rs | 0 {src => capture/src}/capture.rs | 6 ++-- {src => capture/src}/event.rs | 0 {src => capture/src}/lib.rs | 6 ++-- {src => capture/src}/router.rs | 0 {src => capture/src}/sink.rs | 0 {src => capture/src}/time.rs | 0 {src => capture/src}/token.rs | 0 {src => capture/src}/utils.rs | 0 {tests => capture/tests}/django_compat.rs | 0 {tests => capture/tests}/requests_dump.jsonl | 0 17 files changed, 71 insertions(+), 33 deletions(-) delete mode 100755 bin/send_event.sh create mode 100644 capture-server/Cargo.toml rename {src => capture-server/src}/main.rs (79%) create mode 100644 capture/Cargo.toml rename {src => capture/src}/api.rs (100%) rename {src => capture/src}/capture.rs (98%) rename {src => capture/src}/event.rs (100%) rename {src => capture/src}/lib.rs (60%) rename {src => capture/src}/router.rs (100%) rename {src => capture/src}/sink.rs (100%) rename {src => capture/src}/time.rs (100%) rename {src => capture/src}/token.rs (100%) rename {src => capture/src}/utils.rs (100%) rename {tests => capture/tests}/django_compat.rs (100%) rename {tests => capture/tests}/requests_dump.jsonl (100%) diff --git a/Cargo.lock b/Cargo.lock index 91d5f1148bb3a..8ae24941c7f91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -211,6 +211,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "capture-server" +version = "0.1.0" +dependencies = [ + "axum", + "capture", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "cc" version = "1.0.83" diff --git a/Cargo.toml b/Cargo.toml index ffc4b7cf97321..0d56eddfd0815 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,10 @@ -[package] -name = "capture" -version = "0.1.0" -edition = "2021" +[workspace] +members = [ + "capture", + "capture-server" +] -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] +[workspace.dependencies] axum = "0.6.15" axum-client-ip = "0.4.1" tokio = { version = "1.0", features = ["full"] } @@ -26,8 +25,3 @@ async-trait = "0.1.68" serde_urlencoded = "0.7.1" rand = "0.8.5" rdkafka = { version = "0.34", features = ["cmake-build"] } - -[dev-dependencies] -assert-json-diff = "2.0.2" -axum-test-helper = "0.2.0" -mockall = "0.11.2" diff --git a/bin/send_event.sh b/bin/send_event.sh deleted file mode 100755 index 4885cf12db376..0000000000000 --- a/bin/send_event.sh +++ /dev/null @@ -1,2 +0,0 @@ -# Send an event to a test server -curl http://localhost:3000/capture -X POST -H "Content-Type: application/json" --data '{"token": "ferrisisbae"}' diff --git a/capture-server/Cargo.toml b/capture-server/Cargo.toml new file mode 100644 index 0000000000000..04c618286c4cb --- /dev/null +++ b/capture-server/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "capture-server" +version = "0.1.0" +edition = "2021" + +[dependencies] +capture = { path = "../capture" } +axum = { workspace = true } +tokio = { workspace = true } +tracing-subscriber = { workspace = true } +tracing = { workspace = true } diff --git a/src/main.rs b/capture-server/src/main.rs similarity index 79% rename from src/main.rs rename to capture-server/src/main.rs index b7e549fca2da4..c2f22bcb1ac37 100644 --- a/src/main.rs +++ b/capture-server/src/main.rs @@ -1,16 +1,7 @@ use std::env; use std::net::SocketAddr; -use crate::time::SystemTime; - -mod api; -mod capture; -mod event; -mod router; -mod sink; -mod time; -mod token; -mod utils; +use capture::{router, sink, time}; #[tokio::main] async fn main() { @@ -18,19 +9,19 @@ async fn main() { let address = env::var("ADDRESS").unwrap_or(String::from("127.0.0.1:3000")); let app = if use_print_sink { - router::router(SystemTime {}, sink::PrintSink {}) + router::router(time::SystemTime {}, sink::PrintSink {}) } else { let brokers = env::var("KAFKA_BROKERS").expect("Expected KAFKA_BROKERS"); let topic = env::var("KAFKA_TOPIC").expect("Expected KAFKA_TOPIC"); let sink = sink::KafkaSink::new(topic, brokers).unwrap(); - router::router(SystemTime {}, sink) + router::router(time::SystemTime {}, sink) }; // initialize tracing - tracing_subscriber::fmt::init(); + tracing_subscriber::fmt::init(); // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` diff --git a/capture/Cargo.toml b/capture/Cargo.toml new file mode 100644 index 0000000000000..61ccceaeea46a --- /dev/null +++ b/capture/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "capture" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +axum = { workspace = true } +axum-client-ip = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +governor = { workspace = true } +tower_governor = { workspace = true } +time = { workspace = true } +tower-http = { workspace = true } +bytes = { workspace = true } +anyhow = { workspace = true } +flate2 = { workspace = true } +base64 = { workspace = true } +uuid = { workspace = true } +async-trait = { workspace = true } +serde_urlencoded = { workspace = true } +rand = { workspace = true } +rdkafka = { workspace = true } + +[dev-dependencies] +assert-json-diff = "2.0.2" +axum-test-helper = "0.2.0" +mockall = "0.11.2" diff --git a/src/api.rs b/capture/src/api.rs similarity index 100% rename from src/api.rs rename to capture/src/api.rs diff --git a/src/capture.rs b/capture/src/capture.rs similarity index 98% rename from src/capture.rs rename to capture/src/capture.rs index b7653857b817f..d54cf0847c219 100644 --- a/src/capture.rs +++ b/capture/src/capture.rs @@ -140,10 +140,10 @@ pub fn extract_and_verify_token(events: &[RawEvent]) -> Result { }; } -pub async fn process_events( +pub async fn process_events<'a>( sink: Arc, - events: &[RawEvent], - context: &ProcessingContext, + events: &'a [RawEvent], + context: &'a ProcessingContext, ) -> Result<(), String> { let events: Vec = match events .iter() diff --git a/src/event.rs b/capture/src/event.rs similarity index 100% rename from src/event.rs rename to capture/src/event.rs diff --git a/src/lib.rs b/capture/src/lib.rs similarity index 60% rename from src/lib.rs rename to capture/src/lib.rs index 96e98755dee98..0f0b269bdb3f9 100644 --- a/src/lib.rs +++ b/capture/src/lib.rs @@ -1,8 +1,8 @@ pub mod api; -mod capture; +pub mod capture; pub mod event; pub mod router; pub mod sink; pub mod time; -mod token; -mod utils; +pub mod token; +pub mod utils; diff --git a/src/router.rs b/capture/src/router.rs similarity index 100% rename from src/router.rs rename to capture/src/router.rs diff --git a/src/sink.rs b/capture/src/sink.rs similarity index 100% rename from src/sink.rs rename to capture/src/sink.rs diff --git a/src/time.rs b/capture/src/time.rs similarity index 100% rename from src/time.rs rename to capture/src/time.rs diff --git a/src/token.rs b/capture/src/token.rs similarity index 100% rename from src/token.rs rename to capture/src/token.rs diff --git a/src/utils.rs b/capture/src/utils.rs similarity index 100% rename from src/utils.rs rename to capture/src/utils.rs diff --git a/tests/django_compat.rs b/capture/tests/django_compat.rs similarity index 100% rename from tests/django_compat.rs rename to capture/tests/django_compat.rs diff --git a/tests/requests_dump.jsonl b/capture/tests/requests_dump.jsonl similarity index 100% rename from tests/requests_dump.jsonl rename to capture/tests/requests_dump.jsonl From 3390610719bba9c1e3bc46e9ea3016a0b264424e Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Thu, 14 Sep 2023 14:07:41 +0100 Subject: [PATCH 026/247] Add Prometheus metrics exporting (#23) * Add metrics middleware * Organize a bit better * Track a couple more things * Do not install metrics in tests * Add some lifetimes and fixes * Run formatter * Simplify routes * Fixes and feedback * Patch builds --- .github/workflows/rust.yml | 2 +- Cargo.lock | 138 ++++++++++++++++++++++++++++++++- Cargo.toml | 2 + Dockerfile | 2 +- capture-server/src/main.rs | 4 +- capture/Cargo.toml | 2 + capture/src/lib.rs | 1 + capture/src/prometheus.rs | 55 +++++++++++++ capture/src/router.rs | 30 ++++--- capture/src/sink.rs | 11 ++- capture/tests/django_compat.rs | 2 +- 11 files changed, 232 insertions(+), 17 deletions(-) create mode 100644 capture/src/prometheus.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index dbb689dc8e72a..478017af701f2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -30,7 +30,7 @@ jobs: key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }} - name: Run cargo build - run: cargo build --all --locked --release && strip target/release/capture + run: cargo build --all --locked --release && strip target/release/capture-server test: runs-on: buildjet-4vcpu-ubuntu-2204 diff --git a/Cargo.lock b/Cargo.lock index 8ae24941c7f91..0f4cd76bceb4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "1.0.5" @@ -196,6 +207,8 @@ dependencies = [ "bytes", "flate2", "governor", + "metrics", + "metrics-exporter-prometheus", "mockall", "rand", "rdkafka", @@ -255,6 +268,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -494,7 +520,7 @@ dependencies = [ "no-std-compat", "nonzero_ext", "parking_lot", - "quanta", + "quanta 0.9.3", "rand", "smallvec", ] @@ -524,6 +550,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.14.0" @@ -709,6 +744,15 @@ dependencies = [ "libc", ] +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + [[package]] name = "matchit" version = "0.7.2" @@ -721,6 +765,70 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "metrics" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +dependencies = [ + "ahash", + "metrics-macros", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" +dependencies = [ + "base64", + "hyper", + "indexmap 1.9.3", + "ipnet", + "metrics", + "metrics-util", + "quanta 0.11.1", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.31", +] + +[[package]] +name = "metrics-util" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.13.1", + "metrics", + "num_cpus", + "quanta 0.11.1", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -946,6 +1054,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "portable-atomic" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1017,6 +1131,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "quanta" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.33" @@ -1266,6 +1396,12 @@ dependencies = [ "libc", ] +[[package]] +name = "sketches-ddsketch" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" + [[package]] name = "slab" version = "0.4.9" diff --git a/Cargo.toml b/Cargo.toml index 0d56eddfd0815..59ffb47dad7a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,3 +25,5 @@ async-trait = "0.1.68" serde_urlencoded = "0.7.1" rand = "0.8.5" rdkafka = { version = "0.34", features = ["cmake-build"] } +metrics = "0.21.1" +metrics-exporter-prometheus = "0.12.1" diff --git a/Dockerfile b/Dockerfile index c44e53be54fc8..cb158e22abcc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json COPY . . -RUN cargo build --release --bin capture +RUN cargo build --release --bin capture-server FROM debian:bullseye-20230320-slim AS runtime diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index c2f22bcb1ac37..88ffc949eea7a 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -9,14 +9,14 @@ async fn main() { let address = env::var("ADDRESS").unwrap_or(String::from("127.0.0.1:3000")); let app = if use_print_sink { - router::router(time::SystemTime {}, sink::PrintSink {}) + router::router(time::SystemTime {}, sink::PrintSink {}, true) } else { let brokers = env::var("KAFKA_BROKERS").expect("Expected KAFKA_BROKERS"); let topic = env::var("KAFKA_TOPIC").expect("Expected KAFKA_TOPIC"); let sink = sink::KafkaSink::new(topic, brokers).unwrap(); - router::router(time::SystemTime {}, sink) + router::router(time::SystemTime {}, sink, true) }; // initialize tracing diff --git a/capture/Cargo.toml b/capture/Cargo.toml index 61ccceaeea46a..c58bf67ba2340 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -26,6 +26,8 @@ async-trait = { workspace = true } serde_urlencoded = { workspace = true } rand = { workspace = true } rdkafka = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } [dev-dependencies] assert-json-diff = "2.0.2" diff --git a/capture/src/lib.rs b/capture/src/lib.rs index 0f0b269bdb3f9..d4ca041e5e671 100644 --- a/capture/src/lib.rs +++ b/capture/src/lib.rs @@ -1,6 +1,7 @@ pub mod api; pub mod capture; pub mod event; +pub mod prometheus; pub mod router; pub mod sink; pub mod time; diff --git a/capture/src/prometheus.rs b/capture/src/prometheus.rs new file mode 100644 index 0000000000000..1fcdb7d7ca30b --- /dev/null +++ b/capture/src/prometheus.rs @@ -0,0 +1,55 @@ +// Middleware + prometheus exporter setup + +use std::time::Instant; + +use axum::{extract::MatchedPath, http::Request, middleware::Next, response::IntoResponse}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; + +pub fn setup_metrics_recorder() -> PrometheusHandle { + // Ok I broke it at the end, but the limit on our ingress is 60 and that's a nicer way of reaching it + const EXPONENTIAL_SECONDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, + ]; + + PrometheusBuilder::new() + .set_buckets_for_metric( + Matcher::Full("http_requests_duration_seconds".to_string()), + EXPONENTIAL_SECONDS, + ) + .unwrap() + .install_recorder() + .unwrap() +} + +/// Middleware to record some common HTTP metrics +/// Generic over B to allow for arbitrary body types (eg Vec, Streams, a deserialized thing, etc) +/// Someday tower-http might provide a metrics middleware: https://github.com/tower-rs/tower-http/issues/57 +pub async fn track_metrics(req: Request, next: Next) -> impl IntoResponse { + let start = Instant::now(); + + let path = if let Some(matched_path) = req.extensions().get::() { + matched_path.as_str().to_owned() + } else { + req.uri().path().to_owned() + }; + + let method = req.method().clone(); + + // Run the rest of the request handling first, so we can measure it and get response + // codes. + let response = next.run(req).await; + + let latency = start.elapsed().as_secs_f64(); + let status = response.status().as_u16().to_string(); + + let labels = [ + ("method", method.to_string()), + ("path", path), + ("status", status), + ]; + + metrics::increment_counter!("http_requests_total", &labels); + metrics::histogram!("http_requests_duration_seconds", latency, &labels); + + response +} diff --git a/capture/src/router.rs b/capture/src/router.rs index 199cd2095e0a0..0c40658c04647 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -1,3 +1,4 @@ +use std::future::ready; use std::sync::Arc; use axum::{ @@ -8,6 +9,8 @@ use tower_http::trace::TraceLayer; use crate::{capture, sink, time::TimeSource}; +use crate::prometheus::{setup_metrics_recorder, track_metrics}; + #[derive(Clone)] pub struct State { pub sink: Arc, @@ -24,23 +27,30 @@ pub fn router< >( timesource: TZ, sink: S, + metrics: bool, ) -> Router { let state = State { sink: Arc::new(sink), timesource: Arc::new(timesource), }; - Router::new() + let router = Router::new() // TODO: use NormalizePathLayer::trim_trailing_slash .route("/", get(index)) - .route("/capture", post(capture::event)) - .route("/capture/", post(capture::event)) - .route("/batch", post(capture::event)) - .route("/batch/", post(capture::event)) - .route("/e", post(capture::event)) - .route("/e/", post(capture::event)) - .route("/engage", post(capture::event)) - .route("/engage/", post(capture::event)) + .route("/i/v0/e", post(capture::event)) + .route("/i/v0/e/", post(capture::event)) .layer(TraceLayer::new_for_http()) - .with_state(state) + .layer(axum::middleware::from_fn(track_metrics)) + .with_state(state); + + // Don't install metrics unless asked to + // Installing a global recorder when capture is used as a library (during tests etc) + // does not work well. + if metrics { + let recorder_handle = setup_metrics_recorder(); + + router.route("/metrics", get(move || ready(recorder_handle.render()))) + } else { + router + } } diff --git a/capture/src/sink.rs b/capture/src/sink.rs index 20ab3fc70a429..96a6be01cdb72 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -20,6 +20,8 @@ impl EventSink for PrintSink { async fn send(&self, event: ProcessedEvent) -> Result<()> { tracing::info!("single event: {:?}", event); + metrics::increment_counter!("capture_events_total"); + Ok(()) } async fn send_batch(&self, events: Vec) -> Result<()> { @@ -27,6 +29,7 @@ impl EventSink for PrintSink { let _enter = span.enter(); for event in events { + metrics::increment_counter!("capture_events_total"); tracing::info!("event: {:?}", event); } @@ -68,10 +71,16 @@ impl KafkaSink { timestamp: None, headers: None, }) { - Ok(_) => {} + Ok(_) => { + metrics::increment_counter!("capture_events_total"); + } Err(e) => { tracing::error!("failed to produce event: {}", e.0); + // TODO(maybe someday): Don't drop them but write them somewhere and try again + // later? + metrics::increment_counter!("capture_events_dropped"); + // TODO: Improve error handling return Err(anyhow!("failed to produce event {}", e.0)); } diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index 29e344aa7bc62..219a8c7fe691d 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -90,7 +90,7 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let sink = MemorySink::default(); let timesource = FixedTime { time: case.now }; - let app = router(timesource, sink.clone()); + let app = router(timesource, sink.clone(), false); let client = TestClient::new(app); let mut req = client.post(&case.path).body(raw_body); From 76905ecb266a63849e8bf1fe717b51fcac827778 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Thu, 14 Sep 2023 15:27:01 +0200 Subject: [PATCH 027/247] implement error handling with a CaptureError enum (#24) --- Cargo.lock | 1 + Cargo.toml | 1 + capture/Cargo.toml | 1 + capture/src/api.rs | 49 ++++++++++++++++++++++ capture/src/capture.rs | 74 +++++++++------------------------- capture/src/event.rs | 41 +++++++++++++------ capture/src/sink.rs | 51 ++++++++++++----------- capture/tests/django_compat.rs | 8 ++-- 8 files changed, 132 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f4cd76bceb4a..6ef5c6c66b662 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -215,6 +215,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "thiserror", "time", "tokio", "tower-http", diff --git a/Cargo.toml b/Cargo.toml index 59ffb47dad7a6..68569a3ad2f86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,3 +27,4 @@ rand = "0.8.5" rdkafka = { version = "0.34", features = ["cmake-build"] } metrics = "0.21.1" metrics-exporter-prometheus = "0.12.1" +thiserror = "1.0.48" diff --git a/capture/Cargo.toml b/capture/Cargo.toml index c58bf67ba2340..60cca70824cff 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -28,6 +28,7 @@ rand = { workspace = true } rdkafka = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } +thiserror = { workspace = true } [dev-dependencies] assert-json-diff = "2.0.2" diff --git a/capture/src/api.rs b/capture/src/api.rs index 51f8321a85836..9a18a89c4c71c 100644 --- a/capture/src/api.rs +++ b/capture/src/api.rs @@ -1,6 +1,10 @@ +use crate::token::InvalidTokenReason; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::HashMap; +use thiserror::Error; #[derive(Debug, Deserialize, Serialize)] pub struct CaptureRequest { @@ -20,3 +24,48 @@ pub enum CaptureResponseCode { pub struct CaptureResponse { pub status: CaptureResponseCode, } + +#[derive(Error, Debug)] +pub enum CaptureError { + #[error("failed to decode request: {0}")] + RequestDecodingError(String), + #[error("failed to decode request: {0}")] + RequestParsingError(#[from] serde_json::Error), + + #[error("request holds no event")] + EmptyBatch, + #[error("event submitted without a distinct_id")] + MissingDistinctId, + + #[error("event submitted without an api_key")] + NoTokenError, + #[error("batch submitted with inconsistent api_key values")] + MultipleTokensError, + #[error("API key is not valid: {0}")] + TokenValidationError(#[from] InvalidTokenReason), + + #[error("transient error, please retry")] + RetryableSinkError, + #[error("maximum event size exceeded")] + EventTooBig, + #[error("invalid event could not be processed")] + NonRetryableSinkError, +} + +impl IntoResponse for CaptureError { + fn into_response(self) -> Response { + match self { + CaptureError::RequestDecodingError(_) + | CaptureError::RequestParsingError(_) + | CaptureError::EmptyBatch + | CaptureError::MissingDistinctId + | CaptureError::EventTooBig + | CaptureError::NonRetryableSinkError => (StatusCode::BAD_REQUEST, self.to_string()), + CaptureError::NoTokenError + | CaptureError::MultipleTokensError + | CaptureError::TokenValidationError(_) => (StatusCode::UNAUTHORIZED, self.to_string()), + CaptureError::RetryableSinkError => (StatusCode::SERVICE_UNAVAILABLE, self.to_string()), + } + .into_response() + } +} diff --git a/capture/src/capture.rs b/capture/src/capture.rs index d54cf0847c219..e60421759a062 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -2,10 +2,9 @@ use std::collections::HashSet; use std::ops::Deref; use std::sync::Arc; -use anyhow::{anyhow, Result}; use bytes::Bytes; -use axum::{http::StatusCode, Json}; +use axum::Json; // TODO: stream this instead use axum::extract::{Query, State}; use axum::http::HeaderMap; @@ -16,7 +15,7 @@ use time::OffsetDateTime; use crate::event::ProcessingContext; use crate::token::validate_token; use crate::{ - api::{CaptureResponse, CaptureResponseCode}, + api::{CaptureError, CaptureResponse, CaptureResponseCode}, event::{EventFormData, EventQuery, ProcessedEvent, RawEvent}, router, sink, utils::uuid_v7, @@ -28,7 +27,7 @@ pub async fn event( meta: Query, headers: HeaderMap, body: Bytes, -) -> Result, (StatusCode, String)> { +) -> Result, CaptureError> { tracing::debug!(len = body.len(), "new event request"); let events = match headers @@ -43,28 +42,14 @@ pub async fn event( RawEvent::from_bytes(&meta, payload.into()) } _ => RawEvent::from_bytes(&meta, body), - }; - - let events = match events { - Ok(events) => events, - Err(e) => { - tracing::error!("failed to decode event: {:?}", e); - return Err(( - StatusCode::BAD_REQUEST, - String::from("Failed to decode event"), - )); - } - }; + }?; println!("Got events {:?}", &events); if events.is_empty() { - return Err((StatusCode::BAD_REQUEST, String::from("No events in batch"))); + return Err(CaptureError::EmptyBatch); } - let token = match extract_and_verify_token(&events) { - Ok(token) => token, - Err(msg) => return Err((StatusCode::UNAUTHORIZED, msg)), - }; + let token = extract_and_verify_token(&events)?; let sent_at = meta.sent_at.and_then(|value| { let value_nanos: i128 = i128::from(value) * 1_000_000; // Assuming the value is in milliseconds, latest posthog-js releases @@ -84,11 +69,7 @@ pub async fn event( client_ip: ip.to_string(), }; - let processed = process_events(state.sink.clone(), &events, &context).await; - - if let Err(msg) = processed { - return Err((StatusCode::BAD_REQUEST, msg)); - } + process_events(state.sink.clone(), &events, &context).await?; Ok(Json(CaptureResponse { status: CaptureResponseCode::Ok, @@ -98,12 +79,12 @@ pub async fn event( pub fn process_single_event( event: &RawEvent, context: &ProcessingContext, -) -> Result { +) -> Result { let distinct_id = match &event.distinct_id { Some(id) => id, None => match event.properties.get("distinct_id").map(|v| v.as_str()) { Some(Some(id)) => id, - _ => return Err(anyhow!("missing distinct_id")), + _ => return Err(CaptureError::MissingDistinctId), }, }; @@ -119,7 +100,7 @@ pub fn process_single_event( }) } -pub fn extract_and_verify_token(events: &[RawEvent]) -> Result { +pub fn extract_and_verify_token(events: &[RawEvent]) -> Result { let distinct_tokens: HashSet> = HashSet::from_iter( events .iter() @@ -128,15 +109,15 @@ pub fn extract_and_verify_token(events: &[RawEvent]) -> Result { ); return match distinct_tokens.len() { - 0 => Err(String::from("no token found in request")), + 0 => Err(CaptureError::NoTokenError), 1 => match distinct_tokens.iter().last() { Some(Some(token)) => { - validate_token(token).map_err(|err| String::from(err.reason()))?; + validate_token(token)?; Ok(token.clone()) } - _ => Err(String::from("no token found in request")), + _ => Err(CaptureError::NoTokenError), }, - _ => Err(String::from("number of distinct tokens in batch > 1")), + _ => Err(CaptureError::MultipleTokensError), }; } @@ -144,34 +125,17 @@ pub async fn process_events<'a>( sink: Arc, events: &'a [RawEvent], context: &'a ProcessingContext, -) -> Result<(), String> { - let events: Vec = match events +) -> Result<(), CaptureError> { + let events: Vec = events .iter() .map(|e| process_single_event(e, context)) - .collect() - { - Err(_) => return Err(String::from("Failed to process all events")), - Ok(events) => events, - }; + .collect::, CaptureError>>()?; if events.len() == 1 { - let sent = sink.send(events[0].clone()).await; - - if let Err(e) = sent { - tracing::error!("Failed to send event to sink: {:?}", e); - - return Err(String::from("Failed to send event to sink")); - } + sink.send(events[0].clone()).await?; } else { - let sent = sink.send_batch(events).await; - - if let Err(e) = sent { - tracing::error!("Failed to send batch events to sink: {:?}", e); - - return Err(String::from("Failed to send batch events to sink")); - } + sink.send_batch(events).await?; } - Ok(()) } diff --git a/capture/src/event.rs b/capture/src/event.rs index 32c1b5216ca68..fd109f2594ae4 100644 --- a/capture/src/event.rs +++ b/capture/src/event.rs @@ -4,7 +4,7 @@ use std::io::prelude::*; use serde::{Deserialize, Serialize}; use serde_json::Value; -use anyhow::{anyhow, Result}; +use crate::api::CaptureError; use bytes::{Buf, Bytes}; use flate2::read::GzDecoder; use time::OffsetDateTime; @@ -52,31 +52,48 @@ pub struct RawEvent { pub properties: HashMap, } +#[derive(Deserialize)] +#[serde(untagged)] +enum RawRequest { + /// Batch of events + Batch(Vec), + /// Single event + One(RawEvent), +} + +impl RawRequest { + pub fn events(self) -> Vec { + match self { + RawRequest::Batch(events) => events, + RawRequest::One(event) => vec![event], + } + } +} + impl RawEvent { /// We post up _at least one_ event, so when decompressiong and deserializing there /// could be more than one. Hence this function has to return a Vec. /// TODO: Use an axum extractor for this - pub fn from_bytes(query: &EventQuery, bytes: Bytes) -> Result> { + pub fn from_bytes(query: &EventQuery, bytes: Bytes) -> Result, CaptureError> { tracing::debug!(len = bytes.len(), "decoding new event"); let payload = match query.compression { Some(Compression::GzipJs) => { let mut d = GzDecoder::new(bytes.reader()); let mut s = String::new(); - d.read_to_string(&mut s)?; + d.read_to_string(&mut s).map_err(|e| { + tracing::error!("failed to decode gzip: {}", e); + CaptureError::RequestDecodingError(String::from("invalid gzip data")) + })?; s } - None => String::from_utf8(bytes.into())?, + None => String::from_utf8(bytes.into()).map_err(|e| { + tracing::error!("failed to decode body: {}", e); + CaptureError::RequestDecodingError(String::from("invalid body encoding")) + })?, }; - tracing::debug!(json = payload, "decoded event data"); - if let Ok(events) = serde_json::from_str::>(&payload) { - return Ok(events); - } - if let Ok(events) = serde_json::from_str::(&payload) { - return Ok(vec![events]); - } - Err(anyhow!("unknown input shape")) + Ok(serde_json::from_str::(&payload)?.events()) } pub fn extract_token(&self) -> Option { diff --git a/capture/src/sink.rs b/capture/src/sink.rs index 96a6be01cdb72..e6f4b7bb4e8ed 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -1,30 +1,31 @@ -use anyhow::{anyhow, Result}; use async_trait::async_trait; use tokio::task::JoinSet; +use crate::api::CaptureError; use rdkafka::config::ClientConfig; +use rdkafka::error::RDKafkaErrorCode; use rdkafka::producer::future_producer::{FutureProducer, FutureRecord}; use crate::event::ProcessedEvent; #[async_trait] pub trait EventSink { - async fn send(&self, event: ProcessedEvent) -> Result<()>; - async fn send_batch(&self, events: Vec) -> Result<()>; + async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError>; + async fn send_batch(&self, events: Vec) -> Result<(), CaptureError>; } pub struct PrintSink {} #[async_trait] impl EventSink for PrintSink { - async fn send(&self, event: ProcessedEvent) -> Result<()> { + async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { tracing::info!("single event: {:?}", event); metrics::increment_counter!("capture_events_total"); Ok(()) } - async fn send_batch(&self, events: Vec) -> Result<()> { + async fn send_batch(&self, events: Vec) -> Result<(), CaptureError> { let span = tracing::span!(tracing::Level::INFO, "batch of events"); let _enter = span.enter(); @@ -44,7 +45,7 @@ pub struct KafkaSink { } impl KafkaSink { - pub fn new(topic: String, brokers: String) -> Result { + pub fn new(topic: String, brokers: String) -> anyhow::Result { let producer: FutureProducer = ClientConfig::new() .set("bootstrap.servers", &brokers) .create()?; @@ -58,8 +59,11 @@ impl KafkaSink { producer: FutureProducer, topic: String, event: ProcessedEvent, - ) -> Result<()> { - let payload = serde_json::to_string(&event)?; + ) -> Result<(), CaptureError> { + let payload = serde_json::to_string(&event).map_err(|e| { + tracing::error!("failed to serialize event: {}", e); + CaptureError::NonRetryableSinkError + })?; let key = event.key(); @@ -72,31 +76,32 @@ impl KafkaSink { headers: None, }) { Ok(_) => { - metrics::increment_counter!("capture_events_total"); - } - Err(e) => { - tracing::error!("failed to produce event: {}", e.0); - - // TODO(maybe someday): Don't drop them but write them somewhere and try again - // later? - metrics::increment_counter!("capture_events_dropped"); - - // TODO: Improve error handling - return Err(anyhow!("failed to produce event {}", e.0)); + metrics::increment_counter!("capture_events_ingested"); + Ok(()) } + Err((e, _)) => match e.rdkafka_error_code() { + Some(RDKafkaErrorCode::InvalidMessageSize) => { + metrics::increment_counter!("capture_events_dropped_too_big"); + Err(CaptureError::EventTooBig) + } + _ => { + // TODO(maybe someday): Don't drop them but write them somewhere and try again + metrics::increment_counter!("capture_events_dropped"); + tracing::error!("failed to produce event: {}", e); + Err(CaptureError::RetryableSinkError) + } + }, } - - Ok(()) } } #[async_trait] impl EventSink for KafkaSink { - async fn send(&self, event: ProcessedEvent) -> Result<()> { + async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { Self::kafka_send(self.producer.clone(), self.topic.clone(), event).await } - async fn send_batch(&self, events: Vec) -> Result<()> { + async fn send_batch(&self, events: Vec) -> Result<(), CaptureError> { let mut set = JoinSet::new(); for event in events { diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index 219a8c7fe691d..658ae137735b3 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -4,7 +4,7 @@ use axum::http::StatusCode; use axum_test_helper::TestClient; use base64::engine::general_purpose; use base64::Engine; -use capture::api::{CaptureResponse, CaptureResponseCode}; +use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode}; use capture::event::ProcessedEvent; use capture::router::router; use capture::sink::EventSink; @@ -57,12 +57,12 @@ impl MemorySink { #[async_trait] impl EventSink for MemorySink { - async fn send(&self, event: ProcessedEvent) -> anyhow::Result<()> { + async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { self.events.lock().unwrap().push(event); Ok(()) } - async fn send_batch(&self, events: Vec) -> anyhow::Result<()> { + async fn send_batch(&self, events: Vec) -> Result<(), CaptureError> { self.events.lock().unwrap().extend_from_slice(&events); Ok(()) } @@ -93,7 +93,7 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let app = router(timesource, sink.clone(), false); let client = TestClient::new(app); - let mut req = client.post(&case.path).body(raw_body); + let mut req = client.post("/i/v0/e/").body(raw_body); if !case.content_encoding.is_empty() { req = req.header("Content-encoding", case.content_encoding); } From 1474f999f6bb114921bbd5b45656429c75577885 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Thu, 14 Sep 2023 14:44:44 +0100 Subject: [PATCH 028/247] Fix docker build (#26) It's taking SO long to build on my laptop I suspect something is up with docker. I'm 99% sure this should fix the image build though, as the binary has moved. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cb158e22abcc3..c9e3d8c1621d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,5 +22,5 @@ WORKDIR app USER nobody -COPY --from=builder /app/target/release/capture /usr/local/bin -ENTRYPOINT ["/usr/local/bin/capture"] +COPY --from=builder /app/target/release/capture-server /usr/local/bin +ENTRYPOINT ["/usr/local/bin/capture-server"] From af529b171981adbcf5eab53d1f0663b06f68c676 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Thu, 14 Sep 2023 15:30:06 +0100 Subject: [PATCH 029/247] Add graceful shutdown handler (#27) I rolled out the new image, saw the old took ages to kill, and realised we hadn't setup a SIGTERM handler. Add it! --- capture-server/src/main.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index 88ffc949eea7a..e2232d5023655 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -2,6 +2,22 @@ use std::env; use std::net::SocketAddr; use capture::{router, sink, time}; +use tokio::signal; + +async fn shutdown() { + let mut term = signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to register SIGTERM handler"); + + let mut interrupt = signal::unix::signal(signal::unix::SignalKind::interrupt()) + .expect("failed to register SIGINT handler"); + + tokio::select! { + _ = term.recv() => {}, + _ = interrupt.recv() => {}, + }; + + tracing::info!("Shutting down gracefully..."); +} #[tokio::main] async fn main() { @@ -29,6 +45,7 @@ async fn main() { axum::Server::bind(&address.parse().unwrap()) .serve(app.into_make_service_with_connect_info::()) + .with_graceful_shutdown(shutdown()) .await .unwrap(); } From 6ed0609d952758ac1037a5906e1c5a05dc3401f7 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Fri, 15 Sep 2023 11:28:24 +0200 Subject: [PATCH 030/247] get django_compat test closer to green (#28) --- Cargo.toml | 2 +- capture/src/capture.rs | 4 +++ capture/src/event.rs | 42 +++++++++--------------- capture/src/time.rs | 4 +-- capture/tests/django_compat.rs | 54 +++++++++++++++++++++++++++---- capture/tests/requests_dump.jsonl | 22 +++++++------ 6 files changed, 82 insertions(+), 46 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 68569a3ad2f86..117abaa970f3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ serde = { version = "1.0.160", features = ["derive"] } serde_json = "1.0.96" governor = "0.5.1" tower_governor = "0.0.4" -time = { version = "0.3.20", features = ["formatting", "macros", "serde"] } +time = { version = "0.3.20", features = ["formatting", "macros", "parsing", "serde"] } tower-http = { version = "0.4.0", features = ["trace"] } bytes = "1" anyhow = "1.0" diff --git a/capture/src/capture.rs b/capture/src/capture.rs index e60421759a062..d2f106fb9a657 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -69,6 +69,8 @@ pub async fn event( client_ip: ip.to_string(), }; + println!("Got context {:?}", &context); + process_events(state.sink.clone(), &events, &context).await?; Ok(Json(CaptureResponse { @@ -131,6 +133,8 @@ pub async fn process_events<'a>( .map(|e| process_single_event(e, context)) .collect::, CaptureError>>()?; + println!("Processed events: {:?}", events); + if events.len() == 1 { sink.send(events[0].clone()).await?; } else { diff --git a/capture/src/event.rs b/capture/src/event.rs index fd109f2594ae4..5b0c08d1c2798 100644 --- a/capture/src/event.rs +++ b/capture/src/event.rs @@ -13,8 +13,10 @@ use uuid::Uuid; #[derive(Deserialize, Default)] pub enum Compression { #[default] - #[serde(rename = "gzip-js")] - GzipJs, + Unsupported, + + #[serde(rename = "gzip", alias = "gzip-js")] + Gzip, } #[derive(Deserialize, Default)] @@ -26,15 +28,6 @@ pub struct EventQuery { #[serde(alias = "_")] pub sent_at: Option, - - #[serde(skip_serializing)] - pub token: Option, // Filled by handler - - #[serde(skip_serializing)] - pub now: Option, // Filled by handler from timesource - - #[serde(skip_serializing)] - pub client_ip: Option, // Filled by handler } #[derive(Debug, Deserialize)] @@ -78,7 +71,7 @@ impl RawEvent { tracing::debug!(len = bytes.len(), "decoding new event"); let payload = match query.compression { - Some(Compression::GzipJs) => { + Some(Compression::Gzip) => { let mut d = GzDecoder::new(bytes.reader()); let mut s = String::new(); d.read_to_string(&mut s).map_err(|e| { @@ -87,6 +80,12 @@ impl RawEvent { })?; s } + Some(_) => { + return Err(CaptureError::RequestDecodingError(String::from( + "unsupported compression format", + ))) + } + None => String::from_utf8(bytes.into()).map_err(|e| { tracing::error!("failed to decode body: {}", e); CaptureError::RequestDecodingError(String::from("invalid body encoding")) @@ -108,6 +107,7 @@ impl RawEvent { } } +#[derive(Debug)] pub struct ProcessingContext { pub lib_version: Option, pub sent_at: Option, @@ -116,14 +116,7 @@ pub struct ProcessingContext { pub client_ip: String, } -time::serde::format_description!( - django_iso, - OffsetDateTime, - "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:6][offset_hour \ - sign:mandatory]:[offset_minute]" -); - -#[derive(Clone, Default, Debug, Serialize)] +#[derive(Clone, Default, Debug, Serialize, Eq, PartialEq)] pub struct ProcessedEvent { pub uuid: Uuid, pub distinct_id: String, @@ -131,7 +124,7 @@ pub struct ProcessedEvent { pub site_url: String, pub data: String, pub now: String, - #[serde(with = "django_iso::option")] + #[serde(with = "time::serde::rfc3339::option")] pub sent_at: Option, pub token: String, } @@ -160,16 +153,13 @@ mod tests { let bytes = Bytes::from(decoded_horrible_blob); let events = RawEvent::from_bytes( &EventQuery { - compression: Some(Compression::GzipJs), + compression: Some(Compression::Gzip), lib_version: None, sent_at: None, - token: None, - now: None, - client_ip: None, }, bytes, ); - assert_eq!(events.is_ok(), true); + assert!(events.is_ok()); } } diff --git a/capture/src/time.rs b/capture/src/time.rs index e510f5f692fcc..3cfed322d5338 100644 --- a/capture/src/time.rs +++ b/capture/src/time.rs @@ -10,7 +10,7 @@ impl TimeSource for SystemTime { fn current_time(&self) -> String { let time = time::OffsetDateTime::now_utc(); - time.format(&time::format_description::well_known::Iso8601::DEFAULT) - .expect("failed to iso8601 format timestamp") + time.format(&time::format_description::well_known::Rfc3339) + .expect("failed to format timestamp") } } diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index 658ae137735b3..c39f2c7a91dfa 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -1,4 +1,4 @@ -use assert_json_diff::assert_json_eq; +use assert_json_diff::assert_json_matches_no_panic; use async_trait::async_trait; use axum::http::StatusCode; use axum_test_helper::TestClient; @@ -14,6 +14,8 @@ use serde_json::{json, Value}; use std::fs::File; use std::io::{BufRead, BufReader}; use std::sync::{Arc, Mutex}; +use time::format_description::well_known::{Iso8601, Rfc3339}; +use time::OffsetDateTime; #[derive(Debug, Deserialize)] struct RequestDump { @@ -74,8 +76,10 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let file = File::open(REQUESTS_DUMP_FILE_NAME)?; let reader = BufReader::new(file); - for line in reader.lines() { - let case: RequestDump = serde_json::from_str(&line?)?; + let mut mismatches = 0; + + for (line_number, line_contents) in reader.lines().enumerate() { + let case: RequestDump = serde_json::from_str(&line_contents?)?; if !case.path.starts_with("/e/") { println!("Skipping {} test case", &case.path); continue; @@ -93,7 +97,7 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let app = router(timesource, sink.clone(), false); let client = TestClient::new(app); - let mut req = client.post("/i/v0/e/").body(raw_body); + let mut req = client.post(&format!("/i/v0{}", case.path)).body(raw_body); if !case.content_encoding.is_empty() { req = req.header("Content-encoding", case.content_encoding); } @@ -104,15 +108,51 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { req = req.header("X-Forwarded-For", case.ip); } let res = req.send().await; - assert_eq!(res.status(), StatusCode::OK, "{}", res.text().await); + assert_eq!( + res.status(), + StatusCode::OK, + "line {} rejected: {}", + line_number, + res.text().await + ); assert_eq!( Some(CaptureResponse { status: CaptureResponseCode::Ok }), res.json().await ); - assert_eq!(sink.len(), case.output.len()); - assert_json_eq!(json!(case.output), json!(sink.events())) + assert_eq!( + sink.len(), + case.output.len(), + "event count mismatch on line {}", + line_number + ); + + for (event_number, (message, expected)) in + sink.events().iter().zip(case.output.iter()).enumerate() + { + // Normalizing the expected event to align with known django->rust inconsistencies + let mut expected = expected.clone(); + if let Some(value) = expected.get_mut("sent_at") { + // Default ISO format is different between python and rust, both are valid + // Parse and re-print the value before comparison + let sent_at = + OffsetDateTime::parse(value.as_str().expect("empty"), &Iso8601::DEFAULT)?; + *value = Value::String(sent_at.format(&Rfc3339)?) + } + + let match_config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict); + if let Err(e) = + assert_json_matches_no_panic(&json!(expected), &json!(message), match_config) + { + println!( + "mismatch at line {}, event {}: {}", + line_number, event_number, e + ); + mismatches += 1; + } + } } + assert_eq!(0, mismatches, "some events didn't match"); Ok(()) } diff --git a/capture/tests/requests_dump.jsonl b/capture/tests/requests_dump.jsonl index b62f1c61665a7..88d30102039ee 100644 --- a/capture/tests/requests_dump.jsonl +++ b/capture/tests/requests_dump.jsonl @@ -1,10 +1,12 @@ -{"path":"/e/?ip=1&_=1684771477160&ver=1.57.2","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-05-22T16:04:37.164197+00:00","body":"ZGF0YT1leUpsZG1WdWRDSTZJaVJ2Y0hSZmFXNGlMQ0p3Y205d1pYSjBhV1Z6SWpwN0lpUnZjeUk2SWsxaFl5QlBVeUJZSWl3aUpHOXpYM1psY25OcGIyNGlPaUl4TUM0eE5TNHdJaXdpSkdKeWIzZHpaWElpT2lKR2FYSmxabTk0SWl3aUpHUmxkbWxqWlY5MGVYQmxJam9pUkdWemEzUnZjQ0lzSWlSamRYSnlaVzUwWDNWeWJDSTZJbWgwZEhBNkx5OXNiMk5oYkdodmMzUTZPREF3TUM4aUxDSWthRzl6ZENJNklteHZZMkZzYUc5emREbzRNREF3SWl3aUpIQmhkR2h1WVcxbElqb2lMeUlzSWlSaWNtOTNjMlZ5WDNabGNuTnBiMjRpT2pFeE15d2lKR0p5YjNkelpYSmZiR0Z1WjNWaFoyVWlPaUpsYmkxVlV5SXNJaVJ6WTNKbFpXNWZhR1ZwWjJoMElqbzVOelFzSWlSelkzSmxaVzVmZDJsa2RHZ2lPakUxTURBc0lpUjJhV1YzY0c5eWRGOW9aV2xuYUhRaU9qZzBNeXdpSkhacFpYZHdiM0owWDNkcFpIUm9Jam94TkRNekxDSWtiR2xpSWpvaWQyVmlJaXdpSkd4cFlsOTJaWEp6YVc5dUlqb2lNUzQxTnk0eUlpd2lKR2x1YzJWeWRGOXBaQ0k2SW5CM01XeHBaSEZ6TTNGMWJXaDJjekVpTENJa2RHbHRaU0k2TVRZNE5EYzNNVFEzTnk0eE5pd2laR2x6ZEdsdVkzUmZhV1FpT2lKblVVUkliSEI0Y1VOSVoxVnJhRFZ6TTB4UFZsVlJOVUZqVG5ReGEyMVdiREI0ZGs5aGFqSnpaalp0SWl3aUpHUmxkbWxqWlY5cFpDSTZJakU0T0RRek1HVXpOR0kxTTJOakxUQmtNMk0yWXpjd09UUTBNVFZrT0MwME1USmtNbU16WkMweE5qUmlNRGd0TVRnNE5ETXdaVE0wWWpaa016Z2lMQ0lrZFhObGNsOXBaQ0k2SW1kUlJFaHNjSGh4UTBoblZXdG9OWE16VEU5V1ZWRTFRV05PZERGcmJWWnNNSGgyVDJGcU1uTm1ObTBpTENKcGMxOWtaVzF2WDNCeWIycGxZM1FpT21aaGJITmxMQ0lrWjNKdmRYQnpJanA3SW5CeWIycGxZM1FpT2lJd01UZzRORE13WlMwek1UWmxMVEF3TURBdE5URmhOaTFtWkRZME5qVTBPRFk1TUdVaUxDSnZjbWRoYm1sNllYUnBiMjRpT2lJd01UZzRORE13WlMweU9UQTVMVEF3TURBdE5HUmxNUzA1T1RVM056SmhNVEEwTlRRaUxDSnBibk4wWVc1alpTSTZJbWgwZEhBNkx5OXNiMk5oYkdodmMzUTZPREF3TUNKOUxDSWtZWFYwYjJOaGNIUjFjbVZmWkdsellXSnNaV1JmYzJWeWRtVnlYM05wWkdVaU9tWmhiSE5sTENJa1lXTjBhWFpsWDJabFlYUjFjbVZmWm14aFozTWlPbHRkTENJa1ptVmhkSFZ5WlY5bWJHRm5YM0JoZVd4dllXUnpJanA3ZlN3aWNHOXpkR2h2WjE5MlpYSnphVzl1SWpvaU1TNDBNeTR3SWl3aWNtVmhiRzBpT2lKb2IzTjBaV1F0WTJ4cFkydG9iM1Z6WlNJc0ltVnRZV2xzWDNObGNuWnBZMlZmWVhaaGFXeGhZbXhsSWpwbVlXeHpaU3dpYzJ4aFkydGZjMlZ5ZG1salpWOWhkbUZwYkdGaWJHVWlPbVpoYkhObExDSWtjbVZtWlhKeVpYSWlPaUlrWkdseVpXTjBJaXdpSkhKbFptVnljbWx1WjE5a2IyMWhhVzRpT2lJa1pHbHlaV04wSWl3aWRHOXJaVzRpT2lKd2FHTmZha2hqUkhVM2JUTmFkbTVKYm5CclluaHRTbkpMUldKNVNuVnJlVUZhUTNwNVMyVk1NSE5VZUVJemF5SXNJaVJ6WlhOemFXOXVYMmxrSWpvaU1UZzRORFF6TWpaaU4yUXhZakV4TFRCbE4ySTJNVGcwWVRJek9EVXdPQzAwTVRKa01tTXpaQzB4TmpSaU1EZ3RNVGc0TkRRek1qWmlOMlV4WldRd0lpd2lKSGRwYm1SdmQxOXBaQ0k2SWpFNE9EUTBNelE1Tm1FMFlpMHdaamMzTkdGbE9UUTRPVEk1TmpndE5ERXlaREpqTTJRdE1UWTBZakE0TFRFNE9EUTBNelE1Tm1FMU1qQXpNU0lzSWlSd1lXZGxkbWxsZDE5cFpDSTZJakU0T0RRME16UTVObUUyTVROallpMHdOak0wTmpjME9HRTNPR05pWXkwME1USmtNbU16WkMweE5qUmlNRGd0TVRnNE5EUXpORGsyWVRjeU1UZ3pJbjBzSW5ScGJXVnpkR0Z0Y0NJNklqSXdNak10TURVdE1qSlVNVFk2TURRNk16Y3VNVFl3V2lKOQ==","output":[{"uuid":"01884434-96bc-0000-a64d-d01794a3cbbd","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$opt_in\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"pw1lidqs3qumhvs1\", \"$time\": 1684771477.16, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"timestamp\": \"2023-05-22T16:04:37.160Z\"}","now":"2023-05-22T16:04:37.164197+00:00","sent_at":"2023-05-22T16:04:37.160000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?ip=1&_=1684771477161&ver=1.57.2","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-05-22T16:04:37.165076+00:00","body":"ZGF0YT1leUpsZG1WdWRDSTZJaVJ3WVdkbGRtbGxkeUlzSW5CeWIzQmxjblJwWlhNaU9uc2lKRzl6SWpvaVRXRmpJRTlUSUZnaUxDSWtiM05mZG1WeWMybHZiaUk2SWpFd0xqRTFMakFpTENJa1luSnZkM05sY2lJNklrWnBjbVZtYjNnaUxDSWtaR1YyYVdObFgzUjVjR1VpT2lKRVpYTnJkRzl3SWl3aUpHTjFjbkpsYm5SZmRYSnNJam9pYUhSMGNEb3ZMMnh2WTJGc2FHOXpkRG80TURBd0x5SXNJaVJvYjNOMElqb2liRzlqWVd4b2IzTjBPamd3TURBaUxDSWtjR0YwYUc1aGJXVWlPaUl2SWl3aUpHSnliM2R6WlhKZmRtVnljMmx2YmlJNk1URXpMQ0lrWW5KdmQzTmxjbDlzWVc1bmRXRm5aU0k2SW1WdUxWVlRJaXdpSkhOamNtVmxibDlvWldsbmFIUWlPamszTkN3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRVd01Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPRFF6TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFME16TXNJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpVM0xqSWlMQ0lrYVc1elpYSjBYMmxrSWpvaWJuUjVkWE51Tm5Zek1uWmxhbWN3Y3lJc0lpUjBhVzFsSWpveE5qZzBOemN4TkRjM0xqRTJMQ0prYVhOMGFXNWpkRjlwWkNJNkltZFJSRWhzY0hoeFEwaG5WV3RvTlhNelRFOVdWVkUxUVdOT2RERnJiVlpzTUhoMlQyRnFNbk5tTm0waUxDSWtaR1YyYVdObFgybGtJam9pTVRnNE5ETXdaVE0wWWpVelkyTXRNR1F6WXpaak56QTVORFF4TldRNExUUXhNbVF5WXpOa0xURTJOR0l3T0MweE9EZzBNekJsTXpSaU5tUXpPQ0lzSWlSMWMyVnlYMmxrSWpvaVoxRkVTR3h3ZUhGRFNHZFZhMmcxY3pOTVQxWlZVVFZCWTA1ME1XdHRWbXd3ZUhaUFlXb3ljMlkyYlNJc0ltbHpYMlJsYlc5ZmNISnZhbVZqZENJNlptRnNjMlVzSWlSbmNtOTFjSE1pT25zaWNISnZhbVZqZENJNklqQXhPRGcwTXpCbExUTXhObVV0TURBd01DMDFNV0UyTFdaa05qUTJOVFE0Tmprd1pTSXNJbTl5WjJGdWFYcGhkR2x2YmlJNklqQXhPRGcwTXpCbExUSTVNRGt0TURBd01DMDBaR1V4TFRrNU5UYzNNbUV4TURRMU5DSXNJbWx1YzNSaGJtTmxJam9pYUhSMGNEb3ZMMnh2WTJGc2FHOXpkRG80TURBd0luMHNJaVJoZFhSdlkyRndkSFZ5WlY5a2FYTmhZbXhsWkY5elpYSjJaWEpmYzJsa1pTSTZabUZzYzJVc0lpUmhZM1JwZG1WZlptVmhkSFZ5WlY5bWJHRm5jeUk2VzEwc0lpUm1aV0YwZFhKbFgyWnNZV2RmY0dGNWJHOWhaSE1pT250OUxDSndiM04wYUc5blgzWmxjbk5wYjI0aU9pSXhMalF6TGpBaUxDSnlaV0ZzYlNJNkltaHZjM1JsWkMxamJHbGphMmh2ZFhObElpd2laVzFoYVd4ZmMyVnlkbWxqWlY5aGRtRnBiR0ZpYkdVaU9tWmhiSE5sTENKemJHRmphMTl6WlhKMmFXTmxYMkYyWVdsc1lXSnNaU0k2Wm1Gc2MyVXNJaVJ5WldabGNuSmxjaUk2SWlSa2FYSmxZM1FpTENJa2NtVm1aWEp5YVc1blgyUnZiV0ZwYmlJNklpUmthWEpsWTNRaUxDSjBiMnRsYmlJNkluQm9ZMTlxU0dORWRUZHRNMXAyYmtsdWNHdGllRzFLY2t0RllubEtkV3Q1UVZwRGVubExaVXd3YzFSNFFqTnJJaXdpSkhObGMzTnBiMjVmYVdRaU9pSXhPRGcwTkRNeU5tSTNaREZpTVRFdE1HVTNZall4T0RSaE1qTTROVEE0TFRReE1tUXlZek5rTFRFMk5HSXdPQzB4T0RnME5ETXlObUkzWlRGbFpEQWlMQ0lrZDJsdVpHOTNYMmxrSWpvaU1UZzRORFF6TkRrMllUUmlMVEJtTnpjMFlXVTVORGc1TWprMk9DMDBNVEprTW1NelpDMHhOalJpTURndE1UZzRORFF6TkRrMllUVXlNRE14SWl3aUpIQmhaMlYyYVdWM1gybGtJam9pTVRnNE5EUXpORGsyWVRZeE0yTmlMVEEyTXpRMk56UTRZVGM0WTJKakxUUXhNbVF5WXpOa0xURTJOR0l3T0MweE9EZzBORE0wT1RaaE56SXhPRE1pZlN3aWRHbHRaWE4wWVcxd0lqb2lNakF5TXkwd05TMHlNbFF4Tmpvd05Eb3pOeTR4TmpGYUluMCUzRA==","output":[{"uuid":"01884434-96ba-0000-1404-a179647bd08a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ntyusn6v32vejg0s\", \"$time\": 1684771477.16, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"timestamp\": \"2023-05-22T16:04:37.161Z\"}","now":"2023-05-22T16:04:37.165076+00:00","sent_at":"2023-05-22T16:04:37.161000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/engage/?ip=1&_=1684771477165&ver=1.57.2","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-05-22T16:04:37.167862+00:00","body":"ZGF0YT1leUlrYzJWMElqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR0p5YjNkelpYSmZkbVZ5YzJsdmJpSTZNVEV6TENJa2NtVm1aWEp5WlhJaU9pSWtaR2x5WldOMElpd2lKSEpsWm1WeWNtbHVaMTlrYjIxaGFXNGlPaUlrWkdseVpXTjBJaXdpWlcxaGFXd2lPaUo0WVhacFpYSkFjRzl6ZEdodlp5NWpiMjBpZlN3aUpIUnZhMlZ1SWpvaWNHaGpYMnBJWTBSMU4yMHpXblp1U1c1d2EySjRiVXB5UzBWaWVVcDFhM2xCV2tONmVVdGxUREJ6VkhoQ00yc2lMQ0lrWkdsemRHbHVZM1JmYVdRaU9pSm5VVVJJYkhCNGNVTklaMVZyYURWek0weFBWbFZSTlVGalRuUXhhMjFXYkRCNGRrOWhhakp6WmpadElpd2lKR1JsZG1salpWOXBaQ0k2SWpFNE9EUXpNR1V6TkdJMU0yTmpMVEJrTTJNMll6Y3dPVFEwTVRWa09DMDBNVEprTW1NelpDMHhOalJpTURndE1UZzRORE13WlRNMFlqWmtNemdpTENJa2RYTmxjbDlwWkNJNkltZFJSRWhzY0hoeFEwaG5WV3RvTlhNelRFOVdWVkUxUVdOT2RERnJiVlpzTUhoMlQyRnFNbk5tTm0waWZRJTNEJTNE","output":[{"uuid":"01884434-96bb-0000-669a-d0fa1bef0768","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"$set\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$browser_version\": 113, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"email\": \"xavier@posthog.com\"}, \"$token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"event\": \"$identify\", \"properties\": {}}","now":"2023-05-22T16:04:37.167862+00:00","sent_at":"2023-05-22T16:04:37.165000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?compression=gzip-js&ip=1&_=1684771480232&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:40.235238+00:00","body":"H4sIAAAAAAAAA+2dW5ObOBaA/0qXK4+howvXfssms5u5bXZ2kqmtTE25hBAGGxAGgcGp/Pc9wm5sJ+5sestT7Sn0kIrR9ejonPM1SIjfP85EKwo1u5s9iwVTTSXmccYWc86yTESz57OykqWoVCrq2d3H2TMJ/81+Zvzm7a83/4FsSJi3oqpTWUAGRrfYuUU6PazkphYVJP49rUQsO50YiTblYq76UkDGa1GvlCx1Bm+qCsSYN1UGGYlS5d2LF5kEKRJZqzsfIfRCl9NXUOA0R2eUTCUFy3WzQ8F99wfZMKZHyRkrFg1b6OKisN7/qqvUvBKimCciXSTQSeDZh8RNGqkEGnEQgsQ2FZtSVmos69u68TH5vrRNdXKWhtDNRoS6E7g41tet490SnZ4WIJaapxGklmLT+m3ieSu15q1f6HyV6rFh17c9D8O/W+zi57MorVVa8H29xS+v32Rlt371ZvF+lTg1/entb+9/cV7yfyq8yn/LUNe+ZUtSx25+NBlDVezDEJCgduhQzi0UUe5yDwW2jZ3It2xMIsJpZGHXDpFvHYq7EfV1Y43W6v8hRVrPI5HLOZjZUnBQZcyyWkCDi0o25WBzY9YM7fu1KHaFBTOPLAcz14oj13Yd23cDJKBNWS1YkW6Z2il5rEUCFOxq2ZHAVhA4nkcYRrZja0mKWrGCa5M4a3+zTyAVaxQkloOfgPJZCE4yh6HDlM7rNILK9/IzrtIWnOnIqWA0v/8BWSeOVrI+kyzSI4UOSugtkYsTE7Hp4FGVYFmuhYMiIrJ4lvJVIkHvkCdylmaDHHpGWQtXWrRRmjpjfPWV/GfgoQI8ULvrswj8FdQ9pqbFYh5J6EGLc5R7PAzIqUWtRbYqUQtlwQ89LCin5ErommXC58s3/HXj5fRDW3xflKuwy3+ofvwu7H9oVv3LD6+2/Y/iJ1S/6/5GV4NH7po8mKhNiRt6EQ4xtpDwQhf7NiPUd8Amz9norrzAIhqCxCYtIrk5bs4OXGaHFoo9z2YisP2ABO5DbenCDkEU7wLOAvxHfNGaiymHBl1qu57tM8/nIf9Kex7BPtWmJeMY9Da7o8j1Pj0/CsuDI4BpFSqNexOQnyIgB57Xl2i5UQQnxYYvHgjIrgnIJiA/YUAeZujel+8naUxfif7bZ2xfZ4hIH2fajOCvjaYZ7Okbm9g7/2sRsyZTN/8a5QHxhdbYfB/iVNXAiPd2N2pA+9dWFrqJ9+9eHZnDXA24KXQeTJDBy2PxQgxergsvpbvxky5s1j0Xy00VGLwYvFw7Xk50/hBjvj4BXzLmW2vuHV+HmDprNA30T/BcprHCtOkQRKiFHIuQdxjd2fQOge+BIwXOByg6auR+vvZTpVgnC5n3Wqgdnuo5l42OlBAHTnIPcfKohGHUn8Koa38ylUiwx2+l1X3hvzKxQsqWKWbRIkLI3Xrrs8QilBhiGWJdiFhcFrWEgJ1B30AkWUWaU6I4M8DdDcUYOg+ld78OPndabdYOdn6BR2F7HVl07/4n6gTYlDCWw9AMHR5FBxIErqHDNdOBNhEtVCPJmglbbbwH6ABtGzoYOkyODjupLDCLvFS14cPl+eAYPlwzH1pnjSLlL6jjLRvqDC7wJR9AOoMHg4fJ4aGQSoRSrgwZLk8G/3Rp3ZDhysgg2n6L5NrZkl4G22wIvWfIYO4cDBqmiAaeySayWFHAggMXuY5ihhEXZ4S5e7hqRsQbXottR9cbsqz6qjnPCMesPRhGTJERoJ91ZrBwcSx4ZtHhqrGw2GZq0ybuWsUxdeX2PBbMJiqDhWliAcYCHg+vZbByMGSDh4viwexYunI8LLu2wA3KaObE3Zo/gAfP4MHgYYJ4EKzKeotxDhJY+zxDictTAhtKXDMlMkXiREYq7TbuKm5aQwlDCUOJkRKxEFEI47FqLuDNBsMHw4dp8SEmjpPyjb2sk0UW2Ho2zvDBN2sPhg9T5EMl1g3kWcOgLeghjVM+viJoWHFRVthmQeKqWeFWbR9Tr6g8D9myXp5lBUyuYYVhxQRZcTguat+zlRZxWqRKWBn4gwHGpYHhe4EBxjUDo28Q8sMO4y2mee46BhgGGAYYXwFG3eQ5q/qBF3BtkGGQMTFkuH7bMrlssNMsENmePynWIMMgY5rIGHusB0ZYLbH2ghpW/NmsuO/IAOJJX6hbrgjE0UhtVU9owc4DwixYGEBMAhCTCuqeb7vUscDzYofhOAgi14sfbs/zHUKcwTYPQf2zRYbrPSx2cpF9bfOaFQ6raNN3DT6/odVGJrKbyD6FyH56Ou04TZ+dTHt+zsZC+6Noa1iE+FqwmdyRrpcgCTkhCRimjuh6wwCcBT+PwdZ2O44NUZ5uc1PTeSlZ+UiRpl+z5CxRXGKIYogyBaIcuzcr0xdhmmV61aEl+oMDuQCFabv7x3fv4DJqqv28Uxu+lAFTqhrQrY3AmQ0qHocKF52gYpzpG/0M7yYWiidm3eFpUbGMCtalXom7xmdh6Z5FhReYc5oMKqaACq2e+c7+KYbwv1+QPtaRefj0aA7Y1JzEdNUUyNrS7VAXLCqOcG6fO8fVv0WeeVvOUGAKFPh89TliilmiKzO5uzeADa61dkNzsOvFUYGdkxfnIlYnoWRVdKNNVt86DGHJwOIpd7du2NIJWhpm/jqo+bnXIXz4yJ1hhWHFBFixj0s/62dKNfRRaGX5g/nfx67vwfDgXsKg4HEoQK5/jALdh4huRq0aCDwlBDYhwbRa1VHHaxkm9CwEAmQoYCgwBQr8jy+W0lvXcwJC9RdLwZrqhFXwyON+ZGVaFPpyJ/XQlBYtl4NM0e6z21CzZnkJ44c17Xz8VCk4mL4+urwXJYSvm/YwDTAfu4bHyDn4gH64xWr1bxHDPYt+DH8qt3uHCLjyre8TLTQMXhd7qaMRtmHhUEFIjEYpIIRUQunNRCD3Pm14ehYL3nOQ+ZAWNzDYbKxpuPhYLmJKyKc//gu3Hodi74cAAA==","output":[{"uuid":"01884434-a2ab-0000-af34-b060321fb9d0","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"pewv8vh77ktqcv8n\", \"$time\": 1684771477.161, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"session-reset-on-load\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 3067}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0001-adb0-d108b4f12665","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"977yp0jwt21hnwcg\", \"$time\": 1684771477.166, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"project\", \"$group_key\": \"0188430e-316e-0000-51a6-fd646548690e\", \"$group_set\": {\"id\": 1, \"uuid\": \"0188430e-316e-0000-51a6-fd646548690e\", \"name\": \"Default Project\", \"ingested_event\": true, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 3062}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0002-1714-12b5f9fc4941","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"p6w8hxbuqycejwr9\", \"$time\": 1684771477.166, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"organization\", \"$group_key\": \"0188430e-2909-0000-4de1-995772a10454\", \"$group_set\": {\"id\": \"0188430e-2909-0000-4de1-995772a10454\", \"name\": \"x\", \"slug\": \"x\", \"created_at\": \"2023-05-22T10:43:01.514795Z\", \"available_features\": [], \"taxonomy_set_events_count\": 0, \"taxonomy_set_properties_count\": 0, \"instance_tag\": \"none\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 3062}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0003-6004-77b16abfc46e","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"b3aji1adgd006z7q\", \"$time\": 1684771477.232, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"posthog-3000\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2996}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0004-6b5c-d8e7d6e97bd8","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"3ud3ntuo2qae4tw7\", \"$time\": 1684771477.233, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"enable-prompts\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2995}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0005-3e04-fa6fc146673a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"v5q0dt8g357ju35s\", \"$time\": 1684771477.24, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"notebooks\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2987}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0006-5a0e-177fb26860c5","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"evyz0oq5z2yo9zl8\", \"$time\": 1684771477.243, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"cloud-announcement\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2985}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0007-dc5e-bb723cae292a","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"fwcsezx3qw2jryru\", \"$time\": 1684771477.252, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"hogql\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2976}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0008-5f7e-6efafcb7b2be","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"gzltwvh6qtff36oz\", \"$time\": 1684771477.266, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"hackathon-apm\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2962}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0009-c180-af233b44b000","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"gjxvn1u0l3l5fxqc\", \"$time\": 1684771477.267, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"early-access-feature\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2961}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000a-bd12-c96d4565d278","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"lt2fhodtixw6kfuv\", \"$time\": 1684771477.267, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"feedback-scene\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2961}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000b-02bb-7a705b509049","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"f255icw4jshgl94d\", \"$time\": 1684771477.282, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"require-email-verification\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2946}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000c-e137-db1cc0161182","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"6rvyf37nr7704osj\", \"$time\": 1684771477.349, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"session-recording-infinite-list\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2879}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000d-7bab-cce00f8e396c","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"yu008bx11z13mm65\", \"$time\": 1684771477.349, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"session-recording-summary-listing\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2879}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000e-4ed9-e08a5a7ecb60","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"68vvaoju15ug02zn\", \"$time\": 1684771477.349, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"recordings-list-v2-enabled\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"188443496a613cb-06346748a78cbc-412d2c3d-164b08-188443496a72183\"}, \"offset\": 2879}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-000f-15e6-c0c8a84bd8d3","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ejk2l0xdtzty23na\", \"$time\": 1684771477.382, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2846}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0010-a367-b217cfce9758","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"q4csan5ar3uyxu1c\", \"$time\": 1684771477.402, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"instance\", \"$group_key\": \"http://localhost:8000\", \"$group_set\": {\"site_url\": \"http://localhost:8000\"}, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2826}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0011-ab12-d83075b735b3","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"client_request_failure\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"fux7i2k80t2uyqah\", \"$time\": 1684771477.622, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"pathname\": \"/api/billing-v2/\", \"method\": \"GET\", \"duration\": 341, \"status\": 404, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2606}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0012-7b5a-642b99f7c343","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"recording list fetched\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"jdnaxi7p1xu8abp6\", \"$time\": 1684771477.793, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"load_time\": 311, \"listing_version\": \"1\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2435}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0013-9ed2-da153edee668","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"lvp6x0x9grc01m4s\", \"$time\": 1684771478.077, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"data-exploration-insights\", \"$feature_flag_response\": false, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2151}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0014-b658-f962d9c6de38","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"dashboard loading time\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ywaj59v3bl8q9scj\", \"$time\": 1684771478.16, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"loadingMilliseconds\": 816, \"dashboardId\": 1, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 2068}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-a2ab-0015-fdde-825234a6f71f","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"viewed dashboard\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"wb213rksdxcsobh3\", \"$time\": 1684771478.906, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"created_at\": \"2023-05-22T10:43:03.675923Z\", \"is_shared\": false, \"pinned\": true, \"creation_mode\": \"default\", \"sample_items_count\": 6, \"item_count\": 6, \"created_by_system\": true, \"dashboard_id\": 1, \"lastRefreshed\": \"2023-05-22T16:02:16.882Z\", \"refreshAge\": 142, \"trends_count\": 3, \"retention_count\": 1, \"lifecycle_count\": 1, \"funnels_count\": 1, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 1322}","now":"2023-05-22T16:04:40.235238+00:00","sent_at":"2023-05-22T16:04:40.232000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684771480251&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:40.254254+00:00","body":"H4sIAAAAAAAAA+x9i3LbyJLlr2A9M713IkwKb4Dq7ZlrS7JlW7Lkl2xr7oSDoiCJbb4uScly3+iI+ZaN2P2K/Zr5kj0nq4ACRYLiS7Ye6LaIAlCo56msrKzMrP/4x6PkIukMH60/+udBp94bnHWHjx4/6vW7vaQ/bCaDR+v/MG++HNeHdT4Zfu8lj9bDx4/SB73W+Wmzg1R63cHwrHu61kmG37r9r391mFr9e6tbP2Y8G1E69YvmaX3Y7Hbwznm07oSxH0WOH4Vu7D5+5CLK2XDYW19ba3Ub9dYZUlyPbdteQ3Tv0br9+JEvv4H8hvIbyW+MxHCpqYuDzOSKPHhBwo6DK9KQK1LxgwABJKQCSCu0ax4K4SDBNIhUr5QZGUgR15wq6+ci5dDxwghBvKq5CHjINHA8BPCy5jOAfNUrVXxPldxTRff4IaP5LHW+Rf788/GjYbOdDIb1du/qu8fonGQwQLm+NNHAKGzs+54bHkXHzpHjVOwkOgqd2K+7XhzYccV33GO34R1XnNA/wr2JnzjJsY3K/PO3Zue4+y2fnF8L6/5RxT6JIr+e1Py45tbCorQYOXBt1B/F7n5NBBRnjS+/bzc2z6O2d3jRedHpfT26bL/sv9o6+v7y/Ov3J4cbf3x/lezYg/eXT72v+PK4ORg2O42hKsbpm83tVu/y7xvbpx++ngUDb2fv4MOb4Enj9dD52j5o2ZcXe/Xf3cFJ2H6EJumenAwSQNqNvdqfj38CwvvJoHvebyRz4Nv1/DX08LDZWEP7J5fVxmCg8B7YKMVExMsboFNBXiN+FPC8qDFDuPOCNII4VmDnvYKfgLzV7LDtBd4a1hqZLPcoMKM4LgQm35XAvK/A/P0GcUmyPI7LQaPf7LFtJiAzAt0ukfnQkdk4O+98rQTv/M2P0c5mDqKYbgsgijeLQDTUfMIcEA1Aba9CtDYFouiAEqJTIYpBfEchevh089Phh71nNwpRfDcnREMy1SVES4gCop/fbvjh3sdXNwnRCAudeSFKbqOEaAlRQPTdi41P+59ePL1RiM4/0YflRF9CVEP0w/sPb7d3t4ISoiVEbylE/e0d7+3L17slREuI3lKIHr7Z/eBGnw5vEqL+ZGHoNIj6kFOVEC0hSoi63rOtt89udkXv1+aH6ASJfQnRhwnRNy/e7e8/eZlfLnGOnQxRvFlIdM/AnKL7cYjW7GKI4l0J0fsK0e3tV5+3dl59uEmI1uYXOsXju0slRB8oRKNXT7Y+bx56NwrR+Sf62jgvWkL0gUL0yZNXh1u7uy9vEqKL8KIlREuIaoi+2t45wP5ndJMQDai1NydE/RKiJUQVRD/522/fHjzPT/SgeQUQxZsFIBra8/OitQkQdaZA1Ckhem8h6j3fCg6fH76/SYi6wNucEHWcCeulEqMPE6OH4fudg/1P+SX9yjEaoX7zYpTqUSVGS4wSoy823ODZh928xmhJR0uM3iqMvtuIDqOdd3l9vFVjNAjmp6N+SUZLiGqIbn6IPu+/3r9ZiC6wqB8zpysh+lAh+unNc6iSvN+5SYiG88/0NDstIVpClBB9/eLJk+1X/vaNQnR+25AJ5kslRB8oRD+/fLWx+/L1kxKiJURvKUS3Pu692//4bOtGITq/ymjAQpQQXSFEozsLUX/7/Zb3afNGqegCktFSMFpCNKOiO29c99V+eKMQnZ+KhiUVLSGqIfp+/8XHj/sfb3SiX0SreYKyUwnRhwnR50+eH7ze3subL60corX5IUo3OyVES4gKFd19477dfnuTO6ALKd6Xyk4lRDVEN/bD/cPXBxs3CtH57ZTjcYckJUQfKESffHi6Hzzdz1vYrR6i8y+X4nK5VEJUQ/TZy2fh6/0dt4RoCdHbCtGt5y8O3715e6OK9/MvlyZZ2JUQfZgQ3Xn19KO3v5+zsKOH8ckQ5ZuFIDo/FSXhvQrRYsfhfFdC9L5CdPfVvrvlf3xeQrSE6C2F6N7O3qtnnz/6JURLiN5WiO69DnfebJUTfQnR2wrRzb2dg48bh/s3CVFnAeMle4LsvsTow8Tok93wxd72Rs6W/gYwuoARqJxdUmK0xCjd4+193niyu5dz1VxitMTo7cLok4/P3Rfvgs8lRkuM3laMvt5237zffJ9TJSkxWmL0dmF09+nu852ddzmf96vH6EIOH8o10z3HKDYRZ9UaffZhe3Nv89ONYnR+IzvHKe1DVozR8JZhdA77kP03n5+Ebz7eBYyWdPRhYjQ63Hz1/vX7FzeK0flF+I5TyvBLjKYqT683nrx59fxGZfglHS0xupTO06a7vb/5NKeWt3qMhvOf/O0EE/TySow+TIy+PXzx8p3/MedW3CE6JmMUbxbBqL+A7Mmb4N/Jm4JRr8TovcWov/9q+7Mb5lyQrR6jnLdXIR8tMfowMbr3dPfZJ8/NK4+uHqPznyDiyCERJUZLjNJp86fP7s5HL3fiYonREqO3C6OfNt3Xz6NnefloidESo7cKo+H7l0/397yDm8Soy8C8GJ1gElpi9GFidO8g2Hz/fCfnE7fEaInR24XRCL5JPr99mdfNWz1GF9EpmeD+vsTow8To5v7bnfebbn6fqcRoidFbhdFXLzf237z9uHeTGPUWmOu5zioxWmJU9pmehB/ePn1zo2umEqMlRpfB6MeXTzd2t97fqOzJW2Cud8u5vsRoas+0teluHh7mPUCsHqML7Ne75X59iVGN0YPDD29fBs/y+qMlRkuM3iqMbjx//mnr6YsblT15C+iUuKVOSYnRdM301D18uXWYOwi8xGiJ0duF0Wc7z8Onn9/e7Lp+gf162peUGF0lRoNbhlEsimfdC/34KtzcfJ7z4Hx7MepPwahfYvTeYhSnd7559/nwXYnREqO3FaOH+wdbnz4d3qhOSYnREqNL0dFnwcFbN3h9o/ZMC8jwvQky/BKjDxOj4ed3Hzf9VzfKj5YYLTG6DEZ33u8cRk+e5d0431q70BKjDxOj0eaL3c03L/K2yxBcFWAUbxayr2dgToz6E/jRYApG0QMlRu8pRj+/ef56d/tlXqdk5RhdSO+pxOg9x+gcdqF7u+8PN97n/TivHKP0OTL3XD9BN6+c6x8mRvfe7+5tep/z8tHVYxTfzT3XI1BitMSoyJ6eft4+fP4xL3taPUbnPxLc8Sf4eyox+jAx+vyT9zp8tXmja6YSoyVGl8Hoh81nrzafe3l7ptVjdAH/o5RXlRgtMSpnNrx9snsQvczr5t1a2VOJ0YeJ0a0X77aebO3lzxBbPUYXmOsn+R8tMfowMXr4ae/d1qftpzeK0QX2QikLKDFaYlTs69+9e7/3Jszv168eo9jXnBuj5X59iVGNUdf/cPh073Xe5q7EaInRW4XRaP/Dxv7hft52ucRoidFbhVGcIbYbfdzPn3O3eowuYHMXlHpPJUZTHxCu9+bg4Hnej/PqMbqAfDQo5aMlRlPZ0+bGxscPezeq98SjIefFaFjqlDxUjNq2nWK02eh2Bmsn9QsGKsfJRRWBf7/4zbVddKhfcWspaClOnwxavAHa5ApgyZXAVQEDXXWfpkX4RjHlUUgvqmkBvwPcUBca6aZBArk7PEv6KY5ZnzWnyqbI4dmNxuRUqGchnvmuxPN9xXO912sllWH3vHFW4ZNqr3P680Dti64+Ma1DJaQfFqR79SYyLNKdPmn2B8MKUDpEsU7OW5U0OlAV0rstsQBIXEFCLSxGAt6VSJiOBLAzt4q4FSrSk7gdJ43mcbIGEub90uz95vzy5bfs28gJnV8ukv5vTjWIqq7CTRwVUTZ5AwIUR+D/ACy5krKpgKFscax5TRVAYjVb85sqQMrGg7+EsEmAdO2y3WId+snfzwHMAgIX09mokLirFI61KcK1vCtxfU9wPe6I7NWLlxufXhxmCyW0ehGI+WaBhVKNh4vOu1C6qkcSOVGhfYi8KyF6tyA6x1p+w326t3fw6g5AtHApL+9KiN5XiPru5pv3z3Y3bxSiC4ibrkqbSog+WIjuvfz08vXuZ2MdchMQnd+CiYc4lRAtIUqIvt9598H9HJrNzxKiJURvF0RffN799ObJobENKXnREqK3C6LBk8PtD5tP35S8aAnRWwfR/X7396Qx3O62k179NKnsPXvhHXzeNufWxzVqw00GK978oCl/ElhL8dODAGtuW7OfNLr946RfuaDuPfYB8sL+ml8EVHkDhNVoaAQwyZVgVQED15p4fGBaEqCwnwRZQFvTzslqXujrfcw0OA7eK4L+mm/TDo9AHtMtjVy3GMh8VwJ5KpDd+PYCeQAkn/eQfFLVSVYb3faaxquox03EK98sQFgd255s2XSSDBtnkwnrmH1oBBgV4xHvSjzeKB7Rg+mDs35yUkgSzzBjI9FvzePhGe186azzLGmeniHb2OfxBRN70HdK9YjrehCNt0wPggCkDzrdYzxIX2DINc6arePXeIoU/iN9jkmoU0dvsqvbLeZzftRqNl6wFrgbfB8Mk3Z6x7q5LGCW2bB++nrk8/pw2G8enQ9VMVv1zineoU1Ry4n5j6aRgMhdSaPoQ0BumFwON5TSBj7+W8fS/+mSiqbJhFyGzWGL8F0sm5Rbtf77v/63tY8hsd091RlGf/6nCsgRKoXF1LFlrE4oXjtB/11tSN1HF83kW6/bJx60tgoeyjD8DYqDzUZSkZvHVrPTHDbrrcoA4zb5zRlrfl3Q2iwF5aidvaSNs3pfwPzofHhSiYtydqBHMCVrk3kOb+B3RuNZz5TW5MD6xdp7V4E+VzuxcH/SPD3vg2vjmzQdGVjXVlU40wlVbTU7HOWjVe0nLby6qunGUdP8gzFAjexL/OHJVHI6p+JcUYvm8D6lhjlszlpDXSv10aNmG+hfQ6lyFfXcS4+s8OzVTPVd5cvZ6xjNVMeiobVUHZ3wEuzNInWUL2ev4/RBqchcSoyLhuaUmmLW6A8b50NLV3n++kzUUS6ojTvjOHeL5pUpNWnXB1/TITd7LQb1k3q/Wek1O53kuDKsH1UHF2OqqY1uq9tHev/07JkdeBuFtZuJqrgLUJV2vdM8Uepbs1Rt0ASf/y05Sj+brW9mohhuEcWYNlEpKtbuHjVxQbkqeFBJJ14zdaXzZ1EBZxruiutdoIDNhswRFXk0T7GuG6EjY9QrGqPZsn0xRuTdxtsX+++/7O882dja3tvZ3HqbZuimfIh33eDLDUAvB+UZiinfpItV6d3zAYQj0DqUleoKahFktcgNn0KCmGvvIrwWVEST/Bwg1n6vX9R17BVUJM4qkoPzzNR6+L2VDM4ScFRTKEFOqIrFUHJZbQwGRej1pqM3ZaJzLeoXIpiFK2pQJrw2qRyFmVdbJwBQ9xTyjYH1Dws9ktT79U4jWbc63U7yqyXM7boFWcO//GqpBee65fcuf7WOlGyuXz9ung/WLY/PGuf9Qbe/bvW6UOZO+r9af+YzWD/pNs6ZTfd8iPbP8lAprVvtBEm19cPRL9dB2P5AXh3IrIf9euMrUhnL7AjPT/vd887x1JSv5H41I5DOr81hZdCC8nEfDO55+2hCy1j/o9nmsqDeGZp2caQR0iaTm7QEdu9ypExZuwVs2HzJ+6dHf3GCx5YL6Q5+wn+9tllHGucWl7dV/46ESPOh65/0rXE8INc0crfXhX43XoLgYWHV7SCnpIXxdpH1mYqSGhEgKiQCzGQdazGmVznCeP36ay6F+tGg28KgAQh69UZziKjOr9ZFc9DErCm3Em4hAlDWGZx0++11FUTWyV9QvsdWxcEFtTzqXlYGZ3UIQqQR6n+xH1v6X9UN/1Wa0MVfgD+pmKSTlqTVQjRvgLY6ajYqR8kfzaT/F7saoSUrdtV2JRX+1CIkNZhc5+pZ8/g46bAJ0/rYo/VREYqrg3rkO+do2Ml1UK5FT1oJalBvNU87FTA+bSChgQJc7Vx8P2lknjRbrRQp3mMr/UHmKfiEpFxJqVpvsLuRoPk+Rvuov6zkN4wUg3V8DxZVlcOFRb6V/qAoeTzdOuj06sfHzc4pyfjVAbZ6JNX7/e63kd4wbflHRebKdTQEqITKed3626O/PWKbDIddJFqp5WYXme7WLXzfPM4eZnMSWkd+TK1aycmwIih9bI3e8zYtGp9qYtNXlPC8AwGKTkMe5RPJPZD7NBn9raSjkswnw4kW1e1h2fePjKw6KK1URoFIkUz1I7AYJbVMOOu7PK0OJc92vY8NjDzVFIBindX4Omn08rdy3MS2mh4l3W+/Wr+fQ4p68j3FAdoa/Y8Bgb2QhN2dzhW1q7kMkhYS6vZJwK/JCB+ct5HWdXnlUsfGSQ/Tz+hgnVKNEyRZodwAbYyRU0CqxgowQsOu5jwY9rudUxRAt7p/tQlMVOnn/yCT9dvfHslM/LdH/4kvc12U/eUG4dRkOpjJk75KR5cguNrvxYliWUDRJYnn1aaZsbdma8ErXajhKgNPI1gzK2lxUkApDI8AeoQwpeJ2U3sYRxfHOgtyza3ppm6VXAOQSvaTdlEi9Tza9Jwg4xi2aF0l3hxlG8cSWF8/SkAcOWnl6Nt//5//RxKniqfoyUjjXE2GDzvdIVO5WhydxrDbS8l5bla6Orte5bv63dbA+jekeZE11khp9KOxzpuUwro24eQ6w6SWI61TPm3Vx7/MU9M/uQwavJO1zvqwf57oVVG2zPadLHTt0ipdUhWtto+6x9/HV1QUbMs6kVtsC+yIZKt8fyaRj7/YElpWfLm1M2Sn/caiehUFq1e/aA19U9IUv5Z1bk7ONEXsk9+B+tGinyDDZDBd9HO1oEVwxPiYLPnpd7ErOdOm3oQkFv3uUQNjlbL4J50OeOAGRgXmqfxNpaI4w4WLNjGLL18are4gmS1RSJMLE91J2t3OCyWx1rvnj5ykTdGO3j7Xt1xj4IbUHXecrJ52L/HAtmzL9fEPT2Hn22Gienh9+/at+s2rdvuna66MLymHrGrrWERSFaPeQiVQtn6znrYTRi0o2mw169VR4KtVIxh2nZoVVn1nx4mqQc0KMLVjOmCQT3EfqIB6hpeIITHVU3nkeBKWx3yCfyqsniOCiv6HaR2s7PronQ2R1U+iF4N3B8/zJDtQY3nscSahC7It3KCIBHKPWtxGTO7h993e0/p4aeaGn0rny5dBciogv3Jfkbltddmc1dtH5/1TVKzE+GSMtz3Lic8c76LinlUc74+2XQnOHFvd2ryNLlz1/o+2E1s1gLfiVYOYPzUrC1UcgFj9cFzg/2UQHfoTER16mSpC0b7ahJ37K5hodU+7120/LQ6X0d5PgRJXHTcYBYsrt6PwcDzbkg3nmfAhkdYvRco+KapTq9XW1OuZqjNWmSWR5bhVL7JjEjw3DuNGRS5gg7wIQHHsIEzDXs0NLLtVqYZxSBB50NJEdMeuBfjIqblpmBpXlo14tdBrjcRm2rWKJGhJ4lF2o1OX6LyV6EgvlKRVSCVsScI54F7U+3+pVI6w2DyG+Oo8+dfZ0Fu0c3d9W7lB1Q5rSzSVc11TObmmQt+YpuLNeFMh+nhTuatrqtyuzawEqwq9twBNhfI4fq0hdfSCauzFPsmQ7RNzVc+N4xgVC2y/BQIFP3MOqRWujapfs70af/0qtPzS6GwTxPb13RnpGj9D8/outJbZmhGcMPmVqufHYXaDS0BC6WHdG7sNNA0k9OjH2LVBD20/qLou+jKwg1aElENM+uqKpvVD/AU1JOinsSRw5kPG6KGorBopA6uFAqKSgSTFzkQhWxUHgQBaqirAsrJycsGztBlQTel/3kkmQtmrUYwOnlZgJ0Cnxy7YFBVYTZFrVcdzHRRPBbyRUuOSL7YtqM/KjYKrYqM78iW/Wm4Uxo4wt6mrdLkvP7p3mXS+24OLii4/ehmJol+RKgqELPHATaFU0SmmV112xMkVHUnmSo4pUxUcmUIV1Mdgg2rpzN9dSLliFKywthiebuCCj+Rlcl1lOM9Z2YLR/T1ptbrfZhvfUdFSccr4Dqu1WhBwfLsOGohjBe2tLvlWS+E9gm4p+AU4cteveWwy3+NoDWSUyoiV0JkaNmwsDG2SM76Sga1iqWF9UanWPM9nMzEe3oOyRpGHtoqi0AO43Vh6y+YXTmR7FbcaANJ4D9IBkuBHfogIkWODt0Kr11ybc1IU2iHqEsaRD7LioLQB0gyCCOq76kYuziD/DN+ZVyg6vvZkoDEmI2JZIWEdcaBS1A91ivquoG+/Jt9n7NjcQnw6FzH7BLHoRBCgJTyhY+wk6UM362pQKf6epcRcyBMnMDVTsA81gQL2UwqlI+vrrEQVLO9MTVcklZjCHjjVMPIBHIz8ABjELAyU+fo6F/2cRE4mzFNBALVyzFO8zlitInWw4moxR8wnXAyDpM8+eak6zFOFCzQViGM0Y1WUks7Y46JFx5SOA5sRhJQEAIXBtZOLouD6stIpQjVD1qETB38/OZ5x8BdJEa5DMAYrIBs6AZgFzw/9UWYBzZCNxWwo+hjlNHKQy9IFLxKzXl9wNfTCi6CKzhjh8qa19EUljS8dj9kj7Xiy67rj80vlxaqlJD1jzHVRbWei0ROngnkHOPbQXYgJXEzCgReBCDu1qMZG8WxwjC6a1gkbLiZZEDfbQic7RHYEr3/6BowMCB3mSMStORwEMdTC9OOQsMgihWdIF0sbMOMqn0pctb0AfYBp2LZdZIds4gYmQDJ/BB3MGKshTMxUkPOybXMdCDgGIK/VCNO7foqcTZTZyEg8P/MThtWoFkVssADO3huoCPxpo0RosRgLQPBEbuild0Bd6KImqCAqWuNbeJVXj102chbJRRQHizzEVY8tlVJ6pyIhH6YUqpe+eprmpu/AxQDxZHvUPZCNEYKFJzQhnADvwB15SECi6DvQZnwP8Rf6IYaQFJXy3PROx2GjIxkMDqajnkLLNxeHNMEPuPhMS6ASSu9UpBm7poh9Ke6aCGKMWixYlq4BmgBFQS0WAlI49py+86pgTUDZgE2HVDuo+lyeoow1TGb6Drh1pP+qPtg1EMaA7CWW52wi8J3qBtyN5AMWz4lRVTZgFDvC0Qeh05KCgI+p1RwKWZAKOU4s09kbTiy4gK2Yj+zcyA7TO/SxH4YCnlCvDzwR5sldNibBZYUBOx80i/2A4usbkHAuii3wSCFFCIBTFea/5E89NFAaZkWEx60JvYtg485yqVJypLdwyxogR7BnSIaMbgDIITvMlT4xGeAhFk6BR5Ej72bs5vlnbLAYkJuhquxBl1TaFhwGke2jdbHqRZWw/pSaoEXBcrKSoG4qrPpPeHpQGdUs6Y2MA5IjNCmpnx/WXAhW5Xqmr7zH8oe3tShwsX0od2pw4E7i6jtkH9hc5ajHHEFRLUYLSRciG3S+vquGtohtUNSIHJZjc++hGgY1dB2KOCtJm593JUeONoHk2IlD8G6Y3+3AEWIdglxxMtAP9OyAlbUTRmGYmy1qmC7OdHx9h+5BN1xwHS6xZyz//EwqhF4YgWbckyRzYJAqyihKCbI/kR5nBDgXRVNjTYw5rDJarInuKClOaa/KZpQQk7RcpcMkiRkZFqIzRoVHiPBEGjxCgkcosCbAgvuU/upcZiW/RUaM0ySkNuZ6TAVoFqzyMIsjb6kNJJgYlzHWEeARZKWAyTu4kHkEYyMlpXDowmmeRAj0RlbhiiIp0R4Yi5CcWQwaCLqjOFD0N6lTROYP65MqvBPoz5AX+FP0EVDogDSBj0lv0LfSzuQJSQBjH/sm6kZkP2BIsCqAaFXGuIeSxhAcgLYEYGiAcTWXRJzewDmyeUP0aHYTe0SYj46SsoCsqDB4UafmD2Ru0DcoVxhhK0ZugBd5B7zwVvDCCSXGOlemBE6zxKu+oTQYhIwP4yidlMkP6fdqQgKXxRTUQxkOaQxpQLBWBAuTTfmDLItZsRJOXoxN3vQMVWyGijA2beMQSqjflt9+PO9AL7Y/oKEqFMThJ2DpFL9V4L+9Bd3H0YQrUHKd+LACXazj5KgOfaez7rcZ9QagZsbdzWn7n6JqaYKVCpPR+Y4819YAI1GpydqrcyewYhR+y53ZQoqHiREC7DOwFFgHY2DhL2pUaxQHytjHxg1og3BLXAPpmUfmHYxpbBmlfwP+kKWTp5ATqxsJ6D/QU2z5yKRDGXENpAPLLKbcQq7I/IKlCPAY4lKyusLvgqZggDc4nrFIIXkj7RQZN/8G6Q15t/RKro4UUz1Mb9K3y+wa13Lc3phyUKoTVFN7yGPfFi0IRF10hlHx5UsaU6uN6aGvPBdU+PkEylA5aSYtOgeADl8jOeu2qIQBTSh5Wa1WidM6hKV4VmjsrvbKGbLTUJxpd8WZEkhs7PiL6rqU5oboFi5P6ah2YIlCbEXpw2pLByhwiqGDqNIqhdFeHwbN/e/QGh+qB+4lfTXk2xsKns0Tbf82qCCzoZAJidI7qzTqveF5P5HYUpTKeacPTRgqhua+a4BQEUP2CuhoajaRWkWwOiUNnEADJ1EB9JzoZaGszrbjb4AMchuYAmSwIGBxwdB67/hanjLarktVrAPX3vYOnNoOVbQOHGcjgICuZgljzXhYyx4gOXI7IG+w6rDwGIm9Yw58BF7qAJE2wPrLJ/iWj/HPOXCiHclkFy+RNH5jRoFsysJKHvty79QTvj1w4m0pweFsJG0yl1Mr0jC4fgp/Wj+Glw8TBKfQprVL/kkbXx6PPEkNTagTrUc6veHQ8HnasBijwnQaoZxvZMrFMHvMCFihFuzDgr6SNxGXoklVWWpWxM7jRAw5dqZVy1N95ifUSat3haJqojv6RrVMfQAF/HP4TZqjbR5ir3ORCtGQ6N9R4w5eNhqUkMgSlXTKB/uHpZc9YJAQSf9RmsgHeC1BXuUfVD2QZkOkjUhF6+vFlXigAhaCFpey4Pp4owJ8RiURB1Iv7sSTvaPgEVJLyz/jhga40Cp4TQuL0oFrCcnkvgiEG4wX4BJIPHwdWLIxjl+KRJkc16JIiv8vB+7Jq0PIBEpScgOkZPJGk2NnvCU0NhagJe/A8O0rAx0opfdhlHWFpHDaAZt5jvnn9BSttTSbOZIjrLzRJAmN+sgO5qa2fRgIdehyylIuw6zmYDAz+Zo2F1/J0mrPuBCeVqedZAgGGSzxV8zdaRCMOZfZCRcZaa0urf91Cac3Sf+vOR+K/zb7LP4pncXhfipzfJXNJlByMsFCU5FyGC4+DOETa/LzbCkIoXU2uWceR8KizpgGqhedCwyV3e5xvVXs9GwR/uEdBFRYQVr6CsXs+nc4KVjF0NYJKvcSK0wwdaUzU4rnY34DV1cQWmvAAnT2Eat9UqfjFm6p0r7Le7K64r1l5eVXKwphDD82h2eszhMxiNVrjQlvKliRHycnWCnPSCDHDQC0PCZjSCeUJp8/BJV0SAnBwuhTGvadD1Khw+g77URh5BkFtsqMfvQ5WGHx7jX+lCLb1PFXfvKTiY8iEbojYTfSUSmdNV5nxiAOTVck+lUFhZSL5VvhVJWflhaYgeAlK0OyYUIKnXvNXs+5xvrUZLVx80n9vLVEcks3ucjN8oK11ZdlrJs2Va2tK+QHrq/SnoJ/rixoiBI8fGXBTDrg5B2U3Th6Z01zlFxh0qGt/uR5XKqT7U/hxGZTs0X2qqYVgpREE7yZamtW6z+adHa6OJZY/BmMPU9dyUwknymH1qofKY9i35rwyG1pCrmc3OEm4FTyu4vzu9oTwfhzQxvg/i0LmikBrumyIWZoTt695XyjbbN5QZbSan+v0EwP3hbUTg92Netw01I46uFUoWSy5qAUWGpjU+cK43QTDNVxfXB21K33j3k0y1ReKotZ0pAfKg+FiruPLeIaFKehYyfWkDG1l6BzSLuugFqK0I0JoYsd1ULRUfFrsK/B1gzUfn2onEBxhjYGUCZzoV/DA0aovANrGaikhLaLaF6N6mdU8oE6Dc094GBfRJSesjyAQaUtqjfQDKQiEa3/qHNci6DTZiML6iviJoYaG7WHarHYXtXgTIc6PyhxQJmoHULDkjVwYxvSUirCK4VmKCmjFDUERDUy8F2WF5qDtA6IbCjRcFueVp2270IJmeJdqIUGkL+0VEvodoGtIu5FQwB3MByCYYUoYQbQkAshRxZFIujYwxqUSk4RjTpxGBuEtaicmIDaPisJRbMgoBYkaymGMX4IQz0qBUFxh9v2vhPaEOhCrQntQR0pG40oak4weUKxqSeKJsDBGyg61COpGOtxvwy3sN1AzAhGO6KUDO3ICEY80PhkD0CDDXqwLn8jdGZE0w4HhpWwNLdhEwslS9oCBPwKxxycsdfQvLA79Byo6EI5m1pg1LOH/kTNpkYtssN3UUATrQB6pjAvgKaXF0PZOmTUMISJcuxBNZYqcoIuqQQaHQ3EO6rrQ8eLdhuixhmGNFYI0DBoAICFH/qUZ1Oh3YMCOlTPYdDiQ7PCQVzgFDqKsKCEoiIACD1Zz7ahdEE7JWAPqhYOTDnsmh2JeQ/1x5CSmKciNIAymMDYDyN5imrgMex48RUe12xoBaPuNCjwYISAVFE7JNuizSa0OPAVhkmorlQH41Oan0l58EPDNxYIRVOf0qwfFmMqbZUNsoTaL42PVJAFofKvC8VXKRkKicT0K+iYsTb4FN8zCgtoUl6OGygQQsOHTjbf3rn13/jSKT8zKUf4ZlEEDzxZVe/Bogh+erKgWQTmnTrd30XRRFZniUURKokNC6gVkdHR3uBHuJwCydKEmBNXUFeWXfvynZX7rmSP7uISSzsLGn9uiCp8B2VBI7oIzLrKzwWLlljlgufnLngG9Qto5sLPLUB/7Zoni1eO6R84pgMsdmDfclGrxrCJ+QNW0lxt0Hwxhm0MjQagmIGHFk2I4gusXPhkmcFf4G7JMf6WnEKHS3eHo3phUK+qZEibsRqAXdrd56fgVScLGmHYQmq/q+SnfgoRXRUrpdFTSFg1rZyVpOIgzBkNTUqyuipJEtSEzyqhEEz4g2AQWnb4CaFvF57BgnAJEgrvF5OfGxIK9xFZ0Czl4CEnG6AmCJeOK+WfbiX/UjiW+gldGl83lFQsetpdzaxSjqU5bGKt+CJuQSgGr1ZwGClaohDTUbkTmqBUCgWroowvlWKoBaNln0abrBBkqNQapfGu0iPFgwoeUOMV31AW6/MlPlluUE7Wnne0vwwJFqqc/2S+xppqijjG3LxNh4zUCnbvaQXh6SALGvoCDy5ZsIjheRCk5iRBoqAyrfrptcshHfeLilwSnB9JcCAoD2FNA7E+pMgwvRZLdbgEcH3O5xCow1AbLym9xh93OfAaEm+8r4TQo4cTBKrUw7tARO36iN4EWlVU9AwOGRxbvBEsRWu0ed34czPSCq3e7xiteabGgfVMDxqpnDE+xHaBCRpKC9cLWXDF0qG7RXKSS5x4jYPEO9cLYPJR7ye9gZUIDEpuH8XZrdENDGydsWvl7cD3geNvY2OX/+3A8i97deBu65gHvD3cBaWi24Rt7IB5eLpB5wyMLDaG3EDG+Siw08FjbrTiJnZ2sFEpCWKP8wB7ktGOjV1Z/CdGhP4GDpxya6g/NibpHhjfgLORtwzLpwzDrhC2h9h35HM4dnCx4JH4fCgp7dAYEekfIKMdbMXR9Rry30CQxt9iP4SnMX07qGocLkMSteb5+HNDCWCNfS9I4tbImFZVM6QPdo1Z0EwHMFfPgouIZhbXBt/kWfoqa4wrU4oHvehMLmaiyCrWnSHGQlaN03ne3S3Wbxe+8+HziLoFUW0DDJ0NLRTwfXDgA1JGehXDI5EEySN6GxIEWYPbPBUbGkSQ/ahkoB7hRztwvwO3GZCvu8EGFDWgx4IPfHjHgyIFbLThtkYHVYoSFA+GeCqx6XiLCUMfgjc1ZydfSkwCcERIIqwKjQuc+qWlBjWW5FhsFVa5qLAquP5ClVwnpoqO3OAaTZedTrnpqUcVHnfIUpVehXXCElblT79QFWBiVJ+SGoyUmfOYnGvC76Gyow45gaMrelbOMpGwzkTCOhP1AR1ziUm+Cm1wSqElvXqMBsjiS1inI2GdvvpAZ5yWB0XD/kgWX8KYWukemWsDqQBvoJZCX4NoVT/e4SsmEkMDKuRHMmUiClqZtvtQKUJDMKSSRQhNTIfKKipUZzidqlRCHgjDxIGvKN5Q2dJbP/pPigOvl1lY0jvchTs8MBHyAab7EA6iQrAGnNNV0kQT85NaswTSOyyUiqXLqj5V1dhRaaoabuANSq/qTs8Duk0kqFKUoCqaRFYlzhVsmcnehcHt5OfZZI+xeD8m+x3a+myl04WqWjbZY4ibYDbZw4mgCa7Y4upuTbN8iuP8Yb1Ebul6rVukYaLfuen3HshedoVugMhsk6AdcDbZxiN4Pan6pGtwuwdfLLiBe5Wqu40HB66TEmlQc0bewFW8tMDNgCbrvMKjCl7RUaV4q8SP0CtPctqAwz8q8ZIqy0tcJV+8ETrNBZNEUXNGmp/OGlFVMV1VboSXom/a1Hn8uRnYsHu+F/SNywNrNz9GVfUMNdMGyBI09B3G3hmNe9BLCawFB/AbdR1xS6OV1OyHSpKtMN2H0k5LcE6G7EfxT+9E6U0svSFlw9mz8ugpuuE4sQ9yY9gLQAUb6uPwqws+Fgr1Fnwri8sUnMg05jKFjjnhVU/9L95PcFEB7QZF5STuPummhbr2Pi4+z1/jVrW4UhEtfpgJ4M1ynlNw8sJkguaaUewsr1t8Kwjavhpo1i/WczjD6GVcm7EJg466CeYoumHrnKKNwgdB0Rrds27/eulIGq2kaD+QolGacGVnnRRNbahnpEzTOotOpWCmwZ15ujOH1QjsXsS0hZZODugZd+PpzRM79/RoTNseWpUwktpZ43FgsPlIQzgmiQZGONID5yVx342HXcDMRvyHhvhHoyYXvkK5oBVLpblKS//vPt1bgwekf2YxtpK8aTsFcxT+iWUYLbrEezMMn6TAWJbzzDZYILGwdBJYhdw9RonowhScKj1ku9gT5GYhnY+2eSYaPeTjmAh+IW7tSXVFQ4He3nEdiGEQNxr5oAp7MVpbqZLJURLKsz3dUUc4ioppO9R1UC1D7/E8+Ek8IUs8ZkfvzBAh0XUyDnETH6xwdM3ewG4nD/ULcUFpxESLRmiB2ITxzKiQXlYh1qEHLxuC/bAB87UqOWZWkdZUaHo616/EMJcTv804KAlyA8tBwzNQQbDCtoVxGsJy5RPOZZjKaO5Ez7JqQgK7jqkMLv0R4P9iJYeLCuCC/5eblyYrbcDntqHQ90RpYyOjqlIt4wDBNQ4QXOMAAU7iTfBBa23UcYo1z1mfgcnORy2npR84LbnY8bwgzY1waCJJKX+o2YW5AdRQOGVy0TBEvKo5RjeCo5pjLTDN+CgeUxfDB9TeAIFFNEkTOp6T75YhStqRwPhzMyLhzuNeECWeEG/GllTN+FmAKa0JGoJsHLq4cOjyI7cyn/QMOw8vC2kp4DHiAZPHXuv8FIrm18oflK+UtTrb8H7SRqy8b6d6Cc3bKTbYID/o4UgPhKGBgUMnYc7PI0RgXw8xADb26K2Am30QdcJ4HTHB4+HcSn32iUSXc1sQg0RTomRxsxRVSvKLlLKHOlr2nSRk2Rsm5Sw3kz1Ko1My5cwKbhJTYZSLgTTB9Is0WZWaqmH6zMRJv1EJMXC4Cw44xD5wgCzwOMTWlYrN/UtYmWNDiyny8AQkCSN7SVL2XnGiFGzcZfcTO52MqW48xmOQJyYhcvYRFh41kyKnsgA56+xQZjQIz/lTJaGISPbaIm4Yw92CxKHnb8aTCPIBzkFCT+jUeAq1pKazwlFDphzqRsqXBqXc2Ue6UjpFVeMsO90cWVmwg4wfSMclKWyC6m4SJSLdfelHCi06RYMSnR9QoksiIRSPR9erMpvIukY6KVVdk5FujKwQbClVMjZg2vPyOIVF+o2CjE4xxUqanWBFl0XCUj4dkjJnX+ja6ORUXbO8dENkBeG6TtpNNyA7JGtC6Z2sbmn3qJpL55g2TLsHCvS6YxBC+6UwyiKmOFOpCARN82UI1QVQ8GUpGMqaTx6nVUq/UfVVCaZtkWbGdkpLwjA3bkQVQkqaxk7roFJSFUxzSSufFkEBN20xjl5puTQTeZ3mnn6rxptKNy11mqdUKS2QulGDN00x/SBNVCWmqmuepXHSb5iOarvRtLNc09pxpKnE0tKa0ktVcvWK1bBNU0w/SBNVialqmmdpnPQbJqDbTrCsJo08wjPkZyOB6SqSasZMNoZyoE4HHaMxlI1F+cYMVpWiIvlmXJuBrka+GmuYPHLUIaUYORKCYZYmlVGbNCNx05NSJVUyilF0cbMxmY1RpqUrqnPKWiEb7ZoGpJMHmy8PcQP9PF5UombIpENIxlNucEkZdSgdiul8mY1VM1+aQZ2Ncj3s0cl2hj0SB00m0ogKLCoVQ2AyiiODLEeMpFwSMgMxHZh5KKt6prmZAZ4OeAw81XCqDRXVG5mesokrm8gMqlMk5abA3IQkNwp6uUlUT6ojWFYDzkzH2fSsp+t0GlcjeHR6zyb+EdDoFDWisuzITWSMBcpGrStd2mw6zaZXpqPrmU3EuhGyiTqdwEdGMCdfXXbCXFcrzUHXWU9I2USps1AjVaef3rDSEkxTTT9Kk9ZJqikze2pipV/p1NQkPJqLyT8tD+ufJmnKrpJWinlpDVUJJaBSTKOnierEdJXNUxUr/UqnxEl4mbW89hM0/tys5Qtdz9yxtfxTxB4kVn6RbFwHwU+aCZq1vPHiAVdjP3Qtj9BJ8/QcrhQhfMjKaxb1cIBUOhW5jU5Fht1uC2cNturnHTnxbKrUQUe+pwKHnyqMLTzXfSqRnFeuy1OGqYIKkS5UEfiHIwprOKIQrAU9BgY+zl7FWYFgHvAHv3yw1xO/f3R2x403CHtxaDX8EvKH77mvxQvjyMajj70nOeqU0elCERtvstkW8dxl6oGhjfkz2xGiOIJ+/vNmsYcm+tjis9Dn5igKzhLLD7cwlUhbjlSWLUE5AdH1WzxBh54B8SMf8YfHOscubBSxiYdK8CVP3GGN8I+OXDyKyXVscRYpP+J1kmwadxuxH8gf7iX6SEs2AOVHmkp+eGiw/OCMaN9pMToaihuJMaZTOVbawadp58j+onSA6iGRAOE7U8/J3ac7nrVCcth9hP8p/MgGKk6glgdZJ0mN026TxzN2m3aIMN6dk51VucZZlQtnVbdzBp9DzVjIKQ5aVPQybROjqAKfPFlwedXDn+21xoXnnSxoeBC4Giu9AC7nukbPuPDZp+fnqbO4iTaL/z8N0nJSv8PO/+BieDI9NX6kXONHCud3m6BZMhk/gLAsetCbb2pXbYATTjByZ96Ey+KXQ+jHuheHP20aE0A5ip6V4YLaBosWwuqKR9bHUJGyxUU0rvKArBGZwRBqWnDfDUZHOBxfGFp8BzYYPJlP98rU4nLwId2Ou1BSg7toudKEG2/kcGzwjsiAvDNO0KZGHp03k2+kL4tINOjocjwG5+hRZwzZqhIxU5Ux/hDFhy4FTcVsG654eOIisoD+F5XN4A+d7CKVx3QC0MpDQci48gRI5iFZsBAVFoIqgVQuY5WYGVORK3lc8MVSNR8qfgE+on6eeJOmM3A2EzOkF25f2kpkkUxTtAf5OmQbkE8cIFE0rbph9iY2OoY+r8Hys4I6NZUTMkezolWRPdqUv2xRS5VWae2hSXVl0KC0q6SGHk0KdVWztmAJkInkoxoKioCqFaURoQ2HtsBqR7UxP1S9IDqBFFopLUFyxrqndFeiUOSZ2c0QksXyS9EuteRYBTgSZ/tptEjhcWoxQcasFJjAT0MLRhQLmSBaEBWhlIy/kUdw0kM7LrxFF8u58BS9wQW+qnRM9pta6wSty6KKR3uNHIgloenIrmTfxNCBpE4iWpSpcf0hqSu5PMtLHUC6e6oC54Q/bwXk8p4foOWYniSqgBPjKusnUZZ38CEVGrl4UsmzWCyVPKJpj2AfZaHbcRQY7uWZusY9FO1RWzpfV1f6RydUUQW59VrpB0hDEsjSy9KX/ESHlb2OSGgEKS5GP9sLypUyQJhark5pPaviRD/ACsurITd2PZ22q0aSVRsbQD5A+6N7JU+64m9RH1OdV8AEpSjsEdNDGAqi04r+Q+fT47q4mFcGBSiHrDEFn9qyQN8Y+4L0AWwWaIKQMzYYXLE2MJpX6htRq9JBdwDn+RLgef1pSKvlZvYSy3EdBRaj2pWeBO+Jxag+/8rKMQWqfoargiOyLGi4KuMz0M35DISJxvxroqPTyrczcChXz9M/6vZxpE1lBQcv9gAJnVoLWjc6mJ76NHJk/zEPKbXQ6R2csZ+oM/1/Px9gyfK9cpQMvyWY4WdbnS1+NtnW0w/PrTaipb1hvKa58Jo2uYV/5EFZFXCls60I80e4z7pILPnMH6nuBZsxzAN0HABWgRts4s1APZU1nATUM7wUs/70NWcYsW1lWB7zCf6psHqOCCr6UjRZeysbf25kM8aPlws/XllwEZI0mWAc1zunoEuK3Er4CrmYZ/+rDunEupWN8LhoWC9MRi4vklaru9ao9+gTDa5L/36eDFDw83YvzdW4RnONazQXDuNW1WLf6v0OaYI0mb5ZuM3eJhfNAURlptXgqGjVrYbNeJzv0TjiqdQqD4Mq+ITKgsYMQvvylOOGc8GiDp1ph5Knf9NrZpFEEg6s5k+9XW92KlCpTcnv8vPq0z52UuVU9BmlAzMm9uXLUXZzRZxhXlTs5fNc1WmvOKTIQGMe29TZR3OWU8YLePDXkWVaBLeZ5jRrpO3NqYH3bKqzuV5w4CUHVoNYQ9ANE2c+Ln/pFsjNnmMRH9R20vgOvPRgxSxBN3u6lKqHp53PjD9fRLGhYOSM9mpu4Mip0N2ONHjh6HJ+2im8nnFUg6PPlsK1zDttvD8WllVuW/zwHuF6ZdJ2Tx/NP/48Wwx6cKZU0pk7RGfg+WXy80W44rlnaPcH0JBtc+S8qlnGr3naM44EzQoaPi8XYG8VhapkB9wvL5NAKhVhOSqQ2CwhaJiQyUzfnTkzlG0O3zyTablhib1ClnjqWl7Kgrwb/WZP1O8KeGLPuAfBeZuLdDEzUnKSGbnZVYlcBgloI1ZFU2UuV0ZZHpA44uUC8jOIburtSjtpH8GryKqlKNetGSYcasQyWWlx0k4yo9M4NvHg2GSFqhS3SxFy4tny086dn6bZKIekpSf6DCo4qie9STVHfuIpP3ND5HXyzTKnEykkGFZjBb4UfpTmUDHi0hiVCqDME9oLyZfxpuBpRxMSLOJ/75qS0dzDYAkdoxlHzHG/2zvufpPKTVIoKuXQd3ERA28Hk5+bmQeeALKgkaYYzyYefJhkQfMUbgYX5Coog53GusBZw4KjfFNRFav9vcIO7Xelm8zKqjDHRTikYbdXaTUhQ+YAhDARZA1uYpp/8K61PEecJr98So1uG1QXS32kpkXRq020QhOSWRM98xZe3HzunvctdY5Y67vFQYwjjXOHpEpfwhgm7VY4mpjcrTetqWaY1J9ybuvcrMcBmtLC1mTWhtnOtgcTqyxoaEah345pkNFzfzZcViGwHh307jyD3thSeTA1W6A6DWQF/VosWVczTrlpJG1U+V4JV9Q4774mrYTghUNFMPvHFrTPrG+iNfkDpBE7XTEt++//+r8psswR8vB6boILH3l66+pnRo45fNuD0u19qZ9hxrV1hwQXGT+3sn7GVsODrUYWXEQ+eDvrZ0BpjDdw6sC9qZ8BpdF/94z+u2fsaj1jV+sZz1kePGctOrUVTjVQ0F+cvywZwLyIBIzfAMJgy/julhY2yog4keR2c3035E58aabPaDZ6RrPR09qfElxELnnLmD6jnuJBqbNk+n4IUTaqaJ5RRfOgxnlPJh19pqgEzcgpPEb0ztXPKMbhZEQTXGT83M76GVCaw/68hQ77u531M6DUZz1K8N4sSqBjllXK8H/mNEfPHK7tmcO1PaOp78GyctVMnw8luJLpWwHT91YEfpT7oRI5qwxfH2UmwSIs3xKery/HvuPCqtw2zs83Gk2+UTbzzalo/kJ6eLeL8/PNqUd+4alHJee3WsrsmwOofKOJ5EMT6X7MPL45eAYe0Exw4Q2s21Y/s9/nG4Uiv/C8obtXPwNKo4DjL66Ac+vqZ0BpDqXwCw+luHP1M2oivlET8Y3ajG80HX2t/CkTgHlqrAY8OMqfv1lSFUjY8QzOjrr1/vFcfNG0pE2KSjFy1TNpQWe6M1qSrLorzVRhlBN8o5zgQzlh/u4RRVL6+xhUyHqBbzitY+K9YxqNOT2dn848bpzR1s/KwJn1nxl/5sQJXx/IIUFDgKBc8mM1PHwI+OfP0dRx6QG31e4Nv2+uLr0iXD9+NBh+lzZRClDrlmf3k/avy+eolpAZrehVoKF0JpQ/42J90dvGoVRSOgk3uq2V6XGPriyY380SQmfNY+inUEOjnOBrP7gSXDVj6ay5P62K2PRP62U25XyzKecXeve9dmCkiNNDQ6NSDZekcyxjxVmBwaQeE9q8NzX45bhL27niXrYs2q+etLrftI5gwfBAwXLDV7QZ1+Fc+V9+tdKhHNv/ssRAvkoY4KZkmeSuh5emDj+RPTSOJn2jmOEbxQzfeEPzC72h3VsAhOF973+jzeEbbQ5fuyuVTjfUBuosD6z/Pe/e978Z/0YFxDc6ML7RgfGNw1bfaKb50HlcGBUlq/RDOtn4XvLNjo6/0I7ObWWVjBKGbxRdfCi6ZOBdRCKXW0NkTJGG5QonGbhWC90aiFhK0OB5/L7THbPJ6BuFE994EfOhazN7f11t0gAuEmPvoTWpwb3RIfGNKzC/0BXYTE1KZ5CB+8Ca1ODRaLD4RoPFn0uDZYy7gEdM94G1qNEewfE6JmioAdRnlgApPPzH0QMb90Z3BQPUBA01gPLO4k0Kt6I4uBvroAfUpIHxzuQbzRnfaM74c/mEGGtSOEetBc4Da9IMjwF0JbJgRg4CbDMt3qQh3A3CDesDa9IMj4FRSgmMUkqQU/8x+sq+MQDwFzoWa1RQ/+XLCfbySnF9uQadDlaj5xIYPZdg5XouP3ENGhhVl8CougRQdcmCC+/PluL6UlxPBBltnMBo4wTQxsmCGVMdLOZb+y4D4N6L6wOjwhMYFR5wPiZoqM1COkx3uv/vvbg+MHo/ONzXBM34N05JAqP5FRinagF0vEpx/e1mlYyeUGD0hIKFPLDcVlYJfkfSesFRRhY0iF3Iw80PEdfj1FAX55M/rKWm0bAJjOeTwHg+CebaBRxrUpw85TwwsV1glHsCo9wTGOWeAApOSwhEcHIS3EQ9sCY1eDRqK4FxqBJg23oJsR1O0HXdBzbwjfpHYLyZBEZrKCg8nnamJoU/YyjAPqwmNWoUgdG+CYz2TTCXL5WxLRCcHhbHD2zgGy2WwGixBEaLJYAWyxL7dDwO0HtYG3WBURQJ9KGq0o6GHECMvARKcVxbXIsfWJMaPBodlcDoqARGTwdU0bDCOcmp6Qnte1+k+DllrJz3SXPQEXYI52em6QH2ok43lPqUGWvsCZyoKpeeS3fbWNKZ2YXVG8IKtHeJHuzhkIkVmIFOyqvVHSRftLnK7TlLu8jPbXkc2m1xQ3tXjkMLtBrQ+HND5o1KVmBUsoJCH0Bn3JsrkNGcH815GMCb82bjq/VuWO+nnr0Do84UQJ1pchl4MNli1PnDILHoKzWXsXV6DnqAA+CtVoKTx6z6Ufd8aCWQdH4fntGt9D6cFWx3T61GvWMdd62Tbt/63j236hA14Nq3YJB2fN4YVrMaGFZkIU83tOBS4rSRfdHRvaAV7bxmArt8VsunXe/wpMzuaT8ZDNBQ5qbSaPYbrWT0mSZ7nS5O2GpdeXUGYXOzc9IdfawPMV1tQZEPTUjHJdeu3bs0smt9ewJMwf/+HwnkGD4eLE8rJ7TTGCnEXhf/liBfU3KrDPv1Jn1fCImzAvtxYFtty35c8YO/dSzLqlt+8NjnodIgUHhesyc+rtRsacV+9ytdQnQS+LVGisKVmRfZdMDIJHCVLmS8zeF3VjXXCzo6TQ7r/X79O9o7dnFqste7fGyCv1q5eN2TExxUu25JR6FSMBDlvL8+Hseyq97ASuogC/bgcf695DXx9eRv1En7OIwnTN8h1mNL12nkm3GsTKTRhacWzt2v+oMVditSmreLTe/yQJlrO3zd6p8e/SVwH1s1/LlB8K8jXayhEEOXshaEc0OBTQR3g0MkMRZ5HAbpk9HOzjp3JOHjc9jbCtgIDnR2/uWw2ebJlic40lLFISIeF/yOJpvATU6apv4TgM0Op8LZ9I7DaSb6oeAUA0pOLQRYwolwAkpKIF0LpMlnEgaFbuGmLjpGAEZmbXbuMUg5LqMKGxhVWBydZviwIu3iMU5y9LB0ORLQamMpOnupwKVB3gB28rSblc+IUI3vtWAhN4ETVrFn3dPZ+IFme5z5GPQbhb65yJc1G2vIOMFJQOikE/QRDl5HhpUX7w9e7Ox8elvtdZioOTv2spJJXOrnaIXukXhr0C46cstAuIHLLwN5W+CjIDR6uYHRyw0WOn8YfbmCLeQcOM57GP4NTuzy8LLYfe0YUj6e1Yf/c2B18PDfNVRCoyQbQkl2cv3OyaPNf5Bbq7nQZz/7bK0f6mtk9B1xrBm7qUdvsbseqcPVvvBwtS/qNLNU8gDfaAmX7/qoszqQUm/rE89+tNeS+Vah7ctZPcIUE9L2lAOYi06DG20iPR6Mm6Kw8FjK4mLIek0vMfVIFcJu8YRoVnSOUm5ROABJFY5YUSIAyKDqneYfQpdFTnCUdIDxoXXS77ZTAUJWDzPEjROm0Ah2Qyh435UTzQpJthFDh0bvN4S++v04pmyq/FZNDKs6pWyEhLz72uxZw/rgaynuLcW9Yyx4qF0Rjj03ioCh0VYPjbZ6aLTVQ2NIEhae4lyyEjfKSjRwNDJYiZNmfzD8Yk6+HKEEGxJHTT8SMTsis+QoRryyFbZTOmbMKCg8RfPHsBW7dQgXB0mHexQnqsR0fGcdfbcEEZw9wFtcLb8Z20Y3PDRHE4YwHrjz7ITRbw6NfnMIW4iSnSjZiZKduCF2QjumHH9uiIuxTAiNZUJoLBNCY6QUwkipZCd+NjuRd096DUMxp5fPB8xSXPX5Gho9rrDwrNUfw1RguLcggFWlTc9VpeCiPqHUZlybI01Do1wWFjqLvUOshLFpCI1NQ7jQ4XYlK1GyEiUrMRMroT0Hjj83xMWYw4TGHCY0/n5DYxUXFlrF3VFWYgWn8dw0F4E95MbXL8D0sNv+AvW4jpwsnjvcZ4Ch0IOoG9uDVRyHtDbE9h/w3hqsSWxqFzW+ctdaFO7+CZuKXNFWznsVlWglSxRqeacJh+SXo1a98xVP+kJjOl3stUNHy8LcmJwkwGC5fzI6279nG1uqPanDqNpTDS8zpgo9uv4YlmRCIWW/HOpYbXRsnkvB47yOZVoXYzUTGp+1oTFPCgvNk+4Qo2IMhkJjMBTC2K1kVEpGpWRUbohR0W6yxp8b4mJszUJjaxYaj8mhsUINC52n31FG5a7IPDCHDHGQnxZ61Hsys40qY0iE/CqesUp2YlQdY2IjaWwbw7YQhm0/VSHjEoHjVNECOlU4ETOxUu3Ieosa2N8AfJY+Y4jMsd6hcZMXGru8sPBY7zvERBiTntB4WQ5h5VkyESUTUTIRN8REaCO28eeGuBgH3aExxguNMV5ozK9DY34dGn+q4VyeU2+j/usGzkyGwTZYj7T2hiLDiO42KL/+UHanUm9dIT/TWJ5Rbuc0AZujmB1ZUY+zO4yRn8hVtJJ8/Tjy1XZBo3joA/yoVd0QNAeXKK7AZ48Tql8LXqbCWkV+LfhG8CP9W6vU/mjjeS2uWX4VfrIaVYeEC2QLHhXwmQ0PICoExXFU36vWJGnXda2o6lbwx+uAAfVYQnyc3aURG04VR72yDauRR2LoI04cgDbCFU7QQulqIctYCxu44IAg/Nouq+B6DsLQSakEVcfnc4b9ai2MLbvCotuei2ttICELIf6PGHJXqTVsphU5lapTiysuPGZHFTQAkraD5Wgy7J8mPzeEeDELlvvF8E8mE2kDGQJtbG5CY3MTGvfzoTG/CQvNb+43VYaE+7z3ZQCjGTLJI8f0jzKTEHye96D5JRHz5/mX1Lmkzg+COkf6RIDx5xlvHMEK7IFT52JCkTZRRpQjcwhAZNYMUc68zFjShcY+MTRH34RwXpF5AzFWi0ZUAoQbd3ALLEXed+vi0aJIdhHlTjfIiZfGWkWsi+W/v6WkN8qb0ARXP7A2kBE2l2iYiBUM1GtbzYsEs0Efdj1WlkRu0pqQZxqt0BRp0Og3e8KHT5qkmNza7/WLuo42OwY23r7Yf/9lf+fJxtb23s7m1tusJBkjE+W9/U4oO/9X7ZWrxyK6tG+TemO42z2ut/a7fUgKC7sSNkA3mfoi8Js99UXcaM2e+jwmUcJWwAG36IJXzpsgHvpw9PzDnsoRNvWFmc5jejyWKf03LJLpPCpWY5nS7dACmc7lDX8sU2/BTOeRzIxlSrZokUzncXI6lilt6RfJdB6qMZZpuGCm8xCTsUyjBTOdh8aMZRovmOmNkh5Yvtxk6rnzhjLvhBnvgK+aHXikqLf2xFcGs2glJ7jaKEkX3j/sPxEHfi6wHq23ce9AqhphIokiH0k/fvTP6cJKGtuJYxxr5IZH0bFz5DgVO4mOYOvm110vxolJFd9xj92GdwxXgv4R7k38xEmOSUb/+Vuzc9z9lk8OZ6zVfexGnSDPelLDIXVgqovSYmQcvuGROg7h24PLkt5Z48vv243N86jtHV50XnR6X48u2y/7r7aOvr88//r9yeHGH99fJTv24P3lU4+KTsdN8ETYtlLFOH2zud3qXf59Y/v0w9ezYODt7B18eBM8abweOl/bBy378mKv/rs7OAnb7AfldgSdGSvfAWrtvo6m6tR78EpCbkT7K2mqXs3eQC98WDf8Sajgyge91vkpHDegMkq3a60Dn/7d/te/sqK9+ncAGkX9B3yuwGFLMoDkoMH1mZPrMRgmgQtxCx1NHCcNbCCt/fvFb94vzd5vzi9ffjO97bnRL+DLfnOqARY/SNlDyjZ8yTzyBSyB/IbyG6Wv4jRQSwNwGpOGUDTEdVAe/YApYumEEJJ0HC9GCMnqEJL2bTSIg9RVAKlz4crK9JO/nwOhrDCykvqtYR2Gexfpx04N5cWLGurvswS5asWFAOe7EuDTAB6pFcHiAEd/pA80ZIXwgFHWNCxP/cCNa883V6midjKEVMcd91DSoTzrjPzCQdijP9H1aZooyarSRLFxRhoOaUnrcHwsoUJCyqNhS5xNxdltI6SYqIsJab3XpJ8eLmwHa84ayjfodgZrimjWXBRmIs2UNyBqNc8RiilXEkwVMPRS3TMtCQi1dDEoFLUUCopEncjnWySMJbwmlyfJsEG52yQqCSqM/EkmnRro5SiddLGUL8Iv35X4nY5fWf/cJfyeQw4zWPtrO9GwjWsA30TYyhvC1gYNJWx5FdjaxGQOtgJDwlYCAtsgm+QlJLCVz8hDqNB1uIWEWk/vbog0RnHrQHuoCLd8V+L2nuE2R3exQkK/ZnTXB+gm012+IYB5JYB5FQBLIAdguWdacF6oARwS7AJgCRHAKXxnAa/ngEYLdh1OC6M0Fz4FCmku3pXYnYpdWC/9QN50hOkD2uvceXnBxXaMr2TVrRSMNDc4FoX+53jTwbYm7rCMN0VZtZHS/H7vzWbrqLXR+CBU3M4aAeHZiecfBV6jUbGPvUbYgDQfhyQEx5Pxo6KHx15cLM3giMg3HZ7M0XTLb1GlBYH7iysF8ebpw/nOYr5yjB8O08AJgFcPgZQDm4vOW8uK7YwVG/RnJcXGbmGnQYPs9IzAwsPfssJwah4tDHJbbWFEG077KVfOwpWuc3GhvLFCgRBMK9TYzs4z0dgYsKcC6wykA87XMy+kEVw1TOkC1QQLInhw1m92vk7YyswqF4zljcXCaoYPHZM26Za42Rie98fVALIyhGNlwMy3kl7fgS1jgt7tfwXxSoOVij6HhnOSnIsAqVEdahD9v+bMJIuLG40VF0zAXHj4lHU911CFgFdAWxDvhHlKuJVP/KTdPOq2juF0WA2F4hqO0VTvGpo6VsMJ7akTh6uCP6cIPsqF43VMjOx33FUGvNPp0nnxPEy4kvZSCKHEvuSIVSgnL1YPRF4sIXLiWAZqTlyFkDLc1gsnLtdrOXE/XUUGY1LikhNfCsTKq8ddAnHekyykIFqRak05F87AHBSCmW+4ouSVK0peZUUpgdyKUu4FyCL/ECA76ZJShZYA8viSUoh9AZDZTSWQpwL5bmx3yDECy+1uZEnMu5kBVJQomo4ipcNwV8jh1F1hP/bGd4VdGwuaybvC8oqzvAQ4y0tAZnkVys3y6oGkKIRQiCP3SBRxdEk6V7ErLBsyI2QS1SpWe+B5aCXApwCcthwlmbyGTMI1XYmi6Si6bXsPU5Vnks5p/RRkcoxEOmEwRiJjpjV5M41vQMLQHUIf41grzaiAoY7qXtJKNWZcP90DViHuRZBukvDyOjNl9EkRSRm9kJR8dEMNbjcKN9TwrgT1VFArG6y7Auor6/lG9wzKhoN/b+HYq+FvOA8oXdRz23Xyoh5vZFHPpYgs6hlQi3oJ5Rf18kCthdLtNTXJq+le7w8vuBbiFsvVtRCF4UVroVIl594huWBRr3LPFvVhkUqjvOGinlcu6nmVRb0Ecot6uRcgh6TDijZn0ikJEcg4UVGQ7JHvuBbKIisWKEcs4SiU4Z6xEMp4V0L5RqE8nV9NmUsA/qLegoEkTll7JDvBk7jYYhb06imGbHgXXEU8cloi23NWdndxjrlwtxueMUa8ZUzY7YZx2xz7PdfbJqf7GHCQfGUfg/pvM2d0R6yZV+VRK2u1sU1Yf55N2NXtqGNBdLUg12y8rsBMNMt8bFOWHPTMmc9nWJplOrYbSzX4uTbepnoGi+BYe0rfqvZdRddONGjPajm2gcqdntkBdh9M4CfaVMucPdow8xDG0oL+VlrQQ+useMM7gKPKkhebyovB8eftWlYsuleoTs0crOnVMohIumtYqP/PN0utlmVNLIsMCVE47mltVBW4dpGRSn4Al/H18pRFBt6VwJ4O7CV3fR6CUBxrhhJF01FU7kBfiyL4RilRNBVFcNF/xybZnBRaiv1F3GULalJ5dGD/ol6xjL+poDrdQE+8UZEFk7yhZI9XSvZ4FcmeBHKSPbmXHWkRR3PS9WSXmpOuZ+tJ1wtkl5qWdzHkz9db3iE5veFCId+Vaff/t/ctSG4bSdpXQSjG/u34BTbxIgl5HTNyyzPyjLyeGdvyrscTHWwS3aTF1/LRUmtjI/Yoij2KjrIn2S+rQBKFxKvAaj7a3Fm3yEKxMpEoZGVlfpmF3MfcZZfyIs9TvWiqY2e/X9+eZ8q318IEafmqc6+a6w3V7iu43qR7zITnDVliqZ1ssOeElgd2up3IoTub58E8oYGOJ9ScT0+oLpURUxke5T49vAVp4joOxXo+PRRESBMtcSRqHB7URoG3ggcrhfvgDj0c6pNmQieB4/E69GBVpQWj4+k8O/SO0qGHU7ryHXrAQJwNsGIDDAl15x1r8Y61hXqg51lUPIuODQxYvmMVedDdEWK0w6s30f0GYCKQSJnbULoiwNEUfBTgaIHjE+Bo8SkJjt6UzJKfxFaUagjIraj4hJFjiEkVrJQbYnMgN6Hc94uj73I3obh2nr2Fs1cWEDyi2QursXj2XmO5pB3KnRtPWwdXcmu90SVR640+iFpvIgVP1HoTn5K13kSDmLgiSCEm7gbVLz9haNqp0swV/5ZNXUyBeOpyLD9uNn/qnitvlU1dWVj5iKZuueIVZGhvejO8fQhUn8wvFfOWij7IeSs+kcKlrBKhcsWHspkbONgiSKh1BqqPNhR5qD64cM8zd98zdz5/G10DBz1ZTEdRet6OQIv2GHDOYHMhcHhkrf7jyXTyx+5whKII1md82rqef0F+qmHvojdYoXzDz60fXr3+67/92Ph18Qw5mZ6HemyfYzjaOl6NcQDvCBmojdlkNr54EwH3BeowNazPsNVpNGFpdP+AHV0juFD6J3pejIbXyGTtR+9AAZyIxvbFv1xcjeBjunLwaRK9fYWHQP6mxT9kGaW/RPf/vPiXP1Tg/+sXl5c//fjdT4J/H35LzO3Pn8AM3ojqH0/+8xcppV8gt19gWJObDp99uDB/EeLHl1+AdYNTddjHmR0iW+GXJ7jYw12Ji0gtv7qhyg6iuR8tIWJx4etJfzbFmcXwlFHJD/RoiC5kzKMD7cP/C1/nEY4yjua4L/GreMGTPdd3LK4Qz1+tr/7Xk3/mvpMtqoh8fieL3klZzf+8GSzaDJ5TD0pn0SlXA10fq37xe5wagrwwAHcB5IWeHt4Mo/5Vd/mpDGd6zU/lHX/Z/HTRvYv6X5JT6tPr7mLYkx8hydEKOh7KcX5/tR73y7Ub03jIUyQuCLOntTF7WmucUSAKkIrdANWDLI95otxXbPlQHsQ55mnu/QhwZMxZy5ZoWQLSnGdR0SwiMNZpadmSBK95BPJDBGbvK+8KZQKicMUJwIdwxYlPSVecaEihQtrrIi7yE+Xi4k5oC0k6Mv5YqiQ77VacwOg3WW3b8/5wt/l9ckhj8DWewXIY31+tP8bOOfgk8pxz4pJAFdMHgSqmDxJVLD4lUcWigUaUDg2ayr6ANdFUlp/IyUFZueTjqFJX3FuXFRd3mCq0AVMut9AGmXnnKVw0hamIxMlN4WR5ZpTTuofb4fZiOBbH8BBkb9Cl0083bmfjR4xImJ6Y2eQ2ljNbfCLoXlwyX/xbOrM3sL0WVQJLHSySr5zp2nlmF85sVCo93Zm92eJBUYvdHQFUov7vyzdsTpPiHTnTnS6J6U4fxHQXwREx3cWn5HQXDWK6i4iKVOTr+iA+VczXDg9uYiysKIiHU/JzJzuunSf7g052U5XKqVgG3YkHQGdOpXLqYgS/F8F8ubcxXxYom5OPNWO1aQNTDIgy26JCdW86QkltwoJml92Ov45xOl8eny2GgCUXydEhYMtqt69VUv59MmQpVQg6ALJU4BFURkqQpQxn+RpKGcfPxne8HpeBRpMPUt5tHcyoKNA8Rmvfwo2Iw2bJhZF7ewxGqrJBmNJ8NgZ0YGYOH6ICdi5Zhi+lfbCWVP99urIGcFci+kInI6NxdG/J9Y8LmyFJxY3pkPv63QxIPtR5HwwXVrwE/7/FhhKdKtwbDQX22kJ9hOnbxoY47LNc9xAdeH1etAoXLXls5alYaHGEEi9BBLuMatXj0U7nNqjZ3/zw+ptXr/7t74ipEiRYgl0wEfPALrgkwS4xSkt8iMEuKZSWbKAR5d5ZmGGU4yPNMEr4F2CXZqcdbzzWH8kcG46Jn+zalS4hv4VjiJBhachLfo02unae2YUzm97905nZqb0HpRBdT7vz/ibznBBXD+cmEgEgMa3FJzGt10EhUT5Q7C+o7kg5FoZgaWKHQd3PjiJzUxqzaP8RoXYb2s3wOZ3tNsx/42d/nuZ5okpi3yEqXemfIrXAyTsjtUlurmAcZifg5VqqrHgOnf13iI0I26FSxMbIDnU2woFOYuMgjuTO40D4XhQOKGRvJs9Nkn+3kP+K/Us+H2xvSIaMljVvQdlhgiyn1va8JHGAY+6TltJ+8Jy3NtsWUlTPyFOGzxsvy/QWyz/OiUp8sXvDeW8UqW3x2yUPsUpdGuAwsOHkZqo2x4df5d8a22q2dUqfVb01cAY0IbrGelDm9z2z/Obs3ReWTO9bf5MHGA3fR88sx0VDPu9sv0phVTNaQGGepmk+F2wbm+QCl2vnQ2ZMB/RV0x1h3tF/1ZLZ2ixFsb1zimIBs3b8A7JdvrWQFB40rbHVfGr77i8Ty7K6Fiw9nzIGHfyv+bTjZzbbHcrnlDnyGGnz2QYmM+p1YQw+EWd8bS9skkdbaJviQDrEmUhe+EaJgLbatJ6S8sfPrPnt9Wcwui0Hef2W2/r8C+glNT//mYWJSteQ3xgiR1/pIc0u0eULHL+1We1jUws/Zp2fMgrrls0VcUtPrZhzZeC1GQGS2VaE0hs2NGV23OBYMPmjqLsAmey/Kp1oJG5doG7i/ySRRfY7yucfW7aT809YjI94+tFWoOL0Q2qMhYCL5QZB5vRzgibya8PzJKwzCZnJpk7CYotCexKC9+HI7CysOPNK51pyRm2mkeaMyphI4oGIJyceT5pW5uXs38jaHGL6xNfEo47vSflNxYff4dZySfUCZqWSd12OxS3epJdemL+mC0x0mDlM7iEt/v+2gkPc+h6VRsgBIEfl9m2ysESbFZYoEArsyXy/Ogzps6um0FWDUOJxeR91TohYP1hEfoBuh4Zf/J41Xa1Xii8/aX/1iesS2/jnE+85/m56UYt7ib9Ind5cXf9yc1EUXZKXW03ZRDLqAuS5+dHtkj62X3y6ld+XnwRffRK8kPXCNt/i8+LXXyHF6OoGOLsvbdfpfyrzua9WKODzZQzRd5t0sgvd3ZckUeFhDbF65HhY6ZLwsNIH4WGlD9LDKj4lPayiQXhYBX6UPKxBc30Yhvwkynuuq3tWAuKtERwcS+oLvZJ34tXZv1ry0qL2/DEgOHwHWpvuBCLLQXCILntEcCAXIb1e6WzR94bgECEzlU+dqkaHQXCky471p73FxWoRzWWdMYK8r/Vw/o0zl0BHqz67MU8qDoNOM1Li0mVGyKuoO59YYwACNtYIs/WTj1Xe6h7wHDjLrIgNyph6CDxHyAzUUNcl+8MA3nkLA8SAjng6wUc3V0AXa4Hj7L80yRJvNCP5bRfbgQWlS+NYAwkmWQ4i69fuXVdK2lpMhrMZ9hnQvqsZwB7y/AOkZmNLEm0QHiGM2nxLNDzHwUsWNRx9f1yWaCIOTpqPKrrdzjErsJMR6k9+vOi13KiP5xZdt67dVj8KOl7YbDo3zb7n+r1m9PvFl2Hr0/6XwMOu4R5OPuqWLsGykkHxOCKuhsPjWHicL7FOjQTSQtpo1ICRMpMiY8sszoRsOxSeV5G1KDCWi6w9Fx8rmcL4vyObwqWFd7KhHM7F77HqY9iBihYnpEXOvKVLAqZEHwRMScAyBExJfErClESD2G2IwlG024B3Yz2TxSfKYGuimlucwhY0KyX/BN4a1NGmeiipyS3O/MmZ3OcDUksmN2S1f1BHh9RhDrBhjvjjrXBKwkWYAEPg2eSZLzi/iAJ/l5jk1rXIh4eNgX24fTsf9m2y5a3eYoGKqJMFDI7xIr6KFwFpnqjaSWU6o3Tj1s0Zgy3op7KK72Qxwq7+M/BHFVXWzlJil54nWBdvV4J1wrsfG+tPLb+jcQfsgR3DHQD7qHEL0DTHdwthS+MOoFWP7w6c0K18C+3wKB8CilJUvwV9QFq6REEN/JnJYxdNF0pQHENUDl74jnAkoLKDE4BlcQW1XHO8SlpHLWY4dXJ3sAydQuIz4j7amFdiii7st3McnVdQVT1knqzQlCcr8boguDldLa0tc4T2sFEeJvFGxNWwEbPr0AEB+fgY7HHTLOs4tYpYfqiXniwSm0iCRpzFYfdAJOPufXHzlijqIAKR3WuUZQOjcXiS6D6TH5nC+MKK8UcUzcyXH/ONkRY04xtj8uhO+kB7ZTfDOZjPZNpvpnV0ZdYzFmbpDTY08pCCoQ2DP34yyVaRsU0F7/JZS3vfhCrZwam6cf0KhFg0Hl5PR/0YpAjcIP27xruVpLptXrALJ4999EuxjxZdX9a99Xw2s16sqcW+qQ5yDAr0mtQs9V5M4XN9gcx6eiXplAT0nKNaIc3HCPQoNpZ/x9x7l2ALl3dz1uOF6EX2O5zcsPbMX2MTHWEblctQOjCNFpPhgy1HAr6wXQ3iYGA+Y+mFSWEMlw3xlc9AejlCiyHJ/DXGCP1xOELYpEAG6eVFYQGXd5OBEsoZ2J18PtJqGi1Hd/Izymni/JnUQTMZgPg19J23LvBKr4+lSS6TFKi2gfWBmGL1Ipqw7t1GV4toBHeSqEcqz7XoLmbT2Yp26bknWwgJplcVtDxIWKgId91BvcgCLnB5P7GpDmrJFvFBftuHF4aTXjPQcj54RSRDMsHomBm5B69864TimJPOS6fT8F67L51WI3hN33GCCr4Gjbb45jc6l3QMC3W1PBxW0sJHHNnSeu02Lz3LbTbCkK45Fs7r7+B6R3x4ScNf4qqDM1voAs55kX3Fh9cY4BKfxHCyF42/5ulnyZ3bJPKvcSSMI8YDyfgKWBaXMIy8goafdzgJpeOwtVjMex1z6HJ9Hp8ckK+hOrkzj3hCs7Xd0Vnb808SaiNVwArof3awy6k4OHKIPTpTO8uU9QFwR4luZqs/1VTfiZXk6roGTMcrLKCZW1OA4L7weouEG/J+T6bCi7D5Psajx54uPQT9hPrF36kewnpvG/vQi/elcMKzW9ZZnPdj8FTNANwYPKg6LuDpW1sm+RxSVwWcWGyprvI3cA4zHygItX9sS8dl9kOSEVzeh/3gMvuBKjmd1W0HUeK0YIzYD2Ms2Y43sFt3rYHt3tn427qz3QH+tAYuGls76WCXrcdiHumsx8/7fUvZOXRQwz6t2E05XCmLVXoG86coW/hcU5vaxHvy03A5+B6bqedij5/URuqVWF3lc8sWQaq19rDcXl2JbWCspHM5Y2tikjNcNuSf77hsJSLYy5GtRBKnV7YOISpP+iQnE71kOx7/VkJ61M25GmWQfiU4eAWaI/8BspXLfZiVq3DFwKlrBVzg8p6WTxyJUMhHcaKNKWGwVULLmf14l09EBdOC0TpoN3/5bDdagdVqeIE8L1T+8Rq+jxM+PWxhcFl+8ikZEhvnJo4hxVGnYruN40XXXzoLfG1bHRxUGjf0PBxRCqG1Gh0aLAhwqe3RCo2zQzs9u9HB1rrheXSAqG/5ghT+8RpeU9IJsE/H/6E7zl1df2ktxNcW9v5xA45EbbXEsaoO9vF0kCkOSHUa7c5IHKYqzlAdtO/s9k6WgMcWbfFKGNEWxQqsMDO+g8KWab52zc/PJcXWYjrF0IwIRLRpHPWHK4HAyqbPVtwkfVwuL36xNsE8tqaqQ7EkaY6DR9FZazUjL/Am4uSxFUXwpGMtuhaSdkk4VndTn6PjszVCLAumVypmBVGAZJ5vu6170LouYmG5Tw65O+nF5RDRg8JqOhkWDHwNMyAxqLBhuj1eTnPPtK9un9h97LgBmBT3pxM+AIYkJVRKPtu3/SLwpSoXOivT412yfbZgaIEG9uBg9DnaRsmY0d7BnDxyJUbrVUKuAHBYJD6BR/yNiU/CNStKj+N+FOn9BiefxOtWFB/L5FPFV1zl5TGKTwK2K0qPpSyq0iuOeT8+6VUUG2rWp5Y0Sjs4LqxeJ2C2ZpJJXN4XjwUsMstNZbEYAmZSjPksMrNOZZFsvJ3ekNfD9/nEmemkEq8P/KL38lucYxu/m/a1tW6xbdrcvBBn3BaIhe22A6PO8wSo0NUBFQZsa57kC5d3ltgVzsOS6fG5PLDtebBrkJgzMe4iQTCXA7arT3KAywYYwMVr7CuvURQlnw/mBxCkTUjiR1QeoN03ner0zaQ/7BVOC3ZmhsIILhsQSIXc/A470yLFR7FO5hjlSidsXLz6+fm4H7z6Uz5bXA8rbNWH4m7FQ1kyufSZkm2ZQt5uGSBPBfRIvk5jB2EoXOByceGEoKr7kJ18gRbd4hPkdPOaABLfr0+46LATLpRhcbncK/i///0/G88gnVlRNFzxviRzbZYFJOg0lMUKh62/p3qz/Y8frrEov/n4gVw/FurKWjEc2prOP36QVZOsO3JWIZEILZTB28h/hkzzldVq5lUo8FMkHa0HZCqs7NyQ/XvsqsJ9K7jrhJNn1MXxJeRhkcVNkiY3Na0DwppuOlY7WpEkLh9ekNWCt6VizJUA0/5lVasfwlHJilyj5eyoJDGwVUir8Ha+o9JtWv4Ayf0UWUR5TVRabyCYZ7l3jttDyc2G0wAMl/43QB/RxXJtaiMgD/Wh36HBRoPtvh/jN2I8uoifvB/bjm93gPJxEVKky/6gQ58778fwkDY3F+yWvbmyi7+UlfGWL+++5zFbRvUKcj/eecwMgbKyoxXncQsO9/QcXqzn7mbKJiaqg5KyWj9AOFvrBzvNYm7g6OjBAT3bbPvmBbbO98KMQTkpKpG2sD578fzHBaWO5/mvUKW4oLpWhxWzRoumXVPElaTBTR3dCl7fw37DAQ6ooDVZja+xf5/eWKvJEIeQxhSXgy6ixSiiK6poIQnagoWH1RiG7LqWVoeVzZX7M/MZ3x1WVBctphTZ8r20o6z1BxgIZMGKlzyXIbY2JxnC5eqRfVZmNzUUc1xlbzKQDT68GW5D+1RyNzWsbk3gAFb+CnMhEdhnlRulb8vEBnD9kskSkmToFYifaU6twozlFSwF+fh4ilwmmF7SKpJYUkdQpjjmE2d6Jkkcl4t1Ivnqqr15TNeodOoHHrAcIPFHVqOwC71krFAiWgyeRiKeuEjPkM9enqOSzw3TRdr1E9dvE6uLqAyFy7vflw0XaH4l1g6VQ0xxYAoMkWThFsDY8UzEhdZmWouW5Y2V5lPxLNUsawFiR60VzLJKZgSrGoIWjVvlN5pTgcZyaKFWDU5RdH++EkYHeUqm/ewjkzjTTN9q1RfRezq2MHATrIL799F8WpFVppOTrOJyIafRaDScSWu6Or/bH/Vo1nik1Xp0qoFHxVHnmzbEAJ6hAF6122BaXas8Sn7eqA/Up+PhkI5G6Dliq9hqEiSUoKYhEkIB+myLj87LILhDcimqfF8Cg+oBDgrwqNdoIY8UGBpgUdEYDGwf6Z2XyPH0gBgNLKBb0TEI6QPscq/9GlkmDj7RTtX1CNcqSGAb67kxOfwN3c5do+nK/S0Ywt8m/uJCEwmm+KHo1HQGgKq2A1eMRedPiEu4B5QNjz+jm+Pd2fi5wK0WPjghpWqPg61zSuA7rHE4jHgaEKzjNtqtVz7SYQMft+0GnUt88Vrtht+G7oHAA0Lp+nT/bav5Eum4zdADDLjtE9gXuOMW9cSH0CexukH7FXJ7aFTnNRC8Pvb9SLH1s8OG/E75SluC7i+uyzvuogIjIQFj1+ymOC+MfFEzmqiiBiqjqu+HxXnKiM+iEuat2FIQxNQSlQZwKsfcmtHJEt3JFJewkxA+YxzhQkzRVeFUJpfyeluBycZwaMbSHEwF7UNWFEVhEpcPHrQPeZkUlcXiiPg+gvYhL5iislg/bl4etA95sRSVeP1yOzsG7UNWQwUtRxC0hxos4guXHz5oH7IaJGjZa9A+ZPVHFA5w2VzQHmdG5ddBClkBEkl771H7kBUhURjB5f1E7UNW8yPFR7FSrhu197+7XN64737NZyutiFNs1YcmVYrah6ykBlr2HrUPWf0KhQtcNhO1D1kVCrTUidqHStQ+ZBUllGFxWStqH7JqDanhiu3aA0XtQ1Y7AS27RO2BXmcD6uxvz1H7zaNhhRwUSeLy4QX5sFH7kFVsQMveo50hK8CAlnO0k8TAViHXTHrRI4vah6zehXx59z2P2TKqVcfiEc9jZghopYb+RqL28Cuy9UhHDxZE7X+KojcsQP7TTmF7HO/BHqquYVPIliDCKmfIHcle4vYoqPtm42FjpTPkFs184D5k9THQcsjAfcjqUigM4XLlwH3ICiykhmLOq0qB+5DVR5BM7Ra4D1kpBOnf2nPgPmQVEdCy78B9yGopoGVfgfuQlVxQiOOymcB9yEowqHT8+pkUlQP3IauigJaDBe5DVn4ALfUC9yGrIaAMhcsPHbgPWbI+Wh5n4D5kWf9oOfbAfchy7dFynIF7BC+LWMXlkwjc45A2dhtG7PJz4L5O4D5kGevK48DlxxK4D1mSuVxcHzhwz7LGpelwyMA9zxIXVR2OK3DP88STTOLy4QP3PJtcZbE4Kr6XwD1PK1dZrB87rxC457njKvHdc8frBu5ZRjlajiFwz/LMFb5weQ+Be5Zjjpb9Bu5ZcrnCAS6bC9zPVnOymnM5Ycp718yJmpF7llCuMILLe4rcs8TyFB/FWrlu5P7yJ+/PzqT5XT5bbJelsrV7XY/iyD3LQEfL/iP3LGFd4QKXDUXuWSY7WnbPt8fBzUXD4rJe5J4lmqeGq3H80Nqz/eMEyh0erEkfyJ/pW8Scr++lnQULLQ7h48iDp1ivF+SQ7yGeSl+Wq/mEbDgR7++TF2Sy/PhBOITzY/kszRstutIe3kS9+55wD8gxuV7T2fWew/nbp8MVc3JTLjLBH3c4n2V5o2X/YVCWQo2WcxiUxMCWJq2s7t9OOJ9lecuXd9/zmK2tba1Tax/vPGbWQVnC9m8ynM9yzqXxayCc/ydp6XR7velqQscI14/is4R0tGiaNFncyLG5aaMbln0J9CVMM0Atb2R4XobshdtPWnWxIZcw8MQ3eNtiq86Co2agBvNZKrrctT1AMJ9lqqPloMF8luOuMITL1YP5LLE9NRRzaFUL5rNEdcnUjsF8lnkufV77DuazHHS07D2YzxLU0bK3YD7LTFeI47KhYD7LOU/RqZ9gUT2Yz5LV0XK4YD5LXEdLzWA+SwxXhsLlBw/msyxvtDzSYD7LEkfL0QfzWU44Wo40mM/ypRVWcfk0gvksGVpyfg7mHySY7+AVL3oedP2xhPNxL2yx1S7woh/PB1m+vmrDCMwG9METD5ZreQ32EdEHlzxenuCSrh88pg8meMRcZbI4ZL6PqD6Y4JF1lcmdj4gpiOtjeFZ7JUV+9wTzmpF90E4vrNR0BLF9sJFeK1XOqMPDh/edJstGp6a9BvhBMK26VR6ow34q6oNSWp3HxPce4wfdtBZXWaEO+wnzg1JaVTNWinV13Uj/++AvP734Ot+VBMJp/cw4270eSGGwHxTS2pea9h7uB9G0Glb5oA5mIv4YiWtVkfquHfNvJ0P+GITrxOS41EEr6u80WaZ6esQ65/WvfeDfR5HIzRfeYekVls5gC3pmsbpeRLAVYaXRPVrdG7ikYMZZQziou6PYkMsN9IMzrhXLMkqZMP4e0Qd6tzejZii4ozsj8ySC/ZBlhoZWLB+RQ/6o4/24R742aGWIm4mUgihfCdxzwXIpB75CaWWF/2ai/pALX0T1CkGYms18hXW1UPmPeDZzI6Es6/u3GPt3mixxPbaGDUT/k3ZFvag/2OP2TdlR9xXsG5a6Hm9OatQEgC0nB1cj/1s/G8tKj/dk5oP4oMWXWZGqfrAwPujzFTfJEnWoHMlHZ75IpUYrP2ErK5iPH3KdLjjbLZyPQTJ8S6byRjYTm1iA8wRbB3EIvn1v+wVPhKvGsvQyhaNlfApsCU8/ZBwWu3lZbDlILo8sw52adHi8nvbJ/V7ttWG57NSkQ21enRTXPElS1MEYKa54UqSKNx46pLjeSZEqdv3okOL6JEWq2JejQ4ormxSpYpe5DimufVKkit0yOqS4QkqRKo4j65DimiZFqth7rUGKJeqmSYlDe82QKtYW1MEYqTJtUXI6sw4pri1EvmxlHcjM5XxSXFskSVEHY6S4tkiRKtEWGqS4tkiRKtEWGqS4tkiRKtEWGqS4tkiRKtEWGqS4tkiRKtEW1UmxbNE0qZJ8UR1SXFukSJVoCw1SXFukSJVoCw1SxdqCOhgjVaYtSjI1dUiVaQuRG2mGFNcWZel0bLtxOR1M58vNToOlWMb8atWgQ2nh7YD83S9LDMjcJVvN7ZD8HdeG2oshCbcoh2S5jvE81x6SIvPxkPyd1T6FTwxJILl4SP5ulqWFZw9J7pZ4yIx3sE5hQ4tCevGQGe+abshODNnaDpnxTulm/ooh29shM1Za3XdHDNnZDpmxouq+O2LIcDtkxspZ6+1xtq8Py9SJLcDKRhr5O6opKJZvo5KiDsZIZSyRKqmSJVKDVMYSqZIqWSI1SGW8niqpkiVSg1TGa6uSKlkiNUhlvM4qqZIlUoNUxmuukioxqDVIZbz+KqkSg1qDVIZaUEmVGNQapIq1BXUwRYoly6RJlaTL6JAq0xYlGTM6pLi20MqLKXSnq47Sq6sfMCo+5vtEWVoMNWkuKN9250BPb9YTlh4TS+9AN8h1S1muDLvB7VrJslHiSWjU8b6+NdzW0yeL5b2Ie113e29Q42M16QMugUjcMzqE4zMbaF3F+233u/M3ssfnX1hKz7eD4TL6/IsCUXGFVZa2wkT1yVZUXCclR6MO5YJvJETPFY9W4p5J0c9vr7ufwYCwnDb+uAgeWs20uPvRTXc1WhYIHEGb9C2hqa7AEQEsHI066AgcsW0ucFMA5kMJnGk7NNUXOFN0ymjUQU/gXFVpHRB9jAJn+hJN9QXOFJQyGnXQEzhXUcaOBT+UwJmWRFNtgXOIvjIaddASeAbcXpj2JyxwDttHU32BcwWVHI066Amc22Jl9QSOXuBMS6KpvsC5gkqORh30BM5tKK2aF8cocKYl0VRf4FxBKbB4dNASeAbMXrhbTljgHH6PptoC57B7ZTTqoCdwvmMUrrRTFjg35MqA5EUC5woqORp10BM433OWVSnaReBWRpttz6L5cJpd/UAyyU2zslOuikTIVU5yNOqgJ8IMp5UWFquye8fJgL4mSVEHY6QynFYqKVNOKycD25oiZcrF7WRAW1OkTLm4nQxka4qUKRe3k4FYTZEy5eJ2MuCsKVKmXNwoV1FGypSL28mAvKZImXJxOxlY1hQpUy5uJwOSqpIqOXZJhxS3KrTwqEbdsg6HrKKpjt95Ext1ODRVSu9AN8htirIDl/L9zg4HqMpJ+Bj8zg6HxKKptu3AUa/KaNRBy3bg0Fapak7Y3uUQWjTVFjhHySqjUQctgXMorFxGTljgHHKLpvoC54pOCVaig57AuaoSJsIpC5zrS23oTULgXEElR6MOegLnKqosLejoBc61pDbIKyFwrqCSo1EHLYFzaK407U9Y4BwCjKbaAucoX2U06qAncK6iyg6lPXqBcy2pDVxNCJwrqORo1EFP4FxFaR2cfIwC51pS+3iXhMC5gkqORh30BM5VVFkO8bELnIOe0VRb4BzvrIxGHbQEzsHO0pV2VG5QDp9GU30RcpWTHI066IkwwwmllQBb3duQBadWn74xJ1QWzFolZcwJlQW/VkkZc0JlobJVUsacUFmobIWUMVS2k4XKVkkZc1lnobJVUsZc1lmobJWUMZd1FipbJWXMZZ2FylZJGXNZc1Q2mg7muuPAbTTV8E26m4QVhwO0pfQOdIPcdimDUhb4JjlMW07CR+Gb5MBwNNVe0Dn2WxmNOmgt6By9LVXNCRuhHD2OpvoC59us5GjUQU/gGYrqxLdZHPWNpvoC54pOQX2jg57AM1TVaW+zUJwsfUtoqitwl6O+ldGog47AXY76lubfKQucaUk01Rc4U1DKaNRBT+BcRQnT/pQFzrQkmuoLnCkoZTTqoCdwrqK0TvA7RoEzLYmm2gLnqG9lNOqgJXCO+pZb8hMWOEd9o6m+wLmCUlDf6KAn8Ay/0QMG9Gq5ylyO40ZTfRFylaPguNFBT4QZtbO06tlV3vy6GSXNlfg5OhgjlVE7SyVlyqnkZuCyVVIl5c91SGXUzlJJmXIquRk1zFOkTDmV3AycdYqUKaeSm1FIO0XKlFPJzaiVnSJlyqnkZuCmU6RMuaBxJhfTPmXo54dz77gcPY0mTc36fAb/1UYbcvy0FN5h7o9DrNFU130FE4Df26HAGKbdVzj0jYuqvp3CodnKaNRBa5Hl+GupaU7YMOQ4bzTVFzi3apKjUQc9gWfoqdP2F7ocF46m+gLnii45GnXQEjjHd0sL4YQFznHkaKotcI4DV0ajDnoC5yqqLHfu6AXOtaQ2kjwhcK6gkqNRBz2BcxUlLPtTFjjXkmWnWBYJnCso5ZBCdNATOFdRYtd2wgLnCHI01RY4x48ro1EHLYFz8LbckR+VN4VjwtFUX4Rc5Sg1S9FBT4QZuz4dEWpsxTIqJidJUQdjpDJ8RCopU8AjHHFcRsqYjyijYnKKlDEfUUbFZJVUWcVkDVIZPiKVlDEfUUbF5BQpYz6ijIrJKVLGfEQc/oymg7kAOHwaTTVcHJvsQZcDqKXwDnR/3ATRxj5v1S/HPss5+ChcHByIjabaKx/HQCujUQetlY+joKWmOWFrjQOx0VRf4NxwSY5GHfQEnqGnTnw/wmHbaKovcK7okqNRBz2Bc1VVVjH86AXO9WUZQqBI4FxBJUejDloC53Btaf2dsMA5LBxNtQXOkd/KaNRBT+AZe6sTxy9wGDma6gucK6jkaNRBT+BcRZXV4t//jpujxNFUX4Rc5SRHow56IszYxenMWY2dQUaZbtXBZaxMt5tRpjtFytguLqNMd4qUsV1cRg3uFClju7iM2twpUsYi/RmFt1OkjEX6M6pyp0gZ8/lwtDSaDrZN5GhrNNXYBjubTBWXA66l9A5ygx7HSqOp7j7Yy8BKl9WCP5l9sMeB22iqux55HDOtjEYddNYjLwM1XZbWceQ2lMeB22iqL3Cmw5TRqIOewDMU1WmH+j0O3EZTfYEzRaeMRh20BJ6BmtY61+MIBc6B22iqLXCOmVZGow56AufbMq1zRo5R4ExLoqm+wLmCSo5GHfQEzrdlD3n4Sa1tmceB22iqL0KucpKjUQc9EWZsy3Ssq+qWqpcFwE6mzaCDMVIZ2zKVlKltmZcFwFZJmdqWeVkAbJWUqW2ZlwXAVkmZ2pZ5WQBslZSpbZnHAdhoOthWgmO00VQHFb05f9PjQGspvQPdIDdo3NpZ/R4HWctJ+Cj2ShzzjabaiwQHWSujUQetRYLDrKWqOWHDhiO90VRf4NywSY5GHfQEnqGoTtx050hvNNUXOFd0ydGog57AM1TViZvuHOmNptoC5yBrZTTqoCVwDrOW5t9Rme4cu42m+iLkKic5GnXQE2GG4aSz2GsYThxTrZCiDsZIZWAYVVKmMIweB0KnSZnCMHocI50mZQrD6HHscooUr35dmxRXClqnmJg1NzksGk017Glv+4pxELOU3oFukCsQ7bPWE/fGbRathPKjtqc56BpNtTU3x1Uro1EHLc3NwdNS1ZywtcFB2miqLXCOw1ZGow5aAudga7mMnLDAOagbTfUFzhVdcjTqoCfwDFvn2AAzHsdso6m+CLnKSY5GHfREmGEI6fg7NNZxDsdWSFEHY6QyDCGVlDFDiCOnU6Q4cro2qQx/s0rKmL+Zg53RdDCThIOl0aRdmfTeam+mPYdLS+Ed6P64itBGOm9faY50lnPwUZhcHHaNptralCOeldGog5Y25ZhnqWlO2ALgsGs01RY4Rzwro1EHLYFzzLNcRY7KAuBAajTVFyFXXcnRqIOeCDMcPDpud41VhQOcFVLUwRipDLtGJWXMwcMxzGlSxuwaDm9G08GWLY6ARlONZdnxN5ORI52l9A50g1y7lGGbCtZljneWk/BRrMscYY2m2jqOg6iV0aiDlo7jSGmpao5qmeAYazTVFyE3bZSK1OigJ8KMLZXOjqC6lvM5+lkhRR2MkcrYUqmkTG2pfA5URtOhVJvPgc5oqqG7XWc9Q3yOdZbSO9ANMhWCprq62+cwZTkJj0mB+Bz7jKa6CsTnsGNlNOqgo0B8BjzuaPnqCmfHPOr2lljOFsP3YiEbdCf9UWRlN9uLKF+GDM6scEnX98VlEZNprZVmslhpmRRlAZNpfZdmkpRfvTfmm8lieDtYvh6+LyCfVnBp8sUoxiLyl915/9to2bWup/N+NLevrXWLbf80XA5e4NNwVCQarinL8M6VmXsVjaeTF8O7ITizxvc2VTOdT4UZuIhAsbuczgs441pTwU6jw85iu7qazYfj7vy+gA2ubkUhaxMC2rIx7g4nBTxwbZrkgToY4AEXr6cT63oEE72AF66Ly9DdlXn5cRHNn/eWmDHL+28m/WGvZIYwO1BhhToYEAuI9ubDGa2A+axwqHeKlRKod5cxMphHN7gwWC5nzy4uRtNedzSYLpbPOs1m82Iotc7i4vlfg8HLf/36poAzZlamOSvW0NWEtJzOCljg5qYxcN+WhR6WemiXAl3H4eQKH9ShkI9BUNnC5nByNGnaP6+6iyUcEla/e7/YmC4cPK4MTB1KB7b+97//x9qOyBVcasRiT1jmEr4cLoWefz1crLqj4fvIwiL98cM11u43Hz/0p28n1mD61sJCjtdqYU3nHz9Ed6C/oE3+vTW9i9CyHI6jRsHj5FpRG07+A37bT0iXKzct3C/ufwk1waQhem6vKgvkV6LVSny27WU0Xw5JEErrYtldrhb4J+qOloPUtXF3NFKbJlM4UMSNqu2D7sIe4gqxOR927VH3OhqBm/F0HqGt34XxQPzHTfb2pqg3fj2bzlYz2nHOVwWWIQfDK7KkDoeX5RjD9neXZK4QOIYfTaa2Gglerq5K+OCrgNZZyYu722I2vpET6u2wvxygzYnG+DaIaJXafL0ZjmieTaYTmmd3w+jtV9N3tDWzmpbr4//R+m48mtCg8br39u3bxluvMZ3fXuBQreaF5ONm2lstyPZHx5vuCJuo9eQc9vvRpGhqLr5//acnz+jyRjR8ddI6t3vWxR2nZYMfPhm7Tcsf2E6rZzsNx2rartUI8ce9c9xe00JbI7Tw1XIH6CO6WK5NbbZ7Z1Mf+h0abDTY7vsxfiPGo4v4yfux7fh2Z+DeuQN52R906HPn/diBTDcX7Ja9ubJ9DL3VnO74knyrVUWVsYAa8/nrzGa+upblMvxmZjO3D8qq31eczS3LaaZn8mI9gzcTNzFdHdfS+4Hd0vvBbnM5w+zR0YgDer7ZRs/fo5toPh9Obq3+lDZ21meYWxtT7vPUGhvb8nYPlqwth8if+Nzk0c4pKecuppVhC+mG1L6HkbewloPIGmPvgjjOGC8XHDgqBwvrZjq37qerubXCDhDmIMw/8atZgrPGlrOMfd+O0IZcifOkEjSZUnnL99L4stYfYFLQvQtlkM8SX8uVzBR0KN8CbITJ81LSo5WnzomdynjaH94MI4pCxCPz1UL7mIIA24QVZkT3drodN8NlZSo5b/0yRuPZ8l4YiUUPgutarXyUIkYSDNjDySQqcIPwXBU06aj8/PmJIeweRi0kz3WS6t4syV8ZkFuw4svIVVKKVDG8ofDJj0arxRI+SYpBF7vjeG4Lmgw99+4knnzYMK+n4WQ6x54knx+eAIMmzRdt83bxFBdlMOqw+83ZcLreFrxZPLMFTYYMvCQTt/ZiOJ5B2W5tvRat6xtTz6eYomrbtXxLtFaw7apZIjznBk0aN8tvFdN4ssCaOha2Iz6PoEY+w/aBVnfVbqVv9nwl7BbywEz72VG9DLa5Gi5Lzilhu+gJ2cJSTjAL/t9H82lVZrmqTjJLHQqZjUaj4Uxa5tVZ3v6oR5PHI0XXu6dPVMBgvmlD+OHZk3bVO+HaXvC+s4H/re80Wo5nOUEj9Byx+Ww18Tn0bRctltvAiXrio/MyCO6cTgMg08sgsDyv4baswGu0OpYXWIHbaKIxGNh+s+FcOs2G51J7KDoGIX2Aje+1XzvewMEn2vvi3DLaNhEJbIyRbB9/Bjm3c9dounLHDIbwt4m/uNB0qJMnOjWdgeM32oErxsLGQ17CPaCkVPwZ3RzvzsbP27R1KHx0QkoVHwhf/5IPhDrUfCCQreM22q1Xvt/oBD7uHGWGL/EFqRINvw09BJkH2DRZPomgbTVfupBO6PXsRtsPsYtyGn6LeuJD6JNkUTb3VShHdV677sCHMyFsuH529DLjZjNWYN0CYD8Mojn8wfhvMrXG3SXIYisQu39pJ7AcDBfWf6wivBcbuhlLrr6v997qIVh8S+Rod4FdGKLJaIiewgNtzYa9N1h1p7g0jz3TTyVbdFW4rslxvd2HsKStdhLW1OaopgPE51kimMIjXT+C+DxL7Uoz+bDxeZYLliZ/wPg8zytD01HE53kymcIZddhHfJ5noaFpz/F5nrmm8EAdzMXnb+cRnIL5vHANXXZuyUPF53nqm8IKddhXfJ6nxqVYKUmNqxufb/76/Z/ffP3TsoAzvt9Kcfbg8XmeYYemA8TneW6dwgd1MBWf50l3aKoTn2+r4XmePqeMSx00w/M8hS494g7h+RfDRU84W+Gn/fhh3J3cxx5YeGphBsM8osh8fz6dffwwXS3x58bqfvywiGCdTXq4evPxQxzDLwjU80Q5NGkK+48rOMFGW6lwLVeWhnkO1Oc9Hq6mk7KkDoeX5UMH6nluIZoOENrkWYloOoc2hRz4MiWOkTkH6rmoMlbSQwTqeYIoms6zWciBWwrijJ5zoD4tqAwDyFCg/q8IQtB8sG6EdfHUur6Hnwcha8S76gfpeb4vmjTNnWLOYjoZNpBu2OcHcrVF77pk7cW0rIUI2hM8UxiEMPkSoflB9y7aWId9y0OI/jayiNUFmJxP30QTRPUB79wyvHWY8ZxhuQV8iMA9zwhG02ED9zyjWGGJOmgE7nlGcHq0uoF7nh0sOds1cM8Tc6Vrbe+Be57Ti6b9B+55KjCa9he450m+CnnqYCpwzzOA06T2ErjnicNoOlzgPuDZxWiqGbgPeP6wMhh1ePDAfcAzi9H0SAP3AU9cRtPRB+4DnhyNpiMN3Ac8d1phljqcRuA+4Hnbkvdz4P4wgfuA56grD4Q6PJrAfZCRQq99TmGNwH3Az/iSZsUhA/cBy6xP1i5sC5NAb52kYZ/cjKYwQia39moI3896A5lsnU3nS9gDWLRzF1CWT6+UVcRlA7xRmKQWb+nFPcWbfpSe80aLeC3e0mtxijf9ED7njZR/Ld7SC26Kt+KTharxRstRLd7S62uKN/0YPueNjK5avKVXzBRvxYGvarxRUZBavKUXjxRv+gF/xltYl7W0qlfy2RCf3t3lao3n8PmJXebb7nwi9DJ9cd9RiO70vLEBLyWAJiPeWBhKoQUzwAoaWPcbjkshgwYSzwdOA67+BpabEf7C1Gm2HFxpdrDMN5CdSWacQ39hYt11GsjDQEgBJkQYhsGdLRpGsg+ZBy6ZCvitHBsu2kD2RCSi4Q/wn/wZ7A5Bl/71G2KcpjNCZ98WvG1+Jr69H7cbDn3FetaQhhycOESu2QLDxLT4D60BHl8jbKCwEf6GREMYouhOd9BswYgjQ07+J/uTf5h+IP4Nd3ISB1R/4Z/oSinj8L2MEfV0Wh2EmSlUTDUQnj753SJaLGAaXIkXzel0kKDitq7bfecaYm5G7euW0/G7LqpQNjs2asL03Z7Xx5371/i+7R85UZ+0xu/eDicwK5LDwanR9a/t5k277XejEP5OnNycNxZ1Bt8eaccl+U1pGg16V7++7L1Ytcfez3eTbyazN9fvxn+e/+Xr6/s/r97cP//58v39X6JXzcUP777y3pCCGC6gGnpLycbt3168HM3e/cfly9sf3wyChffqu9c//i143vvXpfNm/HrUfHf3XfdXd3HTGpNgpzc3iwivptOBRYTpL2wmDPO7xaQ7gxt4CQIzBP8pMCun9ObKFWkmxWZbNyzg9+uhCWqGdEL8zJJvxj+ksguojp76xsRVAuON8FC4dPor6dshHQEfcxMAEvGuyiFAeLchOgTWzRlio7oA7ul9hjOtZu+s/2+582j8OY2A28JH+H3X99hHKQF8yp+HrfM8LJ6H2CTvfR5ih1ZxBjjN5ieWje0wTQR7OxHiqRTSAdH7mkrheSqVTCVv/1MpJAeCkalESQPVplLQ3nEqoQDCeSoVTyXYD3ufSnTgjtZUEhOBTyVchqOn2kgdZ9e5hJfuPJcK59IBLC1MgarqJJ5MciZkTCZRg6+aYmqHO04mZBafJ1PRZGoDG7P/ySSy7bU0k5gJWZOJEkYqjQRM/K6T6WwwlU0m/xC2N7rkzIC1GykIvNm75I6Nku2LNn0sgBq0MAEphJo/ZP4SmROTnb17agEiXDZuvt4tYLXSyPnmQQHHYatkXAnJ0Xm75avJ326RRFfn3kt5bBN2pIxFqmIof6WtKpAPc1YVhaoCWaX7VxWE+ixFJBtPKdimDuh7jELM2wqLllIa2tzZXwpOO06LyI8a8DqxaKpR7snqxymtm4ERNSx4184usZJ3DfUFjt8lJmZ6hhuD6usewztb3TVX450Vd3mQd5YD0MrK1VZ8ZxGxLnhnO+d3tuSdDU7A95j3zlLy31G8s5V9oI5TzZ2uvLRa+aImX1qOaBQ1u3d/aQGXyH9pkeB4fmmLX9pDBAx0vbxyqmd5ealo+zG8thr+5hprrbzPg7y3HC6qXW4++70FXqfgvT1jF8reW2zXj9+jnrPaynrwR/HeVnft11lvH6zwffmLy7FdZVXLq724VNw//8VFtt/5xS18cVF6+ASiF7krrigQcRxvbuU4Sr0394EqYZS+ufzUCjQZeXMLfVLIUz2/uYVvLiqwH+DN1Y1MsDdXNxJAOZXnmVA8Ew4QCWi3S6J1HPXZ8Aj3qfxttpIw0DbVRzE8Zg2Aa4UxtRGv+WNqvw9kap/fh4L3oUXbtV3eB7gD1g2z0QoZb3QzqCs4mN5eTKLl2+n8zR/oRmfde2SFgNX/xNHASIAYzhdL1BuYrYiUk3hoLbeDGeOiz5vontLgcN3Dki0S8L0Qvv6nT3wMsX3KeBUw3bJnAF07z4CiGUDHMZjWiFAjcSyczmDaptQEOJ0k+0GFTXgVzg+q6EG5yIM3/aDIEF6/xsJmyXs8DiVcnB9PweNxENwy/Xiwd4AuFSuktCaQeOZSSi3y+JFlFT8zKolND+27mJVm7lYhbOMQhPNjLHiMVFzyn/8H43WuUrrzBAA=","output":[{"uuid":"01884434-a2db-0000-c91c-f13ff43dd8c8","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"01884434-a2da-0000-ee43-723c53b20b8f\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAJiSa2QC/+y9C3PcNrIwOj+F16dOvk2VKQ9fIOk9+c4mchJ7YydZx3EeJ1suWZIlbUaPo5GVOFv56/deoNF4kcAMRHJmOBqUStIMAIJAoxvoF7r/v//3fyb/njyY3Ew+TK4mx/TT40k0IZOH9O+DydHkgNYcYClrdzWZTd5PTiZnkwssZWWXkzltd0r/n0we0Zpj+u03+u168uvkb5OEtnmILQ/oe2a05oD2rXqdyr4uaM0t7f0E3nxGW17IpxNsldDxVZN8UtKfBP6TSUpLUmyXyt5OaR839K2P6agewXsPab8zGCkb8WP61BR+Hsm3ZPj0FL/nje9F4ztpfC8b3ys5alFSt0oSCQG9LGn0lMiZsTpRllnKxJhzOtpClhbWUiLXfEpHlkkoJnIm7ZrKc73q1ko8oqV7Gkakck4E5kLoW0WNeLqmn0u5OgIqBbQXpalsm2ulmbWH5opmrTXMWquYaaNRb8i1dXPh5J/0h7dm8DmntDEHmjqn0Fj+7EOkkLHQ5zWM/5K+45rS0vGKaTOF1XqEEGPwO6Tf2NyO6Lt/p5h0SOvmLdot6IhJT/rV+6gabeoWZU5bJW7qnbZod9qi3GmLags6iqpFtdMWtU4tVDoDmP1qpcw2JbZxXq3fYowvYYzdMF48GzB+Mcb/a4fwPemI73MKtWsKtSsKwa44X8ozMuD8JnD+kLZ5D7tWTFfiO9riyeQH+qbn9L+bCsoBqKAcFRUQC6+2Pioo5JmznArqHlRQBypYSgU/Tz6juP8j/f/95JvJFztGBeUGqYBo0mCggs1SwU+Tl5N92oZQGvhh8tVOUUEpJc/NUAEJVDASKvhu8oxSwY+Tb+nvM3ou7BYVFBulgsARjYUKvp+8or8vJ08nLyafU6wIVBCoYPeoIKf4/5y2ejn5++RrSgmBCgIV7KJ0/A+K+9+DpYfJyLtEBXlny8AQVJBLPW2ggk1TAWvxBeWFXtK/u6Yjyul7N0kFaaCCkVDBP6hM/B2Vjb+dfEp5okXScTUAFVQjs5pNN2o186WCmtZ3pQL+bKCCxVTwlP58NfmJngbP6f/vd4oK6o1qSitv23GgglVTQUlx/1NKAz9NnlCpINsxKtgkR1R7ywWBClZNBZ/Sn68o/n9OZeQXlCfaJSrYtFwQqGAsVPAVaEpfo/9EuVNUUGj+25uggjxQwUio4EewGLykP68nXy7kiOoBqKAeERUwP55soxyRPxUkPaggCVSwlAoyivvMZvwz/f/z5NVOUUEq8XATVJDAWAIdjMVuRij2M66I+RJ9v1N0UEo82AwdEM0KE+hgs3TA/OlSihtfUBp4sfCOQTgPwnlwnz1L9+k7foabNt8t9K++b3RQwD2DTcrI4TQYCxU8oadAOfmJ8kRf099do4LNaoqyQAWj0RT9g0rH3KuOyQi7RAVkozxRETii0VDB11Q2YPYz5k3BNKe7RQXlVtw+DlSw+nuXf6f4vw/W468pNQQqCFSwe1Tw+eSHyTfgW/oD+FnvFhVUG6WCOlDBiG6cvaLYn1EJ4cmOnQWbtRkEi8GYzoLnVD5mq/UVPQ/IjlFBtVEqCGfBWKjgFcX9Z5Qb+gF4ot3iiDZ91yYJVDASKviSckFfTl5TyfgplQ9+3ikqqDdKBVWgghGdBS+AI3oJ/qVPdowKNnvjLHiWjoUK9kEa+JaeAl/TE2F/x6ig2CgVkEAFo7lx9v3kM0oFBfz9bMeooNooFQTpeCxU8MXk7/SXgBfRc9o+UEGggl2kgs+pdPyMckTfUfng5Y7dOJtuyR38QAWrpQIWheWzyQ+0HYtM5L6Dn/TOPaX3MRYqqDZKBf4eFGkPKggZm5ZTwQuwlqX0RMgpLXwZqCBQwQ5SwTf0NPiGUsIXk58oFeSBCgIV7CQVfEMlYwJ+FJ8HjihQwY7eNfsG7t//MNmnEvK3O0UFyYZvHk/vYDcLdLBqi8EL+oZnlBqeUkrIdowOyIbpgAQ6GE306m+oVLAP9PDNgqw2gQ4CHdzv84Dph1KI5V5Qigh0EOhgN+8fP6Wt/jF5RSWFVwu86gIdBDq435aDz+jvl1RWZpGJXuwUHWw+QleQk+9KB8nK7hqwGHVPQW/E8oHvFh2UG6aDcP9yPPcvv6Vc0U9UTiD0/w+BDkZKB+E8WHWGm5/pOfAVlQ2+pr/PdowOqg3TQbCjjcfL9GvQmv6D0sKXO2ZHC+dBoAPlZ/qEtnpKKeAJlZhf7hQdECl1bYYOijv4Wwc6WC0dvKR80bPJ3yffgadpuYAOkgHoIBkVHeQb1ptmd4hfmvWggyzQgUe0rm8h89lPtC1ZEMX3PtJBsmG+6C72g0AHq/Y2ZfaDLyY/0pbpwpsH95EO6g3TQRXoYDSZbn6Es+A53EX7ItBBoIMdzW7A5OSv6UlQUir4IdBBoIOdpAOW/+/vEKXlG9r29U7RQbrR2/mJEfEg0MGm5YPXFDOYjx3zMXoa6CDQwY7ak3ncup8mL+m58NOO0cGm/eyyQAejuZf5LaWA5+BznS60Jwc6CHRwf+ngK3oK7IOv3UvIfrNLdJBtmC9KAl80Insy8zT9nv7/jNLC60AHgQ52kg5+AG0RywT4OeWNftgxOtgsX5QGvmhE95M/B8ngyeRn+pPvGB2QDdNB8C8aCx28ptj/PeiKiskXC+8fBDoIdHCf8358SX9+pKfCZ5QKftoxOig3TAfBz248cvJntNXP9Dz4HE6GQAeBDnbzPtpzeh4QSg3MkrZr+qJ6w3QQ/IvuSgfZyvwqfph8Rd/xhP58uSAHTqCDzdJB3oMO8kAHS+mgAFj9Y/IdPQ9YJpxAB4EOdpEOWIzr11Q2+JH+/LxjfnaBDgIdqPPgC/r7msoGTIv39Y7dT843fD85C3QwEjoglB/6jsoIT2i7r3ZMPgh0EOhAxW15RX9/pu/5FPLGhngV44xXEehgtXRQ0pPg2eQF/fsP+n9R3JZ0ADpIRxa/aLN+dvkd5IOiBx0UgQ6W0sFPFP+/pHLBCyoj/32hn939o4PN+5sGOhhPvIpvKA28opzRPv1b7BQdFBuOZ5fdwe868EWrvp/8CjJCPaEtf1poP7iPdJBumC+aBjoYCR0U4FHxlJ4HX1Ku6Osdo4Nsw3SQBDoYCR18CZHsWP7kr+iZUAY6CHSwk3TwPcX+L4AGvqRtv9kxOqg2TAck0MFo8gS+hJyxr+mb/r7Q7zroTTepNw10sOq8UCxj7OdwP/ObhXnE7yMdZFuT/yDQwWrp4GcqH3wDlPAjlZY/2zE6yDdMB2mgg9HEL/qO/ryitPAPeNNu0UGxYToI/kVjoQMW0+17eip8Rinh64X39AMdBDq4z/FNv6V0wCLasZsIeaCDQAc7aj9gecRf0Pf8QOngxY7RQblhOgj+puOJ48XasIiOr4EmdosOqg3TQbAfjEdv+oRyRfv0NPge7uzvEh1UG/azI8HPbk10MIWfJh2wv5d0FnP6+R0tv5UlMR31Mf2+hyX/TT9/AvkyUjqamP7PQbaurZRSDUApVYNS9LLaUqYoxix1U43ZLrOWihmUsBZNCmK41bbAsXqW6Ym0KKldoyjqEvDqmGKDjZ7EGj+ipXsani2iL5Yd3U8fy/GjG32JZwN9LaavA9oDm/MxpZsbGBc7hWLZZo/WXtBZB0rzp7TcuMOm6KxZHqhsVVTGWrC+b3rcG3pHe7iGEcVICzd0ROzvOzr2GS1tvyXTcvplrTWZeq5HTf92XQ/+7G7f5hK7HuMVDumc2P9HuINlk4+AP/yEvuejyRv433x3CWWs/hZo8hOguwK8OtPWWlewN/Tb7/Q+Kq2sbux3epna78xS935XAWya+51ZKmZQAwyb+51Zqva7qYWrMEvVXvc7xeGZXEeGGf9L8YLjd7fdr9IyEug7oM/+J9a6C72pZ8PtyWXReL+aPIPMNT/S/z87pFm2e6W9KakelTRbw1s2Kc1WnlRQdrw7qZ4NOp1lOp192opZfFl2y68CFYySCvIeVBA0OsupIIcsHf+YvJp8AXEldosKNqvZrAMVjOb+8N8pL/R3iCjxk/Pe5H2lgk3eH0695PBABeugAhZj67vJ97TlT/RNLwIVBCrYyZgqP1Hc/5HyRJ9S2fibIBcEuWAnI0l8ChGo2Q1iltM1yAVBLtgNKviWvvuSYjuzVtxQCrgEqLKxn4Bd9hvI5JdNXkOklS8XUEY2AGVkgUvqRBlBezq8p8I10AQb/RFY4uLJrbwzxu15iyxzzMOqL0XofVSyTN2srS1lijLMUjd11EZMrcxaqixztVZaaKXtqL015MTMLb4I7Rp/yllukauBXtQ9/Ezz2Mi8KCqFOPzdKEo8e/8pao409Z5+5vM5Bo8dfcx7QEHntFWbPnSP3670ofoYw4mRwD7S9b7xO1hdxpV2PTFyT4mCcQJVZ/zmz9rxO3fi9ylg4LtO+/IpciUCBr+BX8UNQEpE3VDx65kv0xnFvVPYQThnkcv8bHebbQ5r6ppt6pztBR3xkWzXflKsHVvtMzrro8nX+MQcW/yP5Smxm1/AuI8NWDK/BrUbvJ+8pd/ZOfZM7gcKh+a01znQ67m1/kyWpUvnfgOc4tdLRnQAq8329bd0bDfaPFnPM1rP/PzE08ewB/7ZCUbLxnWMO+TycXV7fybffzz5nf7dN7y3xDh+od+ixk8b+kVH6DMMvwG/yk3PsylPRHTm79GTM6XfvkVqfwqnRRsCJR3dP1ulbi8zf+i331V1hPY57NsHnrjepF3mX35Md7QrONUVz3NoHbna+T5Bb/QzOM1jrYY9z7iIM8CDA/AWnOPuegxcqx9tteFerwTuPjvsEJBncz0Ar8pjbUzvwacyhnO4G1SSBV5Y/nBxwca9AxdefUdUftdvMcxpyUf0l8WDidHz9xwok9e/g7OTcYlcEhLP2MaWrQgf8o74MAO8/9UTHxg/MpPPLvNFV2fn2eQPrSfOw03pLPn/B5IL6c7vrNZjviuWFytabbKB1W6uqvneB8Ah8hPrEULUvf4ZHefv8He1a9+8jaTeu8pVL1e06tWWrzrD299Rxlvnqqv3rnLV64FONDdvm3Y+9ftgwBygfw1yNmsbtXBiPavof5Ow2wqmK+RJ0s5SYZ+VY5Q5B1vdelZsTsvfAb94Brdt2MiZ/u0ITt4DOuI92ubW63Ya63UGeCfG+B+UK/uCtmQayf0eq7waDizdCAd2DlqAM9CD6fcxhlvfOchETEv5G/37tvXGVdBhsaIVIhuRVnX+85yO9y3AgsufDKIxtogtOgi7RNvUAHSFc7kiOFcbhfMZ4LaQw2Kt3TpgOhQXsJgTyDpzAjZb0aY0Xd/R8peTZxT2ryZv6N/nk09pyed0Db6hn5/QTy+tc0+tGq5ssPPbfY5nC06PYeGuv7Vp1VI7xnvQzFzDactPBmHNGuO6FY51ywfmnN1UQ9a2ek05yb47PZr8C7jcg8Ybxrh6lWP1yo3IJDdguz0G6eQYtZPdeR+7nw8b3xHAktlk5zCebudCNsi5EDl+2quV9zgfBGS74bmYxaM7QawrVPZof+9w92M7I7fYz2FsEdLcMUgkjGtlWn/23AX0dTz5K/2sbADKGv2fUKPbRDlMr+hIWM1bw8PlGnbcM9iL+chFu0PQCc+hNathu/gZzuIaWvzpnMFjWnoJPYjZXAL0OdW056GPidWdg9zFRnXeaLnonY+RR/0D53WBHm038O2QUisfy/KZvcX2J/CO90BJ3ce8eO7LZsT57F+Bu2Z2nRnGGLhGbT5731tPnIkm/w/ovITt6QBmbcOXRMOEJpZlVlziVm5W44ZTG98KibEumF/TkreTv4A++iHqkXL5iUw+HgBb3ZgT4NsFvjPwTuIjEhLLAeL/NW3rsz9MJRT0ni/h9xb7+TeOZI62T/YmPqdjGAMru7XQmd5LM94K75XBcC5n8hgtrGJ8MXhbXAI8/+oYA9MYzUEPxM8dvhNcwSqcwXnBV5uV38LbzlC6VrWqfIY93CDmzQGC1xTXHhulM/TL+gvCj61jDCcD+/axxJjfYS85BWy5pBioMOEAnn0ofVbE3z3QQXysYWGK/wv8r1ZMjacJkxn94b1lsPaH4LnCOBYG02NKgWewKn+BNiXiZAzfpnj687GITzVtxUc1v9M678GJfgSUcyGxsLk+04Xro/fQZXU+Xkg5b2nbCwcF2XH0HeAJX4MDGOkJ9HCGvj98TzhECCyjXP5+3zPzHcBm1tpTOC/U/PSxdefLrTTfHtMerJKgbj5C2/srxB/9/8f3bE+x7euxpoHW4ZHK6MPNTx8v3J/CrrNs17mCOR3BWp5IbnzZCbYNexLjda5htdy0YcPLP2DfETLoY8SIv0q/EzHnx+Aj9gB+BZ7cgK/DOT5VO2QXJe2xdvz9bO7tlm05ieOO+mRbqxlI4jcAA7GXsvVfVC9qm1ATbU3O5trgCd/Dehwjx6jGoVq5RmJvETnX0HyvGo8+StdohEzLV/cKrar/tnCrCcJWrYy+E+lcpv7pYy+uVoy4TXcuvppo8zwHiHDfexevqXZQbg1ke6zv2Ss+x9DuGu9a6WcJWwvW8l8wpxuwSX1o7Qccrzn9H+NedQMSoaDuplxRL53LHM4xPqJLONOHmNEhUv85jqvvvOxj53cLrlD6WXSydluNdzjKWPp/CJ+8qztzVcsh4ObDls15Dvv3JeA8h4CJ6/lSLLD1quj5f6Q+6xPYl5VMzL79E99pp6L2/2Tg0VygTM7gpo/HhEGxlN67jJRbKoQPr+A8l2HNsLQ1JA4upkJzd42N86Gwcjdt6DR3KH0fdu/Qbo5pDjbRK/TsubGufWKs/l36OqVP2rG7WIArdgwQvOQ1rNBdR3Lg3NtMOUGdx0fy/uKBgT8ubePyETDN2FtY80uYhZAF7fzbe7Agl7Re8HI6DOMG11F0Wh/R8gK+H3vCyBzHDbylyaPbJaZlEuwyrdc19MrO2P+L47y14JcbNmar5fTnO4bHjYjDwrphG5udd+321hlYC5e/08Wh/qnZi+aT7zR7j5BqWHTXtkXJbnXPwepuKx/O3mWzb3W1w78F29MHb+uW8PXXrZbinty6bzvZfRLyFflL5Ru0lisrot0+LjyM2b3c9d9a72YDzjtbyrffZycH+7cNJtUK/KUW3Q7cXb+pwrGDF4P4TfnAvuuezU++u3tNsfPzssdOPdSoNvt29l7OM6ibGJ+CZzbXdR/iCcdlRldNTH90veiDLZrVG/pzCCfCHN61ipFzv/auI38Oo2W09azhqd+8t89u0bC2D5w395sthFVH1ApZRtQLWfMzsC6IVlPgGblfO/8r2vN8ABfa6M2T9zf4Yf7iGf17CRzpI9BMirNXh5SyYh+gjVbFdjgA7lf3RLqG27lNTBQ8A+dcV7O+V7Cmp54rrPagF3AzKIK7R4xXfg73wxg3UcOeyLUcXI4SpaItry+MEr0df7LAO2ftpxOtVQKrIcpLrQ/9r16ut09lG9H7H0784nbOa6S9fe0ex934JiabsJySi+WSosFZ+D5VOc7C0lFOOkd14NET9FxBd90dXoG0/Rng/xj3XX18bK9lNHuinSmL62NNPt6G2Z1C728By09wXcOJEk6UdZ0o5xQK/DbwKezSt3Df6hRswxndl88pTFh0zlM4K/TaqawtQcLVn2c1rE9m/+JnQAzQZjul+FSDRNAsi0E6zuH2l/pUaKdXMZrTgoBe6u5PZY6YKl1vdR4MthvNwA/zcpCbf5vfx5btLO29q4KIdswDZ9kelhotFu9RCdywiowIBcPuUqq3x/Szugvh33MCMQRr+kl/fhUreLIxKbf7HpmABo/dTpnCrpag9q6CiGXMa0l947o91jbGCIlT4Pia5RlEciwAR2bQA4EeElnPYxny3lkvLNoef1MCzzbLReS0CHblPchkzHab2YK+xbhreEKMMNJGXlpqzLGr3qNW73x8RBu1XqaPONJG7N7hb2Gv+gvweW/RDn6Evnpst/14ZXt913uow2Edg/ge+IDXG8K5pDfOJQ6cq5w4V90B5xJvnEu3AufqteMcOxMqhHSFsSb2IMJWDlinVjkDbKxghXLJuU3B10WtWQoZJitczwLqZ8jt8TzoieQD+fdDWCPmN5UB98g/5zCKHE5Ss/fa6Dtv1J1K3lK87RDxNsVYugJzS8zym8O3DKwJxFLDvxWS+83Qq6ACuuE4xW9qcGqt4BvnZadgX9sDqFZy1AWFSIljJqgf0b8f4twJ/i8ALhmMqNmXKjmFz1O4jcihKlZNcCVitTgE+UoW2qhiA5Iz+M5LCoyzqpcIuIqVi7W1iy3YwFczadWpmSj5Yg9gzum7K4QTKE+wfYp+A6pkTFCuYQwZzDhvlWQLYJ1JSrRBe6qdBG14c4jr0ObU4YL5MnhzyExhF48a3w8NyhafTNoVo3ZRewHSpgn/KZ5WU5DOYjnWWI42xxZpa1eKG2Nsfjfhni+Aer0A5nUL4jXusAnOpoC4t6t5360Grwohdve1JVhaAIbo3+6yrsmaVvbuJ/sH8MmdgR/qqk73srP1uM/pTuCuRA2wEad7CntgjjttBrFcssY3FyY293b3zq7W4xbtDin0kUkczOEs5SdvoZ206gRWZafGWSTwjp/pgrMTT8WWXs2z/Ba+1UBzucQ10V+M7xFPZIh1JcRoz3BfTyETt5jzVL4jgZXOQC+2B3brFPnoEnmSGK0jJZw+vIcSYMQ1cJwSasB9IYGV4J1JcF0IvDtHXiZB2BY4zgJ2lKpVo74lFGNc7SLnUymedxXu8FOjbSJ7rBvlZo9zY4xmy9j5XNKBqtl9gQ8rpOekIz131WcNJZdsUrooEIMyjbcTdKeoNrXsAbnx+bTF5StOLG3xYpWkWpMbyxv7GefHzJ7jhrQyLNfLtdurwc9sAxqMBNahhN0tQo1GDrsg20+J/JYCvPTvq+NyfXkgP6mNfcs1qU18X9065mtfRzF7Lh8Jez6XElYh+d22eL/h1+cW8Y1zruUK16voYCUqO1uD+lBrjfRHpK8H32GLAWQrXUowv41XztHxpU3dd+M+riEezeq4D7LR3b3CmJh7sEMkwKHneIIT4KndmpHKcRa3T+Ic+QcCz+jfxrce5cbXQz9tCYVDAVLEFCA6ND3cgpXe7D/W+rftFoV1tyhGuJZVJzt/ORJ54G5yzzq5iinSN/cGSVH+L8Cawzn6BG6w1RK/MtjTmd41RVxPQApOUX7nfOYUbgOw3SGRp0QJWEcaNTnqHVIpPae4UyXy7Kkwpp/ZmsgdqN0TAbsDt48Q0Lb8gdKPmE8MlDWF2XLqyZAupuBLk+LeWYJ9OUEtTW7I2TVqTAhmi9NLhZZgClySvuMWyGvvAY9RaRoF3jZ29rIqDqnagCaMwFlVwmqUEvcKOLkYPmWokyVSn8Sxr0Jraoo7FwEomXUFRklJcbX42vK1ruWzBE8x1TqVON7uKcVeErSKRuiXpVpHxpiadXpPEXqj8DER48ncaNucm1nHdVMpSuxVa+6HUhtF0HrM47QksPNz3jLDt5i9mHUJUlYpff72kG8opNWY20aadWY/ggaI5j8hxqO3zeWs2v3k0i+tkFbkJgz0EcXOmeUrpKdk7fRUokdMDRxZm55SxKJK4p+yrCiYCmo06zLoucC1mOJem0iJoIBnhRU6RbrO0HNQr8vRo1BR5h5ICwWuEgEeRO2NROIX1wzrNVxvVRnapQK8Hvl6F3L3rAAbIqmvZvg30yCSISbW8LSQhQi0FJDhUliM1vhK20lygEKOs0thtyatuhQ1pwTmJPYk0rDDZJo3pqprn60ZnmyF3BNytPPo0DdrcunDIfTzBYxG+HrwfWsPs+amkvtP0JPFLBcrojTetcZJlpjLWsBLh6XgA2ZYK9ZgD0d9KEeTylXl+x+fHZd+c7mDFtgyRw42k56tom51dL5+HQHX03BfwVzSOZfjC8nxqz20AHzMEau5NT1BHFZ2EI6lHENTyXclyAnq5TplKlsH54x0PGrWqDNE8FocfwUnmcNJn6L/svp+2vgu6rkF7VR+K+HNt406/bS5bfRr1uUIxam0h+mtIynZ1SDZmfSZIY+qfKpF3R5YiUpjj0rg/KmlJ1ei9V/ATGI8w7OV8nzr10YLmwLHoAgt0RXAKEf5mEugicbBE+TVhARitjBllARPJgL8OHHIMDVKMaeN/s26DPG3Qqxr9r26lck3oLmYwmoQB/cguPGywYvnTk48vwMfXjq58HwBD25y4LmT/869ue/SyXvnCznv2pPvTp1cd30HntvNcfvz225u281rm5x27eSz67Vx2cVGPIKnqHMoJJ1M0V56qEFErVqCXmYCkwTGEc0GM8Xz+1aTiA61E0Nxv7XGyyqOK0XNTNmq2TO8LBM8d4UWsEJ+M0V+P29wfTO5lolc3wI4tAxlVf1tEY4jlWck3z8T5M64rqhZk+EM9b0lRWrn483xLpNeo/y0uPamRi6d9au4ggxhWqEPyCHyKDl6lhKg+tzg8sWKZPgmoedKrTUVrI2SigicE7omyCznmt4E6uaafGLWRPhMiTeuVA3fedRzsVGrdh4hCVVoM1fSSCF3PCIlYlUjPL2JhsFcdm7K6kJ3ZT6vS2Kl9Lms5Z7TlIiafeSatzrR9pxUSiqmBqM9i1XuOKSTJbHupL0mjltqxRribrRvqp1gVMgx3ul9DznseIy0ucz3zbMSHMI+P74x/0bH+I625JGYl80gxmjR/i1jjL13BNkyDjB63Cms4cXK7gZegc1jiFvOKpKurTSGqHV8ZOb83e3NfCTuXkUc6CvoU9xqt0UVDzfFw03x9d6C5PIv9+U/RW1MjTfJBL+XQsTJPdBUCB49l6dyKTma1Cjj+g9dTlMyGucg+G225v+5/BS12katmqhVw/8fSjlFl86ER3qN/FAqNUlszDOcK5/5rYRFga1r6feaa/faa+SgCN6sEJxDrfFwheSIE60kx/k2a6JGi0j+Fy2jVstmTfPZsdywrxdoOP2iidlih9WNG/i+Y+lqVVGRk4c8mdh9/XbPzdiHJh/yEDmHAxhpLMfhy8nEgA/HsLqiNx5tlEXIOgVsENFpRHQ59fQe/Kjd+gD9lEXbrrEQ60YcBlU+tZZXjviBlSOOT+WI40M6Y8SmI+Rc9447usr5ifgikRZNOzZiaZv5b+JG9hsVwVuPxnwFI+BRbT9gXoYbo0UKsQ+WUwmPdXwGtKDn35wDn/Yes2VcWHq6grkdQgzlG5jBsda/gk8MlHiN0Z7ihW88RH7wRuN4xslfNzP2NPPtiFULfG3ga1fN1/rzNYKG9Rh8HO7J5CnwtvvI4Yq78sI/PkENrYh5xz5/J59WbUVvL+Avj433GtbjKX3iNXx/LiPzvQYN7D5851qxSLMciP64tf01jm4qeWAR849bWdTIvpNzEK24FvA19rSP1hf1llp7v4j4l8DoSjpaNZMX+CQfNf9cyV64B12Enhb8Nul3RptS67kCiAsY/LwyrrPuxB9WnaMBD6W3+AzOixPYW22lXCtzLjNwudqc43uPFrRp5p0ScfxN7kLM+wyySR4PcEb58f1TC9/Prd02To5n6rXzkN3jbIfzJpw3dl85sUurmHbxiKRufvP4z07PpQ4qIhuRlE6B47xaypc3OfzFz+lYdwC79yVI4Vdrwryws4SdxX0DSETzTLVonSIiZ4y3JYQnk7LLC54xR63tFOl5LksjrVT/GzdaxI3S2ChVcUMj6e2ay9jQU7gXoet/K/idGyURlkbSzs91tKJGLxHtRCyhBH0RRdwLoXUVvro1fssBgsqeze3pMXLLMcQtSMGLVXDSqfTbi7WbEwV+K7T+UvT1i7SoEBH6DOkW50Ra2vmoxO+YzgnS8ZwoAlcV9r6d46qqjtRSOqWWfCN81Xeomf3WyA/HM2NcYzbI5VyWkBe5Zvo9SpAn9Ge2Mvxa1byvwC7PMexYZqgVOl63HPwt5s27gLoDkMrndD5z1HxHgCfzlXKT/bQPi2fO8sgdjXItn0N2TW4x4FaBX1Hb0SzlFhPhdXGs2b7a68lsDv9F/x7gfn89+RtqSm7APnZC92qeXfX/rlgD8qNVA5I4oo8noH1z1STOmmk4x8M5vmPneNIpoguPoes6x2uH3qRwWJ6nGzn3mffBLZ6BLwCSB7AO3ez2/H71pvgXNjpuI48a33kmBJZB9RJ9K8bJiZgjnoMl9WiFOWSGHvFh49xaxZjfaxb97cqq14aXyP7Es3+vlnv4Fjief2HucxsnkTr5hbSzXwzD4O1cLd1KpjTGP8BeeSrX8lMtk3zU4ZkYPUOOICsL95tYFXftm1Wm6XFm05j7wssFE+4ZzXY2LiUxD55FbUVG3vfgIWN6/ix67gBhfduyoJrthOd8rPGbi9pzO0EMktGyeYq2wntePbVcolaStPCcilE+E5R8CrLHMfBnJxqv2CfTj+hz7F79OqS596SA63ZIp4ukzk3IkqnDXzIBv2hXDRmBl4SJB+vgQ/qOXnkp8r3/AHagMYx2fLSjvD9d/qPbCjU/an1iYEjkwctlTsk0c+qLMif/lzn1RZnDyyaBG8xj2xdWez6sZtyLuEcuJV6ht/51D7k9c9wOTeDGs2uFN3NztCucBPdl8rOrwAS7j8mu8NxMvxoDVh5J2XZ5e/PMvgvv3daOzkCXeqxpFOfA19/gjc4r6/65eS+gbd33guY/aP63U/OfNzgb/+emzvOydtZUzhq3dJU7a7INn75PoB+hoWZ24Q/g6SR6vobVM28u8jvwB7A39eFX8s6R8oJGMmgkh+eOuHcHv4F4em80jUfQ3ylEAGEQOQLo9tcvtvsNHFXgqAJH5ZvRpQD+I8GIlzymtZkrupJx/XhEXpGhs5BRfxOMfEswh14NnyIZV7jGnHSJjEHKI72KzA+ZzBy0h9EACcakIxDfT8RuK2TUrhq8HFLsLYP4CwTvhouYGTnmBxLZ+HLNazozsgzxjNVTLW5aLXP71cAhVDKiP++1xOigU5yFiATMayqMAVrJGCWVlumyhmemMlJcIqGuIJlhDqxExrKeoi+5yKGi5x/IMANehu/icd9EfOMCZk5kRJVUZujh8aBLGfduitkfY8y+kmKcf+GtTzAOLPcUmxk4YeILz6ScYLaESsK5gngJlcyMrGI2Fxi5lOD9g1KL8lxhXGwVC7CUmbtrGcNQ5I9TWcGngAe1jIGZQezdWt5l5WuZazH2CObQjeSba8xFkuI9BAJvE3ECU8QPEYlwipio4gHmGMOPxz/PtPiINcauniJGZDK2eSbvzPLaFONGFxhZL5MRkEv8nAK8VDR+QQO1jN1bwMz2MC53Kp+Zyr4qvG1QAQZWMrKyyKRTyHflALlTSWsV4v4hRmfk8d6nGD94T+bBnGJsSp4JppAxoTPAET6iQmbGLDDydoG4KKJUEsRD0SsB3Cgx3uFUi4ueaXuXWgmO6aUWOzmVGWdyjAEuKCCTWExkLqICMSbDEesxGnN5H0PkH8kwv0eMEQ9TeD7CTNK83wzzS/OYiiqOL98BCe5xU5jbHsa6T+W+l8uIlTXiyxSgITLpiQiWOVKjyJfOy+YYWzLR6J7A80TLuCVaH2rviuXbKqSNUsaH5pm11FhjbbQzmVk7lZlkCZ4mxPgu4kuKtiKjqYJPghmNYgNCHGr6WzM5lqQxbn02fJaFkYVQLyUyY7HIj5QhpguY5RhHtP1UYawNf2slYVVqELSNeUxajqqjlsOll8iD1XcUVl9f+6FbjtNXtXDa+QqnvisPdr4tsvMVTutv4bT+5sHOt0N2Pn9N1qbsfGcoq4v4WXp0uisYB9dZLtZf3dXnzrffu1gRl9stv9XeGzneGzRnQXMWNGfbb4ssOnLphZNLL5xePYXTq6dwWhzzBTVZsMYFa1ywxq3UGjeHO8jHyGvN5XkzjE2u3WfgKwJfEfiK5TmvuRWOZ3i7hc8VZpL7A+r3pB1K5IGuMLPcqbRdqZaRzGDIWhGjzVg4FdLgBPyfc2kfSGd/nqBPXL8+8Znj/DFXtHSuNXHWFEGXuDW6ROL0dCROT8dNRdgfvy7xIGgQR6FBNPe2bpz6mbWPIbnzR8Dj/Ra49MClBy7d02+Oe9qfQq4mxVnfwndRyuOrElmaYksyIu677Mh9l07uu3Ta/kqnRbh0WoSJs6bobCvepJ4w6MXurhfrdm5eww0RHqlpiFNT749/Oxy1NBfOzXBuju3cnIJG6pb+ztDbkumquHfuqRZ/O8HsN1OMqa1H177FVir/uB5lO4Izl7dQvUQyv7yIz30r36ZieMfyPeJXxSIncnziSf6WMZ3kdceTvHKeymXQow2oR4t656n206i9tJx8+rpWTg6tcnJolZMPK50auLKzBi5waLvAob2D0YlY0O8AVicDWSDNvt8YvQduLXBrgVtbzq2leJOC59/bw/sSGVglM3l3hmnhT/G2Uy51IRk+O5P3fsRdjBifjrWnY7wTkaOuhOej2cN7ajxLTQ2cXylz15T4VgK5tnPk/Erkz6bwDK8fD4/WzBXs/xxxnstF4NG2kEf7wjifIvq9efrpq1w7ua/KaTernHx95fS9q7bS9y5wbOvi2BhGX8HszmTG8GH4NVfPgVe7v7waz30WS04t8Gvd+bUXwJXV8ifCe6QZXfcEs0Pnk6cYjUD8PMccze2nWIbpp40+X8vanyFvc4pxGCLIRV3gzXXedh9vGyeyZ5V3utLKRG7AHLk9vYZpI57jbXE1Qn5H/TXeAWfZpKcYB4D/qMzRLLt1DLUpnKBT4EMjvIE+xZHvoe5MPSvK60afBPNL78vb5qp9gnd560b/oqUa03OZyZqP/zXO6Dne+M0wzx+f/z6WEtRkqlyKGcIyQg2lvho/j4bjrTtmQ6mdPErdmUcJHO8mOd7PF/IO5gpXzrUvnTXEWZM6a5IR5JdZd+6OJ8jxtSGfOvOcJ5j1PkgGQTJwSQa3A8sDt0EK2LAUoHPjNhlArw/a2s1y/xXGI8uRIxVc7T5qVqcY06rAb6XkQAUvWcHdT71U6H8z2osq5ZwpkdKC6JvHkksws7YaTYURkxinXssYWxHebWHRkPYxMtIU5ZQpxtHJ8Okcs8HtQZwcs1QfoyoVfLRoq/rOMbu2GHEu467lGJntuROWXOopkDMnLUjzbyVwKU1YJxjhpwltvVyfi16uQ9x8hw5zc2Q61PncCpQWdLgnGF2ptkA+kXeITNjr5eaICwOfMglZ8x36CoiRiSiCag3ccBaypxhvId/Po6UljRrxLbXMRJWbM1Hl5kz0N+QYfSrD582yfSmR1ZbWmZQJzf5VuTkeVW6OX3+DOeMmfDjUasd8VXmO40mlTJtpeJfImlLGilQ4zCTT5/IpMZIKY/kR+SYlUZcYE6uEnVHBtUKMEGX6aFMZq7ICfIgavfLoYUIS18dCYHT6yEuM4FbB2NRsK4xHmMveFTwzS7kaH4MzkTEe9Tc8R/rIcJfdx1h/XCehjzprzC9rwSBrQUrvy4Sr/lZ9NZ4b49TXcB+fSVrrnhu6jKxVqo9RlepQUz3rMLZDbCxajBROjz87PWfXYvAzM2gxtk+L8Vzm//vcIvGYK1w517501hBnTeqsSYJkHiTzhXHXbwD6jO54XsdzidXDxF/no7G9IUjywfcqSPN3leYV78S5sqeS23wt5ayn2Oq1bCV4T4Kc81NZ81o+/xRbvIYWTamAyxmi5338nhiySVPyEN9fa09F2pP6J8UrZtqc9mGl92R0eMH3qyf5dzXfVMoB+kj0XnSZqTk/c9a5MWfeuw7vHJ8ZCy+aNG7e+T/n4iCSzhxE4EU3yYsKS05E8dl9tpsrTZw4UDhrcmfN1MmTBmtR4EkXRUBlM5tDD8PYi5o9Bj4z8JmBz/Tx8Y8gq4J5VzIy7lTOG3cnI3lDUr8vad7SNG9PTukbeH6UCKOm7ml5OWq8YZDK/D2HmFshRm2lyPKTYP9E6188EWNOE3FvVGTbEfk4Ish1IUoiLBG/sVETGyWxLMkbc4oxu0mJ76xk3pgcv7EbDfxGqogIcYi3Wve0XDc8Ww9/Jh7RrdIU4Plnp+emTq6hDtzmFnKb3xonbDT5iP5+CW9m8cjtOtDUyTumjvghKfy4MKdy1pSB3wz8ppPfZM+ewj45lH9Ss8fAbwZ+M/Cby/hN4f+yODbH3BKJo81hRq24H+LWqMrfl2C+sAzzkk0xr2Qps5rF8olYiwFSIz/HY4tkyFeKHH8ir5noSb/Pmsp7sVPJQ6oy5nVwK7MdnqL3y612F5Zzh3uY+S7V8s8lMo9eilwzn2+OngKxnPGqYJvL3J2ZNk4Os1zmeCzkmFMJlQozqYn/Kr+oyOdZyFyZpcyyp+4MR/i2EvMl5tq67WHmvT3MWUjk6ueo685kJtIaoc9vHXPo89x8JfauxqDi0JQ4nlTy7SpWTI73lvn3uZZdUNx2Fi32MBdpIm9SK5iVWnZOYpSliGP6uBMZv0bHmQS91mKEhMATvb9Y9qcyEiboacP9OQ7Rc0TkZVS0kaGXTYJv4N9KmZVRZbWMcR4ip2gqS+aYv7PAsaoWvP4QM7XuSd2/WMVa5hJlI8+QLmNYpz3IQCHuwUTo1RehfWCO+MlLYiyNJd4W+Fk9qbcRMmaq5TMliI+ZIe1xq8dclkRYIn5joyY2SmJZko1K9qs7yn6Vk7cPEYW2Ufbbt/LP+spmTqktc8QUSiFfpwtPiLMmxBQKkp5b0juADGeXOOazAS0Mrp6D5BckvyD5LZP8Urz/fSt59hL2BsVhi08iziKXgW5Rv6/4MF5PvOJEph5xImdoAeBvqjwiQsbSG1zYK4SOX40zu3PdWLi+rFOMIp6d23XOhxhF28j1fbrwLNVXOHdycpkjPhHPD++qyZ012Q7e2P6UPnHlhLyL786DX3jglBf64MxoyQlmAxnGB4ftSv+CyOg3wDELvA088n2O4pSHKE6DeX5zDWMhPV72pV43gx9RzmMBlaAXJeiBwvnqHPOlRfI2b4H6XXHrd4o3+lSfXP86xRIh8+yBZzCRvUVydPsaV616affbHqM+JvV539HS7K39vkiD1xR8wttjbs/NNnsOG3NMNni2IW4bmV5OZF+ZBarNdzRHq49NX8NmO1s/iWOkOn79DLED+D1PHiugwFlkMrZV2ehb3M3O0XZAtDHyugJHSRqjVHfiK7yRK+JdCYlPvKNZk8n+RCm3EIie22/ilqraOkYhgRY4Z3N2KVqq+BjbMBGebJX2TUQfIBLiejvVX2Z5Qy1H0xxbLecg+jRnVcvVbMJDr1Hwa5YqeLffZK6UOUZ9jduzM7GjDRciYUfwLoYaFbHseCoim0l9zTfpe4s5RtteYs5vaoHJtAG9qQXOtp7NNTJHpa+ubUYmZrQhIXBKh5nAwCbNq9ZpizrM96RO6sha1GHOTu0rJlxUuYKfWabgnFn3F7U2qXN/Sa37i8KINkRIA3ZNDKzljt2Efa2dNM11a1KPvuZ1Y8VNbGlSz9RCMbyM419zN2r3WC7gAkiDCzBn0d5DTQjou6+ABZGr24S1at1cJbLgjNNH2MSL5swEPjVhIsorDeeaMG323VwHfUz6Cjbn0lz5JhT0HbeJY+LsVTjXnIl6ujn35nuJY7xNWDfnqVapCSG9Rj95m2NsvqE5UuJYXVu7Zj/N94jx6Hi3aNztuTbXLrFQRhO2NtgTKyxUTWWcts0xNt/QHKk+Mn01be2a/TTfQyTMdLxT+7Iuabj28Pae3z4TEoukYTtn2ueQfadunnT7jbL2uajeYztZ9THqXL7tvLad6JWxR+vwm0IvNt6hyWPYuZDCInvYeJvmjIgVEqQBM2KFbvucbJ+jhUXuaM6pjQvtsz1rwMuUPAT2uXZx267v2l/0kdpOmeYppM4n+8ml4GiWNU/FpnzZPldt8qXtpG6f5eZpH2HE330Hl9DkJpo91g4qsHEwbR6HWCHQPuP0vc92IjZPTNeunLZixBVOidd2wmcIL4VxOh7qvJ5bempLXG2JzLZXN/ckuxRol5BUjb7r2SVRU1J178vEoOK2dNyWnisLrejwU2fwIum9LfG7dxrS2mnUHtWeHbHChEi4ESts29JpW3otLJyUTSI2MaEtUZMWzFxnsJB8TbgnWjw9tVrNOZjrbEpIbYnSnIV+pprjb9ZkrTO4Odbmm5qjzpx6jnZbW1+VhXZMGApJeNFcbPNvwqewysJ2uOujJtaxkhYMiRXezd6bIzVHZq6yra3eV/NdpAU7MqIYF3lHH4Tc6YOQBx+ErfRB+AzfMoeYa4ss4YXTByF3+iDkTh+E3OmDkO+kDwIvfQcWRLYXKI8Q+1q4vBKKrfRK6Ldy9h3gB7DFn8p1/RSs5mcLPRIWPcM8A97T/5xOz8Bye7SiXSV4adzdS8On7RzXta9vB+uZndFvAR4z2LUvAAOG8PMwew/eHcEDenc8O3zXdlgOep1+2omMaJLI+4EiSon4fyrvyc7wTmGB+ME/3craArUw/P+hlj+2lvdkU+N7jh7dM9QciVuXIi6MyiFbyW+iH3VrN8e7ezHKd+pOaGrc2lS3M0v09y5lrMEUPcP5pz9WJu0UnSOi9In8U8rbiIkRN5HDLtbWQkA/0tah7bcfaWXiDqyIxMMhOMOaAm88ik/qTeLTrVy3FDMIl3Jt1JOR/MRr+N9TueIza98C64j2SbytljpHcXM3wcg8/NOt7DGVs+H3W9UnhW/qU4a58dSnUzmemew90t7IdRL876lGATNJg4qmYoOq4hZ1KU8p/j7bet6FLs294NYYXYEa51h+Uvera9DFqhZtGouddBivgQ5JQ4L0p9+q43OlU0oiQVsxoLZi1THhFX8dTV41uGMbnrliHxGn3ExGGKN11Zz0Ksa9SIJmc+JvPQR58q77jL5ahXMdcyfNFxvRK3WFk5BUTZl+FZhg0xrsjt6ByWkxYOWR1PUtb2/u1l1uk5yhXMVm9w5kw2ttn9X3uNiqYeijsbD3qMt9B0Cpl/SXRZ9ry37NJ2YgVx5rEpm5awedRtBphNsqd5UiS5QeI7Ah8t8YeP6xWBLLjjw9cdqMiNNmRJz8PHHaJYsF/EAW7m+G+5sL7m/qty3ZXnID82T8wWruc7bfEU7JcEqGU3K5rrXGqIkqX02FkfZy1NelqDHlsfVy7RtrO8folRnWE/yuWsykn2UsvTAPUasbo8ZN6dhyTbfO3zfDCCcZ+ssd4hvE0wQjAor/BfaTG9+nMjbeHs50ito8kZMykxaDHDWLh1p0xT2pD57K+JoqAmatzVZYAERExkKOOMMR5cb/XPZfy++nqFWc4k+OfQhP3VhGHxRxFXNcyVQbQdQYAcHYl4WmG98DHb+ah5pFKseUyBmI1U0lHPWZibGo76UcbYbzEXPMcCQzjFljrm6C9anEJjHDBOeUa3gVaXgVI+6KyKfiaSLxoJSYy0fKsVavmUm9cbvvBEdRyDVUK2iOTZ8Tn/ke5ixN8W+BeCo+CxyNDNjqkT45lporwzF0T2YVmGImVXWnVq1qGy8EDFLN3pTLNwhsbOKiwsQI5yjW38TjmYUWVJTSVPPZFpFMa23U5g6iqJJDStgBBDVncgRT6RspRpghrU3lfhdL/DP3FgX51Nj5crnfqJ2J2w8SHGUs7TyHGg4mMqpooUUYLSEeZyznIEakamf4tlxCKEPrj7pnMkUfYWGdEFkrEhnlVkA1kXuXuefUGJH1UKNKQTcVRqElsjd9bIlchVxSO9HGGsvMZtyP81Tap2aSjmJZq3Zy9bx4w1QbdaaNVN9xKvyea3PPpK1mhnNNpL2uOXoBLQEr1SrV7HKVBpcK8aGU66TGbu73GfY4xXkWEhcLGad5ru2qOVriCnnKzFpviBsjaI+vPX41PxUFWtB6LWPzHmrQzdBOmGtxcxPtTOVjs69Tcz33ZMRljg0JWvBmhn03QxucjklRY7UOtTdUMh51rs0zk+suovOWhkVVcTwmjdhoiJ8KKko0pz9O+YWMypthT3quFw4PZWdW+6eZ/cWssWWCabY4NPLJ2HPEzJdkibFFqdPfo+LLmaUp4GptlESWVu1Y3e1MPWPSxXTLol06dSRlyKK9pblkdM1C5NRemGtdObGgdNYQZ03qrLHHK04wp9H6LXRvaY9Mt3SK2p8IKXcO1BqDHeQMoXxNa99C+RF8i1eGg/3mdIX7kznWGcaRMkuvoN9zqcFT2GvW8FzX15NfaRuxn1yAfotD7R1oF36nn/4F+gZu2foANkyGf7/Rv8crtGKuTifrmy/0c7pbfU933Aj2riNNs6JTQOWkjaqzF8J9tOE2se8A99DhbLlz6J2d2au1+AY9btDjBj3u8th8Nd5RZnLQc02zF8n7p8KbtjDaKvso0aSoqaaLSWUfpeXpRGuVGJJYqfWh/9XL9fapbEOkl/RY5IOqwa35P1c4z6vcWZM5azbD5d2FIzqCnfUE+T2dm1fli7mh1d/aPEDfGl5ig3PVmZfYNCf1O13nY6CVS8gFyCLd3gC1MI849vd/AVfnuCrv6fivrFConbdWK6cHQtU5i876MPQ3wL0LyeMoHDVrNomlL+n3W9hnuF+jC1Nr4Ie3E1N5rI0ENGiH9N1Hjhm69snauU/Wzrw+pQNzE8z9a6+pRnKr+hJom9+a/NDDB7jekD8vkxjOgNPn8bfjtWik+o35M9gzmVR0CDvAOX1uvkUjZXjz1lKz3LPH9lRMafbBSO+x36C8fI679gP5Pqab0rn+31e8e/9o2ckykExc+9U68o6vluuwz9iurcnAqu2CRdUZFsNKx9ECimIS4xXg2QFI/9dBlr5HsvRUWrMSkEafY15dbvWqMbaXkLSFr8FUyrHt9glGF6vhCbN/xl88xxZ6aWppO55ISBlY+v7s9NxmotTc7XyM7tg+hhNU3URS1NHtnE1GoPtcrcbftO6Zdjf7aVI6TxMymjNDSZPn2MeRpn9XtTPtveHMCLdNVr9b1x1368pJdUXg1ALVBU5t9Jxa0pC2/J+bbgGn1pXDSu85h/WUlp7j7ndi9afIQPtq390Th3Y1A3x3ye6b0v7rvGMMN8bMmY/Ry4iPLdb0UzF6uG3aH8h/Zqt4+ynIPsNB9cHI5BiXpSHrYWno69Gi4MahcgjPXmmRVLvZGjL0TLTPdlO7hZir7rc1n+xahJA54OglxGXt51+2/AR27dAx2INu0Xs0xnvN5yA9H8M5fQzepNvmBzaMXctvB3qmQTBqwc1Gka4zP3We+WlnO3OITzzuKARND+ZuO8CqIggz2fUWdWg8G7SQyOfgXcqk62ZNbIVVn2gDzTc/wjeHXcm9K30NEIoasLPvR5Vz1ylDZLk1R5aLBugjBj7yDE6lo17x6jLQM7qwo3bWkBCvbsu40dWfQ5uKVjf06XYEvPQV/c90BRedYtCFWxXhVkWwL2y/VS/rdG85g3gYfzrO28xZkzprEudJXDhrEqdeiGxYLyS85vvqvrKN8yFPDC6MaQc+gE+86PkaMFTfC9vW0u6z34x+7wbOvRhiDfJbDuIMP0DZ/ho1UKzPP2TdbJR2guZsxjjGQ9DmXaH/kxqrefNim0Yeywxmqxv5KaWQTdsIf6Kl72FnuAbdLK+dASctuAvGn0Yt/tQmyefOsyPvrD+8P1E77TrzIfVSQR/lxvTXiM0R3lq34y9x4m/hrMmdNclGzj9TE9M++x6MOg+hnWNJ18Kx5E5NZL4hXoZbYi8BY4WUNW5eQNzEVFgYwwqSkePdd5Nf4d7ssTwDruFs5FYhdgKe4m3S37S4yvfbg+g5LVVZXH+h707hTg+x7p6F8/QvnJJjsaE9Mqz7kOtOnKtbOGvysO5bv+61c3UrZ00Z1n3L1504d3Pi1OmRDXkSh3Ufct0L5+rmzposrPvWr3vlXN3SWePmCaZOGat21kydlod64zJ2d3mTbGj0QaccdMqr9vziOuQ53lOIoBceF/bCoT0unadI91jKQXu8KIeSuSJBd9xHd1w6z8LSeRaWTo6q3JC/ddAddz3LSyf3Um4ofljQHQcZY1tkjMp59ldOrXK18TspYd37r3vpXF3irCnCum/5utdOib5ynqPVhs7RsO5DrrtrN6+dWuU62AjvwboT5+oWzppgI9z+da+dq1s5a9w8QeKsmTqlr8RZM91a3XGO0WSD7jjoju+T7vil5oUsfJL5irnzr+UYic5OJXnQHA+uOb4GvRmPQv/I8BwPWuQ+WuTcGW0xd0Y/zTH6qb0mC1rkLdIi5xhxz76WddAiBy1ykDac0gajHhcf4IpzmGOcwyBlbve6V87VLZ01JKz7lq+76w557ow6mGP0xbDu273uuXPdM2dNsBZu/7qXztUlzppgLdz2dc+cu7krBljujCqXO+NX56046br0lTprEqdv8mbslc3o1zyP5ZyWvwWoX0O8qdXrE/vNwjZmPVL2tmkV7rYLpCvLs7etO4BLssucvEDm5AWyDfECKqo7j9Z7DvpZoTvl2qYTiD20uuwEIR714njU9qh7Qa/tpuR9iDfIc3tHln3bTs+1kzYrZ03prCHOmmRnI3DleG9q/bO34cD4zubPwdLK3vdkK8Z719PjIeITG+HMsPOquJMik9c19PfXUc5bt9m3uaYryJU4Rf5JSE9t7X+uZVWJIUOTgqcqPwRqHHvWlcU2t1juMPeHH04mjyieirLAGet7vOu8zJ3nZb5luu8EotCG1W+vZOHUWBROnUnu5L3yDenK3NmuzNPN3L/1U/AYMEedf8lI89SbZ5n+LW58m8kzukkb7HT/HU6pS4gefg1wuQRK0aMr3+2E4xB0cw0qmvVjiIpcTP6T8gtRi5tgXkL/OQpOYjnnk4A/03+OlO/psluaPFDQJ7t2zdK5axJnTeGsyZ01m/EtCLvMmHYZFgsl7DG7tscQp8WIOHk24uTZCifPVmzIyhT2mDHtMczWEfaY3dtjSudOQpw1hbMmd9Zkzt0ndcqS+Qj2paAtDNrCsGMoqiyd9Fo6uZJyy2J3Bm2he/UL5xrnzprMeSrUI/KiaGsAzR18OyTFarIHkU+YXxvnSpvcakpbFIHX2znarZyyYemUDUunbahcSxzu5RhfAMaXEOUpYHzAeBPjXWdS5TyTKid/U63FE8gH4xk+5xD3JA0YHzDewNHKib2ls4Y4a4pRYDzzwduD+E4B3wO+6xhaO3fr2imN1k4+qFqLPOKzw9cUn3lsw8DTBIw3sZc48bpw1uTOmmwUGJ8DPqfgSUoCxgeMlzhaYJwyO/bWzprKWVOOBOMZTzMFqk0CxgeMNzDevl8XGI/JXpM6a8YhtzLfhD2QWUtpMQ8YHzCe42jlxN7SWUOcNW4rhduq7faC34wP3qLbReyG3ju8fR1uGoWbRsF3IPgOjH+Xd0VXK5zR1Yqti64WfAfcq18517h01hBnTRpuGgX/3HDTKEhG93rXTJ0nY+rUBaROXYAr3mGB8Q7DLYBw0yjsMbu3x9TOXaFy1pTOGuKsKcIeE24ahZtGO7jHZE4+JnPyMZmTj8mcfIwr0meBcZvtNUm4aRS0hUFbOLIdo3bSeOWsKYO28F6sfu7c+3Pn3p879/5sQ/Gq7u9NoxT8V1LQ6mbBmh9o16DD0kmhxFlTOGvyUfivMOmVY3zwyg0Yb+Jo4TyTCueZlDv5m3wtEVl8tIEZ3CbN4PwMGB8wXsfrwonxubMmc9akI/HKZT6KCVhQAlcTMN7E0dqJvZWzpnTWkJFgPAF85tHxA8YHjFc4Spz7NXFK58TJCZG1WJh9btdlcPOigpuvAeMDxus4WjqxlzhrCmdNPpL70xlIrilYVALGB4xXOFo69+vSKbkSJydERiK5FqCdTOBGdxUwPmC8gdeFE+NzZ03mrHHbKdx2bbcf/NR5p8mdc8EdQdOVWdLliZ/i/fL1W0o+hTxqrIcD+M8xn2e9+gxuPUUebeKGN84YKXv5LNr51yK4CRaDrfAKrEQ5fHoL9iKy1fNkXlQsZ9mbRl7A3cpdyO8NMDj1y1y46LkL+iluYZa7/SnMimUyPFzhmqwuh+Gqx31LS/uP+pkc5cOGz59om0BbUa/zOvYW72COM1l7Af0fy3qWTZDlqPyMlv8uWzFPoynMOse/ov3vtO8ZZLNU/Bab7RX99oj+/AY/e/SXyRnMp+kE/Bqm8POoAal3tMUh4O4BhZXOxbEbrTPYCxRUr4Eimrv6Y1yXa9rPqm7AXsGannqu8JF87gV4mkWgV2Mn8HOwGO6B/pCd5Jz35Dc6RKloy+sLo0Rvx5/kfZSWpxOtVQKrIcpLrQ/9r16ut09lG9H7H078Yqt5DfdyL5BHm3XKOzmn58PryZdYI1a3zQs1Y/T5P+eSdSon31U6ZZ2ys6xzqt15vrs33HvaUuWxXm1m1n/Qt7Fd9Ff67TuYzTXU2iCYO2Hb9Qbn1cYli+9hN4pA3mIcqB0ezJPnPXI2EZw/EawNq2Wn/gF4er6HlsfoPf4B9hYusUSTb4EHupk8pf/Z90M4EVku3EvwDbqEt3+AXiKoO5Lfr4ETvIY5vweea8+xQsS5Qpvxyz9v+a8uihew6L7luGMgtL1wXbMc4zwOYHQx4tgJ7POsPnLWsDmd0W+HMO9F7UxO9gLw/By4gMVPneLNC0Y/74BG3K0Z7BhnwTQhN1sFYT6/C8QM3zsijO9iMqLtnohZ+w73vBjylP9B2z5Gzo+3GCPf7Idzfpwuv1Mr/o+Bi+w63xj6PIBRq9z1iiPlGqSH8JdlEI/gWwxed7/ASRMBDfGMLA/hr4AR/2XtWeRM/9YxtNfxls3gVxjtDDD7GM65KzlOpde0P2WTjKYtrjSm7fmNgTNYKbXqbhoy3yOyjx8AV3tA2/J1rkBnXIJmj1HIQ2vpX+GOhq2/SzrGd8BRcGpUlMhX7wLoUOhKHnv1E4F0kMF+fAz1x1A2h9Etm5f/03d5j1on/hSL1NR8jvf1EHgrfbXc7/nryuSJurP30bqou/me8dK2GNf66N5O7cng+8JjsLyc0JX9C/irsWdq/J+CR8/HC2jf3EsqjGhdg81s9XuJwLAr1DJ+8NxdfPaQZptF+0Gb3t0jPgLZ5sDYD8V+w/cD15M3oA/lshW3gV2gLlr0I3aWh3f8vGi0x0DdH1rjNP+rfW/Ve1oW9rR7tqdNV7qnVYgXCdgA+X5E7rCnTcM+FvaxFexjeSddb935Tndfe5R7/xO6xtXqbAur9q92av9qpzdC7dSKd8+UcNUZrmLWMe2Ptz+Cs+EGY4iuEqZc78j9arh+94T+tcO5ckKzdEKzGqnPBbOingIWr0Y7wva3E2+vL8Y72C2QM7ApMr3hKWrSH4NNR9gfb3CGh/QbhwE7Feb0G6fSd0ijl+BDwmccT55NXtG95tnkOf35cfKS7nlXsBe77f5Md/k79ND0xDqAGV3CufF28i/gF240a/wB7PNuG3COOiq3DTi/gxarjYXEGWe/cMbZLzCr0CYsBzejjQRh36fe07dzruVQajVUy9/h+VXuYD8Ar3Mz+T+gWbnAtv9t2cOIM8o8wSjzXdb7vaaZHN6at/jdjEPf1LuH8gLVfVt+AI7tVO7Vn2pcYdThGebrI/xJ36E0c7RzHlhz2JMvAQ6r88ESp4KpJV3ugzXHddO9sZr0+QDW7hati2/gL4P+G1iBc4j1c23xrZmBhfpY8+h4pvXDJWfRV9Toa7s8wg4be+P4KF7ZxM+Bk0hXZpfqPsK4h7XM77Tyx7/2GZU4vHwIelevV2pRNlbTum2e/UqOiWBuMeq9fl8DrD+XHincSzCCXUT3LuEefYwimZX4QDtrhJ/KW+iTnx43YFlmXPV5y7PFvl6Zc71SJx9SO2uqDcn9m/NDXcW4F/ELbE4qKtt1D5nDdQ+BODMCEMwIs8kYX3eDkzi3g3d7F+92XYZa5tl+BbeAjqRX3So94ZfzUOze1hkdUwTzmdNvwX8++M8H//ngP79Z/3mCsf7v/lzi5LpcuV2IM7cLceZ2Ic7MbgQzu3U5+4MOKOiAdkMHdAg7yQHqgN6BzwUb6RtY1bl29i3nYfa1vnSJUPUatXoNGqGgERpeS9EFE20nn+s0Sjvfe7/PuqEXdETc+3cONeI21DtjDY5g5gegB/oAWiGxVkIS41oin/WpnetTOWsKZ00edEFbowvKnHpbV04Hgjmwgi4o6IKCLijogoIuKOiCgi7objqdrKFr8X+ucJ7WubMmc9akzhPeXZMEXVDQBQVdUCddkLj/8BZi511r91T66oNsPQedUNAJbUYn5MZz/SzJnSdT3uNu8P3VC3G+ZIZe7DrsTR3PXHoTHdxhLQrnWuTOmsRZMw06oK3RAeVO/WzutAlmne8iBR1Q0AEFHVDQAQUdUNAB7bIOqOgUT5M4s94RZ9Y74sx6RzDrnf2Ed9eUQQe09Tqgg6D5GVzzw6M7sEilb/B0uIEbGW8wAukFSGfqDLyGdW6eR3M8ka7wFge/Gb0H7z2n5Td4q5mfKjO4Ua36j7VRiCgWKlLqf+At7BtZ9x6yIeujja2j5XFXT+BpMeI3cPYxvudX2e7a4L/Yqc0jh1xgRFEuXR/D7W++34YbbUFXtRqNyStJCZGB4ZEVw82ztHaeslXQTXWEtIrjwaMonuNe4NJg8dauiMu2dSNO7og4uaPCqf/oni8y6LHWr8ciTn0ycdoziw1F4Q56rKDHCnqsoMcKeqygx9puPRZp6In8nyudpzVx1hTOmtx5wrtrsqDHCr5MQaO1JLYRHxl7h+nNdADR1XwjG6leXF4jor+gCwq6oNVEN/LHwfZpUTrPnrLzjaf7HeHodyw7akUk4vEQD6X2pxll+QBOWJ6Z4jc8Z8TazB2rUzpXhzhrMmdNGvQ+W6P3KZ262tJpwyw7+7EHvU/Q+wS9T9D7BL1P0Pvsst6n6pQjgmBm2n9aa1JnTeKsmTpP+KlTV+SOY+mOf5iEmOtbEXN9H+zOV7Cn3aDeyYY3pROjSIi2HjRwDQ1cDKfizUo0ccv0bycAvZuG9k35Vfhq4EQ/Lt2H3mPgMQOPGXjMxTzmOdyGZ5zgHpwaKR095xNT4DEi+Y3pIWKAWQURGPXPEWSl5E/E2ucIsrkVcDvL/FxD1rw/QGvKM1pW0D6HNiy2wCH9lEi+kvOUCbZKIDds0SrjeUY4ZmTQqxp1Cj8R9Mdmyf+L73NZoreOW63bdc0eD2EkOb5tivMuYU6c/82xnwr85/nIE4hGNUPY1QBf9fkQv+XguRfj/FO5CinEQIoRBilAt4Bec9lelOcAF8YjsNHFEupTiKXAv9cUHqoswjLxGzfq2DOH0BcfVwlj2QOuvwLN7R7EXS+hlGEAH/UU1nU80kDd4Or9n3Px/JvLJRdsPffB1nMXbseGfaUTL4mzpnDW5M6aqVMOqbfQOyHIAquTBfjNDSZfv4HPc7AL8L3pGmd3LTXXc0/9MvfEfg86ZrPXyNFrkA2CbBBkgyAbBNkgyAbLefwS81d2ec6u9S8xa2qQDYJscHfZoAvHY8NA4sTNwlmTO2syp3xQO2sqp+TgrnFlHS+debJd/lOJM8vIpu5UvYIIVzxr+Dv61q4eRSVmO7PNzO3p54d9vwCWmT+/WPn9cmFWvcLrbRGUH8kbliIDNrfU8bjxM4D5MUpI15iXMHKMqB5g/raekx47+iHgyJVhCbmLxClG/IjC5wDyspt9rnY32p+8pNz/txR739C/zyef0pLPJ08n39DPT+inlw54pY69JFmgN/NfI/Gr46h77TYTD/4lxCFk+PwC1oPJDd/C3s29fLvTf9LZo3e8Myru3YzIvZvROnJs6hqld8D5HxjRQM6QB3uIPfMsI7Gz7ZUxd95vHxhUI4FBskEY1COBQboxGKSd+YGhYZBtEAbJSGCQbxAG6UhgUGwQBtlIYEA2CIN8JDAoNwiDYiQwqDYIg/vG9aUbyqy+yhnZ9U+uSOKZQ7/0pyy7gPtRPNrdNxTT3qEtVM1vBjfNRclUwu0StJG87E/Z4w1Yc4/Rynsu2yTo116iDFjC3c50AU9KGnSgRsQ8kN/TVTvTbFVmRL9HYI+7mfwGUP918jeN52Ra6g9IUUdar1PZ1zWMn3s2HGrWssQ5F57PT+gAUocdbwaWOQbpUxztY/Cy51a8I9Ag8dtojyb/TfHvE9rjR6Ar+YS+56PJG/jfhmMGmP4R6pc+AZtGgbYdMfZMjn0KNhlemjdWtWh8J43vpbWXylpaW0sTCedmedJ4VyLh2GyZaXrSVCvPtXIWY1WUF45yIqlnKrEtkXM0Syu5ptxaK1aW4cr/gpVlrmnJEjl7hQOP0Gr2QGKJmF8F7cU6iSdriU+5BjP76lcdKTBbSIGZkwJN6jD3Ba7/au9j7l30fxqnYwFePT577xxGrGzc/5aRQnnMOxU14wjGeyBLlFV+j85zDm9b9JnhAvfQ+FODlznmbNRj/qcWRZTZ2W6tpw2/oamXd9vZqwUS1vbt7GmvnZ3drj+DqLMsxqKwFMxhR3iEN3rmsMKsrLlj18CZ9duv9T4qbX9JGnu1XqZ2arPUvU+b7TJraW7s3bl1j9b3dCLLS/BEaO7S3KLU3qXfAZYcav4ld9uT+ZlDWrtyAnZDn305RStTF/pJF1qodpV+3qMdjVHJ3wCWbWqpwO+zH7XofShqmcr9vbaU6dQy1fB6EbXomJtZS3VqKRwcTWHhaBK0iTeppVk+HL1wb7Q2F5MCRvjQS4KxQbrQS7Iwrkg4b/Tz5kCLgGE/b3KJgd3PG9VHZSmrLWU6Bemliygot1JQrfHtOgURjVoLR7miIBv1rIJyMhhDm24S7dRffM7wLPPdzpnUw+9g9fz/Yt7030hrwq/umabfqOSbz4xS3b+tydP69XeBM1G1F+h5/lB6/R41uPjtzRHRPQJMv3h6yzI2+Ox3Ogf9CGizggyIU1rOsve+Bb/bQ/BJndL3ss/MQ5V73eTAVxa0vAJ7DNtfjsArNqP/Y6Ab1scU/VLbvRPos+ql16sNudKOnbzdOrBzXL6cNohlmtZxEcSyNdFzP43xO9AH/A6+ZDcAuznEPzpGWDGvsX9h7H/ugRfDbRPGlxxDq4jCj/vBR7h/XqPd4LeGdz/LEXAAmUlSuLfF38yt8t1XI/FcjWQLVoP7TV/AjsNzsapoajHAPhkQcqkn5NKthJyKT8Ij1h7AWG8AfrY4i30gmXlCMhsIkn4+f19oNyPnkloLoFjORc0hAuMJ/fbAOq+8N22Z+LPJk2EOJ/0Z5DXy90i3r3fhCZdiK85MLq3xGyXHwFedgaT2HmbVB07EE05kC3aY55hj6xh3jGvIi2MrZRznNVAYj5Z9pOXbYqesfjPud/BOPoPn/+bIDtZnBUrPFSjXujP96Nhvqt4nkrnLbvI8EqdPU0pRp88c+K0zyOI9A+/95inWZ+VrT1jWa115P3xvzydHrru7VShotX21dBew1jea/W+d2jrTVl47LOhTR/kiW77ZMnOUK70d1yy39XZmOZF22LSlt9PLhtTb5VZ9d+Fpt1+d3m5XKYnfjD8An6Y/GnTzt8bd3EcAg1uUt+2UVQxAWYVFD15Y9OCFVQ9eeOrBC4sevGkb0ikqsWrCzfIxUZSvJjzrQVHZKDThm/SEEZ4j4/FzaY9onV4sRbDCD+KfmINvpK9/YgrRCfr6J+q9VNbS2lqq8zRm+SKexmyZOXZUfQdONRtlYZSXW+KfmHruyjnGpe4qMWRhVw67srEr5wtwYte8xo/hRsgBxORju/Ly3TiB/Hx+u3Glja+7N5Xqo9LK8sY+XEEMjeYubJa692CzXWYt1fff3Op7aJYrX5DM4guSrWXvzbV9Vu29GWTj9vOqIpr+965eVWTBnaBdo7TF+hqmMzvFOz5zSoczgDPbTz+BGF9Tq+amGkBzU1k0N0piqq2lpuZGL1+sudFbZo7y3MHNmFxONUI5s9I8R5fJmUkPOTMJNNVLc6PDwq65Ib1vOOl9VJay2lKma26I16lltssMn8TccW5NHeeWTRdaW2gq03jFIalKtwjpVFVqa7GYqvIe/vL5yu8xtTluIuN9HgBGKqsVzyrTTbLozsfPof0lpf1j4N8PwNp+ALRzAFG7Hkv841xfRaH0O7yP6wf2QObKoFRZS1cjr+SjlVfcljqiec+78x75+5vyiHLrsLoPGbn6n1bLoJ8HWL4mD7AQLXu1mXPGmLnajpepJ16mwbMWIZF5QizbAn+hrpFZ7ZDJPSGTrwUy64kGa4dE4QmJYq1+Lf0ykZdw47PvbmFSxjj3irtF7rdjQOkJqXJN+2rIFrBN2QLuHrM89/RLzNfEUYf8BSF/Qchf0Jem6x7+pAVoz4J/Rh9d6jmcVW9lXAXTcpE6LBfFABFJilZEkjFZLnTLQ+EoV/4ZmeW2uFk6pJ7VZhFMNX3kcutF1cN6UQV/jOCP0diFg+dkwAkTJ8p75TmwypseKlvmG7wjq6Jy8nPbPJOZV+pHxlMCup8YpYd4X1y/y6NO8LJ3lCS9j8pSVlvKdAupXrrIQqq3yxw+BLnmlaN7YhZGefvkzuA8J5aoYpX0EBg2qhgfedufR9lOl53eRecoSeLZsVhJs5FbSQmuI7PFL7KTrsamWBjRe4eyKZoWuDFbFAvPCB/Fjkbs2X47IudpDiklmqfVG+0cmy+NEDTHM9h+i/URzPkGuIUz1GjqZ26sjUJEVz+B+OoMU/4Do2HrkdevIDKLGm1sHe0NwPZEi6X9gM7rLaw7i33wQNstTN3wJWDPBUZ94bExjuHk4XL7dQ+tfeFppS/WZKUfvzW08LQfF1sRn2VIa2jhaScu1mQn3pw1tPC0CxeD2YX9rKGv5M4WGTtWZN2xzBkVvfcJkybuqx208LQYF2uKrhLsoPfdDlp4Wt6LNVnegx002EGDHbQvTVc97KBkYdb2oIHfRQ08wcznQQNvauBVpOAD8F3mNW9AG/jBcZuoGkBXXrV05eZt09pxB3XqKF8cjcCeLcks13XmiTUagVlOGtyA0panK8knUEstpqkn97Vykx56cnKvsgkkg9DOWzwjhSbslvb3yJpfbNqbXsxeKmtpbS0184tNvWxMzZaZw99Dp5jUQTGphWIqzVJVWsqGoxoevbxNNak31VQ9braGnE+2E0fNTmgl34GkfB/vrZox+XR6yaz3Vs1youW+qC2nTLISm2wB77BFWPC/t1r2uLdaDkQzTF5h0eDfYjSCC8B6xsP60MsMdYJKBuL2Bi71PDTsNceSN30A7/mClp7BiK4h5vdfvCglBf+4R9IyxXRxj2Ad32N06J8pBF5NnlMJ6tvJj5Pv6fr9C6w9Kd5hzuhnttYfyxEKTcYb4K+PQMvBo/vt0dFc0F9mEfoV8rPGEgrHGIP7Lygbst832OpvKOkz6fHRgv7tfT6C8/ItRhs8AqmKz+JRo2VJS/6L/r6BcmYDegPeIf+Fu99vFA5nACthG5pT+Ou5fL4CDvaf8MzfBlqBzydPqPS7P/mBQv8b+letQI4W8MdAKWwF/rkAwxim/Hvyi4FVv0AdL9Wtk7w8R6v4LxpV8Jpf8AYi9ydg0t2bRqSVXwAffgEpna+XepLHVH0D+j4eGVq1PoKdhOOyeuJzaMVOpDPQBkTYR6T1saf1IiRL3oPQtP2JtdcQifo9UNE1rpl6l8lj6X0211k9IyD9WevZP2FV/uwoPy7KCx50CrupU8gCh+fp1XeGq8otItyzns3tCKNuxWh7uZFnyRnwhcewnzEIfmR4/WXg9XepZej+BErmEMmbPfWJ1C9+RPHvAN5+aJSeQUxzthb8DOM7JvMAedMa7ycWu8d98h7MHXwqcfCpxOr3Xxg5THXptdb6GdJ/MNc4Zp1bre+J/2A4E7btTCAL+IRwB6tLPCtWw2Fxhp5rH9aoLTDjttUOj+upo3yxXjq1Rv53+3KX1jjlZrmK0sjXs7Dsx82aIXfkCnCnHU8uh+jR26RBuG88GYMc573OgcdplrZ111yD1Vd3rfdSWUtra6l5tzH11F2bLTOHbi3XOIip9W5jbr0hoUdwLC1lw0ZlJNaojKlnnOlpZ85GPBuoyD9/NM+d8QE1Vicg85zD7K8BSuIW0il8PnFk2UhAz1MMYC/KLSeZXlpbS017Ue6l/W62zBy3jHSaSxw0l1hvJSWWqI3JSmjOfhuJaJlPFtFcBntBN5oTzwaa66JNOEfKEzoE4T3JNAH/PZiknwAPUw1AnZUlTrFeWltLTeqsPPlMs2XmsM6aJ2LuOBHrDfo/2C25iSdlltr9qbtSZjkKf7LuWeFVfGc9Kzy/UdYlK3y+przb/W53HCNvy+548LWeY2aAPv7ttae3cb1VebZV9uZDsJLy/Nnilttd8nCbtefg/9od3sTz1iMJtx573npcdq/Q59RunqV91j3xXPdwM09AIvWEWLrWW1evkROLLNhhm0XWm95NrNjMHTqVafcc27NMunwVD6GfK0133X3N80GglQ8CrVPgLbvCS+Wg7g6NwhMa643C+hPwgRFI/rfgEcT6jWBs4rScwc6tSy2+9EI857zeu5af07orvCVyDHc9TsFfJ2pIdP8HypozjSjWfIBbmjOw2oq76BHGzL6Em2x2eJQ9bo6UEC0lyODiRqDuB8Xp9Rjlbb6K75CrZnOKcaYxxaxXdNd/Rs+855MfJy/R1+zE4RudD+IbnVt9o3Orb3Tu8I3OvX2jc4s0nTt1XZlD15Vbsv5UIK2XFo1Xu6bSItCfa/C9ezbDVLu5rFtscm11lvlJkx5+0iE2lp/eS0TneQvjvgZ5vRmjUnjv33drDnFQG3FQG7FqlomF0lItnvmwntWVhcoy7W2bsuhsylelhFvB6/IM2aPzFN4h7s8si+wibxY25nrLxrxen6DhxpyNeszd/IoWRSrb9VxAXGt4A9Ek+uiyFj93LvNhuFvpWkwujd0ltld3CbbylOaqoPVCSNSeENsGffwVRJRLQV4W+qMZaGP7wKj01KGXa9Kh941MpUPnd9AW2LRtfeCVeMIrWatOJUI7C9+7bkCHdEL/PrDOIe29k5hUc18jVpWeGudyK/J7HWAURy4/nqAUO4csT/YadtadAYc/A02du5154vJIjPwsXfzUKejrYpDU3wHWulvziM8HAM9+e17uuar5Fq8qh+kFRsF8iLRmcsN6rDLhh8vi9P4VNNIqSlmz7h3uQTHw0H/QlsI7mbfoszaF59oUW8HluFdH7Od9YEV6w4r3svloen47kF/8O67lEv9XFYeq9IwtV44ytlzXFYhb71FR6l6AZM74xALgf44aoRi0WL+AfBfBDh+hXiuX0dcS/GXtmX7Kv3UM7fUdTkQsF+Oy1cV4+/cY9K1XmmZX3FO2P2WL7Uhk20uAziF6+yq8bMZOi5e2bO/U+jgeg4XwhK7sX6RWOZHx41O4Zf5X5AkXxW/nOjsVr51gxLsao7i7+9BvV6pe/ooU09aucKy6Qhmev3l5zw895tBs035GrRmrMyHvHnFTH/R4if6nrQty932DOnkeHegdYNyh8aZjmCefzV0+L5rPMdC8WHV1F878r89k3uEs998/q977p64jDrvn/dw9k5XuniJmlfAgTiGes//umcBq8YihddhJw066kZ20HmQnrbdqJ+VrcwZRmMa+l65mvxxqP3Ttbe1dbLV7m99OpihL0aqiumXz8n/6Lu/R87yoXct8Tu0G5mq537O6HaPy1vqTtWqxM6u+uvLWuU8X7m/JlufOqDz19tWavbj/ATmAeO6M7zCbz41jJbPe1hNuv1gH1uW9vEXLexVTOluh59ocT/gztNxc472Va9CMM86O5XVc3upNi9P8ZPKfdAaf0b/M00qsAf/2n3ROn+Lndl+iTUpxhn/mUa/bzzbf2X5SZfPTn2Z+93orwdMfYByT9ptOaI0oLSdPJh81ZIEzgBCbcwFzLqCNnr+mWafHymvXcuw8hgiAPBrEJ2CBTmjNR0Z88Dd0djwP2CeNWFwpRN66xbZiVRKrB2ItOdA+Hoiql8paWltLTQ9EvXyxB6LeMnNEXVEeiAXkXWh7IJrleq5sW6bs1cSTsN2g9Y3Kkhtn0129D6tReB92vUGbAx40zw8eraDLDVq9v127QVt5Wv+qNVn/7vsN2srTglitKSPX7t6gXZaZ8wj4K8YxvYdRXRu5N0UktCbX1AczSk/MKIO3IUKi8oRYtVbp7DloFa4B08/xzp1d3ql77wUmPtzn27W1pwZlMbRqTWOxzbdra08NTb1mr8hX4MV0jb5r1637teZuyf2arhfcSH1gnXvqOffpWuf+go6D6zHnMtvAh9Zt4xuAEOMzDkBO1WmEPXkBn69Q88o1sCyvM7+XK3pkMOP5FObIy+w5YJX10qnUO3IbcN7Iy3kCeoFbnNm1xhfopSwfAeuZ7XgE5EO2IoT+Mp09w6gCNDk1aGUSiu1TWprBnTaWGfAYtCyfAJQ/AlmeR3uy3c1NBol0lbQiXTXvCbZvCPrcDWzfCmxHd7TFri6N6Kqk1bJslfjK4G2Zux2ZuoRZ+kWsmvaIWLUoA+KuZuy6y53aBO7Vcj6ej/p0aWQ4ot3j7E4vqpfKWlpbS8277Hr54rvsesvMkeku12in0u7LFo5yYvSfQaSxJnUVcB+1XMkt28x6z7bU8iUto7xcs2HclfLyBTaBTd2zrbRd+O53Ka/Rf//EsGtzO6/7/maF1vku/OozIx7IPtJlBDQqckZw/o5rt2M4Pdm7Y6lniUCWnmPWd9bfO7yxMG88y+mbRyZ/i54Hpyh1L2vpsr6bd1DFm/Wc2Lxmhvr3vyA8P16QYVxAXODFpVwNtS+5VyP3vLUcVkOtxkPQ1VZrWJsqrE2HteFRVFa/OmVYnQ6rU4P9c9VrQ8LadKKcGuy2q1ydEvINhdW5++rwfEyrXp11xl7xyRO03sgqvpmLQi4l/1xKiyzeudWCnht+Aa5nVQS1M6M062h9z0dnfe+ulc89NdPbcM+5rYuJDbv5b4CnV+iX1MeWUXhCbRt8EOynIL+9cAk9RFbIihu2MWZcPF7gS968Kc6j9uW974LXnt4J9Zq8E/qtxPZxLUITFcu5qpiS+lxi0K9eN24xLMaRXMOQSMvgpm4CHIA16RK0zjet+zhi3o+N0uX80l/BUqXHPJgOgKelJ55uh6/Ecjw6AI+YmQcuma1j1I93h7SPj0U+wvPsTLud9A7tIDfy3hrz44lRA96kPVdrlT1qplmZukO29oJsNtjdgmE8wWxedyo2yTH4Ib+FXeSoERtqZnxvxpbpn++gfao+AptEtxWqDIufe4V4u/X6GTCvuE+B84omT1qzfmCdS9KbMzU5sM2d68qz7AlmejuS3pkf4MbhAykXXTbu4B3jzIXPfx/sSHtDtNKiVI/Db3aON+WOgWIZLNsesW/R5nwMs+0DwcyTvrKt8j22wVDdq7RJUuatkD4QzXtDtNIiqo8Vnn0gVHhCaBtkzW8bkQm+AFjc4N2fPlAivaHEexkTHrl9/k/hJnkfeJWe8Cp3PKZwH//+OYz6ADD8dICows1IwT5t58ht6PGFl8nu4lZfjPEDbowIgbqcIlpey6y0bwBiM/SSutSeO4C1PJBju4Kd4L3mEyJuo/fB68oTr7c9AnLfyKUVZOPsC6tKy+l5v29vVOCP1h9eyUD+/GPGrMRLdqvWdouxX9xJBalnjR3UFnvlGKy0D1saTXsLEZ9F1F5A/75RLFPg1FNNfvyd9j1DntymjxDe4r9BfJZLiHH1CCz5ImeTDql3oL14D7c23xpyaduTVd/dz8CWdqzZArvu7H7RSiq8SeuDcekII23qUYmYZ8UexA7iEXSegu8li/7ympY+Ba/RPcrFvpb1JX5ntQX9XGp1Ofzfx++J7JX9j6CWYGkEzxLoZwpPRPBpD/R+kdZDCr6zFT5faSVP5ej38dkEnlVPpNiD6leVvMYR7GOZGp3elxh/E04/G7BjT4jZv4Y9lz2vxsdnaT7Doaye4qPRn+EtfnZS7yFYwgXe7YMup4vOxh/rM0+sX6++bx/1L5faTmeO21fqr8P5EM6HQSil8KSUYtTnwznd3fnNCLZfi192w/+PEe1KvroYsvUaq3hA6aL0xNBydFBzaxKakUhNjUKsxR90R3nk90eixneeQUL59LTrz3Gf4TboZSMRbxH9mfXzBX4L5v2WIfwF+H0bP2yogn5uoPgbq9DPXaCHlMA+u0bNTT+Ln48tFuQ3vSzIiadOKFkbZzZ2rVDqqRVaBLFKy1J6f7VCqadWKA1aocD1D8KJpp5aoXTkWqFz1H0weZ9Z/ghdDwKfWOTIGD8T+JzSz0SWptiSjEg+SD21FumatRafAn5GS2xeFVCWn3yzDT7zIj+k7kHcZ48vPNd2G7wV7KfhDwCpU4rt3LL6qeYLE3V4xuQv+8CeeMKe3EvYvwHbs7B2m/JGd5iWvWFaaZEqxn0fqIJ7GX7zDXJnd7lTjzrXV+Y8gpNXcGJHa/UfMd/cjGO02L9k0T2tA2N3NKPo9KHl2hO36533Dck8pdp0oVSbjV6qHUoLkHnLtovhlewAZvnJZOu6RRK0APddC5B5ypvZmrz2+2gBSvBJKCAK3x5ERjuEW2Esu1xhfMrgUw4aAd4yRo+GwijLZWY67o8xpWUleGdUsiwDD5PKUlPRsfLaEr5FlhaH0KIEj5IpjrySIytgNiW2iKVeYw/86Njs9mDfjHDMGc5tCvida7Pi3zJoNTXmU6C3Cf/hvRP41KxhOZlULUGvGLPFIUCZQO0UoZOgt0qNfispjDiBWVWQfyuTpeLTKa27BZiORzeTeeo2srXpaPudZH04zf75tyugMT94ji9Xefc5E885k63AIXVb9BzknDP61HkvnCh7w4f3MqT2M7LqOjNPfcCysVYDx6s/AIk6gtsD4g7CkWMGtecMyrVqm1O4/XkGfXMcjAB7L62zyD3lsWwwnfkmJIzl2j1xr+66k6a12YfQn6gbud2pOveWAHf9RtWNzEW2Gr0Z9wu6wohOPCLC8vamJuAuWrTV6rsYhnLfjyPwM7pY632q3FNaz9cmrY9Xs5F7ypn5muTMoNm475qN3FNaW1e0m/vs1Zx7x3xbnM2nDHHGdjbOmBk1d/g4YzyecV8s1aMiByzdPSzV426vBkfLQXA07KS7vJPqkftXg6XVIFhaBSzdWSzVc3+sBkfrQXC0Dji6kzi6GqwsPHXVxZboqsca65ZBMOkNad7LfYdzPying0A53QFs7gPlbBAoZ6M7y14DzPpAJh8EMmOIkSnO9RcQkfWgcb4zr/2o1YbZXIRV7wmWnYEetg9MC0+Ybtc9FXt84XQt8YULT7+PYmEMzmIUuqcmDjILkXlbvA+cSk84lVuAezZInQOFXvSCUdUbRryXcUKIP/0WbeVvgYvvx6PUnvCqtgCnvod96Vr6btzC7v8BztUjsNT2362It5SyKJojGeiGw/AYxiFwCL1cabm5usMrGQReyVpzP/TJzmBK8Syr9/PJzxQnz+m4C/r5y17QTAeBZjpS7BO5BLvDx08aIFsRad8GIeFvxDmyeS9Y5b1hxXsZAlKnkNd9dd7FxJNzJ4Nx7nf1Sc3Aj+EI8qfNrd6cxJNTJgs5ZeLNKft6AP9C4ZJCho/U4Q9MPHnXZSMvN6ALuQEuQpd+XoP/x3s4AZiMfoxRxI4AEuwUOKC/b1EL8qtRKrzxIjg7eHwyPSdGBPyJ/sQxZLu8gLMkop+F9yPPgmm2vQFJ43iy12tfqDzXqlorpbzCURw5qaP2HHcePGk7etIOnZlgKB9a5U83Ay3jsebxdQ4U5WN/ES1jx9qt3m+29JQuyMJcS+VA0sXu+oKvC3u744mfVFVuyX27VfpXl54yUxlujgf/6kF8fktPKbQc/c3xFFY2h/skCdxh5nfFE7whzfj9PbwbHUHUOJYp4RCe4u32MFq/+D3FflQvrDSW7URsOtGPeF8s3yN+/6Djm6JPlxifeJK/hbWIoZ5lveJR7VK8Jy6eZs9WsryC8gRxuv0Eu8cdW54Zi7d56SnTl1t932+oc6HwhFURzoVwLgxCncQT48jIzwWC926WnQfz1v7f3u3tO3sCbVf5hhgjcazqDeM5E3x1b+vhf0+1neCumrcn6MPxQdOb3YL+7T3a/piO7C+03adgDfy4h9dlbLy/+ylTedL8erVp3SFpzs5X55asdXbfoXZ1DpRxCvO6gPgbb9HD5ZLSJZvrBZ3P/wLlmDO/AWn9RpZGdBSXQLkR5GtgmSOOUQP7Qert96xQqry1OtORWaW6Yn3lqZ+otkQ/cUP3c123GLVKuH5H6OQVJ9IdgmlvCPJeVh/tpfKUd5eNNVuRZe0c+juDk9gV76XylJyqwU5Jv1kUaCl6j/uSO9ZL5e2buA3xxZunMpNHrjADj9Do9qMv4rne2xD12g6dGKB40TNCTeXJP1Zr8ju86rGL83GZWdH7QKbqDRneyzB8dbriM70eZLb1KHaXGfhOzvGWk4qi098Htfbk9urRcXuLsqmpHSaylHFb2DVazfrALvGE3XrliQfWsaa9x8p7GeMqx+izftLrlK09OcN6K2IbuaB0gvHuWal+07OteSWa7sOmd80hC5uPVpVgHNtkcK3qarRQtSd/Xa+JOz3x1kE178bq2uL2/VimF/3YW3Mu6mOAm67pEl5fl6ClXt2qFJ6rUoxqVbrTZmxYDlzQ5+v0B/CHlyuEPukN/cq4pd8H9scAC+b1b1pMVrUO9vcdantdpnG0h3B3SpRn2s00e/trrX25whUsPVdw3BadF3CS7IGeNoM9jMcvZ/G6lRWe6SdFXHMRh72G9inGJi+10mTyFO7s3UK22T3ouaT8FY/cnmE8cx4rPYPeKyjnUdx4JPcCS07B6j6FXvbB8rQHqy3a11qPBXzjJYl872vMgpdgmfBBSOE5YakUs+DeCxnUmuV8dinY//fQ51n5OxAccwJ1/DN/Ygr2pRTfWBrlpxgHvoR5q3GlxrgygG6CceqrRjnvjc3xFqxR7O16hPg+lKqwalVUVPWmIt7LZmmowLVPYS3J5DlEG9pDnVuE68ru5uxjTQY+7XsQlYrzcxlmJRBZAnKJJyWUPIU8p3uQqSDDPAclxGKJJJ3mkCuB9xmjD04usTqF24/PMTOkGGtCV4pnF8jRj6aGtnmHO+j+6+6rUajX7Gl+Cic/9+u/RuvRJWiSGTZwKPD4wKZ3PucRufXoDEq4bYmfRk0I1EClfhBYv6/9B4iLwyM1nMj5CmuaiJ4eIRd8givM538F/jm/oq7gEp+6btx2eGjATzyr7k2IuxJ7DtglnlFM6xAvpocUX8Mp1xfSvJcQL2YRlLNBoJyFeDELoZwPAuX8nsWLqYFbHQIyRYgXo0HDR8Ln7UK8GF+Ylr1hynu5z/FiapAR/eBU7Wi8mBqkqL4w4r1sQ7yYE5jbcS+IJV4SiwmTXY4YU4PGqS/EeC+7EDGmBp3EEPBKtzhiTD75hsL6hvaX0pn9qxc0s0Ggmd3LiDE1aF594JNshdfcKiPG1GAV6Asr3sv4I8bUYCPwmy3ZUMSYeknEmBoy//adA+9lvRFjarAZDTHyKkSMWXHEmBry0/itVT2iiDE8p47fuIsQMSZEjBlhxJgarIF9cZj3EjD4/kaMqeGk9cGTdOczctbgU+EHq5CRM0QG6O8JUQMM/DAuDxFjQsSYQTGv8MS8JJwLnhqBdE3WvHAu3P9zofTEuDJEjAkRYwbDOl/dWzb6iDE/gMXzV49IJz+MJGYMg3/tSfXr1af1gaU+v8xb75be86gxvyFE9xxw8tXtJPcibkwNdx58ZpxtiZZi3XFjarjN0heCvJdVx41hb8kHGWu+sbgxNdw/8ptDNsq4MTXcQfHzECh2Pm5MDbet/Na73Om4MQwClSekqp2KG8NmXPeGDO9l/HFjari32X+2+Sgypq0ubkwNN3B94JSPjtvbdNyYGm5J+sEu3XDcmBruePYdK+/lvsaNqeF2rB+M8hA35h7HjWErXHhiQhHixqwpbkwNd8T9VoWEuDErgH7ZG/q8lxA3ZhNxYxjsK88VLEPcmBA3JsSNcVBR3ZuKauM+c4gbM/64MSwa2tRz3af3NG5M4akryDbgcT/2uDGFdzSTKsSN6SXFF94RTRZp/oodiGjSD8r5IFDOQ9yYhVAuBoFyce/ixhSeUUyXQYaEuDEaNPwk/GIrrIzjiBtTeMrcxUJpoRjoRul448YUnlJVsRURPlYTN4Z4SiDFwqgKZBQ2zeVxY67Ai+Sqp2cS8ZRZyFbkxVhH5BjiadVcBLHayOt4vyPHEE/L6jJ4ZVscOWaf8iLZ5O+U1i5oi296QTMfBJr5PY0cQzwtk2Qr/OZWGzmGeNoLycJb6mQgGWHVkWOIJ/dOBuPe7+rZmi2NHEM8uWWykFsm3tzycJFjiCf/umzk9Sgix3wP/AS/X3CD2gweyY/pOHlMjbegc1X6T65ztceQuQDfkIeoO5nLm0WHGONA1NxAzYXU37aj1RxJzy0GWb1G3YjoE1Wm9OSxyzVr+Z+DR/gxWJYPDb8dc/S+3C4JsWVCbJlRxpYpveWPRX4m5UDyR4gtM9bYMqWn3FWuLV7KeGMIlJ5SVbkVnq0hhsD4YwiUnnJquSY5NcSW2ZXYMqWn1F+GmGMUBqUnrMpwLoRzYRDqrDwxrgqxZUJsmcGwzlc7l48+tsyXhhaO+SEdwo37C2h7oe0Km4okU3nq0Ko169D8IWfOxlentt5YC08xLjfXhX6QcWJU7Bc9RozyDNY1saa+1a6fVXXc49XUwEboC3e6NK5M5a3XSe9JXJnKU0NRbYmGYv1xZSpPvcUiCPJeVh9XpvKUeJeNtdhgXJnKU3aqBjsnh44rU3n7MJIQVwYy/Pqtd7XjcWUqTw6yWpN/4njiytTe3N4ir8R6IK/EVceVqT25wWWzTe55XJnak9urR8ftbT6uTO3JN9ZrljgeWMea9x4r7+X+xpWpPTnDek22kBBXZlNxZWpP/roeWQST+x1XpvaUGeo1WUF2K65M7SmD1As9J+uBbkiFuDJdVrD2XMEqxJUJcWVCXBkLFSXIgfSlI9FPiC2zPbFl+Jolnmuf3MvoMhwGvjqDaYgvY4Geb9yTMkSY6SHPc1jnvWEt+glRZhZDuhgI0kWINLME0mQgSJN7Fm0mQT5yGOiUIeKMAQ8f6V+0DFFn/OFaDwBX0dN9jj2TAO899YRWshVxQlYRf4bPPhkATqKnbYhC8xYspMc9oZZ6Qy0JUWgkLLIBoCZ62oVYNHyu+WBQy7c4Is0flN//ivIrTyaf9/JB5JAoBoNpcS/j0vC5EW8okR2PTcOhUA4AL9HT+CPU8JFW3nOuNhSlplwYo4aPrR5gFqKn9UaqSaBuOtD404EsDX3j1XwHNw8izKSr34nQ70DoNx4i5O/YXYe30IZrp7nuVaw/09C+Qw9Jrr2N4KzhURdmDQ3uXq/9IPXmrdM12wBeAtyOcd76vcrmDPz53DLEnwnxZ0YYf4bjcTYAHoueAh7f3yg0fI1z73073/GIAxwKhTe8ihB1IEQdGMS/IvWWTtM1SachIs1uRKThOFV6Y18WzgjwLPSFVxXOiHBGDESltTfW1SE6TYhOMxjmZXfQyBWjj1Dj1hdtJiINh7Cvdi0bqXYtu4N2bb3RGX7AuC8zkP+vW/NZFJlmzzlbfx1Mdi/ixfBZ595Y+v+3d7XNkdvIWT9laqtcZVd2xkO8I/kSn+8SO2VXrmLn8iG5D2tJK20srRRJ3rX860M0CA7nhcOm2IMBRiiXvBKHA/bT6AYb3Y1uUWrG9HJREnAxjHToyjH+OYqMYnW0+jH++RqNRCZZQ8bThs9KzKEX3vY7JvDI54T5+JbX9XPwvj/DjnqaFlq0JMSxpp86qwsVH3/eGrPfxtl+M87XaHo5rwXafhSR8hefIIPuonPW5zBvTIG260Sk3Do/44fFzAgwh5FywczJMPNsMAsyzCIbzJIMs8wGsyLDrLLBrMkw62wwGzLMJhvMlgyzzQSzHGGN7ccsifLrYmCuyDBX2WBmZJhZNpixdpiM5JfDxkemYBYEmMNIuWCWZJhlNpgVGWaVDWZNhllng9mQYTbZYLZkmG0mmBXaDhvCrMjssMNjrsgwV9lgZmSYWTaYORlmng1mQYZZZINZkmGW2WDG2mF0faxxMUR39RpygZ56IocKbU+pyPHPn6AGzmUv3QZN93GyS1zuVx/tFk07PxLtVQ/tGm2j0PVFGks766W9QtNeHYl23ks7Q9O+PBLtopd2/LvfHol22Us7/h1ujkS76qUd/y7WR6Jd99KO922oI9FuemnH+yjkkWi3vbTjfQ3iaO+mZS/1eK8Bi+TtvTiw9WlGeA3YXovbkHkNDo+5IsNcZYOZkWFm2WDmZJh5NpgFGWaRDWZJhllmg1mRYVbZYNZkmHU2mA0ZZpMNZkuG2WaC2ZLZYTYbO8yS2WE2GzvMou2wWL0ip5552ndKwJ2Q/7mh3l99M4l3HM27uJ7PH4ES31flTQ/tAk17VeZ9gyMSzbu4HpFl72wrNMXLLE9Ubc61n+O3jWQ5ym7W6nG9g54wV1BL9DeonOwrWPmz5u6uT8CjL+GE5P6zQ3M4dfRQj9cd46uzf4KuNX1jfga5c6O6O6fJo0bPblzP6Be98mgIKA4j0WrQYo8WYe3AWH3I09ciV5/E3fdl65uqIHL1FrCF0/7LQW25gOrU74B3TxM1pkL2egt3Hl9jKmSHsiGKw0hxNKZC9hTb3A0WjUlRYzha/ngiGiMIKA4jxdMYidYYXTQmaY1RaPlTiWiMJqA4jBRPYwxaY1TRmKQ1xqLlzyahMdUIO9Lu1ZgK7SWdrjH4PlImUr2zojEv0xhsbyt/ZxoawwkoDiPF0xiB1hhRNCZpjZFo+ZOJaIwioDiMFE9jNFpjeNGYpDXGoOXPJKIxloDiMFIsjcH3hTKRIrFFY16mMQztq43dS+qLXtljBBSHkeJpDEdrTFU0JmmNEWj5E4lojCSgOIwUT2MUWmNOJw9ghrzPyfU9VGP9AJimSbRGy4dORKINAcVhpHgSjc/NrE7ijEw1ol+BHHhb80xyM6sRHQSGMVfZYGZkmFk2mDkZZp4NZkGGWWSDWZJhltlgVmSYVTaYNRlmnQ1mQ4bZZIPZkmG2mWAWZHaYyMYOE2jflijnBrZ4x9C8Y0c6L2J79lkCHbeN1eMip3kXaN6JBM6LVOg+AFW07jHlvAjVeZEK3fGgWutocEzflEB708RAZqKImJko0B41XrLfk45PCHREWSSSmSjRmYliIDNRRsxMlGjrkpfs96Q1RqJtXZlIZqJEW7hyIDNRRsxMlGjbkpfs98Q1RqLlTyaiMYqA4jBSPI3RaI0p2e9pa4xBy59JRGMsAcVhpFgao9C2JC/Z70lrjELb1yqRzESFtiPVgI9dRcxMVGhbkpfs98Q1RqDlTySiMZKA4jBSPI1RaI0p2e9pa4xGy59ORGMMAcVhpHgaY9EaU7LfU9YYjbavdSLViDTajtQD1Yh0xGpEGm1L8pL9PlGiOVo+eCISLQgoDiPFk2h8FuXyRLLfx/RSGVp7VDaYNRlmnQ1mQ4bZZIPZkmHOJYtyTMeX/ZhNNlmUYzq+DGGussHMyDCzbDBzMsw8G8yCDLPIBrMkwyyzwYz12MWqu5dPRrBB+8hMZB9ZyARnPX0YK3QHmCpa9bic5t2ieWeTyAS3aN9UqXyWXya4RfvxbCLxYIv24dmBeLCNGA+2aH9YqYaWtq/eov2ENpF4sEXHg+1APNhGjAfbEdZliQenrTEaLX86EY0xBBSHkeJpDN62LPHgdDWGobvTsES60zB0dxo20J2GRexOw9DdaaqTqoZ2mhrD0fLHE9EYQUBxGCmexkh0lLycC09bYxRa/lQiGqMJKA4jxdMYg9aYcgY8bY2xaPmzSWhMNcKOtHs1Jl53GobuTrOey1I0Jj2NwXanYYl0p2Ho7jRsoDsNi9idhqG706xnvJW81pdItETLh0xEohUBxWGkeBKNzXcUkfYZh86nYOg+K8uBii1sratJ6pgtGWabCWZ8n5chzCyTfEeG7oiCwVxlg5mRYWbZYOZkmHk2mAUZZpENZkmGWWaDWZFhVtlgxvrcWKS9eC65fgzdT4Wt9UuJsVf4pv6Gz/F800O5RVNeMnvXecfRPjceOXbbt9fi6NgtK/WwMsvwZOgeMoywrsY0jwVH++D4gA+OR/TBcXQ8l5U86aS91hzt4+OJ+Pg42sfHB3x8PKKPj4+wLUtOdNoaY9DyZxLRGEtAcRgplsYItG3JSk500hoj0Da3SOLcDUP3rtlPcRgpnsZwtMaUnOi0NUag5U8kojGSgOIwUjyNUWiNKTnRaWuMRsufTkRjDAHFYaR4GmPRGRYlJzpljZFo+1omcu5Gou1IOXDuRkY8dyNH5AWakq82SaI5Wj54IhItCCgOI8WTaHw83JxIvpocEQ83AzOlssGsyTDrbDAbMswmG8yWDHMueYlqRF7ifswqm7xENSIvcQhzlQ1mRoaZZYOZk2Hm2WDGWlIqUkQ6nywehfbdqcgx4JC7ZXssUYX236lIUdWcZl2jeaeTyN1SaI+ZKhHB7HK3FNqzqBI5cazRHjQ1kCOtI5441mgvmipRwqQ9uBrtWdSJZDtqtIdOD2Q76ojZjnqEbVmihGlrjETLn0xEYxQBxWGkeBqj0X7tEiVMW2MMWv5MIhpjCSgOI8XSGDMiGlsqJ6WsMQZtX5tEsh0N2o40A9mOJmK2oxkR7S2Vk9LWGIGWP5GIxkgCisNI8TRGoTWmVE6aJtEaLR86EYk2BBSHkeJJND5yrU8kE8WOiFzvnymbTeTajohcD2GussHMyDCzbDBzMsw8G8yCDLPIBrMkwyyzwazIMKtsMGPtIFsq6mzxzqB5d5yKOlVP10SG7s6yLstl3iuok4Oz73j0fii7rXA+oh+KLTV1MsvL4Oi+Mf7O4+9lObo/yn6Kw0hx9rJ8RI8UW2rqJOzP5Oi+Mf7ONDRGEVAcRoqnMXjrstTUSVtjDFr+TCIaYwkoDiPF0hh8jxRbMmiT1hhs3xh/ZwoaU6HtyGpvlJmvdSM5vMZwtMaUbNm0NUag5U8kojGSgOIwUjyNUWiNWZYo8ySJ1mj50IlItCGgOIwUT6LxUWZ1ElFmPqpvi9o7U7n0beGj+rYMYa6ywczIMLNsMHMyzDwbzIIMs8gGsyTDLLPBjPXklW4W27zTaN7pI/Uw4T02CkNbVrH6WuQ07xbNO5tExJWjY8Sl/0J+EVeO9u3xRHx7HO3b4wO+PR7Rt8fRceLSkyFt3x5H+/Z4Ir49jvbt8QHfHo/o2+MjrMsSP0pbYzRa/nQiGmMIKA4jxdMYvG1Z4kcpa4xA29wiiXrZHN13ZT/FYaRYGiPQtiQr8aOJEs3R8sETkWhBQHEYKZ5E4/2Q8kTiR2LEaRc5MFMqG8yaDLPOBrMhw2yywWzJMNtMMMsR8eD9mGU28WCJtp9kJNs9H5+6RFtxMvKpkRBL4b3va4m2i2QkCzSneRdo3okkYikS7YkTpVZddrEUifZY0tW8mbbHkWjPnBzIWsB3XZm+x5Fo75wo9euS9nNJtMdSJlGHm6N7ueynOIwUS2PUCO9cqV+XssYotK2rEjkhrdAWrho4Ia0inpBWI7x/pX7dNImWaPmQiUi0IqA4jBRPovEeQ3EinmE1wmMoBmbKZIPZkmHOxWOoR3gM92PW2XgM9YgTJEOYq2wwY60fHSknKB9vmUbbYTpyBPbHmpLn+jfd8+bSaFtMF9/4Fu8kmncyCR+pRltXunTAyc5HqtEeR51ILqBGexv1QC6gjpgLqNGeNVUiDUl7fAza42gSyQU0aG+jGcgFNBFzAc0Iz5opHp9JEs3R8sETkWhBQHEYKZ5E43MB+Yl4fMyIXMChmVLZYNZkmHU2mA0ZZpMNZotehWzZB69xxKKtKBvZigreD1dX600P7Xh7ypR53+AdQ887S8L/YUdYRyVWn5v/w6ItSZvI6WGL9iDagTiDjXh62KK9iKbE7ydKtEbLh05Eog0BxWGkeBKNj+uyk9jNCXR/kv2Yw0i5YK7IMFfZYMZZaYKw78dpWLgC3YFErHUYibmzcdUG3vTQLtC0l8juJu8kmncpRHYFuvOHIOw29RptMoHuVuLvPL5NJtBdOfZTHEaKY5MJZGcOEy1HeOrq5Wh3u+2nZh/9WH/zj87e+RpGv4C/ZqPuntefXk6SaVw/ETNQX9WPc+q8nsrpiojT1SuQ6mmcZkScZgm8D7+vn+64dVXz4ensb8C3adzhRNzhCXDnW6D+oraKL2H8Wf29O7ji3vlzsBQ273E2wX+B5/H67M/NNcfDqVKHtbqrLCq6/lDfcQvv7j/DOB8ans7qq881B1k78gN4eLsry2WD3j3TzcY0vkoCvoaR0pNYZ6vew/dv4ZPnidxSaG7l4KHdxa1b0NePE/mkCfgURkqTT/7bv4AOz+BN62M+0/hm0HzLIY72n7BaPZx9A7aJX+fc078Ha+RDfZVmDbMEXAsjpSltng/nMMp9uyefwjVsB5thrlF1sHmH5Ng10Pm+/d413H1f//V1/Z+Lhjq5uqmv30Fs9h8hZ9P99zWsbCt777G+8s3ZX+s31/XZdzU9f4FRp/C0IuNplagkOn29n8glrOc8jyrJu7h03vhlvPX2OJFfnIBfYSQKfl3XGnPY6ApD2/sscs7BDzD7T83TZ5Cb8Qx4duOQBDjCSJQ4Zmf/U3OHQXSKwZXd9Csy+tURPCtPYHF0909uj/9Y3+neEG637/wn3jvi+OHeEu/qn18an8qva1cvaso/g713Db/NGq+Lfxs/1n87W6b7jct6XfHcemzzc57hvk9gGXXvfYJdyuXZYuJqodEzFteP/nNDx8UebTFo2uNkf3uZemqsLZzEdUffPcI+f8CfOvfPeq7PIT/MyY+TmSBT/fc+Arqn+upj89clyL/z0ez/3i3cd7P3LsfhefvG87O17/5rwD4H+3+drw+AZg7c+aX+/k3Ln1vQrMv23gvA4+4Ns7F557xn7sIzPBX3YM38BhZNyBx4qP++nKiDlkCOw0hFjnfL8W1D9UUSMvxyaeHonRjPpJ7mbr45u5iGXxWaX7EyMD7VV6dz6/uNFfEzIHerdLi3gnvD59f1X2Evu/uO98Db1Tr6EcZfraPOI3NZP+dP9fXfO/HjGfwwsHBZJ8v/93rsG7B1du++P8N/i/qH1/+/g/OpX4OV6fff65x6D3t1p80hVhVGfQ86+9ihtLtyO65cwIxPXbUfz36qLcJ/bT4Jo+yWOoaWuji71nuQtGuk3F103pAMZlfU352DB/Yc/l3UP27e57AfWNRvHv8bq+fMRevO4Vv+PvfprPnU/Vw346xGcVfn7X1zGGXejhOeN2+fE37+qOlbNpwM9IVv+qe4O+bw+by2F69hZAb3rr7tvmva6wauV41cb39jXo863/GdPi1yMvvQzv63bQ7+IaUP7wOw5R2B7ngmCDuelXdEeUdItNTJxN8RqsnYHno3PG69C7ZX/t2rfAX3HvIJc0BxuCek9H7A++ni2MTXnRVhrI/uPyC+cwk8dBGbK/Cz3rWx6dnZl80Kue2H/QrhI1iPAs0hXvQAZ/S6tEx5+2j0OhDXB0fD2U20eK9d3HPKPzV+2kfQnmvw9d42UUF/dvK2eavOmphiP2fcGO9Bi2c1JXeg4TN4L7nvPHZ8uqtn3fdwctHLSXwkO7UKSC/XFoH2f4hM/B9P9buh68WcbV3xPqQgMSvLZgoXKwIuhpFoI01veihmZBSzA8X4bmG8D/CGv+ycKdtEwtFI4nYkl02s6rdmjXLSfteLAp9jmUO/7823vNvz3MPzV57kqTon0fMuM+DYbg7NgZMfYaWaxi2F5paKtDt6+Qrv6XK242Wjd1O5owm4E0aiseDZwd/8hgyzSWLFcZL5G+iOz88O1TwosmoF2jrMoxPXO6C8uy7PdlzzsbmHJoo3hX8SbWfKyPuUNz30VgT0hpFSnO95k5d/NfE9LNGWpIx2Fn+Kt7ePU1fwXv4AV286fs9tH7DqeF92eYBF53z7fv+ugp1r934q/+6hfGESbY3LSHbsFdoT9gBz/9h4Gm7XfNf+k5vGKvuyiZJ9hfbjh8/nwLuuvy1kqd2Bz/yQMyPQMyOSmpmX6+h8LZbRNwN+rv4AK/LuoDMgCWYgjEQxA5fAEXeqYT2Sc6jZ2P28887Kxzt27zmcGAvXeedU3u77Hzr364POo0LPY9qRph/h3bKAfn8caJYQaXG5O6tMAQUVQRZQUUpA9MXfM4PfGOSyrq5WZ98B7k9womrRVJf/Fq7NYB4XkI88gzfAAqpjzyAmNwMLYQFP81euITNgCaN8CxGxBcx4uN92RpTwl79Stc/9G/x7DWfLF508CQbfC1HUgMJnWHD4dP26R8cgR2HR5G2vcjJUQ3MFn/nf/TeWEPdizRP12vVrkJEFdNJgHbrYGl0cuOvpWgKvutf9aA7jJ4iSLaHexR97sgTx2rqSqsNpkibQpDDScTVJNhLAYEbV2Q+wF1xAVW3RyAaDXrTfNp/4HmsL8Bl4O8/rg2wipzPwrXpp0XDlO4ggLqCOCIfI6gK+a5vYagWjqXbMeZMtJFrZZsCpH+AbK1qrer4Y5N6IJuPHwr3iBSfyx8w+3gNhImfLX4M14M8n+H/dyDPYMT01nPBxqvVTBiFK9QR3uCv/B/j9u2k3F/CehfhnBp4hVudrWFy1mEOM7aKxh2eNhXzVzLTnwT1kE/3aeBbumm89bJzceLvGw/Dd1RmQcO6jL26H67Cr99Zm1CMqM77WKi+4nsB6oAamH6dUednPaUbE6VOs8oLrD4zhTqnyss4PrH9AlSovo/gqCfgaRjr1Ki8KvbdWr7rKi0LvnNRA9E5lUuXlCvBdTuabQfOtVHlZ8cIScC2M9FqqvGh0zHOIazrrKi/Ls/89++ns3+q93F9qu+VpIk8rMp6ebpUXjY7J6lLlBbjACfgVRsqjyotG2/v6aFVe9GCNF422rvWAda2PUuNFo+3dYfrTqPHyZ/B7nncy8v05gG61lVt4wvNGDr8/E+CjD96ruFnR5QK0/X7t2h1QtX7lPfgYu9fcU7wn9iNkDc7gru4d69VkFhPXE42e1bg+3X+Bjlgfm/oeu2k3aNpLHZhSBybVOjAavWfRA3VgdKkDc/J1YAx6r2ZKHRjgQoXmV6kDU87402QMGPS+1pQ6MKUODLn04b0EpQ6MgE4bWG0tdWDKO4JKSyVa6kodmFIHhlLy8L689OvA/LU5GxZWHpfbuPIdvYVMlecmU8/X+fCncI9d/8WgfW8msu9tGkc3UeK9dMvoubM+89Xd9a716K6jndV3rOrDhIreK7+w9+HurvXi9v2fYMRNj/EFUOlrv1zBHYHXjw1/HWd/BYpnnYriu/i+6OU8PhZ+OnViLNo/YkudmD1crAi4GEaKUSfGonfcwxQft06MRe/ebMJ1YuyILM1SJ8ZzTKLnvdSJsWgb1r7COjEWbVvagaxLm02dGIu2NIcxn36dGIu2Dm2pE7PFPwneJwz//J3HrhMj4RzvdHrDSKdbJ0ZCHhSWU6VOzGnXiXFzzNHSUOrExKsT4/gt0DNT6sQcZgYkwQyEkUqdmOPUiXHcV+h5LHViSp2YUiemX5M0gSaFkUqdmJzqxEiQZuye+lTrxEiYSywXSp2Ybf5VKM+C2GNR6TWfTaw9+ofO6vEechi8z+ojrMS/1b/5/LztyHvf3ffAu6fGVxO8Gy/f2VcoH4gYsFWrI/g/cLyVR+UtI+EtS5S34qi85SS85Ynylh+Vt4KEtyJR3rKj8laS8FYmytvqqLxVJLxVifJ2eVTeahLe6iR5a4/KWYPi7L7uyGYtQpVSlr3bTT00ecKrmP5nwPixs9cIn7D635uSm59cbr6Eznk4P06sHnsvz82v2hNY3nszA9+naP6uwE/yqfW1sObk1KI53bRodpM3ze+zxqOp4DyW992YxvuyAM2tWi9r1f7uvZOfwH/qO7FdN1b1AvoZWshp+gQnqcIdN2vjBG8Pa71A/rldun0Wv1wbc9Y8RYAvaNHkM4Snef/UCm/42925osf5Nm+akQV4cAPftp+2+sxRo+Gv8Knf/y7WPK0+Uy6gW4KnrMvp1b+LJvPLa+sCPFqi9WS7n4DjvOM7sy2//OjzZrTtf7vjh/ML4Qmrv21CJxkkYHJ6+vf2CU/g8b1sMs5u27P6FfjlfR2LUD3C97ldaflun1J4S66067HJgTxv711u+J+2UfXr639vvLklzBlGyx+B4puNNSXEdj90MoYugN53nWphYe33+ebLpi5Rd81bp4gnRZHp1FkfT9GuN6mvE3Z+9iWsVLKWg9/r6/8A2vcAb9mvWnr+3sbnvAXwaaccv4P3Vvf6S2VUJSejptP5g5r/FVgPX9RX5000LMzGfOdsbEuHhZXutUiHTU46bCe6laJ0rHow0EuHGzsd6eDglU1NOuTBtHNTOlazgZUOP4I5GH2uykhK8sGTk4+qifnHkJDufOAlRHT8vfQriNtvpSMhvuNsahLS7S596DVkNR9jJEQdTEJ8xdiUJMQmaZ8uX8z/bU+chNyue/Ca9e9JVh11p+ySMNm6Es7sOnq+2uM/xFJtIlG9BFregpfENH9R0K+jc50eg4o+Bxay/CjoXz/Fedj1uLv6Ydfjbo+VuDJCx2PdObVGw+EKdo9i7blx3xw6wd2J6dQ8oalSlX49y131KWN73WyjAbQ2TX/sxkSrbXO42mfnG1mOL4/LMvTpQBa55sBPbc2Vi40OUrtxsEnxAP2qfa0rHcT70lgr/WW1jOeFjr1adue5rJYMfSaUkZ0JPcxqKSauluYVxx5eslpq5C7pNa6W8oA+3+ogMbt962WsnhN5rJcSvV7KpNdLNWm99LX7X2s0rquFY6JxDGn9vr4187Cxytg25vpcl1WToc8Ss8jnKMeummbiqlm96hj1eDuzgsj+sqyaR4jfx7c112e7rJsMnbvPyHL3D7Fu8om5zTbJdTNe5sbL7M1u36yycsbMaznOymnKytlZcbBRIJ50FIhPjAL52tXprZwqmZUzZp5AqF2a1nxooC3WmRh3xiyci+n/3cWbHvfmkGiSDLWYNMc9DUVHM0+a5rj6K/fk+ahe/b2H6tdXTaXrcAr2Dp7p3gburPFHeBt8hrysX8/+uVOR4R66sftT7RedUZedU8cfoI/BU1OT/h7mJ3y/6kWkwKMR5pe147luLc9tdacwDm+tg27dcQ6n8H10/21TQ3e5h4d6rUvwOP6H705fP0VPBlvVnCToO6cuAd142i14BGlijCvrbJf0dd+zL6Gzaqp2TaezauX3bm0FWH8P/d6iWlWjem6vreaii0u1dwZk/17rwHs4d//UviFfZjVZqPko4Nv/D2Wm4z9SDQkA\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": true, \"events_summary\": [{\"timestamp\": 1684771476282, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476788, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476788, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476789, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476790, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476790, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476790, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476790, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476790, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476791, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476792, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476793, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476794, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476795, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476795, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476800, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476800, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771476896, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477161, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477166, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477172, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477174, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477175, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477175, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477208, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477225, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477229, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477229, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477231, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477238, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477238, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477239, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477247, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477247, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477248, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477252, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477252, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477262, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477281, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477286, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477300, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477327, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477328, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477343, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477370, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477402, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477402, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477410, \"type\": 4, \"data\": {\"href\": \"http://localhost:8000/home\", \"width\": 1433, \"height\": 843}}, {\"timestamp\": 1684771477428, \"type\": 2, \"data\": {}}, {\"timestamp\": 1684771477481, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477482, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477483, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771477488, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477508, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477543, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477565, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477569, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477576, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477600, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477603, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477624, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771477631, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477664, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477701, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477773, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771477796, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478421, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478426, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478429, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478432, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478433, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478447, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478449, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478472, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478476, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478478, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478480, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478481, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478491, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478494, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478501, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478552, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771478779, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771479028, \"type\": 3, \"data\": {\"source\": 4, \"width\": 1433, \"height\": 540}}, {\"timestamp\": 1684771479110, \"type\": 3, \"data\": {\"source\": 2, \"type\": 6}}, {\"timestamp\": 1684771479754, \"type\": 3, \"data\": {\"source\": 1}}]}, \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\"}, \"offset\": 2839}","now":"2023-05-22T16:04:40.254254+00:00","sent_at":"2023-05-22T16:04:40.251000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?compression=gzip-js&ip=1&_=1684771483262&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:43.268842+00:00","body":"H4sIAAAAAAAAA+1abW/kthH+KwvB982yRb3LQFGc7669Jtce0ssFRYJAoEhqJS9XVChq1+vD/ffOSLuytF6ndZAPTqwvXnFeOMPhzDyk5J++WGIjKmNdWWe0NYrR2rRaWOdWrVUttClFY119sc4U/Fj/pGzx8dPiP8AGQroRuilVBQziXJDgwkF6ptW2ERqIfyu1yNUtErnYlEykZlcLYLwVzcqoGhms1RrMp62WwCiMqa8uLyW4IQvVmKvYcZzLQq3RoTOkgNCUi4yamqKiIHRlDcJ7N+59JMQbkSWtli1dooqo7M+fUKVhWogqLUS5LMBQEvn3xG3JTQGTBI4DxE0ptrXSZpAN/An5IO17aFOWGZjZigyNwGAct4sgunCRXlbglklLDtTMZwXdZZTdxVGWkA3yTYnrI2HsRxHxY/ci8t1zi5eNKSu211t+9/a9rG9/efN++XlVBI334eMPn78LXrN/GbJa/yCd281HeuM2ebgebUqnSuLY9xzh+VngMWY73GMhi5zE90nAY9snLneZx20S+pkT2/fiIfdinKzFqP4GL8om5WKtUki3G8EglDmVjYAJl1q1dZd7A8ty9nZtj4TCht137IDQ0M556IeBH4eJg5uv9JJW5R01fZAHLTdxkl7L54LYSRJEkUuJ4wc+elI1hlYMU+JkHlpfwatRjaQQfJpJwVNYOmxp2pQclA/+U2bKjUhzQTvhXNIlrOann4E1pqU13UlFOa4UDNRgrVDLSYr4XldZWlC5RudARHCbyZKtCgVxB55Y01J2fuCO0g2M0LXBm0ZStvoV/hlTVaOkSCXY1oIpzctqmYrqxAKNblGjEQ16OJLun+5rbqpmbbo8h5YgoOSxP5xxaBCwrwMVJ+EKloLrHnG7DnVoHt26u9YhRsWNUlKsQQ6D/MUyENl9S6DdFDI14hZz6N+ilnQHNCZpA4sAcesDZGB13RoDc52PR7ZtsAdSjQoTOuSKaRv4gV2BWj/i5q2Udt8FjjgFbewSom1BIlBjdHpYV3Yw31M774A8Ul6cdGxKnbo15d07NaUPLh1sc2qojY9gH0La2qURa1sfAtcLFbBn2HB7MvwwhJFzqzJFyopSQiuAXcGRyvdrJF/PJzsjy6l8eCTvH8m38mnz8xJb52ifP0EmXlOMrqoMujtswiHcDyR+J4ONhCf9K/b2Ar+LOSAO88LeqPbxhS6OBScOAABOHPBOOvBkj3tPOrDQSv0Pm8dTZIrfJ6EpoOa7TEIYnswD6Hg0D8TAqJXA5lIXLL15z9620dr7cVP9o6pX2e36G/3tu2z3Tbvavf7xzd3uW/HBab6/vfa6dnPodwNc+p4bZhEnGSG2I6IsJLFPXS8OAB9P4WUvL4jg3aFlW1ZcbcfT+UlI/cx28ijyqUj8OHGT8LG5UDhwHY/0B6AlYLk4mi2K/dALbADxPKAkTxIeRvnj80Vx4LpBB3MqzxuBxxoSY/SHE+LBzrM6Hk4a0F/zUkJnbP7yKrp+5boPQYq3ujsXAPOV97qXQrv7sesOokhx38DfldgN3LF6x9xQ2e6VQ6cnYWCoUXpQWhp8jN727PvI7QWC61fBntdF+gQdDxOA0A8Z0KrhHKHVejBmu4T35v7vM/NxC/9DH55b4fHc0zkLb0gRxHiye3h4ThzoVPPheT48/+kPzy8J8Sh3EpI5tuOJzAeoc8OAs0d9Q3HqOok7RjwvdMaIN+zPAo0Jvmja9bq/DcwAOAPgswTAW57LOFhlmVC5yRTeBk4AIAlmAJwB8AUAYN+4U6yAdA1xCoIQyuphP+yYUIdaQ62lW6or5DLVIhAAHSwo3Y9R595lfML2Ae9rV53kDLlPg1w/nC+ZM8b+oTB2F25leasjwzS/icjNaYzFRjNj7Iyxf3qMfVmIFxCPeJCmYSZITojLYtfPH0e8wCUiDMeI5yaT16qQJwg/WvzSisakOWz9c/sEP8PfDH8j+GNuRU1cVU4VVHJZd8XyEP6S+Yo5w99LgL9xndO6vNznQ3NJLh86gW1AQlU0oLgWEEtMyb+/+x6Gh8a37wX9t3z4IupAnc8Q+zSIDSeXSqNoYxbdFX7G1RlXnyuuulGsq5s7EuZNqJekW88JXE1mXJ1x9QXg6hos9aX6AQK1GJBzgVckwa8W7ypeq7Iyi0qZRQ6vZvnFfBt9OlS6pz95YqwXuTCsEHxGzRk1nytqlkGQe3QZbqO7LNuR7v99j1DTuyBuNKPmjJovADUxPN3nTsh/D+oM2ziKjmI0g+RTQZJ4/tef/ws4HAtKLTUAAA==","output":[{"uuid":"01884434-ae85-0000-0790-3f208b1ec925","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/home\", \"$host\": \"localhost:8000\", \"$pathname\": \"/home\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"b4chaybacz87b91v\", \"$time\": 1684771482.742, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"a\", \"$el_text\": \"Replay\", \"classes\": [\"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-stealth\", \"LemonButton--full-width\", \"LemonButton--has-icon\"], \"attr__type\": \"button\", \"attr__class\": \"LemonButton LemonButton--tertiary LemonButton--status-stealth LemonButton--full-width LemonButton--has-icon\", \"attr__data-attr\": \"menu-item-replay\", \"attr__href\": \"/replay/recent\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"nth_child\": 6, \"nth_of_type\": 4}, {\"tag_name\": \"ul\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider\"], \"attr__class\": \"SideBar__slider\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar__layout\"], \"attr__class\": \"SideBar SideBar__layout\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 4, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"18844349784635-094f5a1f99d67f8-412d2c3d-164b08-188443497852250\"}, \"offset\": 518}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0001-5a90-cd53a406401d","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"ue3df3rfc6j1h584\", \"$time\": 1684771482.901, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad091b0-03eb47f8265dc08-412d2c3d-164b08-1884434ad0a2092\"}, \"offset\": 360}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0002-988a-cc85b5fb42f1","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"recording viewed summary\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"xdfl85kbbeoftbor\", \"$time\": 1684771482.915, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"viewed_time_ms\": 5563, \"recording_duration_ms\": 0, \"rrweb_warning_count\": 0, \"error_count_during_recording_playback\": 0, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad091b0-03eb47f8265dc08-412d2c3d-164b08-1884434ad0a2092\"}, \"offset\": 346}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0003-e5d8-07b69a479391","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"y6wlixr7tcrdj71j\", \"$time\": 1684771482.963, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad513139-06be1f112c824f-412d2c3d-164b08-1884434ad521e66\"}, \"offset\": 298}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0004-c4f5-64ddb410d79f","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"client_request_failure\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"c2nat8nn0n5nlgp1\", \"$time\": 1684771482.995, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"pathname\": \"/api/projects/1/session_recording_playlists\", \"method\": \"GET\", \"duration\": 113, \"status\": 404, \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad513139-06be1f112c824f-412d2c3d-164b08-1884434ad521e66\"}, \"offset\": 266}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0005-2691-56f4c4e7d4bb","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"toast error\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"278rnjz16fs6rg1t\", \"$time\": 1684771482.999, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"message\": \"Load playlists failed: Endpoint not found.\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad513139-06be1f112c824f-412d2c3d-164b08-1884434ad521e66\"}, \"offset\": 262}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"},{"uuid":"01884434-ae85-0006-1698-9970569297af","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"recording list fetched\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 540, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"i55f3ag6w7zbby1a\", \"$time\": 1684771483.127, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"load_time\": 130, \"listing_version\": \"1\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad513139-06be1f112c824f-412d2c3d-164b08-1884434ad521e66\"}, \"offset\": 134}","now":"2023-05-22T16:04:43.268842+00:00","sent_at":"2023-05-22T16:04:43.262000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684771483260&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:43.265573+00:00","body":"H4sIAAAAAAAAA+19+3LbRrL3q6BU8X7JKYPCHaSyORtbdmJvlLU3vuR8sVMqkIRERCTBBUhdnErVeZbvj+/BzpOcX88AJMgBIIIXWcr2riOSg0FPT09PT0//5vLh94PwMhxPD44OvkjHwSQdxNODxweTJJ6EyTQK04Oj3xdPTvvBNKCU6c0kPDiyHx/kCWk8S3pIMvFunEbTKB7jVVC/Rlq7/fjg5uDIsq3HB1EfCR6yTaNR+OrsLA1Rtu645h+PZWbflZmdPLPvrWS2O16W2XJ8mbmDPCKzZaxmdtwss02EkNm28RJl9r3VzFa7/cevf8jEdBqMJsRr2/F902kbltvGoy/SME1Ru1OicICqOY5teV2/b3ZNUzdCv+uZbSew7LZrtHXHtPpWz+7rpud08XuRPzTDvgFJf3EVjfvxVZGc0/ECp6sbZ77vBGHHaXdQvypalNm1DNsErWl8EY5BZjLonf72ovds5o/sXy7HL8eTi+716O/JD8+7N3+fXdw8+eX4080P4YmRvr1+al/gzX6UTqNxbyrZOP/nsxfDyfW/jl+cv7sYuKl98ur9u3+6T3r/mJoXo/dD4/ryVfCblZ55owOIJM7EZ3U6Nsl67wplZzpi21mzqy0Jrcl1pJPpiIOXKLNNSrOc2RJ8k/aRIiGz6UJlhULRW0uZ/YWmSrU2XRRernymU61OvtthdapVJ6dj3Yk6WZbUEGFpREO6qxrieG5lS5qWB5XglqxuSbPTMe7GMNiO7OtmZuJtGkWWu2R1O7ZpXOF2rGlH2xWD6f57ZBsZqUd20J6yHWs8BrLVlJleKrfZtrQkyOxDH4SC5Hk9xbuwc4fBzTwROxs4VPtutZ08rxw3LBqIRN72KsOwMlle8n4or5/nVTwc05/nzfj1M0GU5AXdKpW27A4IsUpXq3R7DxpNJkc8y/WmqnW8NvwObp3q1nE9+Ej7ah30wnmPKvY0ahNurnvXXHlnIse+qn1gjbl96toHM9h9tU9xMFurO/k2T4Dqm8vc/XR63lzF6MpSc5E7Fo2nYfJWNmtl8zlsDeubz9jSGqJl8oTJcHYeibrE6XQQnx+Ow+lVnFx8S/WcBDfDOACnvx8YyJKEWWM/PjAXDeZ7VhttbCHDYDqdHB0eDuNeMByA3hH8ROcQDTyNeodvZHv+FPbipB+Nz1P9O8d6/fR751XrtxQkbfDlUPzFEV3eFX898dfPH7XFz474a4Il+gAr9IHy6QNU6EPSMEHEg0bhmyRkghJ9gM5B2kuiCcnMBL0DfFqShI2fmOQ5IF/QSdf/o1Jf8Yz1tUZfnfaWrvDO9RUe2q362hvMxhf66x//75vXb394WtDRNtip0FF6tJmOYqbZUEeh1Ss66tXoKJ6xjtbpqL+lTYV1micE50RiEpyHl1F4tayYgyQ8K1U9wzAOk3AyDG7w0QMbfzuLhhgt028e+U8fWVbedngoDehpf5ZAU+MxHj6yn8hcxEz227LmWSnFOsbfi/Bm/rT4unh4GQxn2cueIZOo5sE0TuYvnU/pq/9MPl6IJsvgPn3kZs+EKEvSg56IyKgPIL7w9CyJR/PCdMvsy+IOKjW7Y3CMtFazbRnP2q31TZKrsHvYQzvGw3DV9g5R1hCZwiSJExJAEpCH+AHlwf/7OUjGUEntyzWs7/Nnx8c/v3v1M6wvovaGfWS3vwJBQbg5ATT+keUSgSvw8KQbz6bvJqR1r8bvxqN4Bt+0/13UDZOX42fP3zemj8hcx8JfKiDtDcL+bBjm9AXZ5hS9jmUdOR2iCF2YBNPe4InoPs1J2UbHy2o/CMb9YfgG3To8xvfzsLkoTcO3EUwlapNglob9V900TC7DxNqAFubXiLUWK9mcitnxfe/INInKOO6Hp6OY5J8etibjyejwIgz0qyCanpHeGC2rZZ4i6Vu7Zbbcw6X8hZyHw6h7CDsQXqOEQ0r8Lk5eiy5gHXZD5Ah/QitfC0ke/pX+/+0abP/ivT15//q/3hHbHdfKdLqC6XTWlU4A2UztS7tltIw61pfyL1dgc46LgiZD6RzZXjXPxCV4w0AFU/+t6bVMp2UonBZ46/2W7o4/ywLcshf+soHLA1enwzieWODpPOq1svQPGFvdXxuziyULMGsWsUv28w1G/vX6Y1GNLL/tw/T4RCVIb8a9/9ht1YcYiMBXslT5LO3nJJhgFHmSJMHNJu0FlNvJTMk91aeO/zD1CYP7eGN9MsxMn1iTWJNmyfBtLD2PwxEaCf7bBwzUA9/5dQPd8s0O4LGaQS+BZ0buEknDrhvtZEYhkO4sGvahVySPArsOZDIOr05ytfsgXzk1W/JLJiMKSaYtqgL9kl5Rv7nI7LbRAVTL3Ya7jeg2wvs/ybRqgwmL6WHGIkixOrE6wTANo95FYz1y26ZhIMJFNPDG8Bj/dYPexQZzNcfBJFIwE40vEdf4fhYk/bCfU3wWXm5A0zfxp5Jmc4KugUmVmDSXEnwyxldMMr+LknT6fLNQgmu4WBMmYhHXYQ8DybNs5roBKa8DRKeEVJi+HL8C45uw10YzyVDJKk3U/qdwGAZp81m/41kmlsjeQvZtPDmhANAm5BEHEFyjoz8PKNbRm41mQ9jQ/gbUTBcBFKKWzMbPRTDy5fjpRm3keDaGdaFRROuaAlrgSYYCtiTtI8JjCZnKsMzm4vMRS3BEP+qKNukLzmQEKv3CbB5S8W0HQR6hmiUUm2tlGy0iyeVxHkEPQZWT8Dzo3RTk+eYGg9FoExm0M2MZTEFgMn0b5zoq6Da3nZbp4I/CdHNCpg8bLHRoNkau7jA8hTL9HE0Hr5MoTqLpzQYRx7aRSXSF1AbN3WlDdrL7oaa9JJxm0csNdcctNHaR2gZ6Y63qDdpUkNy0KVyMODANvxbC1h8OPh5koekj7TgY/5+pBi8IhmikBdpP5ApoRDDUpFOnwZ8LxtosjxtrvXg0icdgp6W9HUSphn+BNo71ePJY686mWoR/434EVxAiCLRROIqTGw0W8wLp2g2ASg1+F8Z34SuCSKydRdePtV4w7sEaYOzSlmOBsBaaiPUMkngcz1JtGqQXKHesPUpbj9KPBBd+PAg0RGefn52F4L+H0saziXY2A6YAGlmWjx/HGv6HFzOI/jUwqDD5Me4HsELBFLUnOGoB3Lem6TWW/Hy1eHE1h3jtdRL/hmJfxKOQoDDxll98qx/BW6jK6K6Z0e2sm7FIcSWDeOnJZCIymks5n6LVTsiPW8njFPIIJ+IpFKEfoE0p31KKfMOsegFu2BJl6ys0zK/VgJfLy2Nu2y3EgBcDXgx4MeDFgNe9Dmgw4HU/9enBBsgY8GJNYsCLAS/uNgx4sTo9eHViwIsBLwa8GPBiwIsBLwa8GPDaOeDFuBXjVoxbMW7FuBXjVoxbMW7FuBXHuRi3Yk16sJrEuBXjVtxtGLdidWLcinErxq0Yt2LcinErxq0Yt/pT4VbPgnTQjTFYZlED4DDzJIHGOO1KKGglYxEPmj+qBKNMs5j/3wO24rO/b4Gttjz7m2Erhq0YtmLYimErhq04zPWA9IlhK9Ykhq0YtuJuw7AVq9PDVyeGrRi2YtiKYSuGrRi2YtiKYaudw1bQnhEQmDnMtDhisBRrQuDy3w1rwkV5jDXVYU3i+nLGmhhrYqyJsSbGmu5tLIGxpvupTw82NsVYE2sSY02MNXG3YayJ1enBqxNjTYw1MdbEWBNjTYw1MdbEWNMesKYHDAN5DAPVw0AQEMNADAMxDMQwEMNA93iazzDQ/dSnBxs2YhiINYlhIIaBuNswDMTq9ODViWEghoEYBmIYiGEghoEYBmIYaA8n5U3D76KhmHsph9+ZGFDUHUarefw18pRdD7Wax7g9j1l2KdRqniLMxAcBMirHqByjcozKMSrHqByjcozKcRSPUTnWpIeqSYzKMSrH3YZROVYnRuUYlWNUjlE5RuUYlWNUjlG5PxUqBxiI8IAbicyhpVSYywEcw7Abw24MuzHsxrAbw24MuzHsxrAbw24cpmPYjTWJYTeG3bjbfPZuw7AbqxPDbgy7MezGsBvDbgy7MezGsNs9ht1OgnT6U3iWhOngbXgNwRbhKJh3CErOqjLorAjBoWHg4da/YBaxtvkLJxDF+OlsOs2z+UXACbLDg/KMGDZVsEvJ1S5ibW+T6Pw8TDztSyiQGFi6NxosEBABMHz25ds4Hk6jifdV4Z2yx0vvZ6nFYmSKYCj7LuG8IsfZA6eC7aJ0C09Lc1uI7N8uCsuyy2mS6XgT9cMcXKhtRvjOJZBkMVsZPuqtsZXSWwOLddfAYhnTfeiYLq5xY0y3DtPFRYCM6TKmy5guY7qM6d7jmB1juvdTnx5sDJgxXdYkxnQZ0+Vuw5guq9ODVyfGdBnTZUyXMV3GdBnTZUyXMd09HHDKAB0DdAzQMUDHAB0DdAzQMUDHAB0DdBzQY4CONel+axIDdAzQcbdhgI7ViQE6BugYoGOAjgE6BugYoGOA7k8F0L0cp9H5YPpjOIUTBBAm+32cg1Hu0kbAZShIzVtEeHIkSsm1tMGysLuxkPEl5JuMgyE2OhbhsZfo36nEiIwi+vRTmEafqHvBvSlsjPw+icQrxRo8S4Lzc8p6DHfy1tzztKWcQjXo0UmARpyu1nmtnCtPVt5KoVdpdAnPdylXkbXqTFUyc2vPoy3kK7bP8vMyFNRf4wZKRlMZTWU0ldFURlMZTWU0ldFURlM5+voA9InRVNYkRlMZTeVuw2gqq9PDVydGUxlNZTSV0VRGUxlNZTSV0dSdo6lyypSeROPwe3iBsGYlACiAjzVgUpjfQq7s6fvok0R3VnNjYGGYlmFahmn/xDBtm2HaepjWYZiWYVqGaRmmZZj2XofhGKa9n/r0YMO6DNOyJjFMyzAtdxuGaVmdHrw6MUzLMC3DtAzTMkzLMC3DtAzT7hymZTST0UxGMxnNZDST0UxGMxnNZDTzHkSrGM1kNJOjn4xmsiYxmsloJncbRjNZnf4M6sRoJqOZjGYymsloJqOZjGYymrlzNPMniHQspkOkngKHmScdx+NpgN2oSQablcBMVXmL+JGah3e2MhbMWDBjwTvBgjuMBddjwTZjwYwFMxbMWDBjwfc61sdY8P3UpwcbO2YsmDWJsWDGgrnbMBbM6vTg1YmxYMaCGQtmLJixYMaCGQtmLHhfBxC/gAp9Iqh2+DRgqJahWoZqGardCVTrGQzV1kO1FkO1DNUyVMtQLUO19zoUx1Dt/dSnBxvaZaiWNYmhWoZqudswVMvq9ODViaFahmoZqmWolqFahmoZqmWodudQLaNlf360zGS0rB4tg4C2QcvcBVo2Dc6JBOnIZRReLYNkgyQ8w1PVdBmGcZiEk2Fwgw90iOnfzqIhIhXpN4/8p48sK287PIyTPszVaX+WCKOBh4/sJzIXMZP9tqx5VkqxjvEXc5H50+Lr4uFlMJxlL3uGTKKaB5h0zV86n9JX/5l8vBBNlsF9+sjNnglRlqRn0yL1AZnX07MkHs0L0y2zL4s7qNFs7Ehlza7R7Db2NG+j2fZCs1MMVwTpGqhVeD3Fax8wlsLlSiKMdUTmw+8HxKxpoT8U038/6A2DFF8OTjACjp/OplOMpIXvug5dn0ZkC5dSafCdpfgIg+F0sPzsbDYc6ldRfzV9EKR6BJBaTU2jfigekUJlrNrtO2AVHsEk6FNfVNLBDR1pUV6FAp8+APld8glUfqQ8o955GTYT84JH31dkmU5vhtAYKFMSYN0P9X09tzxg3Ugfa0YKCgsSnW1JuB1nexLQ+u1IwB4oNJYaTJyRcoJhATP91QQ0BIJxWDBX9SBrqIJ6uI6/WlzS02fTaKjD6qPQb4PxVO9jzdn5WDQbMTt/2Vf6q7A5oIApS6ob9KHLcYkmWUkUCL1N4iHVZ5FtEozDYZ6xSF9hroQ+DX4UjkzXKGKRt1iKojslpWBYDcXrFBRao6Tl/IXS2uZqadWk4PWDSDyZTSg8tcwUPVuiq3SilReWhJxVYBh0w+Ew7HepqUubrVjCba2xKuPbCyltj7ZiskrLKW2RmrJWWwRjUIJ+cpkNQJMgQX1foiSrg4pSkbaJ1WIVT8QyqcUTsCzs0HKqA5Miki2v6kHFG5h04oKJiifV71QUgydu5RNxT0PpE3FqR+mTldoXngiHuKSiHeFPlElgpZ7wj3IZu9RGGP9KG2gMV4J+jDHOyOD8wvmw4GkE5/8IRvgOf+hSRnhKTOpThO37vWQ26qanp935D/Ki4Lzo9BJyLR7oFjlJvQHQw3+gxMyVkVW3baHDCzYpqQGfCFaMVxmtLsxRC4NE6woDL+R+0QlY0qn7SfRwhCsyf586bEbdXaGOJtmOtPY///3/tNeYsryIzxfFeCvFiLbfScPqaCvTq2ksX5XftkpVVJlsZkFWNJPA3NYRrpJx+UbmWojpdZ6nmvG2yjgs/06EVsnO6WlCq9UxSA1nI3hulcx1VOYwuO6EudFUt4zqoh1DKdrBhGMnRVNo+EbHNDLF5LyGBVNlAYP9Tlg4G4bXGv2hNtAGwrtGlBCBN516GpzB32aY9p3d5D9HulvDqaVyClbW5jRQ+JSZDvAbTn5BwysnGSmUTAaxamYZhHMsIh8pQh8TmJBBfN5CAPWwH/ewegMrSfXzGWZoKUVAcktWXXXVSpPHvZWVLqvt6Wk2Q6thRbXhNCo2MrQn8OrHGvmCWkBLlLWCDOblrFrz20q5VSuFP3aG4BQCDef6LNLNTj5kFlMncYIdPMhaI4PVIaDAGiYXdQ0hZlalNhh+H2QQkh2eIDgWjvuEedw2ralkUR0qaNLUqJlaKE7EFn5H4DilkeAIwdkhDqLUu4jmXXwNRGAYJ5Q4CAHJfK2d4V30BtTxCMBAMgqGX2si/yAkU3ykGV9rVIYeDFGHI012+yxNTDYJjaB3x+HXGtZao/xgmOfWjRaCLeEoy4+aQVwCzIgBGIyiT0Bnz6NuNAQ69LWGucYnPU6vdcnUKI6nA5H5HKs+UpBFCX/Mq/if2n+gmku8msXn6eV5tRjm2Y6O5AqOYlZZlwWl/IsM2xQyrhL7AP9frL74Fbl6syQlUU9i7C8gkc2z6SnWquUFP16Qp+RqloMxbBXN51HPVNQbM3CDsJWzaAwTnTWf4F4D8ib2M1BgF5/HUQKcBSx8C7U8S6DZoFB8pP3+cYz1GI9QfKFRk5jApS9tz+iH51/h9Y/jHVAgg5G+kQGLaTIL5x1g1eVY8r8zL7emo16Sx1djMF9S6y154imgEhFBpt0XFEKieGo4ouFAaNT8J+Z0ZF9ItPhFcfOn8TXFVTRDsxz8Q+r1aIjuPQ+eX11dta7sVpycH1oUPZfsncW9WUoBEzJswTAVloUmlIOo3xeBWJJIqZFI37z/fkVgqhvkNHGDaLGkatnw4EcTC/oNw9S8k3bL7RhGR/NbjnliYg2Sj2TTmqdjJZLbOcnzm+0TJMiv1jzV+2UhQvQK4vaYzNCa1XRVl6voQ2eO6bb+zhVWmGWeDqzU3M3phtOrMBxr58FEdzR4+061BXdVx2yZUZfctM0YpXGEggwwA/KLPo0pWpMF/g66YiQir41sTnLe/dIy/cda/uerr2vYVr20FbbJZ9uMbYKZ9Gk0RUgkia+EpFdFW8OY6kO5TXyoWxmTbmTNmOyqrlORA8rQlIPqslbdJ0ralccoqosCc6S+psqrntIyG5Shlo0BoUq3KUNN8aoX5G45Ya4uS53jurfMcRWPK5vLIgRBY/bcF8ZRvQrpJt7wfIJzF9OfKuCkSm7Y3LpaOa/JHHiHcxusHKhjhTLsiJVbRKLaUK/JTHct5+VheimAwBXRNLHilV7KyOy0HBuORqvT7rUMR2/ZVsvw9ZaHpdH46LRTvWXY+O3hUyQMscfS1M2W5/ZaZkdvmW7LwnsOki16b6hbut1yvB5eoMd40aL/XORBPvo0PM3AE5Rk4sNHAXip5cATwksuqDigD6i95dtUEKigWLDWRi4UKzmiQmXB+A9ZHGugg2n8z0EOy6UiHKzxBhkHlbQEAS0j4JnEiEmFux1RhiiCmNCJCYOqC8JUJSqMqIhPn8jaKIdKtEEEL2lZdeHaYUkDxEQFonaQBclKE7JCugPhisceycAn4YIoRCt/UPGL3GgYE9ksR1QwoyZLQuEQK6SK4iFT+ksS1SS3VCliFmWLykCgLQdUjTbESdRFVeeyIA5QiChHCmqQS1EIUUOJVP9MxvSibAXIUIhZluei8fKWypoSTEGGopnh87bFX4hPg/hINSA+en+YaYtg3pLKR0VJZfo0QuN0sOii5RNBSBAVcalK+OvbpJwoDYTETzSxSVVFQ7Q7LV9Wuo3S6KtFSmsRq/SFVJ0E22l5bUgbTUlt03ZarkfZJDVk8wR10xNUwSA+bXeAJ9BzUn/6KZRcPKcXIDmiJ4hKxWnj0xGl44tv4sUOROfPyRNbxJVIskgDSPfBSxtyAMMmdQeTKJIK2shooEyXpIhP34EWUR/AFmL6aQ/zF0BDEJjTm9MX5VE5otWRCUIQ7KL3k7xcqi/1VFAr1CmvZ8sS9R7gTwelUdPbbi4kIkvVIoLU7G00ryjTprqj6rJhoFRkeLIWWbQQukIbBVP7ofGh3y70w/J7UDpsTjcc9L9MP/GG35n/cFL6qeFLntCjN6BvJAx6ojup+Kbjmw7NMjR0WnAMilAnegcNM/9qpS1MzugL+kmWpFlaq4M/pDf4gy+ftpqVeaprLMbaJs4Tvp1F5zMAzHOqJU5wk3F0nSlUvqZFH8S0Oa9mUC9xhQvMUIbGzAB7p0e01HoZo5lzNw4ua1hS3WNvVyBlkQOsEKa1ddma3tL03Bmo5lV1r4u8Uobd8EpT9xo2VFfc2xVYtMRG1qD5jLwQBRNfacfIl8bkum4q7quetb8rdGnOa4SV592gRu991akuckEZtucCf2o4UN3pFQ42D0msyYHqta5wUB+NXJeDQjwHa0fydVOVXKlG198WZFlYpNzLTxGVp61EuZ9fYqSI2e6UZiR5yJvmGerqoY0WZlXWXh0caGXYjhYj+Kq1F228N9nmM6uthLvhqrRKEavjCy2OayTiHM0viFYdCUQHvt+i3XIpXqWI1fGIVgY2FrHYhkor0rElRyCU2aq2kuBMk4UbDSNPSyOgWCu4GAEvozTD9o406Sx8rYkwNSF1y02gi6ihGmLIEwMczEfLE48OSLO6CGvUtcet6xmzl8NrtBw24c41prLR2uqQSAsstwt5ZSocjUTsZ76skFaCDIfRJI2oBxUELeDHHCycZ6nmWR1E29vGpBZBJc/R8K/d8ei/ipiRQMlF4GypSusEsZZmJFtEn9rqON7eSfTpRyx517AOMcB8E/8MzTRM08Ln4rcuEj6NrDYicLvLt27NSyZou1r2tjqVWngx+dyK0KlqzVQH8nYTqKN+PBDWeF3zktuqZfC/YuggyqpF2cky4UpRqV5JUVRyFfMDE9WGK7crRaT6LCsiIg9mhyIif2FZQnX+BhFY8vIpIff0KyTUePV8pWxKIK6drhVcWhsoQPIaZkpAsQIzlOHumOmoflKnyVT7MyJ0KZb5kEouLQQOLgGxq6uAM4y7Rg6qZ1OUA2W413JoCl7i4Fmlvo28ot2Bl7iutI4VynAn4GVH9RRok9puRJKECHzRrr1sNRt11BpWVM+g02gRxJ8XR+2onkCjbQx1q72wcOuF6bw3vRem8d5s//KjrXnv2y8s8733wv7lR4Aj9guz/d40X3jvTXu75VwddbgWet5k/vud3Hw+n/R21GFOmK3P0anVQa7TdHr/BsYcx2hoha0cctOmoQ5aYszc2XBOg2b13iPPKJkGF8qnDPdgsCjd41wzWNTtxF4eZekUAPLcoXy560ZJCO+fh1hNSvGnwnR1EbSo7OZSaMqQhKS9aG/tOOAZyni0xAdluKNe5BnKkLTKSv3Cu52JRBmPkMTjkZCDMh4haTfjUQdrFbAMAkNOy35vvaA1xu/pN5B//MT6BvELawyO8WlSVs0Gdu7hq+a2vPeWcWxrltHqdOiZiQWwyIHs4ssLIn+Mp8D+xQMA7TKv+PIeBI7xTZCTuYh+ztMvkjvLoOLfY4WzKeihyOwJWBaPQEY+QcJWA6ZnKAOm1P5G+3eCdEr162MjxWI0UUZN2cNZuT1DGcWRtJNFa1gsgpge/V93t1qB4WHXuMJjE6fn/g3GhQNQqgwyNn7XVZoy1FZaRthrtfZEZKnmQB2tzVtGa6U7PsuP5pgTVYdeUZG7dgGwS7+OD8pwVy4AtuUrrDRFe/9TM7VRNAYjC0mrw5aoExs9Dwc+KKJpsrjoToxe/b70bCP4zvelD8OzfFu6Bh3vhfpNTXDRM9Xhw9zVPK2Oy9pAuWepA4a1q9VFyxvHr+TG8e65fjWgrYniganJTUrYFIhjAulaNrlnqY8jEYHBVrOtmnxrVzFs2tSVt2fNvNdSbX6RBcqwdxbUEWKFhc0XJK3Lgjo4rLDQeDdUYxbUQWGFhfrd3LtgQR1AVlhojsg1ZUE11CssNEe8mrKgzh1WWNh8cee6LKhGdoWFzYeCNVmwSyzqEgs22df9slBvHSnD3lm4zTrae7eO6qFIqyzs3TqqRyWtsrB366icp6SwsHfrqJy1pLCwd+uonrm0ysLeraN6etIqC3u3juoZSass7N06qmclrbAgTk7aLwu3WUdxctJmLKQDHGVyoRs4CyeDOMu2ty8vSpjACyaMxdSEYyxrMI7FDgZADflFyrdCBurJSkjakU+8w/M/K9lXzXWRfXk66Y7OsPPU849WC9vcMJcvO0HQ4bpWK1U7vcLRLWcS1W71R9Ha5AZMyFlXDRuqrXaaRPBr2RBn+HSxwwp3E2DPUI/udhGHAF2n2oi4o/tKLugEJ0RK5jPEOQhpLkm1pg6qsW98apK6GcBTzqLJ2uSuY6vNz3IuCa1iwR710X7Dw53FEd/iRXBY0wLqQFMUFWVgUUlRqSfZrIjKrR8QG5k99TQaJDXsGaamA+qa9wr1pBjJ8V1HqdWDYZDEMVwhB3Wwa3TwSA0qawMjBUrqneDUBMfAN3kmlNHCPQfYRo3znrL07EyoLD/OhMIJAfifPBMqp7IdJqoeliPNzJ1rojqEivNxWBM99SQfJP3pTifz1EOEpD+5M0/8zVVEd7gVvus6rj3rXdTFzNXzh5C0vf8iVjwNiRM9lazQ4FDCLk65zilkS+ll/mqO1UOFkLR7OZ6eyhvtahhRx82lI4WQYTeMpEM4vjWurXqmkCx6JzaucE8EOUHqPEb46tmya/T/8wFO9BxM5y78BNs4qtfxeuqhP0hiuyjkoI7Q3m5GaI/WK+GuqBM68Qb/o6OR6KeHxUz4NGiVEn3ZzuCpR3TIIMruOuoFFlKSe5/jhDgbCYiieWjX6FvJVHYpzrPBUR3rMVUzzVcP60DSnk7q99TTNpDUdJWYPND1f/77/89dfvX0DCnKvdRBPf4CSdvXQT3PQkYeP6vGqkdcLDFFGe5eY9VTL5C0t9ZWDeFth1ms1dqqfWp02ECzOqh2h27C2roOquXY2QUHm2usamGKTFGGz6Cxqn267TSFzVtbPVsBSdu3tnq4gBTlfuqg2p3bTgVYqw6q5djZfSEba6y6BX6JKcpw9xqrbn5H0t5aW7VPYsP4tq2t2ichyv3UQbU7dAnZ1nVQLcfWV/NsrbHqruQlpijD3WusukUYSftqbXV/LpK2b211r60U5X7qoNqd2zbUrlUH1XKI1RSfV2NVC1NkijJ8Bo1V7VOju9yatbZqnzo7sE/qRlIpyn3UwVc3liJp6zogGqsqx7YHvWyrsX7JdssiU5ThzjXWV/deImlvra3YJyTtoLUV+yRFuZ86KHYHSTuoQ8nMa7fh/E00tmTmVWCKMnwGjVXsE5L21drqXjckbd/a6m4yKcr91EGxO0jaQR1KZl67uiVlY40t2UdWZIoy3L3GqjvKkLS31lbtk7kD+6Tu0ZKi3E8dFLuDpB3UoWTmtdPFi5tobMnmqCJTlOHuNVbd+oSkfbW2uskJSdu3trpxSYpyP3VQPRuxKWnbOpTMvD435kWXfdcxRRk+g8aq9klsRtpPa6v2SWw72ra1Vc9GiHIvdVC3ECFp+zqUbAu6jez+NbZko1CRKcpw9xqrbh1C0t5aW7VPje9TL2tt1bNpNKI2q4Pq2dwWhlqrDiXbdj435uWXbeQpYl7I8Bk0VrVPjdDBRq2tbuJB0vatrW7MkaLcTx1Uz+a23TJr1aFk0+Hnxrz8kt0uRaYow91rrLrhBUl7a23VPt22rWWt1lY9m0YLPZvVQfVsnB14NiV7NRp5Z3vR2JJdEUWmKMPda6y6ewJJ+2ptdV8FkrZvbXWbhBTlfuqgeja3bTxYqw4lW5o/N+bll6z6LzJFGT6Dxqr2qdF6+2atrdqnxlfnlrW26tk0207RpA7qinMkbV+HkvXj4lCSz6qxJWvJi0xRhrvXWHWROJL21tqqfbpt2fVara16Ns0u/m1UB9WzuW0x7lp1KJl5fXbMq2Q9c5EpyvAZNFa1T43uEmzU2upqaCRt39rqamgpyv3UQfVsbluMu1YdSmZenx3zKlnPXGSKMty9xqqroZG0t9ZW7dMOVkP76mpoKcr91EH1bG5bjLtWHUpmXp8d8ypZz1xkijLcvcaqq6GRtK/WVldDI2n71lZXQ0tR7qcOqmdz22LcteqgXsWxdEiJe8sZLXSgTcXeQ5DQR0jta6hKL4km4izc6gqqN18scYIMtZwM7GpWomnNXlBfXeeMpIai/UeMA4KyA0a0+R2V8xJUc9N4gfbrYUinq0jaWlAoTlwPOR2EGp0MqqEfiu84y3RR/qpdKsa0IOfmE1mii3NxYhyJhMNdZpFutpFH3KJQTJ3EyTSgLYjV0qfF0L/i6TQahTjHY4SbFGidOTwYB0cOwEF8fPBFKk9DPRWlmu2249iW1/X7Ztc0dSP0uzgx1gksu+3iHnnHtPpWz+7r2GvVxe9F/tAM+7SV+YsrXPIVXxXJOR0vcLq6cYZBJwg7TrtDZVfQosyuZdh0ido0vhB7QyeD3ulvL3rPZv7I/uVy/HI8uehej/6e/PC8e/P32cXNk1+OP938EJ4Y6dvrpzbdHYfDRCCm3lSycf7PZy+Gk+t/Hb84f3cxcFP75NX7d/90n/T+MTUvRu+HxvXlq+A3Kz3zRiTL+OwsDaEWFowQWja8lEryRToOJukgpqafJHSN9zSSDTZ/ckqNtKRneUIaz5IekmDDSPWyViq2+wfZ8GKsXNaH7No4UF1sOe7n519jM276WDNwaK1QQ0kCHvV2JMT1RduSQPU3JAHRJBj9LnNtDvp98a1SleE5sirXqTKcsjtXZXnNeYUKCBuKNsce+MUmdPq1SfNTR+Xmr2l+TLs/Q/MTHlnuNVQewa1VPtHFMWdQj8bagVGQtaNWO6zttAMuTp4wGc7OI6pLklyF3UOcMZHi9JJvqZYTtCOcZNn5L+UtzUkibnuik+xAh1pznuvDwUecK5dGn8JX3TRMLnGixjCOJ1ovHk0wJYLri1NABhrNioY4KjFBwjjGuZFRT4wnaevjwcGvGI2qlMJkpahXCghoxyYDo8EkluO+NBM4S4RivbiV1aKYShYilE32KmNEd7FRAZwU88I5Kc/rYGXHSl7wUZ7XhtNQYzR4SKnXDwBYOzca0I7pID4/HOPk1zi5WDUbhrgNM9OmxwfmosEAGVCIw1qcQzOMe8FwAHpHbTqGJphEh+DrN8zv0kPzMG/X+UTvdH5N8t+G0SiafmMbf5FV/QZf6JaFb3CLMK6IGMV92JiwfxpQVW3UA3cKPj5wxIDoir+e+Ovnj9r5l07+xURFsm+oA32A8SxBUoTymiCJE4wgJhNks28gTTt3TRAXn6B9cBbSsVD4Afqi9odmiwRngSjUCEziAb3toNiFhrdFmKhc++kZa3+t9m85NdzZkHkQj78LouEsCbUvVdW3bOeQDvCMeoe9wQxnLf/ivT15//q/3rV+S48su2PbR2bnK5CjEAXp9mwYpoetyXgyOrwIcXG2OJU41b60Wzjs7RRJ3+KMuJZ7uJS/kPNwGHUPxTXfKAGciET/8K+HpzR4n5r4Ng6vTtAG4RhPPtA5x/H4h/Dm18O/frsG/8+fHR///O7Vz4J/BwvIjmyP+IfURxFCOHhzSPfB/8fnr9Am1TGPLKu6OYhr8AqrhVK+xUl8ptMyFM4LHPd+SwXXOXdezrc1T/sZB2dDX58kSXDz1w3awHQNB5GS3TPdDc/iJPwp7M+u3+BO0BCsb8KfZXb8TEd2y59s6CWRxudRr5Wlf7gIb9zmOkAXmR3ZQgfoQvL+6nQonw2lJ1TYX9fq82+O/V/8kzc/iAI878h0iHxwhkPqfgSqMLXXorLMpmNBVY3dS1VY1NRZFapM3lSmjuEdeT4xO6L6CtE1r7NlOQ7Fe+d0NiBhOxb42b3YZmkomjLsi8pt0o9xLNmRI1q0G8ChOMYpj+chDGVzQp3OkRhTVpjahJTnZuICrffBcLYZQ5ZzZAuOst6ULqILt5BTohH6d471+un3zisijeOgj1zBHmL/GBd+xoT0RRxfNGcSLLa9I3uhoy9BD/0TV+/hduRjTHlxBOMGGmc6ruNnIuyGcDN+hlvdnIpr+dBbYZXmVL4wm9NpW0Y+wmHMgX0fvRtH01dnm3GF5ZHgSqgszRZOYC/e3Iyb6xmtFYWGCEIZW0SHmHo1/imONxA7tjuZ+FvR0UXv1vtAlr7Munhdh5/nPkRnX/xq9ckjjCcj6AWZgLPhLB0Q38eZC/TPWTgLXyJY0twadNo0aIp2mo2RCwd3niazMen36ySKk2h6s0Hjt42stVZIbaBJnbbbBkUiVl3vDah28sG3nOoGFNsYM0T3S3HkMDXouwldeP5q/F3URUirsRRxDg9GcKFZmMngKNPe4InwOJqTso2Oh5kMkZJH+go/Sxr+Dcys4duZIzgJYK77edjO2oCW5aF5BWd5JZtTgePnQ1Q1o61+FURTdHftS5zq2jLrZgNZzuXZACV+FyevxQTO2sJjLc7KOq51ZNf403o6684h/3UmMkv5lyuwGx+bpvmwn3vwsXc2B2h3HuYcgOabb3Ce9AaTe6yzsDJDEaQwZBUzYp5T/jvNKRGaGm+sT/DdKl0a1qR/M02aJcO3sfQ8DkdoJEyOPtDB6r7z6wa6hRNNjtpivK8Y9LD8kc7UF9Kw60Y7mVEIpDvDmijolZggL9il4MJSyE6+cmq25JdMRm8Rp01bVAX6Jb2ifnOR2W2jg6gLdxvuNqLbCO//JNOq5p6xLUIiHCNmdZLj+fh4GPWax07ctmkYR44YzQkyyWe5G8zVHAeTSMFMNL4EKvf9LABc2s8pPgsvN6DpI0wkotSlNJsTdA1MqkQIsJTgkzG+YpL5XZSk0+eEsG1ShHtk21REeB32MJA8y2auG5DyOkeuiJSskArTl+NXBEZvQBPxRdkFFZqo/U9Y+UwroJuSdXAZbBa2rCH7Np6cULBqE/LzME+cPA8o1tGbjWa4LRbLfZpTM10EULII1HMBKbwcP92ojRyc3H9kCY0iWtcEx4InGQrYkjQODYNLtAjLbC4+BDYxO5/H9MO+4ExGoNJNArg4awdBngVKsEyxuVa20SKSXB7nEfQQVDkJz4PeTUGeb24wGCFi2lwG7cxYYiliOJpM38a5jgq6zW0nACX8UZhuTghnZR45Eiq571FW1LSXAJHYSnfcQmMXqW2gN9aq3qBNBclNmwJQDnnpyysQf/8o11t8xAqMj1i7TDeh4ruDs34/ioUc+PGRhpQAN5HBO/gXAKrpxwM87ME3EA+xJvH0jPZuiWSAOlisIR48H/cncQRekUMTObBiEVlowSwy0L6VP/AT6xpnvTDBggLx1nyZksybLzYQz4jrOT788eCP6uWPNpZE8QKf+gU+W66J3WjFtE8H9ZevmH4bY/UZbkU/PZ3SN23lt459SaNQ1+Udc6sPdaHF82TcMj7GtXTTsJBR3KWno7Z0CXk3xtWDIz0Rl8FVrrgubnrCsc872ou5yjvdGxxEmIUotZo/WWW4cjuUujGy0TUMlZu15IWOVsfBvUN+4T7Hz9holTJQ92vedjvErTLILqhExzjH8sy0G9ACteXr/ZYu/RsGXbGWrbheWyM7Re+pssvp6iBcqHwxeS6cflWGClEvE8kX12W7VA4k0eI2pSMNyzSNUfq1tnhGNllckx0eaRL6+lqLJ0EPg+6RZn5d0xbqJtdiW1CG2ra4k4u/sxssb734e50rvmkLI/FK+0ZlFx7GaahnnFbLSd2fuyKn5ucCZDobDFH36h6LTtW/qWFMPRugs+VJNdVlqUcDFMuiDDuzudRequFZTj49/RTD3AjjUyMhdf9vs3tEHuI1pqgAWRsaGAQf+C27pLQva90S6quXoiBpF7ecjkzc3uwOrEtroFufRobexndPfMfN4rrb092Wa2mG3sFPXHzqtOU300jxw8enlv3uIaMNGdJF0KZOObOvyCp+4gv+yRRQFzdI9+gJkW/rdstt46Odyi+a+E+jH/iQj2Xap23uW/XVe2BkX9mFt6Jc+7ti5kSL66JXVXYSbFBYZRBJDRjc1TiQYpnbuN98IKipmeJxLdWMMmx+A/N5El/VFK04Okja4FAHXNScTam0M8zdwv6RVjJ1o8iiLEMZ02Udd3+vfjba1khAGTaR1FAC34dTbRAO6SiKjGjJkLftUTLl9RNOQ3XllOEQSX/2gWUte9dWb+BB0k7uxzY7mtdycD2233I7movrscng4yul4rcrv8g0PEQOkVOmiiTTFt9FMqXgn/wu05FBZt/K5Lfpvp+qvX0IfmBuz8GPmuAHTrm9X6cFzHuh5Wx0RIBt4JIfbvO6NsdB7fdrPyds0ub7OdO/qVs88zn8N4/8p48si9jGxyP7Cf7Oc1GKdYy/wGbnT/M35w8R8J1lL3uGTCIZBQCh5y+dT+mr/+wvC/l988h9+sh99he5nSX/laHB+U8KiZ/SuUPf6JbZ/wteTVGN2Szqf/MXuTfVMv6CdbVUu29IorRlFJdYVW1CFY9oE6r4QptQxRexCVV+K2xClQmCIh3TLzahIqaVbUKV30CaTq6iTaji87ZNqNj0Ijeh0nGhy5tQO2IGXbUF2+UuW9dlTVwVc/dm2qM2r/Gb3pIbQ44zIrarCYiwXUdZfEhJ1qkfXConfKycrocxImNDnE9Q8QzoRtUzeUJAxTNhAyueCWFXPBMnFFQ8gxZXPsOxRZXPxA7jimcCnKh4ViMXXJVT+Qx39lQ/q5GLVMKKZzVywY1Q1c9q5ILbsKqfKXIh05PphEt6VQJdSAI7iQZMdFsTZ/OlODRufkafHgynWlccIqDPv/SDFOh5zfxGCVogqfnBeYjbgT4OsluMjHLGOp/Q0cWQlW6TSdrENrjOBpt//Pq/0C7uGuSxAgA=","output":[{"uuid":"01884434-ae8b-0001-daec-3c60a40202fa","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"01884434-ae8b-0000-78d7-f52731c49734\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAJuSa2QC/+19CXMbt7IufwpLdXJeciqUOPuMktwTx85WV7n2s32c+07iclEStcTUcklKXlL56+89oLEMZiXIGY9G1FcqSeQMlkaj0Wg0+gP+3//9bfDnYGewHHwYXA+m7NP+YDjwBl+yvzuD48GEvZnIpzzdYnA1uBnMB0c6rSPTXrM3i8E5S3/OPl2yz+K9KP+9Th2zH5Hjg3zmsvpc+eyc1alShrpsXuYFq/Hp4IT9LNinpUw1GviDgKX7i1Jma4rYm3xNfmlNEatrdU3eIGHpijXxUqNCTYku06zJHYytauKtKtbkGZR+0M88XXtaE09nU5NL/fHX4LWsLU29oJ6/YP2a9kdMLY3YZ599HrPcAeVWdHYlRV5J33I+hBvzQchAWd8mJX3r69rTmjxDCupr4vJeJq9JoW8d4m9RitL662qKKsdFXFLPeGNp5fRsKkG8H5M7kCCX/UQFLkSlYzaw6lef5Q025APndJjpre5GEpfbvHw7JRrFM3RnvTRsyoM4p1+7k4VYl/7BGI1+KQ+azUlBydhLa19/jPM5yS2pKdJ9ZfaqVzrL2s19XumMFJTMsV6JdrTVJS5JRVk9YaGesFRnxVY9JMZ/sZ7U5kjriUrrsbMZHJLwsnrcknr8BvWI9mwy8lySoqiVkZdqCLOEMnnYhM6Qxsqno9Op5X9933m9bFFQ0aJNafU/Ia3jzrkfkeR/uha5jVqUzmHnbO5aslLng5c5jm3War9G5sLKVl8PZqzVp0SNeKpm2OXgjP0/HeyxN1wrvWPf5oO3g29ZnTu6HRNWz4y9mWhu8FLHuqw50W9yVuR0KtoSEedizT1Xl3TGaFiyGvcZRXtU5xGrdUZUcmr3pdbz2VvBMc6/I/btBdGwkPbCc/btiNpyTK0+ZW9Ggx/IEno2+G7wI/v0dLA7+IMsC0GFp/noG6sUPyfjQe57mPselZYS51Ilue+O5mb6xCk8cQtPvMITv/AkMChKee4UKHc07emTWPfMgvFzznh5TXOXep/o9zu6L/NUejpVpHk6rpXygOa0zUZIUDMf3r8R4rQ4Qo5YmhvWirdsJDwb/DL4P2zMPGN66T/ZeKgeCbHmWpORkJbSn5EQ3+FIcC1HQthgJIQ1IyGoHAn826luy99Itk9ZvltW95RJ/yqZPyNZP1lLXsf0s0c5+UjkJYtvfKzwGfSfrMRz9kbMpYvBN4PPWOu+Y385lxcZ3f9G5jS1/xv2iY+9iV5RipyfsT55lClL8Sn7nn8ulqrSuIPH8vNbluZDSd6q2tOct8SVm1zNfKVkproi2kRJnJJiTafsjXoaDZ5kcl+zHKqEJfXnIldCQHwIcvl474t+sEs/YXxKV+42OYQcTlk/nRCNFyUtG9FK+DjTup0NR0fC+Oq0NE/M2c87lveQdKxo8xVLN7WaI2aSuzNd3pRK5L27o1s2J55OtRdkh/pSWJe/sndz9knI5HDweUtzxPeMv4/Zz6+DfzFb6Vc5R+yTF3XMcu+z33jwhaYypbsLGnjdAX0KDBreSV48Yr3BZ9sly3etpespe/Mv9ntB74RlfsyswnOWmlP/M3v2hNX5qoMWCH8H9wGJz2kbFpR/SjqDS0e+BSm9XVAZEo2cSp99SqnkOnBBUr2kkh4ZY74Lujwmg9zrmJeAM1bSJaOOc+6F1JJTVoJ4zmeyLqTTIQ+5R5/NvuX8uqH56pjlOKRPcxr/c5ayG8qEP2Cf7KHq/nQ76cWEtHJInx2DmktW/jHNBxf0SYyEBSt3l1FzyX4v2Gc+205Y+XzUc1/xidY+Y5bOZb8OK0Gk+pZRyb/vslbv1ZRfXiZv+zlp+HOSrungvWzFnk75A6V8ZswQLnt7SNaQkMLnclS/NyRzb/C1/v22JX7/m3H05eCA6bFng/9mPFf8Tljb3YLeXo/XC/b+MGMDq3me853zeEy/m/K9uvy6PrgLLldJtU9eGj5L7tPO5macVtwUnBM2rbCrviWtvEu2zC7Zzau4Ws65I6K3jxx0SU/xXaX7xMGs1RtKbr0h7nCr25WcOiVO7ObS/ybXDsHgdQf8FTEawoZzDf4q+/SFXHm1OVtWaSWX9rUjaQlFBjUT9v4Dy300+Eeve35Gc+dScmxe0/vZdL+SXX8tV2SPyH7mq4SuRpmIHvALVgr01Dp6itsw0FPd6inhxbi8Az3F/QZ5PQXdBN0E3QTdJPjLPb0zpkGuMj6RPcYVMZ6En+43udY+o13d1x3pr4hkYp98Z5uuAOfSs6Y8UEo+vI3XfGaJqZQcsmfc538s9ZaSkXL++lJWLmmX4qCg9X7L1PKGqDOfZGVJ7dbzWlU/qHemL+m4E0nzaN5J6HOIeQfzDuYdzDsl8066T3CQ01nd7Mg4FEktZABaCloKWgpaqrhy5xbUjMp624FeCmiflceX7FMMQEqJqGXG8oj/h8Svtx3tw/mEO8nrSt73t6wG3lM/slImFG3Cx0qeyicUN9ANpRH5PPinTSjtgsaAMG37FGG1CY2PaMw91ruffI99TpR832lUA29HQO3wMpEV7yn2SKxVnuR2aruhK6T1B6fPli6u3XhExVPJ9a44GMtxFa9FqZCA5xSJM6Wdhy4iBXyK83Sox6PG9L6kWeBARxR1RX9ciGER8/z3NLpEbAhvywXNehMZd9MNdQ6NJyczx88p5/dGVBuX0+86HFGcMk+u6JMSyt7rmC/BKzOq4G7pjmQ0kJuRVzPq5i6kMJKRDX5mjjo0RsyxwTczomox+BuTji5iXSLKK2KCxmtT2YXujOVoydKXjwxKKRQRL7ynT0lWP1TI6Qv2RqyeLjqThrhgeU6o1ilFhi5JSp9UtqwLu9Sl1YH4ZMPtLmhyCDOZj/W7ofUDL+2QRvgbqad+pbinM9brc1plzOn7h46iJ2OiNCur9XR1M9IT8q4X50TRs0e0Slzm4jq71UJB5Sivoq4b7eNaaJ8nOTq7Hh+BXAsJe+J1TXQ3j9L+nf1m47P5m8c0W/4voll4kIS9xLUj9yo8136EoaaSW6Omh28o/XoT+ntTiGoeUiz6BaGYLiWHdtnTl6yd59Sj4i+vj/smRjRr89YcktU5pDEzlN6Ic+lNnOo8F6RJOdV8vAsb9K1MP2TPBNZpKL1kM12C8EYISq7YX47reE/1HlFbjqTNINZnw9p4Q2FzDI04oTNaq11S7TzFkt69le3llH1G/hT+93cDq/M7zQ9DGZv7PeG0p7IHjmTrLon/nOIbqktRki/nd/bDaxI/otYsOvGZxLjwnv+FvEETaSdNqMbnBvKlDMm4S7bfe5JILodlNa4qI63tGfHsD9nen0hqphr3k9YVVdbFS73dqMSg5RIDmrfaLbGKxvoS0poe0QhIS3RqyvxOjrcD7cWrL8evKCf1WnwntcIxaaEPRnnVacw6nLVrcFfQ7FKJO6Q9N0XvBK3hoIHeAXoH6B2gd4DeAXoH6B2gd4DeAXoH6B2gdxADhBgg6Cmgd6CboJugm4DeAXoH6B2gdzDvYN7BvAP0DrQUtBS0FNA7QO8AvQP0DtA7QO8AvQP0DtA7QO8AvQP0DtA7QO8AvQP0DtA724HeAZYGWBpgaYClAZYGWBpgaYClAZYGWBpgaYClAZYGWBpgaRCRAz2FiBxgaRAtCN0E3QTdBCwNsDTA0mDewbyDeQdYGmBpoKWgpaClgKUBlgZYGmBpgKUBlgZYGmBpgKUBlgZYGmBpgKUBlgZYGmBpHgqW5gnRcUZR6WIVmo1cEFiHYqoU9+CTLlkX8VFfYhX2o5hrfdSLQ7MbkDQPAUnjA0kDJA2QNEDSAEkDJA2QNEDSAEkDJA2QNEDSIB4HegrxOEDSAEkD3QTdBN0EJA2QNEDSYN7BvIN5B0gaIGmgpaCloKWApAGSBkgaIGmApAGSBkgaIGmApAGSBkgaIGmApAGSBkgaIGmApMkhPc5pB5Dzq4hU+YV8QhNpLa2LWfEqUTbArGwXZiUAZgWYFWBWgFkBZgWYFWBWgFkBZgWYFWBWgFlB5Av0FCJfgFkBZgW6CboJugmYFWBWgFnBvIN5B/MOMCvArEBLQUtBSwGzAswKMCvArACzAswKMCvArACzAswKMCvArACzAswKMCvArACzAqRHT5AeIZAeQHoA6QGkB5AeQHoA6QGkB5AeQHoA6QGkB+JFoKcQLwKkB5Ae0E3QTdBNQHoA6QGkB+YdzDuYd4D0ANIDWgpaCloKSA8gPYD0ANIDSA8gPYD0ANIDSA8gPYD0ANIDSA8gPYD0ANIDSA8D6/BEcu8H2tFK984ExqF4W4mJd3Ct7x5ZVU7UUjlhS+WMWynHobmqjXKqkCrFfNm4k9Wl+5V9WIfXqS/Rt6Z2kxtvnMryceMNcFDAQQEHBRwUcFDAQQEHBRwUcFDAQQEHBRwUcFCIpkI0FfQUcFDQTdBN0E3AQQEHBRwUcFCYdzDvYN4BDgpaCloKWgpaCjgo4KCAgwIOCjgo4KCAgwIOCjgo4KCAgwIOCjgo4KCAgwIOattvvBF9zTWRiYdaWCN1fIl8AJIJSCYgmYBkApIJSCYgmYBkApIJSCYgmYBkApIJSCbEGQDJhHgo6CnEQwHJBCQTdBN0E3QTkExAMgHJhHkH8w7mHSCZgGSCloKWgpYCkglIJiCZgGQCkglIJiCZgGQCkglIJiCZgGQCkglIJiCZgGQCkqkM63BAdCypZ09IdjlS5SXZ+cta5Iuw8IVcmftdWURRFcJJjJzLxjU4lZilYg0HUjK4ZXpD4zFfWlSJXjk0cqxTYmCNqVpdVlyJWXpJMnhKO39zpgF4qUfS66dG3SGNB2EZvTM4fMLSviQ/3Yw4fE0apLwe29zV9WfTVrXGTJNyKPvcRF6NrUry1+R2leyW57Uv25XIgDakwqVZYx06lQ3ygv3n3o88PmLz0ehV4hDtS7PFBoYt3SYXtoR5DFrCPAKDCQwmMJh3j8GMgMEEBhMYTGAwgcEEBhMYTGAwgcEEBhMYTGAwEckJPYVITmAwgcGEboJugm4CBhMYTGAwMe9g3sG8AwwmMJjQUtBS0FLAYAKDCQwmMJjAYAKDCQwmMJjAYAKDCQwmMJjAYAKDCQwmMJjAYAIdBXQU0FFARwEdBXQU0FFARwEdBXQU0FFARwEdBXQU0FFARwEdhRgrxFhBTwEdBd0E3QTdBHQU0FFARwEdhXkH8w7mHaCjoKWgpaClgI4COgroKKCjgI4COgroKKCjgI4COgroKKCjgI4COgroKKCjgI669+ion4lifrsZl9BfSHYnBsoh+/5xAfUS1NzkVYf4sCnXscS7rC6r+ua58jvXykv8WcowHzNcCr6oRQb9LOf0RQYjMq7EtTwnGT4ffNSzmVtz09uPJHVpLW4l9mhOqJ1TXepj6dVsXnYxXXWZqeZQuQ5IM4l9zlX93F6Z9Xnq61pInbUguqYrynJr+3m9ktaVs6DyrrjVCLzy8gJrbFua3xbnF9GKDbhD4A6BOwTuELhD4A6BOwTuELhD4A6BOwTuEFFBwB0CdwjcIaIXEb0IPQXcIXQTdBN0E3CHwB0CdwjcIeYdzDuYd4A7hJaCloKWAu4QuEPgDoE7BO4QuEPgDoE7BO4QuEPgDoE7BO4QuEPgDoE7BO6wL7jDRxlPzAE9nRL6ifPnbA10n09xce0gBYV9/IUFUvIVIfXcNcoW6xogJYGUBFISSEkgJYGUBFLyviIlYyAlgZQEUhJISSAlgZQEUhJISSAlgZQEUhJIScRbQk8h3hJISSAloZugm6CbgJQEUhJIScw7mHcw7wApCaQktBS0FLQUkJJASgIpCaQkkJJASgIpCaQkkJJASgIpCaQkkJJASgIpCaQkkJLA3QF3B9wdcHfA3QF3B9wdcHfA3QF3B9wdcHfA3QF3B9wdcHfA3SEqBrg7RO9BTyF6D7g74O6gm6CboJuAuwPuDrg7zDuYdzDvAHcH3B20FLQUtBRwd8DdAXcH3B1wd8DdAXcH3B1wd8DdAXcH3B1wd8DdAXcH3B1wd5UoqKXsBbXDpLRpinUopnpMf5cUYXop96lMJJEdUmXdcoMaLNeqcnDXIu5aBOYTmE9gPoH5BOYTmM9+YT4TYD6B+QTmE5hPYD6B+QTmE5hPYD6B+QTmE5hPRI5CTyFyFJhPYD6hm6CboJuA+QTmE5hPzDuYdzDvAPMJzCe0FLQUtBQwn8B8AvMJzCcwn8B8AvMJzCcwn8B8AvMJzCcwn8B8AvMJzCcwnwbW4VHGE/OT1EkfNWJyRlYcEJNATAIxCcQkEJNATAIxCcRkvxCTIZubgJgEYhKISSAmgZgEYhKISSAmgZgEYhKISSAmEXcJPYW4SyAmgZiEboJugm4CYhKISSAmMe9g3sG8A8QkEJPQUtBS0FJATAIxCcQkEJNATAIxCcQkEJNATAIxCcQkEJNATAIxCcQkEJNATAI3BNwQcEPbhhtyKnFDQSVuaEl9oxA+f9N9dcvq5vvVq1BCZzTfnegSbOa5Mf3sUc5rWnd/kN+O5HzwT9K+M7kDvhh8w/RixLj6Ga3xFsSThdRyb2TOKxohYi57Q37suaHXRc7P2Az7KFOW4lP2Pf9cLFWlcdk8KT4Lv3Uxb1Xtac5b4spNrmaO/TJTXclZeCL3Ioo1nbI36mnENIaZ+5qkUZSwpP5c5EoIiA9BLt/UWMfbpM96821yKFvhDevnOWmJYstGtA45zrRup8Ho8CpHh1c5OhbSWkgRbmNd+5R8IQttU73OrObmtGdyIy0TleZP8v0ea0pdPTKrc/1JOwIzmgsWepwdSCuHe1luKK+wusqfj9jPUsuB0nrVaZVtx62OkVwhT2g8ntXmOyE/2oxQKrydq9KfUatGpNVX0a/SLqjkqZErlYo8f3l8yjbxV9jD16SDjzWuclX6I3n+hVqN2PdHNWcjjUXtM2cFFvXCIp/SYbdkp3w6Ca/iaEQxvDYcXZD2mmmN9KfG4YpzDJQmHhVmIdUDY5buS9JlC0lPFUVJryjid8n6vaPI6xVFfPbzLWmqG5svtU/tQNp8Yp9sVQoxlkT8yWSjHNmxWK2BAprjbdrJ5+8RPT+nkSusNtXmb6kPlsT5qTxt5tIYsYtaGiLLGTzNpegRXkuuG8b62yhjBaf+4TnpvlSTzwnHv6gt8ZraNaUWZ0uta03UamuUda+ilBYttqms7LqWJa22TKxPpgYdase+vRbW1VHdUr5XYtPSzei7kDFRI/LfXZPnKY1qOLbKW0993Eo/VUt/vndmlG9Kszr3Tx4ausF+pNa1KGq9RTbjatN2rTeuYks7cJ3WrTO2Nm3lqrH1Wu/biNnrtrCOuyYqRP//rFvnUuxefjx4FOmV8nDdvO7KvGEhX7Qyj6+tKTMX30HYNOdmdYq9i7hB3ib1hg3yBg3y+g3yeg3yug3yOhtKRsJ+NpWp2GrchKXjJjBGslgzrzuGL6WfJ317KVeA2ahc06o3fUuu9hdxv+p/kadqqjWU8MDvbGitfyfjeo9pD4pHMXE994b9HJa82cl5uUa6PlVeWS7uhdvRPrcj2qPjyKf/ku0u+r+yMuORrK7qQZWyC54v5Fxjw/Um7fat2+231G4v55t8nPH7qPY/N+yWofxmerkXJfacoDOwaFHYs7YMB7+zel2KwnVph0rsQ/xEu0lVLQ0tWmpqnT5riZEcx3ymazaSI2uJ7p/2rNZ62d2kUcm+z6LEKk5j9fP8fpEpr0wmnxXKatIrsXWvxPdAWtfn3hvaAVRnBIvV7Ixmr8uGnE2sOZvcA85eEHe4JmzCFZ/WMTZc8fVeWZ+5oqIuP5CeFLIndsCbccmx5pJzD7h0Qmvl9xRjJD6pcTZkvEl3RURslogeGumZWXiO/6A9Ge6DPSFuZ99esCdBQ5671jx3O+H5xJLf2TJ3ZJql9P5XzTPr76MtpDY1I18220lLo+vLoz8WMv7jWlpbZ2Rt7crIvz1WP48LEWcmLOQuwCmdqnAssYfzUru0iXR41tLRv3WIff/z2fAoZys34ZpvzbVuVzEHcrfoknSHOkl/ok/XHVZIUHkrbVY27bWwPU1telVPZHTWUrZ2RONpRFGVxbV/Vepr4pi4K0OV3EyCQiveVnE2aGmspbuv660WhA/5SiMFVYnXMnptSto0jfJvvtO4Oacj67EadTpWd2W70yifP2X06UKvq/ZlFOWM/nK//KGMHnw7+ErGjM+oH1TKM+oPbm98RXbJleRt2tf7MoJ8TvPWjNKl5fP8atUgIru+orhs0ZIR5RA9tE/1p9ZKNl26i69i5VW9vB6e9lbHr4hYyHzZIzqZVkSDcZ2eLX8uZWwqz1UXkfpXMuL8gm4qEVjLU5Lmcyqb8+ArKptrx48kwwuy20xOXRA+f0mypko+lSdGLCS1og1/lfTif7Dff8jerOarU5l/wThzupE0FEvbZz/mOQ5VpZr9UkZT/slopdSuouw3ueeUnp7wWpZ1RJEeCy3VXPeeZ6TsrxJ6FvKkrnyLvyylXqXehMsTwoZcGLEooj8XRn+LGI+xxjGc0N9ziQQxR1/K+6FEQ6U3A6i4YvH9MWHcjwzZ+1Zq0hOSzQuJ9qjOxVvL47PF+QqfydaXj9Q5jQARj/o5+f7G1E+ngy9k7bykftCQ2i8LOp/EjChaElenFXNDbOnRrJ4Z8j76ZvPxreF93dzy/VmPzeq9kIVEIKRR1OmdDCqCT5XsUMnp6ibVZOUpxC5yaikpCVfvVRQ9xwO8NyK5hvTLz60Qf1X696zsmbQEyuLp39HPLvvlp7vwtpyyp66OqM/y9YTG8g15NA4NSeFvJhTpMS3ZUT8jrhyTdKj0SrbWtU+4lL4a/GgloYm19dKNt02dgmhrMap8v9CI35V94tCeyAEbf7sULTim8/L4zuYu+X4OaDd3V+JbHRkVXkwvTu3hTw4K5fO98wOZwnzqlqQNB/+ulF4xHymuP9YW16fr9cDam+it1ExxT/1k7+SJXFnf2FQiBPNesUNCj/Jbe6ak104p74g0hdjH8RutEgJrz+QqjgeG5/LuOK5WVCrKZ1h4MiKtf23cYpOdN3fIZzDX0ajZlQbXsIdsTnZphH5Jrcl+4jN0sx5xW+sRtwc9ohBkI5LqpYyy4hbOO2NcrJL8Zhy19fUFHfn62uKo6RleNOSR3wKPVEnd8qhJqwPrVgf3wgucyoZofx6N3kxGwha4pUpqg1tnxr11beilZtyJrLnTt3iIJq2OrVsdd+plzEYOiGgg5enYqWhLYt2Wbnz/ZTt/27Z3uC5ablNJDa3XFmFHkQr3YccvtF4f1HFNldRvrrUjZa41v9yOpKxd3xq8Zf3zloXWK5ywoxXOpt6yC9qf3qWoC+Gz2qVzyo7Ia+UzDu/SaXm7dG8p/xbKk6vFN552QZ/4zb/ifSi/pylm8s5Nh/bDd+k0+CN6klAazqldkq0R0eLK/ThV34wi50YkR3xH+0jWoHKLGl39P5Dl+JnvY/Z+SCikXdnSMVEkvH+iBaKmXfJycn6ImgJJiy/pH7M28e8RtVm0KDFaK7gWy7JEa00euZqytMUjWav6fka+JuU75H1xpFujWuHLs8RjnVvkTSkY5igIiVrBEUe3PKB6VTvSVriaJke3QPWuq/lotkzRkn6PNLWebI9qoycpETUNc73ryPeulibVQke2yTfkamjI1UjKrpDcNHeo5SDSkisoFVJrvlGtLyvbkVQEug/THszSZrZJtHxX+old+TeQcqo+KxkdZniremqkpTTbM0JCd0lDK/kU0jks9GpRLhQPXEmHr8dgKlFnBVlMJXEo26j6PyvHs5KxIOQwlWazfYEcefkxlR2VglNCDtPR7GkKxtqrryj05Fgba3030vKX1S0p592M5vO1vkk100eKnHQkleJTpCkUMih6JNC9JD7z3lnosZRSlL6dydp8zSFOY0z0R5mejmXb1FNXa1pXc9XRuiurcxJqSSxlW4xKNW5iakFAfetoGVW0OboXfD3aQ4NWwcGhvAMqII3mUOqZHkcj/TbV5Gl+VcPYoNozKDU1Tiy/+0bbPckVR9aYSKmLSqhX3FK8SlO5WgeMdV2+7AmVTvRTSntW33uyxLFsZ6BlMZAl+FIXuUYvpG89GkXZGkY5Cor0FelP2zfUsqDGeiJlWsz9vqbcNeQr0P3rZmgr76d8f+5S61JpEJ8S2bbEkJS8JA1zvXVk1BDLvhc9P9Q9r/pd9Lo5YkJN4bAwRsrGkJgVYtliNf4+ypjpXYpfHMn2RdTzrswv+CHmv6z+HMq6kpI3XBbU26F8kk9xpOsQ+s3XuieR33zSLmYt6pdTPqa9yCPdO4JGN1PPSEpF/qlLsppkngxLUg3pd5coEt9G+pt6/7FHu6GhtU8+bA01YOdbE09PaH13I89wKG+BvX/dvQc7L8W9xKynZUSR7TMdG9rEjxC2wDlVUtecEyeHTI0VsFhDr8LKFfl7yT7dNuRkZD2Konspg4JHo8zpkNmzh+3T5z0YTfget8B3VVKf+a4iOppxK7HmVnKvpTSvBYrxF+UxkenTmY6MHLPWvG8ccRFZ7wxE9wLDWOT9uTwv/pBa1oxXTgu8UiX1kVfiUzMeua3xyN1aHnmt8cjrMY+GlW+K5xpuzkvfmpf9w6yV24rFnZWFRLWom6aKuyv2VqXqgUP2Kd3RyuMy0h0n2xPtuj21cXN5CazlJbgXZ9FE1qulqIcRfk1HQHE/sg9joPuzPjcfD5G19HSLlHxWycN8C2LrFriQ/07k/+5PhN18PCTW0pTcyXhIb+xV90UMNS/KR0i8RvRXvFWxfdUr8fT83aqV+C3pnSyKeJ/OeEl9RF8R+ljhJRTCd9V4GmUid+2iZ4qpJxQNlp47LHIobXko44aajLZ2zjfOU8A5cS1vBavSZJuP39h6jR53dNrQejF7+fmAx4qe1p71m57eNKPevyapXawcFSmWO4+TLpbUpD9c6/7oX0xgeRReSPt+4m9MJx+q/5vEz6Xna6QxmPU92n70YfX+Xj9i/GJrj0nc8xi/X+R9MUN5hvBE7uMHOtbIoRgjh/Zkx/pN9v3ISMF3kF2a4/m3Ppb3KeXCfpc43II91qHFLqxCeDbR2YH1aAvuyYoqXTl8OuusaEPWnbmy/npNtcHWBuvXPQKbS2PYgjRm72eANN6tNHZ/Z8fm0he1Jn3RvZO+saXsNfFCKWqGK1Pkd43Wl7tubr/ZXNbs8aT366Tc6pNw0zM9mnEuaYFzqqSHxLnE2jeYdBT/8dBxvwt5RtzM4paSBXkzp9Yn9Y9yvGkiN04LcqNKgty0KTddocoTa99e0plvr/+o8sTal5WsiG5OWjuBsM+o8sTax5N0FOHTlF9zsjEnxh3L2ZNH1ezfjGuBNdcCYPGBxW/FH5tY+2q6ui+q2cmV4jzJnwjt94pwcD/R81f0/N8slUc7Qa8oFUeAvaI0Hr0TqDmP8sSUx2GfQ/rk9ersycTay5G0tr9iF/XwA7VTrHmqIoAS63Vz0ss937uyQxJrriUdn141kfPihFA8VfeNZW96H1uvZON7gSJJfQBqpd/k9smQdu2a80eVhBVb1YptQVRPSGudfYIVm03ahbw9yFznrfIlHEtkkdrHWUocZ9HLqlIKTBI/R/BNJj6xKnKgPGJpU6skK5F2K1GR8n7PAM3XVCGdhdCcX6qkhzFjhoSlb4tr/oOQssCaX1iDYg3axlompDNFbKWu/2vQRJ7bI04b+kmeseIxXrj0TdyE8Eq/j+R3/lacD5S+E+fkPJbfHV3qkFaxvjwJSnwL6NsrkojH9MSl2xMSOjMkLcGlM0timT82nvykqX8s8zryvBHHSJktN33ySlLwWD5LqTPLUvTn+fTvDO94DtX6V/IWCMegT7Qym0dwOc0lqDHziBT9WcWHdDaOreR3fUPhhOxi1cPH8latqpVcbN0OD/MG5o2WRk9iLXVJz88RjWQc7pBO9BO//Hysjz3SVo6lvybrb0DkQV/8EWU+hs3XCo6ld6peGlRJbUiDibBpqvcPjLKa8MjWy+G05uWwm9+fkPaYGDdtlNPvWdPvwEtDu07N+aVKeiheGsfa6+B0fMLDf1C9Q7qn9lLyq2qkhNZt8GEBwwJuySKLrKUuggXcAr9j69vu4hU3PvYhajl761DZCTb5Mz245p/Rvc1Lfdsgv69W3Ih9TfH+3ML80DDOOSS/ja1s34fd4U153RwTE5Kus+Oley/ODCy7qVTI4pBpeb4feiNXRYesbn7m5Zm+1TvN4dD79DQIcZP1jT7nYJi7W/OYqDiTZyA06Q3Hujece3HDo7jhNT/+3YZcclvgkippe7nktcYlb4u55LfGJX+LuRS0xqVgi7kUtsalcIu5FLXGpWiLuRS3xqV4i7mUtMalZGu55K1h49dzyTPWANvHJac1LjlbzCW3NS65W8wlrzUueVvMJb81LvlbzKWgNS4FW8ylsDUuhVvMpag1LkVbzKW4NS7FW8ylpDUuba/t7bdme/tbbHv7rdnefi9sb74XMNe3FY1p/yB/TkC6U/EH7SnzNyfESx5dtBy8Y3+ntHtWfW7QtdyrUKhCh3Y51I6G2UecG+n9Xgo/Nqe2nxb2s9vAcfnWKwX/XtyZmMY9vNT8OmDfJ3qPaFWKESH1roneyUY52rprJ6RbWpv3jiqp63ikJu32W2u3f89OFxPxSu9b0NZBazwMehHLdi25wjXqB8kpcze4GbdCa26F9yJS8kRGxI1oF3xG++E3UkvxsxUnFAk11BF0I/Z3QbKneCvuQOSnyYoIh5GMHSvuwheR806NrDfrp8i6n6I7uE1q9f1RnLLYug3BA4/DFxKzpCi9u43CF2f5qvn/uDYt743iSeLrnBH3pXF+cFqn4mOzEZS0IH2qJEgfpG8d6QusV9qrpC9oaaXdjUUbWK+d2ztp2G7e4jP1SGJ2dypod61pHwNFQtHBtvwCmhZYgnZi2wPrNXPQEYKlySkMnjyhQJxSEDIZdOhEAJ+wwPxZRN8cejOmMwNc+YZj8Yrp1ckNCb3Lls9xAQdUonrmGOXkaenTCQTBGqv8BLqZ7iWyHSUhdDN0c0ujNLKWuugenNK6K/vEkbo5lpp1TNo2q5s9OvslkqmFTs2nz+rmbPlCN4sU5lO3JG3fdLO9n8e/V3srL0i7LIljw4rnI1qDnpFX7G1jzE1g7bUIOprn1vdamCc6zjTPuAfA5NrYyoNhcprPjWXU5O8iMutp0heh9Ro+vBdotGqunslbc2cN9/FC69V/WHt3hyipzxxbEOLxuLGfP7T2OYT35OSKpTy9dUH8OpdjX3n+bHYI052a7B0lwqrhltxMW3TFfZ1reQ/c+4b94ln3C3wbsJ/bsaRCa99G2HvfRqhPVHTJz3AgT5j05Y+wbdO3oTyLUXwf69MT1ZM+WbzhGt4I555ZBm/ledtq1yiPNT+TMXfvaPd7j43BZno2bIGTqqT7zUm3IScja90R9c6SaNLu2Lrdccdnk4rYOREl8TurwSW9FlbsP4XWa7+wh7E5m/dgZL3OilpbZ32aHoys1z/+vVsxdjsvRGvE69avJKNeIPvubl6IrFcyUWcrmW60im/dbr/nWiWwbom7VT0YWrc77HkP2tpm3r04I+wu54W4BU6qkh72vJBYj65ki7RKbG1txj23NmNrazNqbeXTjx50rXvQ7XkPetbaLMa8UCsTfgucVCU95HkhtrY2487u1upGq4TW7Q57rlUi65b4W9WDsXW74573YGKtzSLMCzUykVhbe3WcVCU95HkhsbY2+3gXdpN2u9bt7re1mVhbm0kP8S1NetC3brff8x4MrLVZiHmhVibCFjipSnrY80JkPbqirdIqsXW7455rlcS6Jduz7xxRtJhNu0XK/vagwBzYarMA80KtTLgtcFKV9HDnhYhu0LUdXd5WaRXfut1+z7VKYN0Sd6t6MLRud9jzHrTfd/YxL9TKRNwCJ1VJD3teSKxHV7JFWsWxtjadnlubtjcqZ2V9G3rQte5Bt+c9aL/v7GFeqJUJvwVOqpIe8rzgWFubzhbtO0fWNzKLlP3WKpF1S/yt6sHYut1xz3vQft/ZxbxQIxPuGvvObu284D7ofefI+hbaqLNbaLvRKq61ten23Np0rT2B7hbtO0fWt7lGmdta+9mD9vvOwDvXy0TYAidVSQ97XoisR1e0VVoltm533HOtkli3ZJv2nT1rT6DXc0+gt8a+M/DO9TLhtsBJVdJDnhc8a2vT26p9Z8/a2vR6bm161p5Ab6v2nT1rT6DXc0/gOjd7Au9cLxNxC5xUJT3seSGxHl3btO/sW1ubfs+tTd/aE+ht1b6zb+0J9HvuCfTXuIkeeOd6mfBb4KQq6SHPC761telv1b6zb21t+j23Nn1rT6C/VfvOvrUn0O+5J9Bf4/Z24J3rZCJY44b3qHZeCB74vnNgbW0GW7XvHFhbm0HPrc3A2hMYbNW+c2DtCQx67gm0v/XLBd55hUyELXBSlfSw54XIenRFW6VVYut2xz3XKol1S7Zp3zm09gSGPfcE2t8z5ALvvEIm3BY4qUp6yPNCaG1thlu17xxaW5thz63N0NoTGG7VvnNo7QkMe+4JDNfYdwbeuV4m4hY4qUp62PNCYj26tmnfObK2NqOeW5uRtScw3Kp958jaExj13BMYrbHvDLxzvUz4LXBSlfSQ54XI2tqMtmrfObK2NqOeW5uRtScw2qp958jaExj13BMYrbHvDLxznUzEa+w71+Od4we+7xxbW5vxVu07x9bWZtxzazO29gTGW7XvHFt7AuOeewJt70AJaiNuxXuvpXu4N9VkijMjVp5Iz3WW6M8jKueaPT2Xd8I36f2wFZ5FmftVmvDsjOa0Tbl2zn5nG9z2nm1JZD0iok5HBG/BkGg5orvS1dgYsjaJ2e5IykrVCImtW9atFf+MqJ9QO7KtGbKn5W0+oTn9gm59P6N8M7oPnucRc376fMpSLyu5YmNP+rUe0yATb9uVBZS2Y4e1b0b6cin5M2Lpz8m2iXVpx/R+wp5Wpb4mHvM0M11yk7GkbnJ5rcvgNV5QziW1/Dp3s1EkT5qKCRslPNpf1kqWalXKlwXZenNGr0o7zklhke5qTv+W47W5QqrvnwVRPMvJwpxm9QXpKqHBR6zsG3qe6nTB+zFL9yXRv6B+MOU0S1HSK4oC41aO/lDk3SFFSs7mco1yWzp6uL1znHm+2bgRHr9+jRuHYqucjfsg1eKKy9eMJqXb3lFNXOPn398F/8Vqon/89yw1V9G2ekEpFlLSnxfm4wXN4RPqNTULr59nxH5ET05lz3Xbb2K+Ku+3sLLfrhn9N6w954YOmLOfdyzv4WCPWn1Jvctl+FtWn5Laa9l6sdLJyvotWUGqvCmVyLm3o1vGdU8qKyk3ykr9jf3/nf0+J16cDz6y/08ZdQsq+Zb+DinPFcs/JJovqGVTvf54R9rvjH1SnpQZWStT6p1jsg2viOMn7PfI0IyLwS7VvkM9s2nfOK2MKUdz6Sqjz7Pj5r3udWWvfdD2W+rbz+8aZ9v1lPHhhDistNaIotZDox319fgb1+NL7LRdPd7G9Xhyrtl8xHktjTjRn1w+r9ibPbLTl0xm+Zh5azXqxsboneYkR+R0KtoSScymki3XWM9yPXvNvu2xH17nEVnXZ5LaffIn8Z899vyatWiP/eUj/Q+5AlqwJ9xDucho0zcla6I3xBdTny4G/6TPFyTn39A5NH9nedJ+/EY+mdN4nrPvI6n7l6y8C7LzxXjmI/wN9YTihqf7KTRu2/Fzc1eQ+x7mvkelpcSlT5PSp47uufxzJ1eXo3smnzLbFlc/943naZS9o1uVf65al95W6ugWms9iY+XG++GILBf1NinID5eBXUOGXd0S3zi53NM5U5p8zZ2yERhnfNfrjV6V977OlztUzw/s6TlRNCdvwedWY9alE3n2JJ+WNN/tUS/yufEtG0X/Zhx4OTgYvGJWzn8P/sV67w+Wep9yJnSO0T719BeaQuU5UOPuhtrFx/8uo+aS/V6wz2/JUzLSXOAtXRDdHks3pt83MtW39IxLTkAasar88jL3SHfwHjmX8/172Yq9XMqIPfma/b7RFsQb0llfSz38jvFBaKQpfed5fqPeULPvf7KnH9gswvN821IPfD94MnjMfn5l3H/K/qY94MtzDfbpVJ4vDPkSPqVz6YkStc4YDzitbwf/QB+xPuqqfxz67G44QhS3BWfFfCla8y1ps13SY7s0867iejmXj4jelNt57oUFjrsl6X4lvXRNmnM+eER6i2u5rzsaCw7ZpD59ju8Jrw/JF3BF9Twny+SGpXpBu7382Z7kXjccdEmPRwVt0mcOmuO6WlKvaJY+Yrmy6X+jmj4wSl93IqEhWWj7dHZcyt8FK+mWen+Vp+FZwS4+0G37usU5/wWjP2Izf8TKf8H0ZdoGbifuUx9+YXj3T0gPzAe/yOiEJauhPWrqOOrT333aU70fEissvEvqPX+lvJqpu5ZWn9b4+4RsSHl7ofs4lb0u+tql+dQn6YsyGj6lqBs6PJI6waP7IXM3ZAemo5PrmrT3upqfHak9/MxYPaSSeBmP2e+E9NxUWpld0MR9N/uZ9Us9t7qiiyPC8jImKHtFdd10yiWXRp6X4VN2tlqUesab0bfa8z5i614+Gp8Nvhv8yD491XSHdDbKPv1NaRY74sfSYhV+4J9I977thJuCkzGNA69Cr/4sKRRz6gW1dULfHktv9hXZ3N1oPIfOHvPJLsxKI7dehffjV+md7IKagDy9bmFNVaTmbyx1FxTFZH0UV3liJSTs+wuW65LkjXufu+RXJKOe8jaS8igfSMvjBbMsLjvRb+r8HqFRxpUcUxQpbj0lHXBFezPdUOmSxS4+rzvTp3M632EXET6f52b3TWf+Ytl7csYve7dLpQi/4hVRJ3SHsg5OaDZZMJ4ojj/O+Yv+N802/PdnuZvWhc2QUExTVFgv3dBsw0s7JI5wLt7QuBf6/BntB1/RX+7L7UYH8P2P/Birp6sb/ZRQRGksqUxp26TXu6E2KVkjr0NtN1TGci3iZFbyRxSrp8bqvxgFx3Lufko++nOap+adyGRInAwL+uuYvAfX2u5+ZPhEuqDLo9uVQxotgUGXsP6PiXOpD8xcFXRj7Y5pJznvR+T8Evb3cSHiwO2IMpd2r4QVVN2fbie9mEiUXH4MrLez8I52rZZyzud0810Fl+bBTXcWsmXW7SyolD9QymfGfp57R97Zqt22hCymfYp82pTXC/b+sCTyvq0dnery6/qgTz5wn3bTlGUaYhdhw12EmHwq2EXochdB7ZK+oNiAy45iAAR6xC3YGDzyRthp/8BuJ3Y7sdsJPSX5e0Vj5vIO9NR4Y18OdBN0E3TT9usmHjs5YxrkKuMT2WNcEeNJ7Dr9JtfaZxTd9boj/RWRTOyTN2/TFaA4PWKpPVBKPryN13xmiamUHBIOc6Zj9tJ96DL++iuj98xa3hB15pOsLL2U0bu8VtUP6p3pSzruRNI8mncSGTuDeQfzDuYdzDvFeSfdJzjI6awuPMueEW+COGVoKWgpaKmylTu3oGZUVhexKwHhssa0r+5nVu0Kz5LfA+5mH86nU1LyupL3/S3FkU0HP7JSJhKLeFyg8glFY3RDaSTjkvyNKO2CxoBwivnYQnsaH9GYe6x3P/ke+5wo+V5j57pqR0Dt8Ix2cF3AoxbFWuVJbqe2G7pCWn8EmciUerq4duNRiE81nrYbSmM5ruK1KBUS8FyetqROX/r8k+8VhhJ7kI3j3IzelzQLHOg4ra7oL4sL4iPmexpdIjaEt+WCZr2JPkOhC+ocGk9OZo4XEVXfG6gILqffdTiifLpDRazokxLK3muEruCVGVVwt3RHNGMJb1J51M1dSGEkIxv8zBx1aIyYY4NvZkTVorMI34jyeoX4Pjsqu9CdsRwtWfrykUEphSLi5YBiprmsfqiQUx5tJ1ZPF51JQ1ywPMUpP1OKQlySlD6pbFkXdqlAR4lPNtzugiaH7rLYp/uMESvbbqys6NkjWiUuc3Gd3WqhoHKUV1HXjfZxLbTPkxydXY+PQK6FhD2x6pSpP+mkp/T8jd/pnXiq6ufxoOK5Tz6dL+mbOj9EvPldr2YmtHY+lt6G/5EIriWlEDmPpKchzSlOonpDVpk4rSpNLTBC4tSPNMf3lIpjhM4lby8lYiItY9coRZ2gJkpQp6L+Jd/OpbfniDghTk5I6yqe02OWmz9xIc2nuP2sNP9fG5+v5ckziPp3Zh0/2dLd8My6l8SthTyR7AOThjfkeZvI8+nq34/k+bGcgyM6d+OczfJnVjlHhvwXU/P3E0I1XUhtU17iQsr9lMq7lLt2PPchSeaSUCoj4oigrMmZedVn30YU2d3/mxRW9cqRPHt4QiN8Wto3q/Os4v/mZ+Xa3iGQdHSHwKZnEc/0PQwjWuv5hISP6BwsxR2M1Way4lrLSv/uQJnLk73Sk/vm5MGfS9TyIbUmLYuXw2MmzogLx9TO9PQ0bh0sjDP60vQzst7NU8Gqzqcc6rlyvqaM5mkfSeqHa6UuStnx2iWsNwLqKCmenZY/j3cnQ23VecjiFDhhWVywMr6i09yL+ZQ1NNL24ZTymsihr+hU92uyjsTKRkj4Vw3HkdfCOFIltTGOxHhZyvOkbUZTtvTyEuruyvnOSD+seD4i+ZrS2fDnxKMPtWlNO39Esib6fHW+C7lLUZeKczidmdVKqDr9GbV9RGPe5Ep6Hr7icHqThGkBzGglNaUxYvK2idz5rcmdf+f6e0Lf5sbJoOvbZodE/YeGXA2sudq3G+abtDpsodWqpP5b8WoU29hqdal5uR8piuAiY7M1k8DIui+ijm6RumVPm9+o9nNOdxbP23cobbo7MjUs3bIUJ8TbrG12adhxt3Tm8Dumza/06dX8jOYh/bp0e5tL5/OK9O9Z2TPjHO/82cvv6GeX/fL4FW7dnNK9b+r05SynTsiPdkNnzB1mtF2dxVlmoS7JQzs1zqFVdlq6Xko5pdJk53TTIltPOheDF4NXgx/lG0VLuezG1rLbzY3JKlrcTnrT9ecFIavFjQcc1X3Lfs9oLfqRvRuzT7F8HhrPd+lEK35C+xH93aVdiiGlT+TbIcU2cY+Z+cyhmz7Em0h+H+beH8kSPSm/jsQMO3RWuygz+1SUOjJqdIiWfBpB+1i2OCbqRR5FfUzntO+Sh34kfxeZJ0Pj/1C/Geo35hOV7mPlSD6idYCSncdkLXxq2U2sZTe8B56sE9I479nnc7l/tyBL1JylVtmwqc4YGfPn5rNbLD3Eq7kcGyerf1ou93e1tJBnrl1SD3369VKzfnVa6FdVUtd3X5b11ylJ/ruGXHGtuXI3d5sOS/aRhmSfnJP+OKb0dvtaOxU88Kx50L97be1HMl8LHOV43URyfGuudXun6o+08zkkC3Rm3E+ap99+He3f815PfUJN+ju07u8Qq74Htur7NDZvTHuytlIX9Xq99gvhUoaEHuG3PhyQbtmlOxv5zY1DOsFLrY6CTFrxPsg8MdOJnIHUV8XcjpHKIRlRzyOjDPOv+dxM7+o0qvT+rJC4DMQb38QqokPcrbtRskybcd1yN/dGemRDbdNtaO6d3obG7z6zuTMtv2v5zeAz1oLv2F8e9a36QHz7jPXPI/m5WJZK47IRLT4L9Fkxb77OYs5bfVa5mZvvppqpriQacyIRhcWaTtkb9TQaPBn8XfLzWu/mcQ7xNgfU5oDSmLcn5N9lEXb5typ+8o2+n/sb8rA57M3fZa0L2Rs3dNrBMUvx98wtdS7dSXcr06pecUrunYtoxDS9d84sJS59mpQ+Ne+dyz6vu3cum9Jsi6tvcTbvnRORM8V757LPQ30bnFe4d8581t69c+I2h/y9c6FeEdTfO5dkduvWvzUy6OFsFBqc3tzmfqltzwPj/q7hyhQicuM9jaH1Uo/0mL61vHe3zt4U6K6iDRKSTbTaXq3LP26UP3sj6ib544b5o4b5w4b5g4b5/Yb5vYb53Yb5nYb5m8mf21D+3Iby5zaUP7eh/LkN5c9tKH9uQ/lzLeUvnRfz+iswNKt9/HWWpj7vW13TXuFQz6dih8L8fiHLOaZZZ0Zz1aFx//Co5Mkx1SKwgs38dIm1xyTp1C/LWzCUUSuiXXw9MSxd1Zh++3L/rSP36TZbhTpST74e/H/5UeeJVioFAA==\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684771480258, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771480759, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771481261, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771481896, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771482397, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771482683, \"type\": 3, \"data\": {\"source\": 2, \"type\": 5}}, {\"timestamp\": 1684771482683, \"type\": 3, \"data\": {\"source\": 2, \"type\": 1}}, {\"timestamp\": 1684771482684, \"type\": 3, \"data\": {\"source\": 2, \"type\": 5}}, {\"timestamp\": 1684771482739, \"type\": 3, \"data\": {\"source\": 2, \"type\": 0}}, {\"timestamp\": 1684771482743, \"type\": 3, \"data\": {\"source\": 2, \"type\": 2}}, {\"timestamp\": 1684771482757, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771482767, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771482881, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771482901, \"type\": 5, \"data\": {\"tag\": \"$pageview\", \"payload\": {\"href\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\"}}}, {\"timestamp\": 1684771482953, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482953, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482954, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482955, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482956, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482956, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482956, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482957, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482957, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482957, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482958, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482958, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482959, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482960, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482961, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482963, \"type\": 5, \"data\": {\"tag\": \"$pageview\", \"payload\": {\"href\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\"}}}, {\"timestamp\": 1684771482968, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771482974, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771482983, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771482990, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771482991, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771482993, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771482995, \"type\": 6, \"data\": {\"plugin\": \"posthog/network@1\", \"payload\": {}}}, {\"timestamp\": 1684771483000, \"type\": 6, \"data\": {\"plugin\": \"rrweb/console@1\", \"payload\": {\"level\": \"error\"}}}, {\"timestamp\": 1684771483002, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771483013, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771483131, \"type\": 3, \"data\": {\"source\": 0}}]}, \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\"}, \"offset\": 2993}","now":"2023-05-22T16:04:43.265573+00:00","sent_at":"2023-05-22T16:04:43.260000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684771486268&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:46.270510+00:00","body":"H4sIAAAAAAAAA+1VXW+bMBT9K5OVR5BsbMDkrWsnpR9aVKntQ6sqMuAkFDAMOyks4r/vOmRq0ynZHqY+5Qn73nPONZwj87RBci2VQWM00krUelkZ5KC6qWrZmExqNN68dWapMMJWTFdLNKYO+l3Q1apJoIQdZGRrgPb07CBhTJPFK2Nl7L6RZbXebjaoFg3MvUzRmATMc1A2rGhveWm6ZfSglpVSG1HWtstZGBLGKaU+tEZaap1VamapiHDOGPWCOExJTIiLZRgHhDPhUe5j7jLipV5CUxeGxLB/w0siUwwvPXrNVFq9vpdjUSBY7OJ5GDIhI8YjLwoOaVmw72FKQMtUuVQgUy+T2cskuViFJX1cq0tV53FbXjXX3+LuapV3Z4/nP7treYP1XfuV5sBMM20ylZjhGIvbi0lRtz/OJ4v7fOlrejN9uL/1z5LvhuTlQ4Hb9VS8eHoelAg+STWfawlmehHFvfN53m4G+8IIaO8bQDZdAdwNEiorhQG73LoQnQueGqijZqVUphao7+15dyoQh32VpBAaFuiuEvB15t1sZuzqy4e9a5aylK5bZIvln01XNk3VwKS9JP5D3EJCT3E7GjffZ/89bgS4lc5sZIaItWgcAK6DeSE8h6wEg2fT3Ulwf9DFyD+5eNxFanP+CS5y/y8uusyHC36L9ThcBhbMQWn3j4DKPhpv0wdohnfSdsQOHX5AU8jqoZAw5vNTSI6FhHDM++dfwg/U6TcIAAA=","output":[{"uuid":"01884434-ba3e-0001-712d-9832c78876cc","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"01884434-ba3e-0000-51fd-ebb6a55b6144\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAJ6Sa2QC/8VU22qDQBA9nxJ8rhA1RtM/6FNf+hZKsbk0QqKipkSCv972zLpuYy6thEAYVndn5syZmb18f02xh4USFTIsOHvEAB4e+LUwR0RLpLXiVyDFFjlmxneofUtqdvwWWj/Fq7ZIjJKYGO/Eit+pT07thrE/O1ZhzIgXa0LkEzNqbA7GGMHV6PhI76E+YJ9TDhlrk3FMTuGTGjdk+o0RMkpAcfgPGU/E18h79mt/VG2AiWG/jGsykZzXJo+9QiSqB5HqRcqVzarWXFecNX0pDUJ2aUsfwST44LpW0vbkOC+3V14zxVcYnYUXZhIp9hhLZvJGKY1u8I/d5lip07Tg3Gb0mLmueiFtYnJKytFU9/f5vO0Zk5l3kzPmaN+MloIZtLvbPUc7vRqb2JXWuMzFO3O3Avp263pmH5eMu6CmPd/1lfVPeMPuUX9I3tvVb7Mav/NW7EzU0NyJ6kDnXHjF3F5cQ45TLtH7Z7j8C1xBDy55AUdX7u5IdSVU6B+N+AMddAYAAA==\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684771483335, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771483713, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771483953, \"type\": 3, \"data\": {\"source\": 1}}, {\"timestamp\": 1684771484458, \"type\": 3, \"data\": {\"source\": 1}}]}, \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\"}, \"offset\": 2930}","now":"2023-05-22T16:04:46.270510+00:00","sent_at":"2023-05-22T16:04:46.268000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/s/?compression=gzip-js&ip=1&_=1684771486823&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:46.831635+00:00","body":"H4sIAAAAAAAAA+2US0/jMBSF/4vVZTLyK85jx8BIHUCDkIBFEaqc2DQmTeKJ3TaZqv99bngIWHRTqbvuknPvOb7Kd53HLdJr3XiUoYlrpHVl61GAbNda3XmjHcq2n5W5kl6Oih+sRhkL0Ifg2lVXgESD95oIkFEoI7HYgWRq7bysLQgi4XFMeCJihqE0cdo50zbzsRuRJOGcUZHHiuSEhFjHuSAJl5QlEU5CTqiiBVMhETyH989+TbTCMPlkYxrVbr7G8VRInof4OY651ClPUpqKfVljc0QxI5Dl20o3EGPLYv4yLS5Wcc1m6+Z3Y6u8ry+7q1/5cLmqhrPZ+b/hSl9jd9f/ZBU4lXHeNIV/G2NxezFd2v7v+XRxX5WRY9c3D/e30Vnxx5Oqfljifn0jX6h7FjX6/rUQxZSFOAopvSMiwzzj4kfM6Az6jk1uPzd64nYYN3YEbhgO1r0H2+NTgKT3nclXfoyBu/12BVOwfS2A2Q9L8G6RbEwtPXAM7VIOIYzvQUdWrpxWaLfbQWan63b9GjgeoNTr0/7tSE/bcdh2pEfYDh6gjVG+BEScQb3UZlFCesLZXoLR6b98GMEIA8Gn/+KghNVTBwAA","output":[{"uuid":"01884434-bc70-0001-6e8d-1718c6247aff","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$snapshot\", \"properties\": {\"$snapshot_data\": {\"chunk_id\": \"01884434-bc70-0000-a5ba-6c77cd70a82b\", \"chunk_index\": 0, \"chunk_count\": 1, \"data\": \"H4sIAJ6Sa2QC/71Syw6CMBCcTzGcNUFBRH/FcMBHhETE0PoghF9Xh6VUjfFCjGlStrOzs9Mt99sSFRxolDhiy2iBATwMuTvYIGYmNmjDU8hxQoG15U4M910hMGhKjRYZY0a0tuwUGdlK9DNWdqwAIXxyZ4x9xgEjD66p/JfXvj4nP/Hp2u5bXLkrgy8RmUyjoVmTYsVaLQ47TvUx97nt/r2uddJ43lsflVQcZAaxzCLnacRb7XkuGbVz0bbCYS6msiLSOKhlda4Lohk1zm9+n3facL3ifV9h/pNX8A33ItPUSGxXnzqdUkJ+ih2/2uRDyffzPpU/PcIDAvlBHZgDAAA=\", \"compression\": \"gzip-base64\", \"has_full_snapshot\": false, \"events_summary\": [{\"timestamp\": 1684771486730, \"type\": 3, \"data\": {\"source\": 2, \"type\": 6}}, {\"timestamp\": 1684771486732, \"type\": 3, \"data\": {\"source\": 2, \"type\": 6}}, {\"timestamp\": 1684771486739, \"type\": 3, \"data\": {\"source\": 0}}, {\"timestamp\": 1684771486750, \"type\": 3, \"data\": {\"source\": 4, \"width\": 1433, \"height\": 843}}]}, \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\"}, \"timestamp\": \"2023-05-22T16:04:46.732Z\"}","now":"2023-05-22T16:04:46.831635+00:00","sent_at":"2023-05-22T16:04:46.823000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} -{"path":"/e/?compression=gzip-js&ip=1&_=1684771486824&ver=1.57.2","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-05-22T16:04:46.833134+00:00","body":"H4sIAAAAAAAAA51U2W7cNhT9lUCw3yKbpKhtgKLw0sJN0hpB4qBwEAgUeaWRhyOqFKXxOMi/91IzmQWd9CEvWs65Ow/v568BjNC6YBacdaIGDWKE4HXQWdOBdQ30wexrcGbwFfwp5Kv7D6/+RhqBYgTbN6ZFgpILGl8Qj5fWrHqwCP7eWKjMswcVjI2Ewq07QOIW+oUznSfkYC0mLwarkZg7180uL7WRQs9N72YZIeTSQqfFGl8SLX+tGu0w7y/n6fU5Yz30voQCSWNV09aFGqxwCCF5Hl1trHze7T9jO1OPsBt8LmC9Yw/dJ3IUetg6J2QD+cEIZ+zOqXb+M73d0PvJbQ3i6/N4y02jPoEL6ZOeIJRwUFTWLHfJQkbVJp0foB8TTu54ZJ7ohJu3YunnfTzBg0PanyCl0QGsRVsPqAX0hTZ8+OBdemkB2mIOTT3HjHnK9+CqUW6OQWJCEBwbWHXGup1txn3wHfzdmkce1k2JaVZQ+iT4c6iqizi9YB5vWizLFY1CVKRYXTTq1Yq6l7KtPO8a3yhNMp6mlGfJRcbY60A1vWtaufWr39/e6e75n5u7+mExj/vo3f2nh/fxlfzL0cXykybP4714Yn2VLA8kO7nSDFsgEPEyjqQMiYpkIlOSc05jlYWcMsVkpEKa8JJk4d48UVHmgw1+qj9RRdMXCpamQEk9gcRRVkL3gAFra4Zuupk7KiDbvGFEEwhRBiSMqUjCSiU8iXEqOfEX29hatM3LpPJDL5aTfOPFFdAwz+M0ZYISHnNfSds70UoviZO3NPiGVYnBIdi5wUKBwxelBlVg63ikRd8odP5ev9f7iMIGMRlXWtTYzecvSB1iRSfW2gjlO8UEHWabm/pIIjya9o4FoZe+ODQBFUrdyMXc4NyRg6Vo9FSHP1Ex4p8vbVdNr4Vc/A9/JvFqGg2Fxtz7TQPtiQadHbzHf/fS5mt/547dgnHSOS5MwIXot+eZwvWJ57pDp+VmsBXf9551ZgEe6eayeLqTt0O6jB7H9o+2W5TPyzf27W/l+s2wWF893rys38I70n98vo4W05XeFrnTOI9YUqaKlpSGBNIyoRkXLMpiFPUpkW/sgYKaVs6qaZVZHYbjeSJ4GZIqTbmAnGc5y5MfxfLGMSMR3ayvGi8gHEcTKqYRjVCmSQm0opTJjPHqR/HQnFFIEq9NvyFQwcsOozHCopDEIWMfaTIjfManhfEYfPvyL6II7K8OBwAA","output":[{"uuid":"01884434-bc71-0000-a710-92ebd1e26e1f","distinct_id":"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"event\": \"$pageleave\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/replay/recent?filters=%7B%22session_recording_duration%22%3A%7B%22type%22%3A%22recording%22%2C%22key%22%3A%22duration%22%2C%22value%22%3A60%2C%22operator%22%3A%22gt%22%7D%2C%22properties%22%3A%5B%5D%2C%22events%22%3A%5B%5D%2C%22actions%22%3A%5B%5D%2C%22date_from%22%3A%22-21d%22%7D\", \"$host\": \"localhost:8000\", \"$pathname\": \"/replay/recent\", \"$browser_version\": 113, \"$browser_language\": \"en-US\", \"$screen_height\": 974, \"$screen_width\": 1500, \"$viewport_height\": 843, \"$viewport_width\": 1433, \"$lib\": \"web\", \"$lib_version\": \"1.57.2\", \"$insert_id\": \"a7ang3vlww1tzbnf\", \"$time\": 1684771486.822, \"distinct_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"$device_id\": \"188430e34b53cc-0d3c6c7094415d8-412d2c3d-164b08-188430e34b6d38\", \"$user_id\": \"gQDHlpxqCHgUkh5s3LOVUQ5AcNt1kmVl0xvOaj2sf6m\", \"is_demo_project\": false, \"$groups\": {\"project\": \"0188430e-316e-0000-51a6-fd646548690e\", \"organization\": \"0188430e-2909-0000-4de1-995772a10454\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"posthog_version\": \"1.43.0\", \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$console_log_recording_enabled_server_side\": true, \"$session_recording_recorder_version_server_side\": \"v2\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k\", \"$session_id\": \"18844326b7d1b11-0e7b6184a238508-412d2c3d-164b08-18844326b7e1ed0\", \"$window_id\": \"188443496a4b-0f774ae94892968-412d2c3d-164b08-188443496a52031\", \"$pageview_id\": \"1884434ad513139-06be1f112c824f-412d2c3d-164b08-1884434ad521e66\"}, \"timestamp\": \"2023-05-22T16:04:46.822Z\"}","now":"2023-05-22T16:04:46.833134+00:00","sent_at":"2023-05-22T16:04:46.824000+00:00","token":"phc_jHcDu7m3ZvnInpkbxmJrKEbyJukyAZCzyKeL0sTxB3k"}]} +{"path":"/e/?ip=1&_=1694769302325&ver=1.78.5","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-09-15T09:15:02.328551+00:00","body":"ZGF0YT1leUoxZFdsa0lqb2lNREU0WVRrNE1XWXROR0l6TlMwM1l6WTJMVGd5TldVdE9HSXdaV1ZoWlRZMU56RTBJaXdpWlhabGJuUWlPaUlrYVdSbGJuUnBabmtpTENKd2NtOXdaWEowYVdWeklqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR1JsZG1salpWOTBlWEJsSWpvaVJHVnphM1J2Y0NJc0lpUmpkWEp5Wlc1MFgzVnliQ0k2SW1oMGRIQTZMeTlzYjJOaGJHaHZjM1E2T0RBd01DOGlMQ0lrYUc5emRDSTZJbXh2WTJGc2FHOXpkRG80TURBd0lpd2lKSEJoZEdodVlXMWxJam9pTHlJc0lpUmljbTkzYzJWeVgzWmxjbk5wYjI0aU9qRXhOeXdpSkdKeWIzZHpaWEpmYkdGdVozVmhaMlVpT2lKbGJpMVZVeUlzSWlSelkzSmxaVzVmYUdWcFoyaDBJam94TURVeUxDSWtjMk55WldWdVgzZHBaSFJvSWpveE5qSXdMQ0lrZG1sbGQzQnZjblJmYUdWcFoyaDBJam81TVRFc0lpUjJhV1YzY0c5eWRGOTNhV1IwYUNJNk1UVTBPQ3dpSkd4cFlpSTZJbmRsWWlJc0lpUnNhV0pmZG1WeWMybHZiaUk2SWpFdU56Z3VOU0lzSWlScGJuTmxjblJmYVdRaU9pSTBNSEIwTVhWamNHczNORFpwYkdWd0lpd2lKSFJwYldVaU9qRTJPVFEzTmprek1ESXVNekkxTENKa2FYTjBhVzVqZEY5cFpDSTZJbkJYUVd0SlZIbFJNME5PTnpNek1sVnhVWGh1U0Rad00wWldPRlpLWkRkd1dUWTBOMFZrVG10NFYyTWlMQ0lrWkdWMmFXTmxYMmxrSWpvaU1ERTRZVGs0TVdZdE5HSXlaQzAzT0dGbExUazBOMkl0WW1Wa1ltRmhNREpoTUdZMElpd2lKSFZ6WlhKZmFXUWlPaUp3VjBGclNWUjVVVE5EVGpjek16SlZjVkY0YmtnMmNETkdWamhXU21RM2NGazJORGRGWkU1cmVGZGpJaXdpSkhKbFptVnljbVZ5SWpvaUpHUnBjbVZqZENJc0lpUnlaV1psY25KcGJtZGZaRzl0WVdsdUlqb2lKR1JwY21WamRDSXNJaVJoYm05dVgyUnBjM1JwYm1OMFgybGtJam9pTURFNFlUazRNV1l0TkdJeVpDMDNPR0ZsTFRrME4ySXRZbVZrWW1GaE1ESmhNR1kwSWl3aWRHOXJaVzRpT2lKd2FHTmZjV2RWUm5BMWQzb3lRbXBETkZKelJtcE5SM0JSTTFCSFJISnphVFpSTUVOSE1FTlFORTVCWVdNd1NTSXNJaVJ6WlhOemFXOXVYMmxrSWpvaU1ERTRZVGs0TVdZdE5HSXlaUzAzWW1RekxXSmlNekF0TmpZeE4ySm1ORGc0T0RZNUlpd2lKSGRwYm1SdmQxOXBaQ0k2SWpBeE9HRTVPREZtTFRSaU1tVXROMkprTXkxaVlqTXdMVFkyTVRneE1HWmxaRFkxWmlKOUxDSWtjMlYwSWpwN2ZTd2lKSE5sZEY5dmJtTmxJanA3ZlN3aWRHbHRaWE4wWVcxd0lqb2lNakF5TXkwd09TMHhOVlF3T1RveE5Ub3dNaTR6TWpWYUluMCUzRA==","output":[{"uuid":"018a981f-4b35-7c66-825e-8b0eeae65714","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b35-7c66-825e-8b0eeae65714\", \"event\": \"$identify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"40pt1ucpk746ilep\", \"$time\": 1694769302.325, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$anon_distinct_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"$set\": {}, \"$set_once\": {}, \"timestamp\": \"2023-09-15T09:15:02.325Z\"}","now":"2023-09-15T09:15:02.328551+00:00","sent_at":"2023-09-15T09:15:02.325000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?ip=1&_=1694769302319&ver=1.78.5","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-09-15T09:15:02.322717+00:00","body":"ZGF0YT1leUoxZFdsa0lqb2lNREU0WVRrNE1XWXROR0l5WmkwM09URmhMVGxsWWpFdE5qZGhaVFZpT1dWalpqZzBJaXdpWlhabGJuUWlPaUlrY0dGblpYWnBaWGNpTENKd2NtOXdaWEowYVdWeklqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR1JsZG1salpWOTBlWEJsSWpvaVJHVnphM1J2Y0NJc0lpUmpkWEp5Wlc1MFgzVnliQ0k2SW1oMGRIQTZMeTlzYjJOaGJHaHZjM1E2T0RBd01DOGlMQ0lrYUc5emRDSTZJbXh2WTJGc2FHOXpkRG80TURBd0lpd2lKSEJoZEdodVlXMWxJam9pTHlJc0lpUmljbTkzYzJWeVgzWmxjbk5wYjI0aU9qRXhOeXdpSkdKeWIzZHpaWEpmYkdGdVozVmhaMlVpT2lKbGJpMVZVeUlzSWlSelkzSmxaVzVmYUdWcFoyaDBJam94TURVeUxDSWtjMk55WldWdVgzZHBaSFJvSWpveE5qSXdMQ0lrZG1sbGQzQnZjblJmYUdWcFoyaDBJam81TVRFc0lpUjJhV1YzY0c5eWRGOTNhV1IwYUNJNk1UVTBPQ3dpSkd4cFlpSTZJbmRsWWlJc0lpUnNhV0pmZG1WeWMybHZiaUk2SWpFdU56Z3VOU0lzSWlScGJuTmxjblJmYVdRaU9pSnpjSEozT0RVM2JHVnVOM0ZxTXpSMklpd2lKSFJwYldVaU9qRTJPVFEzTmprek1ESXVNekU1TENKa2FYTjBhVzVqZEY5cFpDSTZJakF4T0dFNU9ERm1MVFJpTW1RdE56aGhaUzA1TkRkaUxXSmxaR0poWVRBeVlUQm1OQ0lzSWlSa1pYWnBZMlZmYVdRaU9pSXdNVGhoT1RneFppMDBZakprTFRjNFlXVXRPVFEzWWkxaVpXUmlZV0V3TW1Fd1pqUWlMQ0lrY21WbVpYSnlaWElpT2lJa1pHbHlaV04wSWl3aUpISmxabVZ5Y21sdVoxOWtiMjFoYVc0aU9pSWtaR2x5WldOMElpd2lkR2wwYkdVaU9pSlFiM04wU0c5bklpd2lkRzlyWlc0aU9pSndhR05mY1dkVlJuQTFkM295UW1wRE5GSnpSbXBOUjNCUk0xQkhSSEp6YVRaUk1FTkhNRU5RTkU1QllXTXdTU0lzSWlSelpYTnphVzl1WDJsa0lqb2lNREU0WVRrNE1XWXROR0l5WlMwM1ltUXpMV0ppTXpBdE5qWXhOMkptTkRnNE9EWTVJaXdpSkhkcGJtUnZkMTlwWkNJNklqQXhPR0U1T0RGbUxUUmlNbVV0TjJKa015MWlZak13TFRZMk1UZ3hNR1psWkRZMVppSjlMQ0owYVcxbGMzUmhiWEFpT2lJeU1ESXpMVEE1TFRFMVZEQTVPakUxT2pBeUxqTXhPVm9pZlElM0QlM0Q=","output":[{"uuid":"018a981f-4b2f-791a-9eb1-67ae5b9ecf84","distinct_id":"018a981f-4b2d-78ae-947b-bedbaa02a0f4","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b2f-791a-9eb1-67ae5b9ecf84\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"sprw857len7qj34v\", \"$time\": 1694769302.319, \"distinct_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"title\": \"PostHog\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"timestamp\": \"2023-09-15T09:15:02.319Z\"}","now":"2023-09-15T09:15:02.322717+00:00","sent_at":"2023-09-15T09:15:02.319000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?ip=1&_=1694769302319&ver=1.78.5","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-09-15T09:15:02.321230+00:00","body":"ZGF0YT1leUoxZFdsa0lqb2lNREU0WVRrNE1XWXROR0l5WlMwM1ltUXpMV0ppTXpBdE5qWXhOalkxTjJNek9HWmhJaXdpWlhabGJuUWlPaUlrYjNCMFgybHVJaXdpY0hKdmNHVnlkR2xsY3lJNmV5SWtiM01pT2lKTllXTWdUMU1nV0NJc0lpUnZjMTkyWlhKemFXOXVJam9pTVRBdU1UVXVNQ0lzSWlSaWNtOTNjMlZ5SWpvaVJtbHlaV1p2ZUNJc0lpUmtaWFpwWTJWZmRIbHdaU0k2SWtSbGMydDBiM0FpTENJa1kzVnljbVZ1ZEY5MWNtd2lPaUpvZEhSd09pOHZiRzlqWVd4b2IzTjBPamd3TURBdklpd2lKR2h2YzNRaU9pSnNiMk5oYkdodmMzUTZPREF3TUNJc0lpUndZWFJvYm1GdFpTSTZJaThpTENJa1luSnZkM05sY2w5MlpYSnphVzl1SWpveE1UY3NJaVJpY205M2MyVnlYMnhoYm1kMVlXZGxJam9pWlc0dFZWTWlMQ0lrYzJOeVpXVnVYMmhsYVdkb2RDSTZNVEExTWl3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRZeU1Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPVEV4TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFMU5EZ3NJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpjNExqVWlMQ0lrYVc1elpYSjBYMmxrSWpvaU1XVTRaV1p5WkdSbE1HSTBNV2RtYkNJc0lpUjBhVzFsSWpveE5qazBOelk1TXpBeUxqTXhPQ3dpWkdsemRHbHVZM1JmYVdRaU9pSXdNVGhoT1RneFppMDBZakprTFRjNFlXVXRPVFEzWWkxaVpXUmlZV0V3TW1Fd1pqUWlMQ0lrWkdWMmFXTmxYMmxrSWpvaU1ERTRZVGs0TVdZdE5HSXlaQzAzT0dGbExUazBOMkl0WW1Wa1ltRmhNREpoTUdZMElpd2lKSEpsWm1WeWNtVnlJam9pSkdScGNtVmpkQ0lzSWlSeVpXWmxjbkpwYm1kZlpHOXRZV2x1SWpvaUpHUnBjbVZqZENJc0luUnZhMlZ1SWpvaWNHaGpYM0ZuVlVad05YZDZNa0pxUXpSU2MwWnFUVWR3VVROUVIwUnljMmsyVVRCRFJ6QkRVRFJPUVdGak1Fa2lMQ0lrYzJWemMybHZibDlwWkNJNklqQXhPR0U1T0RGbUxUUmlNbVV0TjJKa015MWlZak13TFRZMk1UZGlaalE0T0RnMk9TSXNJaVIzYVc1a2IzZGZhV1FpT2lJd01UaGhPVGd4WmkwMFlqSmxMVGRpWkRNdFltSXpNQzAyTmpFNE1UQm1aV1EyTldZaWZTd2lkR2x0WlhOMFlXMXdJam9pTWpBeU15MHdPUzB4TlZRd09Ub3hOVG93TWk0ek1UaGFJbjAlM0Q=","output":[{"uuid":"018a981f-4b2e-7bd3-bb30-6616657c38fa","distinct_id":"018a981f-4b2d-78ae-947b-bedbaa02a0f4","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b2e-7bd3-bb30-6616657c38fa\", \"event\": \"$opt_in\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"1e8efrdde0b41gfl\", \"$time\": 1694769302.318, \"distinct_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"timestamp\": \"2023-09-15T09:15:02.318Z\"}","now":"2023-09-15T09:15:02.321230+00:00","sent_at":"2023-09-15T09:15:02.319000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1694769305355&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:05.358129+00:00","body":"H4sIAAAAAAAAA+2bW2/bNhTHv0pg9LFMSVGUqDytS9fLgHbJ2rTbisKgyCObsSLJknwt+t13qDiO3dpt1g2d7ejJEG+HOvyfn0mKfP+xMxpZ0znpUCZVJFlC/JgLEuogINITQGRMkzgw1KeB6TzswBiyGos/qKDGx6LMCyhrC1Xn5GPnQY4/nZdKH/32+ugPzMaE7hjKyuYZZjB6zMQxdelxmU8qKDHxqS0hyacu0cDYaujWswIw4wlUgzovXIYelSWa7Y7KFDP6dV2cPHqU5lql/byqTySl9JEr556wwHqOyyhU3c/UlWu2Kbgwf9s3xsKV5FRlvZHqueKQkYvXrkqlS4Cs2wfb66MRRoV3mzqxpu5jYuBRTBxbmBR5WS8LR4ytJt+UFr7E5NTGaGcCsbOCD6sOOw7lsXDpNsN+1d1mpMZsMg/KNLyazKein4Uuv7bu5VgQ+WEQceodc0887Bhb1TbTi3rFu8eDF29m5/z0Vci5dzE8n2bPg4I/fSvf/mrC4s/AD38xrwbTd3plND4Xh2dIKBUQtBSTGEysFPUUTXxXZ+S89z3GUAOAY+wE8cCgIrQT1yLVZr2uya+UdS5ZyXUKRNEB5jhZTBU6uPypwHHv571jnV91Pl2X6uaZRu98xMc6H4Brpujr7rB38bQQk7n38+Wp/3v19PLls+Kcnz17gu4PzunpM3p65r96rDR9cW2ucsPypUOAhLHhJI45JUHAwjjxpZRB5CpNbGbyyTfrSEYTMIFIXJfzJGleDYcx+PRwY4AGJAwjSZSvgSRcxxRQjonhqwHaK/NRYQ0+2WTWhur/EqrzXmX6UVoPKBUssb0toRrsUajaqmvgKu+inC5dIJ4kKq0AG2z01mhrmXXTGRSpjD2CI0yJEWBIxCUFEEoZP26i9HvivzF4I8Ebm8v0Aczu2oFlnQVRnFdQBetx960mFpp9AokapfXR2bI/2H2oajDdRWQu/LXw4/LZyWKeZ66NizenTb2qVgiubq16mJi5vIMlmPY9IzhVTmEtwXaIYH0GLI3nMzHspUWEIdgSbFP052VPZXau6mtf3tYSQl/XUlIY4gnNBAghjDD/DffWDG+F31d6sQl+d625ULkLviodOUq50EKVKsc75fznUY8TGhEm3tDohPknPj3mQgSM/YVF1RjnbipOoZtgnVHpIvv9hwNnn5/g8ioxqGcvJiGTgmsOKuCwyr4CieBieKewd/1XtlDanfi3VmOvQWhZPyzLAa6Jk8DMi8aZX4JQoMkWhD8GhDtPAzepq1On5zOMj+d5b50PlG3mg+AEBy8gSquYCAPGSENjzdkqHxa87Cap6nUxAlNwQG5RsQuo8LQSGaRT4EWmxWjLBo3Ypw2aXUKFGtUopKKRPzrQTR8Mzl5KHJZuhcuF284pXdvxcm7RxMpigrEeP4Wapbky7jW+d1K22p7z7vUuFMEob4S+Zg6nOUWOerld++06yVa45UXRlnmNwG1jRjVBBQIxOMrKeFQatrZt3HJrd7nlX05HqR+MBtmYpzattnALX6jl1oFyCzLXLYI+uSrqRgAHRS6xhVwhVgefRCZQxAeqIw06AuFMtuTaA3LNSlxHc8nKS86VhiZ/A7millyHSy5VpjNcM2nEClnkHRy/+N34ZVhiQknDtR2lll+7y69hXQ2CcTaJtT+Xk+mg5de941c1wu7MDm/K5W1BlnRnjHyiRCwIU9SEPAAd83axuCfImqZggZfWCiamVX/LYrH9Lni4xDKqVmSiSujn6PL7Ci7gPOTATLs7vyfgupqPC0vzYT4Qlanslt35FlyHC64EnOP1gFQa8GDBfQFXhOCKYxJBiPpTeKbb8xL8CBm04NoPcA1tzEwUTwHm6WSa9reAqz2BcLjkyvIa4jwfHN4ycctZiACrc8BzojKQhBrfU9LTHmNOdS209gBaJdOsEkLW03qY4T/OZmhJ7F4LrQOFVp7FuSoNliZjjzSH3g+MXiHdTC/p9uVlQiQNGMEPispPDNWeWf2uuAz1Iwxk56SjCoO05deO8Gs4HA1nw3CWign3yqHcyC+fYtstvw6EX/vFHt/fzB6Fyz0Pz95HgdTEeJGMEo/jn7Hajxs295k5mZGXytqJikyPeoU7SLyBOX47Z/rnzLm9WrJNeT8ES3hDJr1yPUCr+GI6tXpws4ve3GdujLnRWd6QWZqsUty6+kr+v79TtHTRZ/eJNvtrWWhxgaiyNXwttPftIo8XMfrpw986qs3ZKUEAAA==","output":[{"uuid":"018a981f-4b35-7c66-825e-8b0fb6d0406d","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b35-7c66-825e-8b0fb6d0406d\", \"event\": \"$set\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"v1wz6rl7mwzx5hn7\", \"$time\": 1694769302.325, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$set\": {\"email\": \"xavier@posthog.com\"}, \"$set_once\": {}, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 3026}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b36-7798-a4ce-f3cb0e052fd3","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b36-7798-a4ce-f3cb0e052fd3\", \"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"zgsdh9ltk0051fig\", \"$time\": 1694769302.326, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\"}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"project\", \"$group_key\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"$group_set\": {\"id\": 1, \"uuid\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"name\": \"Default Project\", \"ingested_event\": false, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 3026}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b36-7798-a4ce-f3cc42d530ac","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b36-7798-a4ce-f3cc42d530ac\", \"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"h1e1lbzy5qglp9in\", \"$time\": 1694769302.326, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"organization\", \"$group_key\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"$group_set\": {\"id\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"name\": \"X\", \"slug\": \"x\", \"created_at\": \"2023-09-15T09:14:40.355611Z\", \"available_features\": [], \"instance_tag\": \"none\"}, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 3026}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b4f-7cfd-be2b-71853c3ea63e","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b4f-7cfd-be2b-71853c3ea63e\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"i1h7rrk825f6dzpx\", \"$time\": 1694769302.351, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"PostHog\"}, \"offset\": 3001}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b53-7336-acab-5dedd8d0bc31","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b53-7336-acab-5dedd8d0bc31\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"2ca5nelxe3pnc5u7\", \"$time\": 1694769302.355, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"posthog-3000\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2996}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b55-710c-b2de-daadad208d1d","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b55-710c-b2de-daadad208d1d\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"4jxul46uknv3lils\", \"$time\": 1694769302.357, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"enable-prompts\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2995}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b57-7be4-9d6a-4e0c9cec9e59","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b57-7be4-9d6a-4e0c9cec9e59\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"yr718381rj33ace5\", \"$time\": 1694769302.359, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"early-access-feature\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2993}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b57-7be4-9d6a-4e0d1fd7807e","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b57-7be4-9d6a-4e0d1fd7807e\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"qtsk6vnwbc4z8wxk\", \"$time\": 1694769302.359, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"surveys\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2992}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b58-7c64-a5b5-1a0d736ecb3d","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b58-7c64-a5b5-1a0d736ecb3d\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"xleie3rii515xshs\", \"$time\": 1694769302.36, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"data-warehouse\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2992}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b58-7c64-a5b5-1a0e3373e1d1","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b58-7c64-a5b5-1a0e3373e1d1\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"mzvpi0oqok5sdsi7\", \"$time\": 1694769302.36, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"feedback-scene\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2992}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b59-7cbb-9e7e-9ab6d22f0016","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b59-7cbb-9e7e-9ab6d22f0016\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"qib1d9bxeezlwxlh\", \"$time\": 1694769302.361, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"notebooks\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2991}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b6e-73e8-a868-0d42a82c2114","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b6e-73e8-a868-0d42a82c2114\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"r1c1s558txtqnd22\", \"$time\": 1694769302.382, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"onboarding-v2-demo\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2970}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4b87-7b8f-8061-c9ea4fd0c2d9","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b87-7b8f-8061-c9ea4fd0c2d9\", \"event\": \"ingestion landing seen\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"qquqyq7yl5w32rq8\", \"$time\": 1694769302.408, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2944}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-4ba9-7223-968c-d2989f231c1a","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4ba9-7223-968c-d2989f231c1a\", \"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"nd8jaiiwa9dg02pf\", \"$time\": 1694769302.442, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"instance\", \"$group_key\": \"http://localhost:8000\", \"$group_set\": {\"site_url\": \"http://localhost:8000\"}, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2910}","now":"2023-09-15T09:15:05.358129+00:00","sent_at":"2023-09-15T09:15:05.355000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1694769308363&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:08.365426+00:00","body":"H4sIAAAAAAAAA+0a2W7bRvBXCMGPWpk3KffJdmPHQePYcY4WQUAsuUtxLYpLLVeHEwTot/TT+iWdpQ6L1OH4AGq3fJI0B+cezgz05XtrNGKkddDSDR93fSNGTkhM5FkORqFnh8h1YsfyiedYLm61W3RMMwnke3gkeYRzORIUwLngORWS0aJ18L21x+Gj9RZH2rsr7XdAAyAYU1EwngHC0DuG09EVPBR8UlABwBMmaMynCkjomEU0kDc5BcSvtOhLnitENBICxAcjkQIikTI/2N9PQY004YU88HVd32dZjxZSCQIGBQbKKolC5FgmGR6o51c55grdamsY3go4xVlvhHuKj2bo45ViKSJBaRYklPUSkGbojnkLnTAiEwC6pg7AMaOTnAu5JO4axip4Qe3YPoBTFoKcCQ2VFPix6sKO53ccBWcZ6CWDMoaUSpfwadwLmesYuorXnmTKSsPt2p7btXS348CjCQODs2jOln8+7J99uLm0js89yzI/Di+n2Ws3t04++Z/eEC//w7W9V+S8P/0crYSnmjV2aBLk+ZgiEBSikJIQY93EemwrnpFy3gOEsSIgdMADyK9rGoHDYpwWFB7YE3yUl8m2RC2UoSj2QxNBpHVEHEpQ1/J1Sh2Mia1cyUUPZ+wbLkO+yuU40YwL+w5BphMZDnUchzhEaZIVEmeRivzGxGv9AK1WiiIAH+MwpSQA0yFwQcEIMC/0x5FkYxrEFJfEcYp7YM2Xr4BahQU5vkk5JspSECAoTgdKA5AKhkUpi/oJB+eqyhxglpbCVHTwGH4p+UuRRYqj/g78HtQfhfpSxbhHoBrBp0so1EhAOEhQHlvBlu1gUamlOmWd0pX6UVQpHQCdMvB7S4JV89JTRR2luChU2/jSinkmUcbFAKeAkHQq0bRogUuwlCIISkrgWiHTFkTtViaTIEpYCjkG5ad+8XiumCpHmgaKFtgPF5ZrMRfaG3DEVSRYLtvaYUYEZ6StsXdXbe09hRhp55AmY9rWzjmhnesCwKPwpq2d8raGM6INuKAdCH3FLMLGVcMGEqkqGIRIXzdHIbUSVTEC/FYxwqgJKXKsGtaKlN+gULKjkZQ8g6eDk1SnXhO3kepeksOSuYz+0qlbNQHEyi+EcsEGWNzUwVBbclRsw6ZYQMutAeNRmqJZw6xhElwgVW2IgX0zv9s7HaFtUrEKrCpYxZXqVUG3ylXhFdVU1FVezPSaF9HSu/eIyFrCxSlVb1L1gSKuqglEuRtKCQi0BZVSx60KtuuVtFHwYzQ9Eoz06AW8Unfk7Cai3RX/ALFoInAOQ8xPiV8SP531ZXPdKbukeCqBAD9bDD7HS58mZebuUkOrc2lznscFZJ6x2+SXifqUglZKI0GzWU2NBvDailM+QQkjBCB3FMyCU6vzPS5IKs4I0gtkLOJSB0GXSjfmS51Q28b5OBWvoIcdYSV3W82uUexsLPZPCgTg/Bv00i0xmhNodcLHWbySJfUUWtNgmReVdHmIArPnloOz4LzmROuuFzUn6lU6f70kMIOpXahcPHbWkPEDLJK8D8bCjJZEwbD38SR3Jt/Mo+tj+31xcv32NL+0Lk5/hRnPvdSPT/XjC/v8EEf6WbkQ0ULNfusbAkVeSCwUhpaOXNfwwtj2fd/tKqYJywif3MnjG3pMCSylat7mcVxQtXJ5XlfZvmGV7SIvCm3UjYmDYJqHjTbsGn65Ui1X2RzamlrBnuceuw/1KmFUHdx/oa2wvujNlqXxeCSH7nA8TByfq1Bt2mydZrVtVtunWW2fff8DFZlURt7OcdoEYi+I9veff2kX4MfXvFdrk+V7dq1NUs9BHtY9hL1ujAxKqWnrhukZ/ou6+DWdEqCWN3UNPBzKrNs3fVzia53S65gODGBNp2w65TM4Am46Jw14yECr53RUikB3KqjKAriTmM/knrTQSt1uzJ85JT3BvtyckppTUnNKak5JzSmpOSX9905JMLpv2ZHglBQbHky1BkZ+aDu6aXZ9S1eb2Es5JS3nqvuuR7eML3o54ngyFL3BiFtuNxqOe1uWIzCoWY6a5ag5I20+Ixl6ORrVW6SrU4yghnwUmm6EQgv+RBZjC+S+sDPS/71L6oY3zgx/0MsyogsYQzZ1Sd+HmaHpkk2XfKYnpNW/bzWHpCc6JM2gBEuM1FdAFRCTSKJYQCQmXPTR+1eHxx+C88MPZ59e7d4xrLsuQHctJWuL1clCiwuc0Q3LVA3/4CWqkDflG3OApzNfHmhQuvn0l+aC1VywmgtWc8FqLljNBevfumDBgL5lOwNudbuC+TVGvm4YGGPbw6Z63b+sA9Y+zK+wdGSL4e5he9raU1700mZOfOGIgho2hROXpxb7TUub2yxtzdLWnLa2nLZsz/rx9R91GxYGFDkAAA==","output":[{"uuid":"018a981f-5bd2-735a-b74b-65f538d7536a","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-5bd2-735a-b74b-65f538d7536a\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"eet6doxfgbi6510a\", \"$time\": 1694769306.58, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"p\", \"classes\": [\"font-normal\", \"text-xs\"], \"attr__class\": \"font-normal text-xs\", \"nth_child\": 2, \"nth_of_type\": 2, \"$el_text\": \"Available for JavaScript, Android, iOS, React Native, Node.js, Ruby, Go, and more.\"}, {\"tag_name\": \"div\", \"classes\": [\"mt-4\", \"mb-0\"], \"attr__class\": \"mt-4 mb-0\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"button\", \"$el_text\": \"\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--has-side-icon\", \"mb-4\"], \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--has-side-icon mb-4\", \"attr__type\": \"button\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"mb-6\"], \"attr__class\": \"flex flex-col mb-6\", \"nth_child\": 4, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"h-screen\", \"overflow-hidden\"], \"attr__class\": \"flex flex-col h-screen overflow-hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1779}","now":"2023-09-15T09:15:08.365426+00:00","sent_at":"2023-09-15T09:15:08.363000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-5bd9-7cb4-9fd5-5e574bb918eb","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-5bd9-7cb4-9fd5-5e574bb918eb\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/platform\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/platform\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ilfvutq6qvqh58ow\", \"$time\": 1694769306.585, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Ingestion wizard \\u2022 PostHog\"}, \"offset\": 1774}","now":"2023-09-15T09:15:08.365426+00:00","sent_at":"2023-09-15T09:15:08.363000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-5e75-7a07-a79f-1eee24012718","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-5e75-7a07-a79f-1eee24012718\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/platform\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/platform\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"37x61aqqtn9k28a5\", \"$time\": 1694769307.254, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"mobile\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"attr__type\": \"button\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"mb-6\"], \"attr__class\": \"flex flex-col mb-6\", \"nth_child\": 4, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"h-screen\", \"overflow-hidden\"], \"attr__class\": \"flex flex-col h-screen overflow-hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1105}","now":"2023-09-15T09:15:08.365426+00:00","sent_at":"2023-09-15T09:15:08.363000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-5e79-7f17-be1a-8b450229830f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-5e79-7f17-be1a-8b450229830f\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/mobile\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/mobile\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"oawqrgmuo369cqvg\", \"$time\": 1694769307.257, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Ingestion wizard \\u2022 PostHog\"}, \"offset\": 1102}","now":"2023-09-15T09:15:08.365426+00:00","sent_at":"2023-09-15T09:15:08.363000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-60ea-7698-b26c-b3735fa37bd8","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-60ea-7698-b26c-b3735fa37bd8\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/mobile\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/mobile\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"017vn18mgnnd0rss\", \"$time\": 1694769307.883, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"React Native\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"attr__type\": \"button\", \"attr__data-attr\": \"select-framework-REACT_NATIVE\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"FrameworkPanel\"], \"attr__class\": \"FrameworkPanel\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"h-screen\", \"overflow-hidden\"], \"attr__class\": \"flex flex-col h-screen overflow-hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 476}","now":"2023-09-15T09:15:08.365426+00:00","sent_at":"2023-09-15T09:15:08.363000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-60ee-7e1a-aadf-8011aaa47a22","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-60ee-7e1a-aadf-8011aaa47a22\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/mobile/react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/mobile/react_native\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"2w8r5rse14eqrg7f\", \"$time\": 1694769307.886, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Ingestion wizard \\u2022 PostHog\"}, \"offset\": 473}","now":"2023-09-15T09:15:08.365426+00:00","sent_at":"2023-09-15T09:15:08.363000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1694769311367&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:11.372150+00:00","body":"H4sIAAAAAAAAA+1XW1PbRhT+KxoNj1mQrKvpUyAlpdOk0DRJO5mMZqU9shYkrSKtbAjDf+858gXJNiYFXjLDk61z2fOd6579cmO2rRTmoWnZIR+Hdsr8cMRZYLmCjceew8C1wxAciF0rNV+ZMIVSo/geb7VKeKXbGpBc1aqCWktozMMbc0/hj/mOJ8afH4x/kI2EaAp1I1WJDNvat719i+hxrWYN1Eg8kTWk6oqIAqYygUhfV4CMN9BcalURI2nrGs1HbZ0jI9O6Ojw4yBFGnqlGH4aWZR3IcgKNRkMHhYplDgc18ERHJddySkj3SBS1h2rEqLjOSl6QzYdPWQC/88q2gx455+Wk5RM6C0r28QOpNEkNUEYZyEmGCGzLG91RZ1LoDIn+yELiVMKsUrVeCY9tu09eSntuiORcxmhnBjFZwY9+qPeDcN8juiwRl466XPtpkI7TYppdX6RZk02JryV5bvtjN/DHjjXeD5zxK1NIjEKJnnd61efXl6d/X587x+8Dxxl9/HZ+Vf7mV87Jp/DT7yKo/vXd4Ffx/vLqc9LL47C83HgkWBByYGgpZjGImHNrxK3UJZ2WovcIY7KJBBQqwkK8gAQjlvK8ATxwUqu26qpyxVqCAZaG8Yhh+i0mPMCCd0ILwONcuBRLVU94Kb9jzrtY3ml5XjLX4qEn2MhLbA88zxOeICRlo3mZUOq3Vqh5i6h63RNhjHmcg4jQdcxc1EiBykv8WHZYclEKvBNOcz5Bb758RVafFlX8OldckKdoAMs1LwgBWkXHklwml5nC4FILF1zmnTHKDp/iF9lfmWxynlzu4O9howI2InXtnsC2xZiuqNg4kVBogSLW43ZzY9nSHZyuoaHXQCSVQ4Fy5OCNqdGrRT82FS9RPsl509CI+WL+gckuj1qtVRlFiSo1jSUMCte6RgIJot5WqVdmqbMoyWSOZYZW6UulC2xzFJGGKyqU03IqNRjc0MALo4AihtrQysggr4yZ1JmhM9kYGOMK0zqAHHdWO9cfddx93iKj98WYprHL6+t1OhahbhtW1bLYws15jcNpjZi2ec7mo2WNk2DcoAYq7wE94w2jcmUSg4vMQjN/ZxaMrdiH1CHyIa/DPSTdoR7Sl5iH1AFio8O7hLsozlXielWCg3pQJaO1ZAtJM/R/K/QyjAUOOUuVQsybERxwd9fvQ8hwqA8U3K0Kc/ONvqa+Nwt+NY/woYETrLr65TEYes6e4oisWxxsqmzOyDUqnZiFm45vSBqd3NPMH9VSTOAMr+cdo2Ob0NNSvOVENqt5hYvTD5lfCT+f992c3mm7k3gug0g/XS5Wx6uYZl0L74JhrGsZC52nJSTNgfbN++wT+1kN0Q9LFJ2WsfneR1sG3oBprmYsk0IgZTuOpS4Cmmsa63pPSxLlmWF5oY1lXtZJjFX51npZFzTu03waxA84to842b2vZzckBgbdH5p8mwaRuPiH18c9OVoIGOuCT/O4VyXrJbSBYFUXg3J5DID5ud0OXuONMzzCeeCIWAlaNhY3aobrHL21ukfMzh6yb9EjrS7RWbzusiT6Nvl4Unmz76Oji2P3r+bk4t3b6tw5e/sG10X/3Dp+ax2fue9f88Q67R5X0NAaufnYABbEwmFx7FjM9+0gTt0wDP0xKc1kKdTsQZ3QtlIQvpfS6q7StAF6vvkjn3zf8nzG9w0PAsaTUDBuBUnMExGMfbq2ls9nOd8E5ytgY8x3DqPbi7st6+U9/TO9p4M6m1qz70kynUGbTyfb39Muwnt5T7+8p5/jPf2TTUrn9ut/K0nIYGoUAAA=","output":[{"uuid":"018a981f-682a-704d-9953-e4188e3eb40f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-682a-704d-9953-e4188e3eb40f\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/mobile/react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/mobile/react_native\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"6f7f9fmvhyjfhshv\", \"$time\": 1694769309.739, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Invite a team member to help with this step\"}, {\"tag_name\": \"button\", \"$el_text\": \"Invite a team member to help with this step\", \"classes\": [\"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"LemonButton--has-side-icon\", \"mt-6\"], \"attr__class\": \"LemonButton LemonButton--tertiary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered LemonButton--has-side-icon mt-6\", \"attr__type\": \"button\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"panel-footer\"], \"attr__class\": \"panel-footer\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 9, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"InstructionsPanel\", \"mb-8\"], \"attr__class\": \"InstructionsPanel mb-8\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"h-screen\", \"overflow-hidden\"], \"attr__class\": \"flex flex-col h-screen overflow-hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1626}","now":"2023-09-15T09:15:11.372150+00:00","sent_at":"2023-09-15T09:15:11.367000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-682d-7a77-ac8d-a07cbacd7968","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-682d-7a77-ac8d-a07cbacd7968\", \"event\": \"invite members button clicked\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/mobile/react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/mobile/react_native\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"7rhv0wzccvweulvg\", \"$time\": 1694769309.742, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1623}","now":"2023-09-15T09:15:11.372150+00:00","sent_at":"2023-09-15T09:15:11.367000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1694769314370&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:14.373644+00:00","body":"H4sIAAAAAAAAA+0a7W7bRvJVCMG4X7c2v0n5UBxq95L6cEmd5tIPBAWx5C4l2iSXIZeinKBAn6WP1ifp7FKiSIqS7EgFkpRAAEfzsTM7Xzsz0tsPk7KMyORyomounrpaiGwaBMhxLQ1h6oTIp5ZPNGIZxA4n/5zQBU05kJ/hkrMAZ7zMKYCznGU05xEtJpcfJmcM/kxe4ED57rXyE6AB4C1oXkQsBYSmnmvWuSrgfs6qguYAfBblNGRLASR0EQXU4w8ZBcQ3tLjnLBOIoMxzEO+VeQyIOefZ5cVFDGrEc1bwS1dV1YsondGCg6CLhPlRTC9yigPupZhHC6HpmSAF7i6bQGSYz1OcCJmHT1kpvrmVpjktcIzTWYln4iyaojevBUsR5JSm3pxGszlooKmWvoFWEeFzANq6CsBFRKuM5bwhnmpaG7ymtkwXwHHkg5yK+kIKfGib+txxzy0Bj1LQi3vS1yTHD0VcvlcL866a6cKvZzwSN9fsqenYU0PTzk0DLkQisEIKN5d82Y9f39/8/+GVcf3SMQz9zbtXy/RbOzOe/eD+8F/iZD/bpvMf8vJ++WPQ8mM3vExfJxBemCKQ5EN4ER9jVcdqaAqeUljvI4RFhUdowjwIxDsagMVCHBcUDpzlrMxkVDaotTIUha6vI3C/iohFCZoarkqphTExhS1ZPsNp9B58Lm254bKsoObCrkWQbgWaRS3LIhYRmqQFx2kgXD8YoZNfQatW9nhgY+zHlHhwdfCcV0QEmNf6Q9hByHkhxZI4jPEMbvP2F0C1YV6GH2KGibgpCIBwjROhAUiFiwVxFNzPGRhXpHCCo1gKE97BC/gk5DciixgH93vwZ5CoFBJRZO0ZgbQFmzZQSByPMJAgLNbCyrqxTmmpjkxo2kogQRXTBOjEBT9MONxqlY/FYgbkQYyLQlSYt5P/ga/TmwD4wBCY89zzJBZoN6g1ZpUtE40mDWydWB1gGMWisKQsFXaqYSLnrqAugf8VVdFN+NfglkmcCpErP1dVdV4Z5xA2F7ooRbXWq6NZUEo3A7m0Y4PBeYTRPCKECpPxvBSolM+9YB7FkAhgF/GJhSvr1XbyOF0K9SGaupbKsLh431RXJecs9bxot8k6JPs16Mn0Jaf08kavXToAovUJIS6eDZw/9OGQRLws4A9EMnivj00wuKoHTBkC3bl4oHqYOS7Qwasrg3p1oV2tejihUxe00agLb/RZq7NKjMaSrdCA5KMiKoOYyah5gmNItBjwwwtGcCwsIM4bNkiH5IQiwRZQGaBy7BfbkB0n+nvxZq/OvG4CYwCKEA7B3wh6mCY+JcVGz4I/yNyVteRSgUKeLf/VOGp9i4GzlQPy6sCopTXRgP0oJVQUHaQ10JxJDUiEY7YpLDJGEsn+qOrxFKN9B8U5xiI1B6A7jeZ5bMW35eWBYzr2GTi6ZZ/Nwae64i00U20vb+u5ougItHsCjX49ZESouMOoV4BFtdUauXwO755oSuWjtM9qLe6OTtBK9owAh3N2Lx+VbB5472ZvnmVW9V6/urs2vy+e3b14nr0ybp9/A8+v/Uq9fq5e35ovv8aBeiObVVqIZ3m7eaPI8YmBfN9QkW1rjh+aruvaU8FUQdSy6iCPq6khJbYVilaIhWFB4cXQoekUduyPI45GVOQYpoNcHVuI6ET3CTZpYIq2dhxHvoBxpExS3TLKZenPVVrNZL/YG0f0c12Fs8dxZBxHPoFxZKjhFU97lMrn93GNb5ZHye6+dwc2xjlkcQ8YlnGM6hzsYQLQnuZUxEHiI/0J7e9K/mD3O4iTmnVBG7268LVWitRpVwv85Bd+z2uoH2oJYGqiMQoZA8W2rdTBHqfZtMdgDjJ0e84EL2sztvrOY/qfGygJeQmJzNLiVlytDg93++JblMJl7pHir/KIzOgtPEd1qy9e8C3JQ0THuXjgRFTlOING4VHiG+LT3V7Wpb2yJcWpBAL8Zt1IbOahuczTfWoofS5lxXOcQ8IYJpzd8gX6pILEHxjHxWlzVPc54lWFih/GrFqvYIb1WPOCQjWn0uc7zknCzwjCq7XA6IOgJseD8dInVHZxHqfia+gDrrCQuytntyg6As1HVb5tgQBc/Q/2Jjt8tCJQ+oTH3bgVJf0Q2tKgiYtOuHyMAvW5sufM4cXpHmEcWsXVo+djpsovbX7UbHXH/GggR9VshPEU2A2baFgNTKP7dVYGdU7MM5/o8AgCo/Dh35DHPGR58lU9Bv4jzMHtFcvvvzpyrKzP/+wnyfd5kjiL2dQ28QNf1C369iSpgXrjJDlOkqeYJD/5EgkqRlyOEU0Xp1Tg+5wof/z2u3ILdvwW1smdSmrZg5V0GprIMXULYVFTIZem8CsBX4VY+7w2cWMxfVQx5TqJgtA3ZuldrJOcDRRT8xzWtmMxHYvpJ7GWO/Dd9865ZZDqCd/Bs1wRXGL7B6WVgwW5UtcAuGf/C/qh3eH+A070TfqOjeIjlocn+tJ8cG14aEf4mPWg9dSFQFOqUQyVi6bwGbZ7OZLhV2zf9wD9cVOmiIHVdbclt5EfPUv+VXvEcZE3LvLGRd64yBsXeeMi73SLPMfaMXzCr9JDaiMXGm9EMSF4alsO1ozPaI1XlKBSxiqQ+9eNny0hn/0MuiTJMkjM7J2uz5YLXY4DWzPoOIGOE+i4ztu1zrPdHfUUwtvWMJrq1EUhMWyqadbUUMUvMtb1tCkq++fTsdR+AaVWS+nSv4sSuLB7p7+T+8vtUquCyLHYjsX2b1Fs21XU+fWXPwH99tceLTkAAA==","output":[{"uuid":"018a981f-6ecc-7851-ae7f-be5bd1d53d6f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-6ecc-7851-ae7f-be5bd1d53d6f\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/mobile/react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/mobile/react_native\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"draysluz0s4jwg2f\", \"$time\": 1694769311.437, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"svg\", \"classes\": [\"LemonIcon\"], \"attr__class\": \"LemonIcon\", \"attr__width\": \"1em\", \"attr__height\": \"1em\", \"attr__fill\": \"none\", \"attr__viewBox\": \"0 0 24 24\", \"attr__xmlns\": \"http://www.w3.org/2000/svg\", \"attr__focusable\": \"false\", \"attr__aria-hidden\": \"true\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"\"}, {\"tag_name\": \"span\", \"classes\": [\"LemonButton__icon\"], \"attr__class\": \"LemonButton__icon\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"button\", \"$el_text\": \"\", \"classes\": [\"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-stealth\", \"LemonButton--small\", \"LemonButton--no-content\", \"LemonButton--has-icon\"], \"attr__class\": \"LemonButton LemonButton--tertiary LemonButton--status-stealth LemonButton--small LemonButton--no-content LemonButton--has-icon\", \"attr__type\": \"button\", \"attr__aria-label\": \"close\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonModal__close\"], \"attr__class\": \"LemonModal__close\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonModal__container\"], \"attr__class\": \"LemonModal__container\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"ReactModal__Content\", \"ReactModal__Content--after-open\", \"LemonModal\"], \"attr__style\": \"width: 800px;\", \"attr__class\": \"ReactModal__Content ReactModal__Content--after-open LemonModal\", \"attr__tabindex\": \"-1\", \"attr__role\": \"dialog\", \"attr__aria-modal\": \"true\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"ReactModal__Overlay\", \"ReactModal__Overlay--after-open\", \"LemonModal__overlay\"], \"attr__class\": \"ReactModal__Overlay ReactModal__Overlay--after-open LemonModal__overlay\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"ReactModalPortal\"], \"attr__class\": \"ReactModalPortal\", \"nth_child\": 6, \"nth_of_type\": 3}, {\"tag_name\": \"body\", \"classes\": [\"ReactModal__Body--open\"], \"attr__theme\": \"light\", \"attr__class\": \"ReactModal__Body--open\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2931}","now":"2023-09-15T09:15:14.373644+00:00","sent_at":"2023-09-15T09:15:14.370000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-71d0-7347-82a5-d2d2bda4ec45","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-71d0-7347-82a5-d2d2bda4ec45\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/mobile/react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/mobile/react_native\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"umn253uxubh0ewgt\", \"$time\": 1694769312.208, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"Continue\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"attr__type\": \"button\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"panel-footer\"], \"attr__class\": \"panel-footer\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 9, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"InstructionsPanel\", \"mb-8\"], \"attr__class\": \"InstructionsPanel mb-8\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"h-screen\", \"overflow-hidden\"], \"attr__class\": \"flex flex-col h-screen overflow-hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2160}","now":"2023-09-15T09:15:14.373644+00:00","sent_at":"2023-09-15T09:15:14.370000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-71d3-7016-aa90-636d1a0c436f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-71d3-7016-aa90-636d1a0c436f\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/verify?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/verify\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"zrmm7vg964aytvon\", \"$time\": 1694769312.212, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Ingestion wizard \\u2022 PostHog\"}, \"offset\": 2156}","now":"2023-09-15T09:15:14.373644+00:00","sent_at":"2023-09-15T09:15:14.370000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-79f4-7425-a1d3-9319851b0f8b","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-79f4-7425-a1d3-9319851b0f8b\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/verify?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/verify\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"t2dicfb3gnjl2dro\", \"$time\": 1694769314.293, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"or continue without verifying\"}, {\"tag_name\": \"button\", \"$el_text\": \"or continue without verifying\", \"classes\": [\"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-primary\", \"LemonButton--full-width\", \"LemonButton--centered\"], \"attr__class\": \"LemonButton LemonButton--tertiary LemonButton--status-primary LemonButton--full-width LemonButton--centered\", \"attr__type\": \"button\", \"nth_child\": 5, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"ingestion-listening-for-events\"], \"attr__class\": \"ingestion-listening-for-events\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"text-center\"], \"attr__class\": \"text-center\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"h-screen\", \"overflow-hidden\"], \"attr__class\": \"flex flex-col h-screen overflow-hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 75}","now":"2023-09-15T09:15:14.373644+00:00","sent_at":"2023-09-15T09:15:14.370000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-79fc-7fe6-8aad-eadda9657a13","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-79fc-7fe6-8aad-eadda9657a13\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"xdmxcm4pq22gxv2k\", \"$time\": 1694769314.3, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Ingestion wizard \\u2022 PostHog\"}, \"offset\": 68}","now":"2023-09-15T09:15:14.373644+00:00","sent_at":"2023-09-15T09:15:14.370000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-79fd-761a-92e8-fd36e1159302","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-79fd-761a-92e8-fd36e1159302\", \"event\": \"ingestion continue without verifying\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"1nexbjim0008j2qy\", \"$time\": 1694769314.301, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 67}","now":"2023-09-15T09:15:14.373644+00:00","sent_at":"2023-09-15T09:15:14.370000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1694769317374&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:17.382070+00:00","body":"H4sIAAAAAAAAA+1X627bNhR+FUMI9qt0JOueoRiadMkyrFmyLO2GoBAokbIY06JCUZadoO++Q/kS+RKnSzJgW/3L1rl+50byXN8bVcWIcWCYVoDDwEpRYJIY+ZZtoSAkBKUBdpMU2z03MY03Bh3RXIH4Hq6USHChKkmBXEhRUKkYLY2De2NPwI/xASedXy87fwAbCNGIypKJHBiW2bXcrra2F0tRl1QC8ZhJmoqxJhI6YgmN1KSgwHhPy4EShWYklZTgPqokB0amVHGwv88BBs9EqQ4C0zT3Wd6npQJH+2UFkApRg98fCo5VKuTw7VDEjNPvUomHtBZy8FZSnKgox4qNdCB72hIYX7aqGQVWWQ5awNzspBXPQ7CW5bfIHOf9Cve1DZqjq0utUiaS0jzKKOtn4Nky3d4DtWZEZUD0eiYQR4zWhZBqIRxaVps8l3adAMicxeCnprH2Ah/tCnT9oOtqOssBl4qaFrDDO280roY9yUzlxDear5iO2PJCx/dC2/K6phu+MQiD6HPIW6NXfHo3OP19cmEfnfm23bu6vRjnP3mFffwx+Pgz8Ys/Pcf/kZwNxp+SVnmXu86JewT5AaYIPMUopiTG2OxhM3W0TqWz9wxnrIwIHYoI+vOGJpCxFPOSgsG+FFXRNOuCNQdDoefjHoKym4i4lKDQDkxKXYyJo3MpZB/n7A46psnlg5brJlMtHLgEwcBYLnVdl7hEI8lLhfNEl35j4xpfAFVrqCLIMY45JRGEDpWLSkZAeY4fmhYaNkopboRTjvsQzfVnYLVpUYEnXGCiIwUH0Ox8qBGAVwgs4SwZZAKSqyd7iBlvnOnq4BF8af8LlyXHyWALfw/ml8J86mHeIzDNkNMFFQYmIgI86Iy1uM1xMp/0Bk4z57Q1QFqK0yHI6QDvDQVRzeaQsBGIJxyXpT54ro1foNb5Zc1UkkVRhnMC+CAlWCkZRY0caG0SemPkKouSjHHoMZg//SXSGTA9j5RHio51l0ChlkDElVIAdFnmcVQz8QWqpqkhPpGjshFB+rTZgnjhbyojha6BMdVdUAlWGOm/wBKFQgys01JnFEHqhSRQkJm/7bFbK9FuSzkwWl8IJRlNBlQ3/xI5rThH06NqW206m2wtE1uWloKAnvl7QQwnKFgH01C3d8ZGw1MrpZo0lRni8RTjQQfmvBh//0Ksh5KRPj2HWwSQilzp+3gN+iahZ0Sy3S2qJS7g+vsq9wvh14u+OU62+m4kXssh0E/n9/7RIqdZ04XbYHRWtToznZcVJOVUv5Ye86/Zr+pI/6BEaGsZmj5P9GUIB3XKRY0yRghQNuOY6wKgqWZnVe+F8wt1RtBe4GNel1USQvAG3NQvq4KdxzRfBvESru9DrP0+NrNrEksOnRWHzlc6BOLsH3q0RjOBzqrgyyJudclqC60hWPTFUrs8B0DrVpVCrCTRfsJELMhkYUNl8OrQl3Pz1l5F/NS1CSEqMYDo4cmaJdFt/+q4cOu73uHNkfNbeXzz4aS4sM9P3sMzx7swj07Mo3Pn7B1OzNNmKZhe1uuPZIr8mNgojm0TeZ7lx6kTBIEXaqWa5UTUT+oElplS4rmpfnKKNC2pXjtsq0nG2jbo4AD5vTBGAbYwCggJ4YEdY+Knu23wf70N3kLeLdtzxze15RB/0zbod00b4O22wd02+C/YBl99EbP+mUWsVdZvfQXbcjXbu9Vqt1rtVqvdarVbrXar1X96tbLt8MvnvwD69vHlaRsAAA==","output":[{"uuid":"018a981f-80db-7131-89dd-f8a5cfa325c0","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-80db-7131-89dd-f8a5cfa325c0\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"39z6vxum2ri0t4bj\", \"$time\": 1694769316.059, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"div\", \"classes\": [\"LemonSwitch__handle\"], \"attr__class\": \"LemonSwitch__handle\", \"nth_child\": 2, \"nth_of_type\": 2, \"$el_text\": \"\"}, {\"tag_name\": \"button\", \"$el_text\": \"\", \"classes\": [\"LemonSwitch__button\"], \"attr__id\": \"lemon-switch-0\", \"attr__class\": \"LemonSwitch__button\", \"attr__role\": \"switch\", \"attr__data-attr\": \"opt-in-session-recording-switch\", \"nth_child\": 2, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonSwitch\", \"LemonSwitch--checked\", \"LemonSwitch--full-width\"], \"attr__class\": \"LemonSwitch LemonSwitch--checked LemonSwitch--full-width\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"my-8\"], \"attr__class\": \"my-8\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"h-screen\", \"overflow-hidden\"], \"attr__class\": \"flex flex-col h-screen overflow-hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1311}","now":"2023-09-15T09:15:17.382070+00:00","sent_at":"2023-09-15T09:15:17.374000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-84a8-729b-8a1a-8dd9647bad7f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-84a8-729b-8a1a-8dd9647bad7f\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"qevi1365xjw14d7j\", \"$time\": 1694769317.032, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"\", \"classes\": [\"LemonSwitch__button\"], \"attr__id\": \"lemon-switch-1\", \"attr__class\": \"LemonSwitch__button\", \"attr__role\": \"switch\", \"attr__data-attr\": \"opt-in-autocapture-switch\", \"nth_child\": 2, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonSwitch\", \"LemonSwitch--checked\", \"LemonSwitch--full-width\"], \"attr__class\": \"LemonSwitch LemonSwitch--checked LemonSwitch--full-width\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"h-screen\", \"overflow-hidden\"], \"attr__class\": \"flex flex-col h-screen overflow-hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 339}","now":"2023-09-15T09:15:17.382070+00:00","sent_at":"2023-09-15T09:15:17.374000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1694769320381&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:20.384514+00:00","body":"H4sIAAAAAAAAA+2deZPbthXAvwpH48m0HWPNG8R2Mmm8PpJO49p1nLT1ZDQgCIpc8RIPHevJTD9LP1o/SR+gi5So9cYrJysF/9grnA/Aez8cJPHefxg0TRwMLge64VHiGSHyXMdDWDcY8jw/RCH2fd/FoYM9PHg84FOe1ZD8EW3qnNGibkoOwUWZF7ysY14NLj8MHuXw3+A7yrS/v9X+CdEQMJzysorzDCIM/cJwLnQR7pf5rOIlBL6ISx7mcxEY8GnM+LBeFBwinvFqXOeFiGBNWUL1w6ZMICKq6+LyyZMExEiivKovPV3Xn8TZiFc1VPSkakCkIp9BvV8VCa3DvEy/THM/TvgXYUlTPsvL8Zclp6weZrSOp6Ihj0RJUHi3VBFR0DrKIBdE9lfSas+2sYaBW8EJzUYNHYkyeIbevRVZKlZyng0jHo8iqNnQHXMbOouDOoJA19QhcBrzWZGX9SYxMYx28Dq1Y3sQnMQ+1DPjvqgFfrRH4AJ7F44IjzOQqx5KFZhlzEgWllXESUI8Uov4OhYtNlxiY5dYBr6wXfvxIIih9Rn0m8xX/Pj1+NvvF2+sq1fYssx3kzfz7Bu3sF784P3w1wAX/3Jt/Dx4NZ7/yFrD29U62zcDhD3KEdTkI58HPqW6SfXQFnka0XufUFlcDQOe5kPQz2vOoMdCmlQcChyVeVNIZd1ErYXhKPR8E8Gw6yhweICI5emcO5QGtujLvBzRLL4BjZF9uc3lOGyZi3pOgEyHGQ53HCdwAiFJVtU0Y2LoexV38DNI1TKqIfQx9RMeDKHpMHLDKg4g81p+UFpQ2GHIqUwcJnQErXn/E0S1w4YFXSQ5DURLoQJQ9iQVEkCt0DCWxGwc5dC5wrJTGieyMjE6dAq/RP2bKquEsvEt8Y/AfjnYpzDmRwFYM/TpJhQMZhjkUIPosVasxMna0qU40s55y4BEqoSnkE408MOghlat7NBv6hrSyBTDms/FIMIvltCqEiR6P/gbDH72dhbXLBoOV8mhj2hdl8OlPkHReYYqmQQZkHsZJ8uA6L4C1mnKXDR/sMy7CQ1oTZH4E6LyokZxhlrDuqoJUmd1NGRRnIAQYPDiVx6uesL4+XGnnUE8PdgsiGj9QihskgQtQbBpaE9jtEOZOoJB599BsFYGayeD1ZthKVVVL2T/pXS+rPtSA0Mo5n/+FBlanfO0jIMRfw2YhZbnWS0mrL2u6Et065iYn1AtmpW0gPnhTtVvEh+v9dLebq1bpjhWhRD+7XpivNr0aSS16zYxtN1c2irP/QYkTLhYThyqX0QftSLxH2K5KC1Cy/lbzBZAsjDJZyiKgwBC+uVY5wWBljm13Xz3GyQxzgjUC+pYj8tuEEKwSOrTl92E2qGc9xPxLcxvT6mo95DN7qXoVAjLkk6F9h0rhMDVX+jgGK0SaLsJ79filpbsqtCeBBu96KjLpwjQmvvKPN/pxF1+7xbh58FiU0YdwbQsplC5GN2V+GNTHDSxzsfQeljTRWw4Gb17UTizG/Pp9ZX9j+rF9XcvizfW65fPYB3gvtGvXupXr+1XX1OmfytXzbwS64P9VSRH2A8s5PuWjlzXwH5oe57nEpFpFmdBPvtoHs/QQx64TijWZHkYVhwWFiYxJAj2tkueD9k920LU5D4ivhV6geuZNhO7BrVdOt/tUhoaThHeXDdpHl8nROyOd7dL3oWOidouqe3Sg90uXeVpkfBatGh3f/F0nbz1C6b6Mk5pKSaBTjCMVt1Uh2ITWsrlYSews+noxDCQnpdc6EHqI/PARmaZermR2ZGuG9iVrRsnJesGbeXqhq+l0qRMm0lwORibvr3P7ukXrzwLmvEEhTCN920yOrH3k+xuiyu1r1P7OrWvU/s6ta9T+7qT2deZBPfv65gJ2U2MKDYd5ASUMcelRkhpa1+32dlosETLywB+VxosJzNYJkANaqt3Hlu9giYzYwHKYLqWPh0J++zZ6nkgntrqqa3eMbZ6aw5usDKEZ0lDkWhV6LozYM6s4DnUMMkPJgE7FBYu+ng3yYmRWs79e6Qmpo+wFbjIc3T4BxNCOHcZtoSBrEl9qEO1mtNUgwrAbEdaU8DjO7nxVNw+A25HUeT7syDHaUAyfy4e7+5z2/CgbMVtxe3jcFuC5BbgQKIpTZptNSfFYDCWAwxmCDObI2oR+IvYgWNbsF7CXovBh+csReEzpvA8n2NWFDPGJnF83Uhk9lBYPShRFD46hW9ZJp84h/tPLYhQdxc47FGTIMcwKASbhPlCw3Y5vL8xUBw+Yw6T3BxV03lUBCHnzBKP3RSHFYd/VQ73nEWcKYchO/Zd5Jt6iHwMem5bNDBsobVrDrfVQXRG3tQKwGcMYDxJDRxWXqHHVWDnBwCs+Kv4e3T+9qDm1MHr9oPXdhF2WIAoCUPk2l5gmDoOQ699GMxWbz3B4GcxvJwB/ZH5OZXHNIrAZ0zgyKYu86aLmVWVKZwN9xLYNByFYIXgoy+Bb4HOlsV12Zwcil35Osk+ih2MMBgT8gMvRLptMztkoesGbnsNLEZZKpomht+Ht3GqKJ+JbcHDIa4UtvpqO15wnrQazC/FgN2ZqsuCTh6j1zdN2SSR02A7p5bb/+q7aUGVCqMKo8fA6KoVNa3GFRhfI9gBs/SWqZ0YsItVjCAhQES8u72JhA3WafHVdvr56oYIw5GCUDsDOY5LTGLBIzfWXup2Bxn0CBRFsfUBs9XOkngyHY2uOTeK/EbG77PVcRVbFVuP9FlRW1b5YVHewP45y4CXTH5KtJsIXmqoCniidqpnB6beD1QPFqyWayPIxpEZ2PAOGWHwQ3zuo4B6okAl8Zh6N76RFiQfZVS8kNIDVPWdpgLq5wJqlI8mCVwEUwn1lgZ1RjDVD3w/QTy4RowyDx5+mTYKbZ3oBqVeoLd3/wqmJwbTpMqmi3I0D7I8Gd2M5PfH+zBVL9Qqln4mlk4aXi6GZZNlIoNUvXPj6YEHW7s8pTT0wRywWpyeME91HJNZ6Mcsn7AAk/53YxVPFU8/K08RqJz4dPX3glIKH4xRG1DqYh3KMLkeYsdlTvvlLKDNCMyGzxQ/HzA//XThzEYTs1g4WYkrebvNHj8tQ52WKoAeCaAPnoAgYlzLG3ieCxvXns+LJIc2av/7z3+119CL3+SjLibdA8ehgYNw6FK4Etw1kcMNhinRddPs7OBFGYqQD5eQhsHHzqSwxpQYkb28/2ufkI5aYipCHmuJKbHyYVk3xMwp6HL5lwLEg5PQCzBH2S2QCt6AEn0nGvHgsdohpnHgK1i4hACbvg13xhgYwfMjS7cs7lEmqtwQU2opjHtWx6G4dUex86GyMzViuLytmsdxPC8tKrYCip2KnZ+TnXIY1ua7HolN+Jgv7j4sqzwrGgtdAdvoMutjRazs+RkPaZPU2uuNPMvXw2EsVlRbvra6Uq7taQHYyk2eiSLefX/VGnN4XUscPmQi7mzhb+t+GNoYS11S8D81+I8mhZctZrPJ/GZ0rcsrchX8Ffx/Nfh3OvbgDHBrL+/PAHfNubJlgaAqaQSrBWDACsXXUEMq9MPUTQvpBBnO9zq5NOxLWwdzcFzD+Dck3fTIelBW43HWMwCnCBuOAxeRwQdoPvWxY1KL+07Hc5o6MHnY3NdvUqt0R/aCZYtRNLf6uY8V9xX31YHJ3YhpEHJHYnq24YXEtcVVfWrNfHLsLKyaTq4LPS1S23cNcS+1Yqdipzow+R0fmNwd/sS0QiswPQX/04Q/qQgrKM/go2AyHacK/gr+6sBEHZgcngEoIXBLmgm3pPnQLxyuq8SeTijvvIgn31DUNmhVU8ADngLGzCckDxdmneLCDPpvOHMccEejpgA1BRxjCpB0EIMwBkBBjHznrXojQ8XtO4kcmPeDP22IAvwALZZTBMtpwivG/7AlykXb7h9r7YiVQQmT/aOGkPauTLQn2tu1r6d2UmE4y9U8DFlaCL9PcAk8L5+CqO+34dqz52+vpFOoUPjouxwg047EMAZNuVIOE4MDrJOCvWdKr+97sA8DHZb78GgUqB8ignXXt1hggKuOU3CUq0jfJf0C35hTHfxcecEYNzPZoB3SkwtQKEV6Rfr9+IhWQ7ilMBF2BBcXbh0Sre7qEU4smwr+vYAeDxpQHprRZFHHDIQD7d5Y7C1p9oraeuSAKgHEe+X0JNgrZKcT98rYj/8NPP0KF3UtJ7+v5XxXaV9oL5ea3HX2G2ei8Ft8/sLcVMeHnf6CetBk34vvYf++MPoIvnjrccgoZGl73e164F3L0evedyXF3Xz5biRYCxDBcAg6L5cGooeW4XD7J0XiT4iErm5QXPMUbVP9AjeNSbyTftd3ItnJ0HySH8g+15tVAn/dyQXnbsqjCvDReo9T3cFqbncsKlctyq/nmfj1tPsvZic6hs+oTfDoCbXYSIfDZm5YmBH+8L/921LnbivNVvqTXmraxFjozZxYxdzl1xOB0Z2lpqlf6OpGNLXUVEvNs/6ecXcde+iLRuBBP/vhMSM2wa+zTzFH3OPcg18h5W0ndQ/5uPn3OgPEWZhO42tr5DHs01R25N4MYNjK/7KaAdQMcO8j9BVmX+UwrJ2zaEPc6P3g54v2m9qW/fNP/wen6voL5JoAAA==","output":[{"uuid":"018a981f-8658-701c-88bf-f7bbb67f5787","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-8658-701c-88bf-f7bbb67f5787\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"wnc1ly33pill989t\", \"$time\": 1694769317.464, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"\", \"classes\": [\"LemonSwitch__button\"], \"attr__id\": \"lemon-switch-1\", \"attr__class\": \"LemonSwitch__button\", \"attr__role\": \"switch\", \"attr__data-attr\": \"opt-in-autocapture-switch\", \"nth_child\": 2, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonSwitch\", \"LemonSwitch--full-width\"], \"attr__class\": \"LemonSwitch LemonSwitch--full-width\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"h-screen\", \"overflow-hidden\"], \"attr__class\": \"flex flex-col h-screen overflow-hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2912}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-88be-7843-a2eb-9b3f8d6824c0","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-88be-7843-a2eb-9b3f8d6824c0\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"mf15pfzjumoijl97\", \"$time\": 1694769318.079, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"button\", \"$el_text\": \"Complete\", \"classes\": [\"LemonButton\", \"LemonButton--primary\", \"LemonButton--status-primary\", \"LemonButton--large\", \"LemonButton--full-width\", \"LemonButton--centered\", \"mb-2\"], \"attr__class\": \"LemonButton LemonButton--primary LemonButton--status-primary LemonButton--large LemonButton--full-width LemonButton--centered mb-2\", \"attr__type\": \"button\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"panel-footer\"], \"attr__class\": \"panel-footer\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"attr__style\": \"max-width: 800px;\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content\"], \"attr__class\": \"BridgePage__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__content-wrapper\"], \"attr__class\": \"BridgePage__content-wrapper\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage__main\"], \"attr__class\": \"BridgePage__main\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"BridgePage\", \"IngestionContent\", \"h-full\"], \"attr__class\": \"BridgePage IngestionContent h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"h-full\"], \"attr__class\": \"flex h-full\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"div\", \"classes\": [\"flex\", \"flex-col\", \"h-screen\", \"overflow-hidden\"], \"attr__class\": \"flex flex-col h-screen overflow-hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\", \"main-app-content--plain\"], \"attr__class\": \"main-app-content main-app-content--plain\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\", \"SideBar--hidden\"], \"attr__class\": \"SideBar SideBar--hidden\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2297}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-88c2-7b27-a725-5dacc56a1faa","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-88c2-7b27-a725-5dacc56a1faa\", \"event\": \"ingestion recordings turned off\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"palw1yb272630vgl\", \"$time\": 1694769318.082, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"session_recording_opt_in\": false, \"capture_console_log_opt_in\": false, \"capture_performance_opt_in\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2294}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-892b-73d6-8506-87999ee6c734","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-892b-73d6-8506-87999ee6c734\", \"event\": \"session_recording_opt_in team setting updated\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"hhhbbwdo7md9nbx1\", \"$time\": 1694769318.188, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"setting\": \"session_recording_opt_in\", \"value\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2188}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-892c-7c4e-a39c-794d54327278","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-892c-7c4e-a39c-794d54327278\", \"event\": \"capture_console_log_opt_in team setting updated\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"xox7cppwccqiijuc\", \"$time\": 1694769318.189, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"setting\": \"capture_console_log_opt_in\", \"value\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2187}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-892d-764e-8a29-511a81f29cbb","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-892d-764e-8a29-511a81f29cbb\", \"event\": \"capture_performance_opt_in team setting updated\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"9o2gsvxhpdfeec3h\", \"$time\": 1694769318.189, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"setting\": \"capture_performance_opt_in\", \"value\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2187}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-892e-77b6-b20f-b70f443ad14c","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-892e-77b6-b20f-b70f443ad14c\", \"event\": \"autocapture_opt_out team setting updated\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"7qm17fs8p0isd4oh\", \"$time\": 1694769318.19, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"setting\": \"autocapture_opt_out\", \"value\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2186}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-8946-75cd-a9ff-648d1207ff84","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-8946-75cd-a9ff-648d1207ff84\", \"event\": \"completed_snippet_onboarding team setting updated\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/ingestion/superpowers?platform=mobile&framework=react_native\", \"$host\": \"localhost:8000\", \"$pathname\": \"/ingestion/superpowers\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"h4a6c8vyw3srmhbb\", \"$time\": 1694769318.215, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"setting\": \"completed_snippet_onboarding\", \"value\": true, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2161}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-8957-7215-bd8f-044c4fcf66d6","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-8957-7215-bd8f-044c4fcf66d6\", \"event\": \"activation sidebar shown\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"jzurulh5u74oa367\", \"$time\": 1694769318.231, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"active_tasks_count\": 5, \"completed_tasks_count\": 2, \"completion_percent_count\": 29, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2145}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-896f-7d14-9381-5569293727c4","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-896f-7d14-9381-5569293727c4\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"4nliqvggjee1poz5\", \"$time\": 1694769318.256, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"cloud-announcement\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2120}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-8987-7364-810e-2d4c739c8102","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-8987-7364-810e-2d4c739c8102\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"9ika8zb1mp9ogna8\", \"$time\": 1694769318.279, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"hogql-insights\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2097}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-8988-7ac8-b724-f40901aa8d06","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-8988-7ac8-b724-f40901aa8d06\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"lsnvyrgxdnolgzgk\", \"$time\": 1694769318.28, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"query_running_time\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2096}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-8988-7ac8-b724-f40aafb78a72","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-8988-7ac8-b724-f40aafb78a72\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"07i9wfbicoqcd79c\", \"$time\": 1694769318.28, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"query-timings\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2096}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-89ab-7a48-b670-bb2e0f756c5c","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-89ab-7a48-b670-bb2e0f756c5c\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"bmy5wgq2py5nr7sn\", \"$time\": 1694769318.316, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Event Explorer \\u2022 PostHog\"}, \"offset\": 2060}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-89d5-7f6a-8662-5e1c7a900226","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-89d5-7f6a-8662-5e1c7a900226\", \"event\": \"$set\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"11ek5qp3ka91h4on\", \"$time\": 1694769318.358, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$set\": {\"email\": \"xavier@posthog.com\"}, \"$set_once\": {}, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2018}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-89d6-72b4-a717-d4c3033e8ac9","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-89d6-72b4-a717-d4c3033e8ac9\", \"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"m1irapsxiiixr3aw\", \"$time\": 1694769318.358, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"project\", \"$group_key\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"$group_set\": {\"id\": 1, \"uuid\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"name\": \"Default Project\", \"ingested_event\": true, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2018}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-89d6-72b4-a717-d4c40bff477e","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-89d6-72b4-a717-d4c40bff477e\", \"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"gqp8nywwqxzgj0te\", \"$time\": 1694769318.358, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"organization\", \"$group_key\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"$group_set\": {\"id\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"name\": \"X\", \"slug\": \"x\", \"created_at\": \"2023-09-15T09:14:40.355611Z\", \"available_features\": [], \"instance_tag\": \"none\"}, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2018}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-89ea-7155-85cd-bab752a3eb57","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-89ea-7155-85cd-bab752a3eb57\", \"event\": \"$set\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0zm3r6g4ycnyghx3\", \"$time\": 1694769318.378, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$set\": {\"email\": \"xavier@posthog.com\"}, \"$set_once\": {}, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1998}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-89ea-7155-85cd-bab8418f9641","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-89ea-7155-85cd-bab8418f9641\", \"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"p3taqjp0mpm4b61v\", \"$time\": 1694769318.378, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"project\", \"$group_key\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"$group_set\": {\"id\": 1, \"uuid\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"name\": \"Default Project\", \"ingested_event\": true, \"is_demo\": false, \"timezone\": \"UTC\", \"instance_tag\": \"none\"}, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1998}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-89ea-7155-85cd-bab923f3d281","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-89ea-7155-85cd-bab923f3d281\", \"event\": \"$groupidentify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"p9s9cpaen4oa9vkm\", \"$time\": 1694769318.378, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$group_type\": \"organization\", \"$group_key\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"$group_set\": {\"id\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"name\": \"X\", \"slug\": \"x\", \"created_at\": \"2023-09-15T09:14:40.355611Z\", \"available_features\": [], \"instance_tag\": \"none\"}, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1998}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-8a99-7726-bb09-e5117809ae5c","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-8a99-7726-bb09-e5117809ae5c\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"kcb99ofy2tm7p2dh\", \"$time\": 1694769318.553, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"EventsQuery\", \"select\": [\"*\", \"event\", \"person\", \"coalesce(properties.$current_url, properties.$screen_name) -- Url / Screen\", \"properties.$lib\", \"timestamp\"], \"orderBy\": [\"timestamp DESC\"], \"after\": \"-24h\"}, \"duration\": 274, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1823}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-8fd0-7117-a99f-9706b3cd1ee6","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-8fd0-7117-a99f-9706b3cd1ee6\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/events?onboarding_completed=true\", \"$host\": \"localhost:8000\", \"$pathname\": \"/events\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"y7z2v0ned8dk7uws\", \"$time\": 1694769319.888, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"a\", \"$el_text\": \"Persons & Groups\", \"classes\": [\"Link\", \"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-stealth\", \"LemonButton--full-width\", \"LemonButton--has-icon\"], \"attr__class\": \"Link LemonButton LemonButton--tertiary LemonButton--status-stealth LemonButton--full-width LemonButton--has-icon\", \"attr__href\": \"/persons\", \"attr__data-attr\": \"menu-item-persons\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"nth_child\": 12, \"nth_of_type\": 9}, {\"tag_name\": \"ul\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider__content\"], \"attr__class\": \"SideBar__slider__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider\"], \"attr__class\": \"SideBar__slider\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 487}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-9078-7225-8884-0281e137c9ec","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9078-7225-8884-0281e137c9ec\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"491y0ux93px6ejqi\", \"$time\": 1694769320.056, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Persons & Groups \\u2022 PostHog\"}, \"offset\": 320}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-90cd-72c2-ba7e-e8ee82c2fae8","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-90cd-72c2-ba7e-e8ee82c2fae8\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"infmvij3g8c7bamx\", \"$time\": 1694769320.142, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"PersonsNode\"}, \"duration\": 131, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 234}","now":"2023-09-15T09:15:20.384514+00:00","sent_at":"2023-09-15T09:15:20.381000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1694769323385&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:23.391445+00:00","body":"H4sIAAAAAAAAA+0a7W7bNvBVDCPYrzLV90f+temSdli7ZF3bDUUhUCRlMZZFRaLiOEWAPcsebU+yOzm2ZdlOlq7YklV/bOnueHe8b5P++HlY15IPD4aGGdAwMBMSuokgvkfhKTE54dQVruXFjhGYwydDcSFyDeR7tNaK0ULXpQBwUapClFqKanjwebin4Gv4mrLBT28HvwIaANGFKCupckCYxr7p7hsIj0s1rUQJwCNZikRdIpCLC8lEpGeFAMQLUY21KhDB6rIE8VFdZoBItS4Onj7NQI0sVZU+CAzDeApqVCqvkByBQLdOgIiC6jSnE+Tepr9RZqWpafotcEbzUU1HuErk5N1bXFKxUog8SoUcpSDLNFxrBZ1KrlMAepYBwAsppoUq9ZI4NM02eEHtOgGAMxmDnKmIUQq8tM237wf7LsJlDnrpaO6/sjqvrMC6HBX2lDsK8VriHk0vdHwvtC1z34H9cFlpmbObZcWHZ+NXv8xO7cM3vm1b785PL/OXXmEfvQ/e/8D94jfP8b/nb8aXH1jLNesR48QWJ35ABQFBMYkFjyk1LGokDq6p0XhfIExWERcTFUFsnQkGBktoVglgOCpVXTSBtkQtlBEkCWKLgJ8Nwl3BSWgHhhAupdxBU6pyRHN5RfXclKtVrsvmq2jgcmK5zISwd13uctQkrzTNGXp+a9ANr0GrVkJEYGMaZ4JHsHVwXFRJDosX+lOm5YWIEkEb4iSjI9jNx0+AasOigs4yRTnuFASUgmYT1ACkwsZYJtk4VWBczMoJlVkjDL1DL+AN5S9FVhll41vwKa2iWGaZzEEqxPkSAdnBIOMg6qO6gs99sDivIXhoTrOZlgyUg+BeJuYtNBusKlFhREelAJGzLXy2EGww6Rhxg8cmfg/qjAASLDp7HKoOxM8SigbgCqyJ0dHCNmVvUZEa0zf1SLRqBVJlYgJ06MzPQ1AyuikyVQEmBc0yWsGeADvU4lITLhJaZ8gfIno6BP9TrcsoauhgWZto0JA8GeY6jVgqM0gnkIhvKrnRa65BhKtg8aFKoapgXZszTWGDWO/YDfz6yR0q/gi5lz+vtQYPREzlGgv/ho5bqdbUhIK4pqbVkUwb+27Ru62LzNHgLWHrb4Ro7D60nHXhkLi6ruALsgfKaweb1FlG5oW3g4GMIBL2s2XHoMugRdt+XumxDl3XYh230mEdvtRglwcXcE41JfgISAi/mkgtJmRFdVvIdHyRyQ693V1gdFbU2f0kcHmx7tq3UBqfU9hGlcET2nlXqO2k/KoK3Cn364jbKWadvdNhb9/FPiXz4QOgSSZwnMIviIZsU+KCdoAkzUdD9wX7m/NtenypVMcjGzHUYRErjmk756FTKKFNJOKM1NX49spiXsMWtRrD7mHWSFl0Pnp3VLjTK+v52aHzc3V09vq4OLVPjl9AzfZOjcNj4/DEefOMMuNVM8zd9JuN6Qbm4ZjbJI5tg3ie6ceJEwSBF+Kiqcy5mt65JjCNRHDPTXBWUElSCRwXQ7Nx6MYI7nkO8allkjBwLeJYTJghCwKDee0RfG1YgHkEBo4HNYqvStDfG8Vb9I96FJ9NEq8Yu06dz/K0DrC3bI7irg950c/i/Szez+L3nMXbbHByTdWU3GyTyFyXiojLJq9GC3BTUtbcWYqqgF/9K6s/qrYB0B1tIyY+Z4zQWEAeOWYchB71HYYpvGgbYEZVcrDOAOud4IOphNEzVwN0LJapQVVPJvNRuu8l/3Uvqa9c93xmqTgurSvbG23vJcC6byV9K+lbyf1aybwARphP0QTY2FYAmYRarWDAelkxI16XTYwtMSWkcjSlZY5YpmqssAAHkaqcv+MaRK6YIP8YHDenzEewHzw1iiogAN8B8HE1I2PXbxhYbvkWCeKQkzhJOHNZ4Bjcav+GKWD36Ia+2TyAZuNObR1YM31GZ0Z+LtEp25oNqNd3m77b9N3mft3mwVd1UFFqdOjiEH7w5+9/DE4gVF6qUafmN4d4GzWfWwnUfBGTIAxDyGEWJhYNE9dZq/kP9er4Wy37emzqfKbklTOK87HVRHKn7Nv7Norsy35f9vuy/6/eHXeuR0/mf1cZfDc4nkdyf0+665509c+e2+5JV1T/5J60exUV9tek/TVpf036dY4Y/B3Dpu0T3wQOsQEHDJ7J7CRgie1ZODk87AOGb/VPip4T5GcTd6Snl5ZIzs52TJphP2n2k2Y/af6PDxi6Y+yukwbPuf70FzjrKd2kLgAA","output":[{"uuid":"018a981f-95fe-76af-9f1d-da5e526b4081","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-95fe-76af-9f1d-da5e526b4081\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0rsqs282xgp3wd4o\", \"$time\": 1694769321.47, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"classes\": [\"text-default\", \"grow\"], \"attr__class\": \"text-default grow\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Cohorts\", \"attr__href\": \"/cohorts\"}, {\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"a\", \"$el_text\": \"Cohorts\", \"classes\": [\"Link\", \"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-stealth\", \"LemonButton--full-width\", \"LemonButton--has-icon\"], \"attr__class\": \"Link LemonButton LemonButton--tertiary LemonButton--status-stealth LemonButton--full-width LemonButton--has-icon\", \"attr__href\": \"/cohorts\", \"attr__data-attr\": \"menu-item-cohorts\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"nth_child\": 13, \"nth_of_type\": 10}, {\"tag_name\": \"ul\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider__content\"], \"attr__class\": \"SideBar__slider__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider\"], \"attr__class\": \"SideBar__slider\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1913}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-9664-7a21-9852-42ce19c880c6","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9664-7a21-9852-42ce19c880c6\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ymf6pk54unynhu8h\", \"$time\": 1694769321.573, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"show-product-intro-existing-products\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1810}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-966b-7dcc-abee-f41b896a74cc","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-966b-7dcc-abee-f41b896a74cc\", \"event\": \"recording viewed with no playtime summary\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"uz55qy2obbr2z36g\", \"$time\": 1694769321.58, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"viewed_time_ms\": 3288, \"play_time_ms\": 0, \"recording_duration_ms\": 0, \"rrweb_warning_count\": 0, \"error_count_during_recording_playback\": 0, \"engagement_score\": 0, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1803}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-966e-7272-8b9d-bffdc5c840d2","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-966e-7272-8b9d-bffdc5c840d2\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"5w3t82ytjay0nqiw\", \"$time\": 1694769321.582, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Cohorts \\u2022 PostHog\"}, \"offset\": 1801}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-9d2f-72eb-8999-bec9f2a9f542","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9d2f-72eb-8999-bec9f2a9f542\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"tk1tnyoiz4gbnk2t\", \"$time\": 1694769323.311, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"a\", \"$el_text\": \"Persons & Groups\", \"classes\": [\"Link\", \"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-stealth\", \"LemonButton--full-width\", \"LemonButton--has-icon\"], \"attr__class\": \"Link LemonButton LemonButton--tertiary LemonButton--status-stealth LemonButton--full-width LemonButton--has-icon\", \"attr__href\": \"/persons\", \"attr__data-attr\": \"menu-item-persons\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"nth_child\": 12, \"nth_of_type\": 9}, {\"tag_name\": \"ul\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider__content\"], \"attr__class\": \"SideBar__slider__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider\"], \"attr__class\": \"SideBar__slider\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 71}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-9d37-712e-b09d-61c3f8cf3624","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9d37-712e-b09d-61c3f8cf3624\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"648njm5gtwx2efjj\", \"$time\": 1694769323.319, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Persons & Groups \\u2022 PostHog\"}, \"offset\": 64}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1694769326396&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:26.539412+00:00","body":"H4sIAAAAAAAAA+1b6Y7cNhJ+lUavf5pt3ccsFtjEWTtZbLx2HCe7CAKBIqmWptWiRqT6GMPAPss+Wp4kH9Xn9JV08scz0PyYaRWLVcW6PrKp+enjsG0LPrwZWnZE48jOSMxTn4R2FJHIDmwS2yLOeGA5GQ+Hz4diJioN9rtWNMsBk9O6FFpwjNSNrEWjC6GGNx+HzyT+DL+lbPDv94P/YBiEZCYaVcgKA7Y1sv2RZehpI+dKNCC+KhqRyYUhcjErmEj0shYY+EqoiZa1GWBt08CCpG1KDORa1zcvXpSS0TKXSt9ElmW9gBlKVsqwGyL4HjKYgZrqvKJTI32ff23MzlLbDvfIJa3GLR2bWaIiH96bKYo1QlRJLopxDl225Ts76rzgOgcxcCwQZ4WY17LRW+bYtvfJG27fi0AuixR65iI1WvCw775RGI18Qy8q2KWTLoTMX97Hd9mcRsEsX9Buni7MGu0g9sIgdh135Hn+8yEvlC4qtp5X//jF5Jvvl+/cl29C13U+3L1bVF8Htfvqh+iHf/Kw/m/ghf/gbyaLH9lebB5mjZc6nIQRFQSaUpIKnlJqOdTKPDOnNd77A8oKlXAxlQmS61YweCyjpRIQOG5kW3eZth3aGCNIFqUOQaAtwn3BSexGlhA+pdwzPpHNmFbFPdUrX+5m+T5bzaKRz4njM9sXvu9z36Q3HK1pxUzoT2bd8BOsoq0GsdZtIxL4mKal4AmWjsglquCYvLGfMl3MRJIJ2jFnJR1jNT/9jKF9WlLTZSkpNyuFgkbQcmosgFYsjJUFm+QSzjWVOaVF2Skz0aEzPBn9W5WqpGxyYTynKkmLsiwqaEWibwdQHgwlh7RPWoXfI3ict0geWtFyqQsG45Dd28q8wHMkSgllUjppBFQuT8g5wXAk5MCJRzKOx5+h0QiwmK7zjKPtIH+2VOMALuFNkx270a7jmYSbFJVJ5LerrvFGIqwIDW+bdUbZFhqAlhNh5tc5S+7GH17V/vze+fL2pfedenX77ev6nfv29Vco5+Cd9fK19fKt9+YLyqxvuoayXvJRgQkSptwlaepaJAjsMM28KIqC2Eyawyg5/805kW1lggd+ZkyWWaYECseJPf/T82MkoA7KIPRcfIpslwQ8tmI3tjKfGZUbJHiYrygJ5PxnCAcvfm+rugIyfrfMRw0rhW1Zd7RN/ZyJtuTdgg5gxRsFLhbUw0oPKz2sXAcrD+JiNnGKcKryVNIG4TmMWyNUDdDZufcx4Ywd+u5pnAk8ElKHosjdmNhxFIcuY2EURns4A3/JhsOLA9PYBB/MC50PKjkwETT9aKDa6ZQCo3vweTrgM79vZpbj5WFqTUMadUhxDD79maYHnx58rgefVSdNTEElU4ixnRiVZKza0SB623qTzSlnO9KglpM5bSozymRrWjXoUCmb1bOZYwZ3Qoz8FIFbcVZjrGdqVqbAgNiB+LhQ7dzpKQhI6HKH0CDIiJ2ljLkOjzPL2T891Vi9CUOPWk8ItSZK6ftFfNuWY1nPQ3oGtfojU49aPWpdjVqfPTrAxEKbgG6+JBz88r//D94iVb6W4wPsOH0i8lIXwj2LpB6Ex17scztFKcfuHnasOur6ONTjxxPCD+Vk97f3AQu1JSZjPT2BH/7IcYMeP3r86PHjWvzY9cnNicVGlRqvdZ4f3uimXbtx1XzW7mOt0nJqMvhgugeZiGMu4e9j2Rj97BFrH5PQtE5iUhDiPBM6DloCE8TJ4tTlQRp5qKS988xe4fSQ9IQgqaqXE2RZE+Tav7tP85OQFEQ9JPWQ1EPS9bdAXf/ctLbO9V1jE3vNwnCV3TdlJpgfhzByjU9DVcOlz4dU6yZJONWUmI/bIwJZL5Rsv4NTRHfvB1U6T1helCgeyDdPMltbsdKXaLEw6f/ddiZy8YFuXswgiJVUQQsMG/4LNVV9T1OVJFBCmKy0wQZk38q+jhUTz/BdNupAeVlc0v1bOrcua2R3Wtsn0aag8FuJ+OBwdTPs8nU7Ck5ArFiYEn9osXtgcXe+27O4Lc9anNLmosVm/NjiEu32Oqedj9gF9VvNSi871YQgGZFTMEERVaIBNaTDmJtBEI3sGD8AhsjxHDuuF38dnOJf7TluBhYYLuRvl63Ggr1FYlf1YJHdnuXSIk3NEVrX5xPyiOPPefU9lvglNfLPKTzi+HMrXIs7q+ey+MNEPRKfk9W+A9SsROqv/sBZ5bHGDe/AsHS/Or4/4NCV3A7dGykPXHRYbIciUsnNdfC6ZnM0z65nmO3RocUPBWOXdSAYS3xMu/jQOreJT3HVjvuIGG/skJRRygRzvDgwG6Incynxl9UOCH3rb2vP7+FXv7s/3t1nuWqDOfazy1DPZq0pmhO7+xgq+919v7vvd/dP9sJiQdFSmr+vv84a4R88VncXl+8xgjg+iTZhhGuM2PFIikIiLp7jLLSF45sb0eMXu8xWdpAJzfLHe5/RQ8+10MNEOrsNdDtO51447869R9AT+/0bXj309NBzNfSYMHTvcpmLCNSQabGGdVeV5kb5sweoPazxXP/Tz78Ca5NvR7g4AAA=","output":[{"uuid":"018a981f-9db5-7188-8161-91e9fd602fd7","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9db5-7188-8161-91e9fd602fd7\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"c5yz9qfwa86vhxab\", \"$time\": 1694769323.445, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"PersonsNode\"}, \"duration\": 102, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2945}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a25d-743f-a813-6d909390f5c9","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a25d-743f-a813-6d909390f5c9\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"i100qaub5hceuld4\", \"$time\": 1694769324.637, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"cs-dashboards\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1753}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a264-7a2a-9439-198973cc7878","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a264-7a2a-9439-198973cc7878\", \"event\": \"recording viewed with no playtime summary\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"wzrv024h7b0m7a8c\", \"$time\": 1694769324.645, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"viewed_time_ms\": 1295, \"play_time_ms\": 0, \"recording_duration_ms\": 0, \"rrweb_warning_count\": 0, \"error_count_during_recording_playback\": 0, \"engagement_score\": 0, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1745}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a266-73d2-a66f-1fbcc32d9f02","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a266-73d2-a66f-1fbcc32d9f02\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ksstzx9julgopw7a\", \"$time\": 1694769324.647, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Persons \\u2022 PostHog\"}, \"offset\": 1743}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a4b3-7b40-b430-9495d1bbed93","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a4b3-7b40-b430-9495d1bbed93\", \"event\": \"person viewed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"s2fzjz6c7t0ekgtm\", \"$time\": 1694769325.236, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"properties_count\": 18, \"has_email\": true, \"has_name\": false, \"custom_properties_count\": 4, \"posthog_properties_count\": 14, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1154}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a676-7722-bece-2f9b3d6b84ce","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a676-7722-bece-2f9b3d6b84ce\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"npyk617r6ht5qzbh\", \"$time\": 1694769325.686, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"attr__data-attr\": \"person-session-recordings-tab\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Recordings\"}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 68.19999694824219px; --lemon-tabs-slider-offset: 0px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 704}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a67b-7a6f-9637-bcaacec2496c","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a67b-7a6f-9637-bcaacec2496c\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"fhsu6w5edy7tvvuy\", \"$time\": 1694769325.691, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 699}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a783-7924-b938-3a789f71e25a","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a783-7924-b938-3a789f71e25a\", \"event\": \"recording list fetched\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"fcebvj6tugbw47wk\", \"$time\": 1694769325.955, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"load_time\": 145, \"listing_version\": \"3\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 435}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?compression=gzip-js&ip=1&_=1694769329412&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:29.500667+00:00","body":"H4sIAAAAAAAAA+0Ya2/bNvCvGF4/bEPo6EVSyjBgbdq0HdauaZp2Q1EIFElZqmlRkehXiwL7Lftp+yU7yo7jV2KkHYoikz/Y1t3xXryX7u3H7miUi+5R13FDFoVuihhLMaJp6qAoTQUKUyzdNMTUpWn3oCvHsjBAfo+NjOasNKNKArisdCkrk8u6e/Sxe0/DT/cZ453fzzp/ABoA8VhWda4LQLhOz8U9x8KTSk9qWQHwJK9kqqcWKOQ45zI2s1IC4qGsB0aXFsFHVQXi41GlAJEZUx4dHipQQ2W6Nkeh4ziHoEati8OlOUHiCURDJlEU0AQlUiSMOR5z0uA7xk0+lq9Y8nMta6vcS8l1JfKiX1txlinIWRdgESUzWcGGVrvbyFsx+MobrktXwIoV/RHrW86yQOdn9kjNKymLOJN5PwN9XAd7V9BJLkwGQOI5ABznclLqyiyJI9ddBV9S4yAEsMoTkDORiZUCD6tX1KNhD1t4XoBeJp7HiB6LQTRVxThKs5QNLN7k1g8uAWtJ5HukF/pgkMhrkxd8ca58c3/w9NXs1D9+Tn3fO784nRZPSOmfvA5f/ypo+ScJ6CPxfDB9w1fufz0s9/h1ZL33GcLyOhZyqGMI4PeSg8dSpmoJDPuVHpVNNC9Rl8pIlIaJhyAYHCSwFCjyQ0dKzJgIrC911WdF/oGZuS+vTmHM56dYiAXyMHexxBgLLKwmRW1Ywe3V74zs7ifQaiXrYvAxS5QUMZgONxfXuYDDl/rPYztOJWuIU8UgqI/evgPUKiwu2UxpJqylIKCSTA2tBiAVDOMq54NMg3Nt6g9Zrhph9nbYGJ6s/KXIWjE+uAGfsTpOcqUgveISAn2JgBTikNYQ9vGohu8eeFyMIHhYwdTM5ByUg+heZv8NNFusFokdVxJEznbw2UGwxWTDiVs8tvH3oJhJILGV7Z6A0gbxs4RaBwgN3rTRsYJtautl2Wtc3xQ9uVIsLJWSQ6Czl/mxC0rGi0pUl+DSgy4zpopjwQxD9q9NiKZC1aiSitlLbbREhtlQLUwW8yxXkDjA2z7pdKHBXFZs5NSG/sncxM7cxE8Ha6JFPgZeXLEavAl6dX+DlCqgrtZxDHIQ14WxbQOCb65eQwoHr6G7Wa8N4Sq/SfY+mUuPVdqGKjC+ArEqZ6gGf3NwG+CacF1igTIvhJzaDF/XGG9ojDc0HqlrNU5YdaPGFr+tsYJqezunXX9jN4hfSq7NrBGNEMSiLmws1ahWUH8q1LSYow4Neh6Bj4cjSjD8lNOfOrvodZrW0hx1XEp6XhRFIaWRF5GQ2hM3xrPlsm51sGF1sM9qm4OIleX1EbpF8WVuPgObHzDL/zqBWxRfZuGC3bVybmbv72OfofkcAtBUQS7Mf8BZalviJS3UEDltvhq6z3DonG/T7SutN1zk72GRaDG7SuIMimlTROy4tKnxOmOYujYYg4lGD8B6CMqMxxf985MSTz54D94fBy/rk/fPHpen/ovHD6F6k1Pn+LFz/CJ4fp9x52kz1y06z9acIxFNhI+SxHcQIS5N0iAMQxLZQxOoOXqy90zoOqkUBKd2apinGBiAqWedsWPiZ4hGMkQMRKFAklAmJOC+szbxl9Dq7CR5B8b9RcM+aZpZO+lvT/ozf0qJZGR4cZF7Fw7bPekHoF476beTfjvp327S/+abBqiYm2bEmzIoKdUvJURJpvs9roedf/76u/NiPoPN/wPuie6vdxpCd3aa1IfUjkQAWR0wJKgXcQqFxhG2wtzR3VLbbPY0G2UmtFJBmuVB0r8g/o5mQ3uRD8Np22zaZtM2m6+6Vvq6u521ndPmmunb2/RsvhI2r1f/s01PCHsbHyqi5wY+xgGJ9m16fEJ7Dmx6KMYhjvx20dMuetpFz3+66HED2uyct8fvgMCw43soYiRCHBNCscvtlvaOLnoam9qpe9fUTWbKxxNhxnjsupo0I/L21N2ueNqpu5262xXP1orHDXaveBLHCRH1KQOZAchkhIVuwjwvsKl92WMuRrKadUBUqaQdudtOc4c7TdgfTcbD0Jtlk1mWarKj04Q91ydtp2k7TdtpbttpmlJqA24ALQAwj5pCdNpAwbvNUsO+wf24LL+NqlB97BueZkrWXH5/VX57q0XyoLOKWBQPW8N+gFf8znmlOoeds8sXw1VSWyRsixlKCM9haV8S50Kfrqcnj9g8tkmYwqNLAwBTjhNbRVhqGlcgL8hsHItRtcgOl0KB+uZ77Gq79Kj/6d2/ToySO28rAAA=","output":[{"uuid":"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0ovdk9xlnv9fhfak\", \"$time\": 1694769326.837, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"attr__data-attr\": \"persons-related-flags-tab\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Feature flags\"}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 74.26666259765625px; --lemon-tabs-slider-offset: 176.29998779296875px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2572}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-aafa-79e8-abf4-4e68eb64c30f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aafa-79e8-abf4-4e68eb64c30f\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"y3x76ea6mqqi2q0a\", \"$time\": 1694769326.842, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 2567}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af3d-79d4-be4a-d729c776e0da","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af3d-79d4-be4a-d729c776e0da\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ltw7rl4fhi4bgq63\", \"$time\": 1694769327.934, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"\"}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 86.23332214355469px; --lemon-tabs-slider-offset: 367.0999755859375px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1475}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af46-7832-9a69-c566751c6625","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af46-7832-9a69-c566751c6625\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"6yl35wdtv5v11o6c\", \"$time\": 1694769327.942, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 1467}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-b008-737a-bb40-6a6a81ba2244","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-b008-737a-bb40-6a6a81ba2244\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"8guwvm82yhwyhfo6\", \"$time\": 1694769328.136, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"EventsQuery\", \"select\": [\"*\", \"event\", \"person\", \"coalesce(properties.$current_url, properties.$screen_name) -- Url / Screen\", \"properties.$lib\", \"timestamp\"], \"personId\": \"018a981f-4c9a-0000-68ff-417481f7c5b5\", \"after\": \"-24h\"}, \"duration\": 171, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1273}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} From 9ad1d643c117d886641bef37fb2d0e28704136af Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Fri, 15 Sep 2023 13:00:02 +0200 Subject: [PATCH 031/247] drop the unused site_url field (#30) --- capture/src/capture.rs | 1 - capture/src/event.rs | 1 - capture/tests/django_compat.rs | 5 ++++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/capture/src/capture.rs b/capture/src/capture.rs index d2f106fb9a657..d9831882550be 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -94,7 +94,6 @@ pub fn process_single_event( uuid: event.uuid.unwrap_or_else(uuid_v7), distinct_id: distinct_id.to_string(), ip: context.client_ip.clone(), - site_url: String::new(), data: String::from("hallo I am some data 😊"), now: context.now.clone(), sent_at: context.sent_at, diff --git a/capture/src/event.rs b/capture/src/event.rs index 5b0c08d1c2798..45a1bf128c15c 100644 --- a/capture/src/event.rs +++ b/capture/src/event.rs @@ -121,7 +121,6 @@ pub struct ProcessedEvent { pub uuid: Uuid, pub distinct_id: String, pub ip: String, - pub site_url: String, pub data: String, pub now: String, #[serde(with = "time::serde::rfc3339::option")] diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index c39f2c7a91dfa..41a024286651c 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -140,7 +140,10 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { OffsetDateTime::parse(value.as_str().expect("empty"), &Iso8601::DEFAULT)?; *value = Value::String(sent_at.format(&Rfc3339)?) } - + if let Some(object) = expected.as_object_mut() { + // site_url is unused in the pipeline now, let's drop it + object.remove("site_url"); + } let match_config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict); if let Err(e) = assert_json_matches_no_panic(&json!(expected), &json!(message), match_config) From 7e74df8b2ba3919f27e799f5259926bfb8adcf97 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 9 Oct 2023 19:01:59 +0200 Subject: [PATCH 032/247] fill the data field in analytics events (#31) * fill the data field in analytics events * fix todos --- capture/src/api.rs | 3 +++ capture/src/capture.rs | 26 +++++++++++++++++++++++++- capture/src/event.rs | 20 +++++++++++++++++--- capture/tests/django_compat.rs | 26 ++++++++++++++++++++++++-- 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/capture/src/api.rs b/capture/src/api.rs index 9a18a89c4c71c..319056c993fd2 100644 --- a/capture/src/api.rs +++ b/capture/src/api.rs @@ -34,6 +34,8 @@ pub enum CaptureError { #[error("request holds no event")] EmptyBatch, + #[error("event submitted with an empty event name")] + MissingEventName, #[error("event submitted without a distinct_id")] MissingDistinctId, @@ -58,6 +60,7 @@ impl IntoResponse for CaptureError { CaptureError::RequestDecodingError(_) | CaptureError::RequestParsingError(_) | CaptureError::EmptyBatch + | CaptureError::MissingEventName | CaptureError::MissingDistinctId | CaptureError::EventTooBig | CaptureError::NonRetryableSinkError => (StatusCode::BAD_REQUEST, self.to_string()), diff --git a/capture/src/capture.rs b/capture/src/capture.rs index d9831882550be..98a61d3078913 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -89,12 +89,20 @@ pub fn process_single_event( _ => return Err(CaptureError::MissingDistinctId), }, }; + if event.event.is_empty() { + return Err(CaptureError::MissingEventName); + } + + let data = serde_json::to_string(&event).map_err(|e| { + tracing::error!("failed to encode data field: {}", e); + CaptureError::NonRetryableSinkError + })?; Ok(ProcessedEvent { uuid: event.uuid.unwrap_or_else(uuid_v7), distinct_id: distinct_id.to_string(), ip: context.client_ip.clone(), - data: String::from("hallo I am some data 😊"), + data, now: context.now.clone(), sent_at: context.sent_at, token: context.token.clone(), @@ -158,6 +166,10 @@ mod tests { uuid: None, event: String::new(), properties: HashMap::new(), + timestamp: None, + offset: None, + set: Default::default(), + set_once: Default::default(), }, RawEvent { token: None, @@ -165,6 +177,10 @@ mod tests { uuid: None, event: String::new(), properties: HashMap::from([(String::from("token"), json!("hello"))]), + timestamp: None, + offset: None, + set: Default::default(), + set_once: Default::default(), }, ]; @@ -181,6 +197,10 @@ mod tests { uuid: None, event: String::new(), properties: HashMap::new(), + timestamp: None, + offset: None, + set: Default::default(), + set_once: Default::default(), }, RawEvent { token: None, @@ -188,6 +208,10 @@ mod tests { uuid: None, event: String::new(), properties: HashMap::from([(String::from("token"), json!("goodbye"))]), + timestamp: None, + offset: None, + set: Default::default(), + set_once: Default::default(), }, ]; diff --git a/capture/src/event.rs b/capture/src/event.rs index 45a1bf128c15c..81eb754fd5bd3 100644 --- a/capture/src/event.rs +++ b/capture/src/event.rs @@ -37,12 +37,26 @@ pub struct EventFormData { #[derive(Default, Debug, Deserialize, Serialize)] pub struct RawEvent { - #[serde(alias = "$token", alias = "api_key")] + #[serde( + alias = "$token", + alias = "api_key", + skip_serializing_if = "Option::is_none" + )] pub token: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub distinct_id: Option, pub uuid: Option, pub event: String, + #[serde(default)] pub properties: HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub timestamp: Option, // Passed through if provided, parsed by ingestion + #[serde(skip_serializing_if = "Option::is_none")] + pub offset: Option, // Passed through if provided, parsed by ingestion + #[serde(rename = "$set", skip_serializing_if = "Option::is_none")] + pub set: Option>, + #[serde(rename = "$set_once", skip_serializing_if = "Option::is_none")] + pub set_once: Option>, } #[derive(Deserialize)] @@ -51,14 +65,14 @@ enum RawRequest { /// Batch of events Batch(Vec), /// Single event - One(RawEvent), + One(Box), } impl RawRequest { pub fn events(self) -> Vec { match self { RawRequest::Batch(events) => events, - RawRequest::One(event) => vec![event], + RawRequest::One(event) => vec![*event], } } } diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index 41a024286651c..119777d238de6 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -71,7 +71,6 @@ impl EventSink for MemorySink { } #[tokio::test] -#[ignore] async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let file = File::open(REQUESTS_DUMP_FILE_NAME)?; let reader = BufReader::new(file); @@ -107,6 +106,7 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { if !case.ip.is_empty() { req = req.header("X-Forwarded-For", case.ip); } + let res = req.send().await; assert_eq!( res.status(), @@ -140,16 +140,38 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { OffsetDateTime::parse(value.as_str().expect("empty"), &Iso8601::DEFAULT)?; *value = Value::String(sent_at.format(&Rfc3339)?) } + if let Some(expected_data) = expected.get_mut("data") { + // Data is a serialized JSON map. Unmarshall both and compare them, + // instead of expecting the serialized bytes to be equal + let expected_props: Value = + serde_json::from_str(expected_data.as_str().expect("not str"))?; + let found_props: Value = serde_json::from_str(&message.data)?; + let match_config = + assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict); + if let Err(e) = + assert_json_matches_no_panic(&expected_props, &found_props, match_config) + { + println!( + "data field mismatch at line {}, event {}: {}", + line_number, event_number, e + ); + mismatches += 1; + } else { + *expected_data = json!(&message.data) + } + } + if let Some(object) = expected.as_object_mut() { // site_url is unused in the pipeline now, let's drop it object.remove("site_url"); } + let match_config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict); if let Err(e) = assert_json_matches_no_panic(&json!(expected), &json!(message), match_config) { println!( - "mismatch at line {}, event {}: {}", + "record mismatch at line {}, event {}: {}", line_number, event_number, e ); mismatches += 1; From c03638b18e1c9c088c9cc4f49919cf47e624044a Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Thu, 19 Oct 2023 17:21:49 +0100 Subject: [PATCH 033/247] Add billing limiter (#33) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Redis lib * `cargo update` * fmt * Add base implementation of billing limiter Supports 1. A fixed set of limits, with no redis update 2. A fixed set, subsequently updated from redis 3. No fixed set, updates from redis I still need to figure out how to nicely mock the redis connection that stll leaves enough not mocked to be worth testing. I really don't want integration tests on it :( Also still needs connecting to the API. Reading through the python for this is like 😵‍💫 * Rework I've reworked it a bunch. Honestly the background loop worked but it became really horrible and the locking behaviour a little sketchy. While this will slow down some requests a bit, unless it becomes measurably slow let's keep it that way rather than introducing a bit of a horrible pattern. * hook it all up * Add redis read timeout * Add non-cluster client * Respond to feedback --- Cargo.lock | 257 +++++++++++++++++++++++---------- capture-server/Cargo.toml | 1 + capture-server/src/main.rs | 27 +++- capture/Cargo.toml | 2 + capture/src/api.rs | 12 ++ capture/src/billing_limits.rs | 188 ++++++++++++++++++++++++ capture/src/capture.rs | 24 ++- capture/src/lib.rs | 2 + capture/src/redis.rs | 80 ++++++++++ capture/src/router.rs | 9 +- capture/tests/django_compat.rs | 11 +- 11 files changed, 527 insertions(+), 86 deletions(-) create mode 100644 capture/src/billing_limits.rs create mode 100644 capture/src/redis.rs diff --git a/Cargo.lock b/Cargo.lock index 6ef5c6c66b662..7ea2f9d255934 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.5" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -61,7 +61,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.38", ] [[package]] @@ -104,9 +104,9 @@ dependencies = [ [[package]] name = "axum-client-ip" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8e81eacc93f36480825da5f46a33b5fb2246ed024eacc9e8933425b80c5807" +checksum = "1ef117890a418b7832678d9ea1e1c08456dd7b2fd1dadb9676cd6f0fe7eb4b21" dependencies = [ "axum", "forwarded-header-value", @@ -165,9 +165,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.3" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bitflags" @@ -183,15 +183,15 @@ checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "capture" @@ -212,6 +212,8 @@ dependencies = [ "mockall", "rand", "rdkafka", + "redis", + "redis-test", "serde", "serde_json", "serde_urlencoded", @@ -231,6 +233,7 @@ version = "0.1.0" dependencies = [ "axum", "capture", + "time", "tokio", "tracing", "tracing-subscriber", @@ -260,6 +263,42 @@ dependencies = [ "cc", ] +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "crc16" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" + [[package]] name = "crc32fast" version = "1.3.2" @@ -298,7 +337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "lock_api", "once_cell", "parking_lot_core", @@ -452,7 +491,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.38", ] [[package]] @@ -562,15 +601,15 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "http" @@ -658,12 +697,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.1", ] [[package]] @@ -704,9 +743,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libz-sys" @@ -756,15 +795,15 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -812,7 +851,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.38", ] [[package]] @@ -929,9 +968,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] @@ -1034,7 +1073,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.38", ] [[package]] @@ -1109,9 +1148,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -1227,6 +1266,40 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "redis" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba" +dependencies = [ + "async-trait", + "bytes", + "combine", + "crc16", + "futures", + "futures-util", + "itoa", + "log", + "percent-encoding", + "pin-project-lite", + "rand", + "ryu", + "sha1_smol", + "socket2 0.4.9", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "redis-test" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aba266ca48ae66978bf439fd2ac0d7a36a8635823754e2bc73afaf9d2fc25272" +dependencies = [ + "redis", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -1238,9 +1311,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" dependencies = [ "aho-corasick", "memchr", @@ -1250,9 +1323,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "5d58da636bd923eae52b7e9120271cbefb16f399069ee566ca5ebf9c30e32238" dependencies = [ "aho-corasick", "memchr", @@ -1261,15 +1334,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c3cbb081b9784b07cceb8824c8583f86db4814d172ab043f3c23f7dc600bf83d" [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "base64", "bytes", @@ -1291,6 +1364,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-util", "tower-service", @@ -1343,14 +1417,14 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.38", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -1379,11 +1453,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -1414,9 +1494,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "socket2" @@ -1430,9 +1510,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys", @@ -1451,9 +1531,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -1466,6 +1546,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "termtree" version = "0.4.1" @@ -1474,22 +1575,22 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.38", ] [[package]] @@ -1504,9 +1605,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ "deranged", "itoa", @@ -1517,15 +1618,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -1547,9 +1648,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -1559,7 +1660,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.3", + "socket2 0.5.4", "tokio-macros", "windows-sys", ] @@ -1572,14 +1673,14 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.38", ] [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -1597,11 +1698,11 @@ checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" [[package]] name = "toml_edit" -version = "0.19.14" +version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.0.2", "toml_datetime", "winnow", ] @@ -1694,7 +1795,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.38", ] [[package]] @@ -1755,9 +1856,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -1848,7 +1949,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.38", "wasm-bindgen-shared", ] @@ -1882,7 +1983,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2006,9 +2107,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" dependencies = [ "memchr", ] diff --git a/capture-server/Cargo.toml b/capture-server/Cargo.toml index 04c618286c4cb..6378532482883 100644 --- a/capture-server/Cargo.toml +++ b/capture-server/Cargo.toml @@ -9,3 +9,4 @@ axum = { workspace = true } tokio = { workspace = true } tracing-subscriber = { workspace = true } tracing = { workspace = true } +time = { workspace = true } diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index e2232d5023655..9d60f89258e50 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -1,7 +1,9 @@ use std::env; use std::net::SocketAddr; +use std::sync::Arc; -use capture::{router, sink, time}; +use capture::{billing_limits::BillingLimiter, redis::RedisClient, router, sink}; +use time::Duration; use tokio::signal; async fn shutdown() { @@ -23,16 +25,35 @@ async fn shutdown() { async fn main() { let use_print_sink = env::var("PRINT_SINK").is_ok(); let address = env::var("ADDRESS").unwrap_or(String::from("127.0.0.1:3000")); + let redis_addr = env::var("REDIS").expect("redis required; please set the REDIS env var"); + + let redis_client = + Arc::new(RedisClient::new(redis_addr).expect("failed to create redis client")); + + let billing = BillingLimiter::new(Duration::seconds(5), redis_client.clone()) + .expect("failed to create billing limiter"); let app = if use_print_sink { - router::router(time::SystemTime {}, sink::PrintSink {}, true) + router::router( + capture::time::SystemTime {}, + sink::PrintSink {}, + redis_client, + billing, + true, + ) } else { let brokers = env::var("KAFKA_BROKERS").expect("Expected KAFKA_BROKERS"); let topic = env::var("KAFKA_TOPIC").expect("Expected KAFKA_TOPIC"); let sink = sink::KafkaSink::new(topic, brokers).unwrap(); - router::router(time::SystemTime {}, sink, true) + router::router( + capture::time::SystemTime {}, + sink, + redis_client, + billing, + true, + ) }; // initialize tracing diff --git a/capture/Cargo.toml b/capture/Cargo.toml index 60cca70824cff..7c84c6ce111ef 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -29,8 +29,10 @@ rdkafka = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } thiserror = { workspace = true } +redis = { version="0.23.3", features=["tokio-comp", "cluster", "cluster-async"] } [dev-dependencies] assert-json-diff = "2.0.2" axum-test-helper = "0.2.0" mockall = "0.11.2" +redis-test = "0.2.3" diff --git a/capture/src/api.rs b/capture/src/api.rs index 319056c993fd2..ff245b5072ac1 100644 --- a/capture/src/api.rs +++ b/capture/src/api.rs @@ -52,6 +52,12 @@ pub enum CaptureError { EventTooBig, #[error("invalid event could not be processed")] NonRetryableSinkError, + + #[error("billing limit reached")] + BillingLimit, + + #[error("rate limited")] + RateLimited, } impl IntoResponse for CaptureError { @@ -64,10 +70,16 @@ impl IntoResponse for CaptureError { | CaptureError::MissingDistinctId | CaptureError::EventTooBig | CaptureError::NonRetryableSinkError => (StatusCode::BAD_REQUEST, self.to_string()), + CaptureError::NoTokenError | CaptureError::MultipleTokensError | CaptureError::TokenValidationError(_) => (StatusCode::UNAUTHORIZED, self.to_string()), + CaptureError::RetryableSinkError => (StatusCode::SERVICE_UNAVAILABLE, self.to_string()), + + CaptureError::BillingLimit | CaptureError::RateLimited => { + (StatusCode::TOO_MANY_REQUESTS, self.to_string()) + } } .into_response() } diff --git a/capture/src/billing_limits.rs b/capture/src/billing_limits.rs new file mode 100644 index 0000000000000..44a997c0b4e4d --- /dev/null +++ b/capture/src/billing_limits.rs @@ -0,0 +1,188 @@ +use std::{collections::HashSet, ops::Sub, sync::Arc}; + +use crate::redis::Client; + +/// Limit accounts by team ID if they hit a billing limit +/// +/// We have an async celery worker that regularly checks on accounts + assesses if they are beyond +/// a billing limit. If this is the case, a key is set in redis. +/// +/// Requirements +/// +/// 1. Updates from the celery worker should be reflected in capture within a short period of time +/// 2. Capture should cope with redis being _totally down_, and fail open +/// 3. We should not hit redis for every single request +/// +/// The solution here is to read from the cache until a time interval is hit, and then fetch new +/// data. The write requires taking a lock that stalls all readers, though so long as redis reads +/// stay fast we're ok. +/// +/// Some small delay between an account being limited and the limit taking effect is acceptable. +/// However, ideally we should not allow requests from some pods but 429 from others. +use thiserror::Error; +use time::{Duration, OffsetDateTime}; +use tokio::sync::RwLock; + +// todo: fetch from env +const QUOTA_LIMITER_CACHE_KEY: &str = "@posthog/quota-limits/"; + +pub enum QuotaResource { + Events, + Recordings, +} + +impl QuotaResource { + fn as_str(&self) -> &'static str { + match self { + Self::Events => "events", + Self::Recordings => "recordings", + } + } +} + +#[derive(Error, Debug)] +pub enum LimiterError { + #[error("updater already running - there can only be one")] + UpdaterRunning, +} + +#[derive(Clone)] +pub struct BillingLimiter { + limited: Arc>>, + redis: Arc, + interval: Duration, + updated: Arc>, +} + +impl BillingLimiter { + /// Create a new BillingLimiter. + /// + /// This connects to a redis cluster - pass in a vec of addresses for the initial nodes. + /// + /// You can also initialize the limiter with a set of tokens to limit from the very beginning. + /// This may be overridden by Redis, if the sets differ, + /// + /// Pass an empty redis node list to only use this initial set. + pub fn new( + interval: Duration, + redis: Arc, + ) -> anyhow::Result { + let limited = Arc::new(RwLock::new(HashSet::new())); + + // Force an update immediately if we have any reasonable interval + let updated = OffsetDateTime::from_unix_timestamp(0)?; + let updated = Arc::new(RwLock::new(updated)); + + Ok(BillingLimiter { + interval, + limited, + updated, + redis, + }) + } + + async fn fetch_limited( + client: &Arc, + resource: QuotaResource, + ) -> anyhow::Result> { + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + + client + .zrangebyscore( + format!("{QUOTA_LIMITER_CACHE_KEY}{}", resource.as_str()), + now.to_string(), + String::from("+Inf"), + ) + .await + } + + pub async fn is_limited(&self, key: &str, resource: QuotaResource) -> bool { + // hold the read lock to clone it, very briefly. clone is ok because it's very small 🤏 + // rwlock can have many readers, but one writer. the writer will wait in a queue with all + // the readers, so we want to hold read locks for the smallest time possible to avoid + // writers waiting for too long. and vice versa. + let updated = { + let updated = self.updated.read().await; + *updated + }; + + let now = OffsetDateTime::now_utc(); + let since_update = now.sub(updated); + + // If an update is due, fetch the set from redis + cache it until the next update is due. + // Otherwise, return a value from the cache + // + // This update will block readers! Keep it fast. + if since_update > self.interval { + let span = tracing::debug_span!("updating billing cache from redis"); + let _span = span.enter(); + + // a few requests might end up in here concurrently, but I don't think a few extra will + // be a big problem. If it is, we can rework the concurrency a bit. + // On prod atm we call this around 15 times per second at peak times, and it usually + // completes in <1ms. + + let set = Self::fetch_limited(&self.redis, resource).await; + + tracing::debug!("fetched set from redis, caching"); + + if let Ok(set) = set { + let set = HashSet::from_iter(set.iter().cloned()); + + let mut limited = self.limited.write().await; + *limited = set; + + tracing::debug!("updated cache from redis"); + + limited.contains(key) + } else { + tracing::error!("failed to fetch from redis in time, failing open"); + // If we fail to fetch the set, something really wrong is happening. To avoid + // dropping events that we don't mean to drop, fail open and accept data. Better + // than angry customers :) + // + // TODO: Consider backing off our redis checks + false + } + } else { + let l = self.limited.read().await; + + l.contains(key) + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use time::Duration; + + use crate::{ + billing_limits::{BillingLimiter, QuotaResource}, + redis::MockRedisClient, + }; + + #[tokio::test] + async fn test_dynamic_limited() { + let client = MockRedisClient::new().zrangebyscore_ret(vec![String::from("banana")]); + let client = Arc::new(client); + + let limiter = BillingLimiter::new(Duration::microseconds(1), client) + .expect("Failed to create billing limiter"); + + assert_eq!( + limiter + .is_limited("idk it doesn't matter", QuotaResource::Events) + .await, + false + ); + + assert_eq!( + limiter + .is_limited("some_org_hit_limits", QuotaResource::Events) + .await, + false + ); + assert!(limiter.is_limited("banana", QuotaResource::Events).await); + } +} diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 98a61d3078913..65e64c9135b4b 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -12,6 +12,7 @@ use axum_client_ip::InsecureClientIp; use base64::Engine; use time::OffsetDateTime; +use crate::billing_limits::QuotaResource; use crate::event::ProcessingContext; use crate::token::validate_token; use crate::{ @@ -44,7 +45,7 @@ pub async fn event( _ => RawEvent::from_bytes(&meta, body), }?; - println!("Got events {:?}", &events); + tracing::debug!("got events {:?}", &events); if events.is_empty() { return Err(CaptureError::EmptyBatch); @@ -61,6 +62,7 @@ pub async fn event( } None }); + let context = ProcessingContext { lib_version: meta.lib_version.clone(), sent_at, @@ -69,7 +71,25 @@ pub async fn event( client_ip: ip.to_string(), }; - println!("Got context {:?}", &context); + let limited = state + .billing + .is_limited(context.token.as_str(), QuotaResource::Events) + .await; + + if limited { + // for v0 we want to just return ok 🙃 + // this is because the clients are pretty dumb and will just retry over and over and + // over... + // + // for v1, we'll return a meaningful error code and error, so that the clients can do + // something meaningful with that error + + return Ok(Json(CaptureResponse { + status: CaptureResponseCode::Ok, + })); + } + + tracing::debug!("got context {:?}", &context); process_events(state.sink.clone(), &events, &context).await?; diff --git a/capture/src/lib.rs b/capture/src/lib.rs index d4ca041e5e671..fcd802b43f7cf 100644 --- a/capture/src/lib.rs +++ b/capture/src/lib.rs @@ -1,7 +1,9 @@ pub mod api; +pub mod billing_limits; pub mod capture; pub mod event; pub mod prometheus; +pub mod redis; pub mod router; pub mod sink; pub mod time; diff --git a/capture/src/redis.rs b/capture/src/redis.rs new file mode 100644 index 0000000000000..c83c0ad89a8ac --- /dev/null +++ b/capture/src/redis.rs @@ -0,0 +1,80 @@ +use std::time::Duration; + +use anyhow::Result; +use async_trait::async_trait; +use redis::AsyncCommands; +use tokio::time::timeout; + +// average for all commands is <10ms, check grafana +const REDIS_TIMEOUT_MILLISECS: u64 = 10; + +/// A simple redis wrapper +/// I'm currently just exposing the commands we use, for ease of implementation +/// Allows for testing + injecting failures +/// We can also swap it out for alternative implementations in the future +/// I tried using redis-rs Connection/ConnectionLike traits but honestly things just got really +/// awkward to work with. + +#[async_trait] +pub trait Client { + // A very simplified wrapper, but works for our usage + async fn zrangebyscore(&self, k: String, min: String, max: String) -> Result>; +} + +pub struct RedisClient { + client: redis::Client, +} + +impl RedisClient { + pub fn new(addr: String) -> Result { + let client = redis::Client::open(addr)?; + + Ok(RedisClient { client }) + } +} + +#[async_trait] +impl Client for RedisClient { + async fn zrangebyscore(&self, k: String, min: String, max: String) -> Result> { + let mut conn = self.client.get_async_connection().await?; + + let results = conn.zrangebyscore(k, min, max); + let fut = timeout(Duration::from_secs(REDIS_TIMEOUT_MILLISECS), results).await?; + + Ok(fut?) + } +} + +// mockall got really annoying with async and results so I'm just gonna do my own +#[derive(Clone)] +pub struct MockRedisClient { + zrangebyscore_ret: Vec, +} + +impl MockRedisClient { + pub fn new() -> MockRedisClient { + MockRedisClient { + zrangebyscore_ret: Vec::new(), + } + } + + pub fn zrangebyscore_ret(&mut self, ret: Vec) -> Self { + self.zrangebyscore_ret = ret; + + self.clone() + } +} + +impl Default for MockRedisClient { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Client for MockRedisClient { + // A very simplified wrapper, but works for our usage + async fn zrangebyscore(&self, _k: String, _min: String, _max: String) -> Result> { + Ok(self.zrangebyscore_ret.clone()) + } +} diff --git a/capture/src/router.rs b/capture/src/router.rs index 0c40658c04647..757b975c2948f 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -7,7 +7,7 @@ use axum::{ }; use tower_http::trace::TraceLayer; -use crate::{capture, sink, time::TimeSource}; +use crate::{billing_limits::BillingLimiter, capture, redis::Client, sink, time::TimeSource}; use crate::prometheus::{setup_metrics_recorder, track_metrics}; @@ -15,6 +15,8 @@ use crate::prometheus::{setup_metrics_recorder, track_metrics}; pub struct State { pub sink: Arc, pub timesource: Arc, + pub redis: Arc, + pub billing: BillingLimiter, } async fn index() -> &'static str { @@ -24,14 +26,19 @@ async fn index() -> &'static str { pub fn router< TZ: TimeSource + Send + Sync + 'static, S: sink::EventSink + Send + Sync + 'static, + R: Client + Send + Sync + 'static, >( timesource: TZ, sink: S, + redis: Arc, + billing: BillingLimiter, metrics: bool, ) -> Router { let state = State { sink: Arc::new(sink), timesource: Arc::new(timesource), + redis, + billing, }; let router = Router::new() diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index 119777d238de6..d418996d6b06a 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -5,7 +5,9 @@ use axum_test_helper::TestClient; use base64::engine::general_purpose; use base64::Engine; use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode}; +use capture::billing_limits::BillingLimiter; use capture::event::ProcessedEvent; +use capture::redis::MockRedisClient; use capture::router::router; use capture::sink::EventSink; use capture::time::TimeSource; @@ -15,7 +17,7 @@ use std::fs::File; use std::io::{BufRead, BufReader}; use std::sync::{Arc, Mutex}; use time::format_description::well_known::{Iso8601, Rfc3339}; -use time::OffsetDateTime; +use time::{Duration, OffsetDateTime}; #[derive(Debug, Deserialize)] struct RequestDump { @@ -93,7 +95,12 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let sink = MemorySink::default(); let timesource = FixedTime { time: case.now }; - let app = router(timesource, sink.clone(), false); + + let redis = Arc::new(MockRedisClient::new()); + let billing = BillingLimiter::new(Duration::weeks(1), redis.clone()) + .expect("failed to create billing limiter"); + + let app = router(timesource, sink.clone(), redis, billing, false); let client = TestClient::new(app); let mut req = client.post(&format!("/i/v0{}", case.path)).body(raw_body); From 30b49944fdb0c5fc0a37072735b6a2998213996f Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 23 Oct 2023 16:56:28 +0200 Subject: [PATCH 034/247] align envvars with plugin-server (#34) --- capture-server/src/main.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index 9d60f89258e50..1a78f22ca0e02 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -25,7 +25,8 @@ async fn shutdown() { async fn main() { let use_print_sink = env::var("PRINT_SINK").is_ok(); let address = env::var("ADDRESS").unwrap_or(String::from("127.0.0.1:3000")); - let redis_addr = env::var("REDIS").expect("redis required; please set the REDIS env var"); + let redis_addr = + env::var("REDIS_URL").expect("redis required; please set the REDIS_URL env var"); let redis_client = Arc::new(RedisClient::new(redis_addr).expect("failed to create redis client")); @@ -42,7 +43,7 @@ async fn main() { true, ) } else { - let brokers = env::var("KAFKA_BROKERS").expect("Expected KAFKA_BROKERS"); + let brokers = env::var("KAFKA_HOSTS").expect("Expected KAFKA_HOSTS"); let topic = env::var("KAFKA_TOPIC").expect("Expected KAFKA_TOPIC"); let sink = sink::KafkaSink::new(topic, brokers).unwrap(); From 9ceabe77c00d2cc5d8267e16515115b443358be6 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Thu, 26 Oct 2023 17:12:55 +0200 Subject: [PATCH 035/247] add more metrics (#35) --- capture/src/billing_limits.rs | 2 +- capture/src/capture.rs | 13 +++++++++++-- capture/src/prometheus.rs | 5 +++++ capture/src/sink.rs | 7 ++++--- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/capture/src/billing_limits.rs b/capture/src/billing_limits.rs index 44a997c0b4e4d..4309c2174dd09 100644 --- a/capture/src/billing_limits.rs +++ b/capture/src/billing_limits.rs @@ -51,7 +51,7 @@ pub struct BillingLimiter { limited: Arc>>, redis: Arc, interval: Duration, - updated: Arc>, + updated: Arc>, } impl BillingLimiter { diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 65e64c9135b4b..0413d44ea252a 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -10,6 +10,8 @@ use axum::extract::{Query, State}; use axum::http::HeaderMap; use axum_client_ip::InsecureClientIp; use base64::Engine; +use metrics::counter; + use time::OffsetDateTime; use crate::billing_limits::QuotaResource; @@ -50,7 +52,13 @@ pub async fn event( if events.is_empty() { return Err(CaptureError::EmptyBatch); } - let token = extract_and_verify_token(&events)?; + + let token = extract_and_verify_token(&events).map_err(|err| { + counter!("capture_token_shape_invalid_total", events.len() as u64); + err + })?; + + counter!("capture_events_received_total", events.len() as u64); let sent_at = meta.sent_at.and_then(|value| { let value_nanos: i128 = i128::from(value) * 1_000_000; // Assuming the value is in milliseconds, latest posthog-js releases @@ -77,13 +85,14 @@ pub async fn event( .await; if limited { + counter!("capture_events_dropped_over_quota", 1); + // for v0 we want to just return ok 🙃 // this is because the clients are pretty dumb and will just retry over and over and // over... // // for v1, we'll return a meaningful error code and error, so that the clients can do // something meaningful with that error - return Ok(Json(CaptureResponse { status: CaptureResponseCode::Ok, })); diff --git a/capture/src/prometheus.rs b/capture/src/prometheus.rs index 1fcdb7d7ca30b..0cb4750995b7c 100644 --- a/capture/src/prometheus.rs +++ b/capture/src/prometheus.rs @@ -10,6 +10,9 @@ pub fn setup_metrics_recorder() -> PrometheusHandle { const EXPONENTIAL_SECONDS: &[f64] = &[ 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, ]; + const BATCH_SIZES: &[f64] = &[ + 1.0, 10.0, 25.0, 50.0, 75.0, 100.0, 250.0, 500.0, 750.0, 1000.0, + ]; PrometheusBuilder::new() .set_buckets_for_metric( @@ -17,6 +20,8 @@ pub fn setup_metrics_recorder() -> PrometheusHandle { EXPONENTIAL_SECONDS, ) .unwrap() + .set_buckets_for_metric(Matcher::Suffix("_batch_size".to_string()), BATCH_SIZES) + .unwrap() .install_recorder() .unwrap() } diff --git a/capture/src/sink.rs b/capture/src/sink.rs index e6f4b7bb4e8ed..6f0cbefd4659b 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use metrics::{counter, histogram}; use tokio::task::JoinSet; use crate::api::CaptureError; @@ -20,8 +21,7 @@ pub struct PrintSink {} impl EventSink for PrintSink { async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { tracing::info!("single event: {:?}", event); - - metrics::increment_counter!("capture_events_total"); + counter!("capture_events_ingested_total", 1); Ok(()) } @@ -29,8 +29,9 @@ impl EventSink for PrintSink { let span = tracing::span!(tracing::Level::INFO, "batch of events"); let _enter = span.enter(); + histogram!("capture_event_batch_size", events.len() as f64); + counter!("capture_events_ingested_total", events.len() as u64); for event in events { - metrics::increment_counter!("capture_events_total"); tracing::info!("event: {:?}", event); } From c9e835eee2c2e81e3707320ad88afacde1c7c5bc Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Thu, 26 Oct 2023 17:57:04 +0200 Subject: [PATCH 036/247] add envconfig to parse envvars (#36) --- Cargo.lock | 21 +++++++++++++++++++++ capture-server/Cargo.toml | 1 + capture-server/src/main.rs | 33 +++++++++++++++++++-------------- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ea2f9d255934..2caeb20b2faba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -233,6 +233,7 @@ version = "0.1.0" dependencies = [ "axum", "capture", + "envconfig", "time", "tokio", "tracing", @@ -379,6 +380,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "envconfig" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" +dependencies = [ + "envconfig_derive", +] + +[[package]] +name = "envconfig_derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "equivalent" version = "1.0.1" diff --git a/capture-server/Cargo.toml b/capture-server/Cargo.toml index 6378532482883..fa7151ed497da 100644 --- a/capture-server/Cargo.toml +++ b/capture-server/Cargo.toml @@ -10,3 +10,4 @@ tokio = { workspace = true } tracing-subscriber = { workspace = true } tracing = { workspace = true } time = { workspace = true } +envconfig = "0.10.0" diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index 1a78f22ca0e02..46d8e28496cc7 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -1,4 +1,4 @@ -use std::env; +use envconfig::Envconfig; use std::net::SocketAddr; use std::sync::Arc; @@ -6,6 +6,17 @@ use capture::{billing_limits::BillingLimiter, redis::RedisClient, router, sink}; use time::Duration; use tokio::signal; +#[derive(Envconfig)] +struct Config { + #[envconfig(default = "false")] + print_sink: bool, + #[envconfig(default = "127.0.0.1:3000")] + address: SocketAddr, + redis_url: String, + kafka_hosts: String, + kafka_topic: String, +} + async fn shutdown() { let mut term = signal::unix::signal(signal::unix::SignalKind::terminate()) .expect("failed to register SIGTERM handler"); @@ -23,18 +34,15 @@ async fn shutdown() { #[tokio::main] async fn main() { - let use_print_sink = env::var("PRINT_SINK").is_ok(); - let address = env::var("ADDRESS").unwrap_or(String::from("127.0.0.1:3000")); - let redis_addr = - env::var("REDIS_URL").expect("redis required; please set the REDIS_URL env var"); + let config = Config::init_from_env().expect("Invalid configuration:"); let redis_client = - Arc::new(RedisClient::new(redis_addr).expect("failed to create redis client")); + Arc::new(RedisClient::new(config.redis_url).expect("failed to create redis client")); let billing = BillingLimiter::new(Duration::seconds(5), redis_client.clone()) .expect("failed to create billing limiter"); - let app = if use_print_sink { + let app = if config.print_sink { router::router( capture::time::SystemTime {}, sink::PrintSink {}, @@ -43,10 +51,7 @@ async fn main() { true, ) } else { - let brokers = env::var("KAFKA_HOSTS").expect("Expected KAFKA_HOSTS"); - let topic = env::var("KAFKA_TOPIC").expect("Expected KAFKA_TOPIC"); - - let sink = sink::KafkaSink::new(topic, brokers).unwrap(); + let sink = sink::KafkaSink::new(config.kafka_topic, config.kafka_hosts).unwrap(); router::router( capture::time::SystemTime {}, @@ -58,14 +63,14 @@ async fn main() { }; // initialize tracing - tracing_subscriber::fmt::init(); + // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` - tracing::info!("listening on {}", address); + tracing::info!("listening on {}", config.address); - axum::Server::bind(&address.parse().unwrap()) + axum::Server::bind(&config.address) .serve(app.into_make_service_with_connect_info::()) .with_graceful_shutdown(shutdown()) .await From 7cd2d6132d39150608bd3720f510c9734bafc6f6 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Fri, 27 Oct 2023 16:33:32 +0100 Subject: [PATCH 037/247] Create LICENSE (#38) --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000..d1e439cba370e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 PostHog + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From e6ea52f494331793342ebd2b9e9c49059c05247f Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Fri, 27 Oct 2023 16:35:06 +0100 Subject: [PATCH 038/247] Fix the redis billing update (#39) Turns out we never reset updated. --- capture/src/billing_limits.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/capture/src/billing_limits.rs b/capture/src/billing_limits.rs index 4309c2174dd09..5f1540009d0d7 100644 --- a/capture/src/billing_limits.rs +++ b/capture/src/billing_limits.rs @@ -114,6 +114,10 @@ impl BillingLimiter { // // This update will block readers! Keep it fast. if since_update > self.interval { + // open the update lock to change the update, and prevent anyone else from doing so + let mut updated = self.updated.write().await; + *updated = OffsetDateTime::now_utc(); + let span = tracing::debug_span!("updating billing cache from redis"); let _span = span.enter(); From fb6fa7cdd5673435526b6eb052861baf7cc2ecb3 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Fri, 27 Oct 2023 18:04:02 +0200 Subject: [PATCH 039/247] kafka: check reachability + collect metrics (#37) --- capture-server/src/main.rs | 7 ++-- capture/src/capture.rs | 8 +--- capture/src/sink.rs | 78 +++++++++++++++++++++++++++++++++++--- 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index 46d8e28496cc7..1f659c87c9d89 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -13,6 +13,7 @@ struct Config { #[envconfig(default = "127.0.0.1:3000")] address: SocketAddr, redis_url: String, + kafka_hosts: String, kafka_topic: String, } @@ -34,6 +35,9 @@ async fn shutdown() { #[tokio::main] async fn main() { + // initialize tracing + tracing_subscriber::fmt::init(); + let config = Config::init_from_env().expect("Invalid configuration:"); let redis_client = @@ -62,9 +66,6 @@ async fn main() { ) }; - // initialize tracing - tracing_subscriber::fmt::init(); - // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 0413d44ea252a..e3c2716a8306d 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -31,8 +31,6 @@ pub async fn event( headers: HeaderMap, body: Bytes, ) -> Result, CaptureError> { - tracing::debug!(len = body.len(), "new event request"); - let events = match headers .get("content-type") .map_or("", |v| v.to_str().unwrap_or("")) @@ -47,8 +45,6 @@ pub async fn event( _ => RawEvent::from_bytes(&meta, body), }?; - tracing::debug!("got events {:?}", &events); - if events.is_empty() { return Err(CaptureError::EmptyBatch); } @@ -98,7 +94,7 @@ pub async fn event( })); } - tracing::debug!("got context {:?}", &context); + tracing::debug!(context=?context, events=?events, "decoded request"); process_events(state.sink.clone(), &events, &context).await?; @@ -169,7 +165,7 @@ pub async fn process_events<'a>( .map(|e| process_single_event(e, context)) .collect::, CaptureError>>()?; - println!("Processed events: {:?}", events); + tracing::debug!(events=?events, "processed {} events", events.len()); if events.len() == 1 { sink.send(events[0].clone()).await?; diff --git a/capture/src/sink.rs b/capture/src/sink.rs index 6f0cbefd4659b..8de9ff128b0ca 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -1,11 +1,15 @@ use async_trait::async_trait; -use metrics::{counter, histogram}; +use metrics::{absolute_counter, counter, gauge, histogram}; +use std::time::Duration; use tokio::task::JoinSet; use crate::api::CaptureError; use rdkafka::config::ClientConfig; use rdkafka::error::RDKafkaErrorCode; use rdkafka::producer::future_producer::{FutureProducer, FutureRecord}; +use rdkafka::producer::Producer; +use rdkafka::util::Timeout; +use tracing::info; use crate::event::ProcessedEvent; @@ -39,17 +43,81 @@ impl EventSink for PrintSink { } } +struct KafkaContext; + +impl rdkafka::ClientContext for KafkaContext { + fn stats(&self, stats: rdkafka::Statistics) { + gauge!("capture_kafka_callback_queue_depth", stats.replyq as f64); + gauge!("capture_kafka_producer_queue_depth", stats.msg_cnt as f64); + gauge!( + "capture_kafka_producer_queue_depth_limit", + stats.msg_max as f64 + ); + gauge!("capture_kafka_producer_queue_bytes", stats.msg_max as f64); + gauge!( + "capture_kafka_producer_queue_bytes_limit", + stats.msg_size_max as f64 + ); + + for (topic, stats) in stats.topics { + gauge!( + "capture_kafka_produce_avg_batch_size_bytes", + stats.batchsize.avg as f64, + "topic" => topic.clone() + ); + gauge!( + "capture_kafka_produce_avg_batch_size_events", + stats.batchcnt.avg as f64, + "topic" => topic + ); + } + + for (_, stats) in stats.brokers { + let id_string = format!("{}", stats.nodeid); + gauge!( + "capture_kafka_broker_requests_pending", + stats.outbuf_cnt as f64, + "broker" => id_string.clone() + ); + gauge!( + "capture_kafka_broker_responses_awaiting", + stats.waitresp_cnt as f64, + "broker" => id_string.clone() + ); + absolute_counter!( + "capture_kafka_broker_tx_errors_total", + stats.txerrs, + "broker" => id_string.clone() + ); + absolute_counter!( + "capture_kafka_broker_rx_errors_total", + stats.rxerrs, + "broker" => id_string + ); + } + } +} + #[derive(Clone)] pub struct KafkaSink { - producer: FutureProducer, + producer: FutureProducer, topic: String, } impl KafkaSink { pub fn new(topic: String, brokers: String) -> anyhow::Result { - let producer: FutureProducer = ClientConfig::new() + info!("connecting to Kafka brokers at {}...", brokers); + let producer: FutureProducer = ClientConfig::new() .set("bootstrap.servers", &brokers) - .create()?; + .set("statistics.interval.ms", "10000") + .create_with_context(KafkaContext)?; + + // Ping the cluster to make sure we can reach brokers + _ = producer.client().fetch_metadata( + Some("__consumer_offsets"), + Timeout::After(Duration::new(10, 0)), + )?; + info!("connected to Kafka brokers"); Ok(KafkaSink { producer, topic }) } @@ -57,7 +125,7 @@ impl KafkaSink { impl KafkaSink { async fn kafka_send( - producer: FutureProducer, + producer: FutureProducer, topic: String, event: ProcessedEvent, ) -> Result<(), CaptureError> { From 53017d7505e0b368c98c9ed832b8b722bb017835 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Fri, 27 Oct 2023 18:39:26 +0200 Subject: [PATCH 040/247] kafka: add ssl support (#40) --- Cargo.lock | 13 +++++++++++++ Cargo.toml | 2 +- capture-server/src/main.rs | 5 ++++- capture/src/sink.rs | 23 +++++++++++++++-------- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2caeb20b2faba..3fca31c1399a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1042,6 +1042,18 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "openssl-sys" +version = "0.9.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "overload" version = "0.1.1" @@ -1284,6 +1296,7 @@ dependencies = [ "libc", "libz-sys", "num_enum", + "openssl-sys", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 117abaa970f3f..b8b4d08b05474 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ uuid = { version = "1.3.3", features = ["serde"] } async-trait = "0.1.68" serde_urlencoded = "0.7.1" rand = "0.8.5" -rdkafka = { version = "0.34", features = ["cmake-build"] } +rdkafka = { version = "0.34", features = ["cmake-build", "ssl"] } metrics = "0.21.1" metrics-exporter-prometheus = "0.12.1" thiserror = "1.0.48" diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index 1f659c87c9d89..c2d342d2bb13d 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -16,6 +16,8 @@ struct Config { kafka_hosts: String, kafka_topic: String, + #[envconfig(default = "false")] + kafka_tls: bool, } async fn shutdown() { @@ -55,7 +57,8 @@ async fn main() { true, ) } else { - let sink = sink::KafkaSink::new(config.kafka_topic, config.kafka_hosts).unwrap(); + let sink = + sink::KafkaSink::new(config.kafka_topic, config.kafka_hosts, config.kafka_tls).unwrap(); router::router( capture::time::SystemTime {}, diff --git a/capture/src/sink.rs b/capture/src/sink.rs index 8de9ff128b0ca..f9172dbdb4c7e 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -4,7 +4,7 @@ use std::time::Duration; use tokio::task::JoinSet; use crate::api::CaptureError; -use rdkafka::config::ClientConfig; +use rdkafka::config::{ClientConfig, FromClientConfigAndContext}; use rdkafka::error::RDKafkaErrorCode; use rdkafka::producer::future_producer::{FutureProducer, FutureRecord}; use rdkafka::producer::Producer; @@ -105,12 +105,20 @@ pub struct KafkaSink { } impl KafkaSink { - pub fn new(topic: String, brokers: String) -> anyhow::Result { + pub fn new(topic: String, brokers: String, tls: bool) -> anyhow::Result { info!("connecting to Kafka brokers at {}...", brokers); - let producer: FutureProducer = ClientConfig::new() + let mut config = ClientConfig::new(); + config .set("bootstrap.servers", &brokers) - .set("statistics.interval.ms", "10000") - .create_with_context(KafkaContext)?; + .set("statistics.interval.ms", "10000"); + + if tls { + config + .set("security.protocol", "ssl") + .set("enable.ssl.certificate.verification", "false"); + }; + + let producer = FutureProducer::from_config_and_context(&config, KafkaContext)?; // Ping the cluster to make sure we can reach brokers _ = producer.client().fetch_metadata( @@ -180,9 +188,8 @@ impl EventSink for KafkaSink { set.spawn(Self::kafka_send(producer, topic, event)); } - while let Some(res) = set.join_next().await { - println!("{:?}", res); - } + // Await on all the produce promises + while (set.join_next().await).is_some() {} Ok(()) } From 9cf09592ab639323e93741eb16270008e8954cde Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 30 Oct 2023 16:55:53 +0100 Subject: [PATCH 041/247] metrics: consolidate into a labelled capture_events_dropped_total metric --- capture/src/capture.rs | 5 +++-- capture/src/prometheus.rs | 4 ++++ capture/src/sink.rs | 10 ++++------ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/capture/src/capture.rs b/capture/src/capture.rs index e3c2716a8306d..647594005eaa5 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -16,6 +16,7 @@ use time::OffsetDateTime; use crate::billing_limits::QuotaResource; use crate::event::ProcessingContext; +use crate::prometheus::report_dropped_events; use crate::token::validate_token; use crate::{ api::{CaptureError, CaptureResponse, CaptureResponseCode}, @@ -50,7 +51,7 @@ pub async fn event( } let token = extract_and_verify_token(&events).map_err(|err| { - counter!("capture_token_shape_invalid_total", events.len() as u64); + report_dropped_events("token_shape_invalid", events.len() as u64); err })?; @@ -81,7 +82,7 @@ pub async fn event( .await; if limited { - counter!("capture_events_dropped_over_quota", 1); + report_dropped_events("over_quota", 1); // for v0 we want to just return ok 🙃 // this is because the clients are pretty dumb and will just retry over and over and diff --git a/capture/src/prometheus.rs b/capture/src/prometheus.rs index 0cb4750995b7c..d9dbea8831703 100644 --- a/capture/src/prometheus.rs +++ b/capture/src/prometheus.rs @@ -3,8 +3,12 @@ use std::time::Instant; use axum::{extract::MatchedPath, http::Request, middleware::Next, response::IntoResponse}; +use metrics::counter; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; +pub fn report_dropped_events(cause: &'static str, quantity: u64) { + counter!("capture_events_dropped_total", quantity, "cause" => cause); +} pub fn setup_metrics_recorder() -> PrometheusHandle { // Ok I broke it at the end, but the limit on our ingress is 60 and that's a nicer way of reaching it const EXPONENTIAL_SECONDS: &[f64] = &[ diff --git a/capture/src/sink.rs b/capture/src/sink.rs index f9172dbdb4c7e..e0c89e3ccd4a0 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -12,6 +12,7 @@ use rdkafka::util::Timeout; use tracing::info; use crate::event::ProcessedEvent; +use crate::prometheus::report_dropped_events; #[async_trait] pub trait EventSink { @@ -152,18 +153,15 @@ impl KafkaSink { timestamp: None, headers: None, }) { - Ok(_) => { - metrics::increment_counter!("capture_events_ingested"); - Ok(()) - } + Ok(_) => Ok(()), Err((e, _)) => match e.rdkafka_error_code() { Some(RDKafkaErrorCode::InvalidMessageSize) => { - metrics::increment_counter!("capture_events_dropped_too_big"); + report_dropped_events("kafka_message_size", 1); Err(CaptureError::EventTooBig) } _ => { // TODO(maybe someday): Don't drop them but write them somewhere and try again - metrics::increment_counter!("capture_events_dropped"); + report_dropped_events("kafka_write_error", 1); tracing::error!("failed to produce event: {}", e); Err(CaptureError::RetryableSinkError) } From a841ac4ea11833971de33f96d6868d143b13c150 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 31 Oct 2023 09:01:09 +0100 Subject: [PATCH 042/247] metrics: report capture_event_batch_size in kafka sink too (#42) --- capture/src/sink.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/capture/src/sink.rs b/capture/src/sink.rs index e0c89e3ccd4a0..cc686eee44cf4 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -25,7 +25,7 @@ pub struct PrintSink {} #[async_trait] impl EventSink for PrintSink { async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { - tracing::info!("single event: {:?}", event); + info!("single event: {:?}", event); counter!("capture_events_ingested_total", 1); Ok(()) @@ -37,7 +37,7 @@ impl EventSink for PrintSink { histogram!("capture_event_batch_size", events.len() as f64); counter!("capture_events_ingested_total", events.len() as u64); for event in events { - tracing::info!("event: {:?}", event); + info!("event: {:?}", event); } Ok(()) @@ -173,12 +173,16 @@ impl KafkaSink { #[async_trait] impl EventSink for KafkaSink { async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { - Self::kafka_send(self.producer.clone(), self.topic.clone(), event).await + Self::kafka_send(self.producer.clone(), self.topic.clone(), event).await?; + + histogram!("capture_event_batch_size", 1.0); + counter!("capture_events_ingested_total", 1); + Ok(()) } async fn send_batch(&self, events: Vec) -> Result<(), CaptureError> { let mut set = JoinSet::new(); - + let batch_size = events.len(); for event in events { let producer = self.producer.clone(); let topic = self.topic.clone(); @@ -189,6 +193,8 @@ impl EventSink for KafkaSink { // Await on all the produce promises while (set.join_next().await).is_some() {} + histogram!("capture_event_batch_size", batch_size as f64); + counter!("capture_events_ingested_total", batch_size as u64); Ok(()) } } From 4b3634b8e77b974a0b3f515bbb72c7430f0488fb Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 31 Oct 2023 16:32:13 +0100 Subject: [PATCH 043/247] add cors support (#43) --- Cargo.toml | 2 +- capture/src/capture.rs | 6 ++++++ capture/src/router.rs | 14 ++++++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b8b4d08b05474..318b18098c46f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ serde_json = "1.0.96" governor = "0.5.1" tower_governor = "0.0.4" time = { version = "0.3.20", features = ["formatting", "macros", "parsing", "serde"] } -tower-http = { version = "0.4.0", features = ["trace"] } +tower-http = { version = "0.4.0", features = ["cors", "trace"] } bytes = "1" anyhow = "1.0" flate2 = "1.0" diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 647594005eaa5..45fc5f70f889a 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -104,6 +104,12 @@ pub async fn event( })) } +pub async fn options() -> Result, CaptureError> { + Ok(Json(CaptureResponse { + status: CaptureResponseCode::Ok, + })) +} + pub fn process_single_event( event: &RawEvent, context: &ProcessingContext, diff --git a/capture/src/router.rs b/capture/src/router.rs index 757b975c2948f..8fc080d95d6fb 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -1,10 +1,12 @@ use std::future::ready; use std::sync::Arc; +use axum::http::Method; use axum::{ routing::{get, post}, Router, }; +use tower_http::cors::{AllowOrigin, Any, CorsLayer}; use tower_http::trace::TraceLayer; use crate::{billing_limits::BillingLimiter, capture, redis::Client, sink, time::TimeSource}; @@ -41,12 +43,20 @@ pub fn router< billing, }; + // Very permissive CORS policy, as old SDK versions + // and reverse proxies might send funky headers. + let cors = CorsLayer::new() + .allow_methods([Method::GET, Method::POST, Method::OPTIONS]) + .allow_headers(Any) + .allow_origin(AllowOrigin::mirror_request()); + let router = Router::new() // TODO: use NormalizePathLayer::trim_trailing_slash .route("/", get(index)) - .route("/i/v0/e", post(capture::event)) - .route("/i/v0/e/", post(capture::event)) + .route("/i/v0/e", post(capture::event).options(capture::options)) + .route("/i/v0/e/", post(capture::event).options(capture::options)) .layer(TraceLayer::new_for_http()) + .layer(cors) .layer(axum::middleware::from_fn(track_metrics)) .with_state(state); From 4f85f1ab32317c15c272d6ca430fea3e574fc585 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Fri, 3 Nov 2023 10:50:53 +0100 Subject: [PATCH 044/247] cors: enable allow_credentials (#44) --- capture/src/router.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/capture/src/router.rs b/capture/src/router.rs index 8fc080d95d6fb..58fef22a61e3f 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -6,7 +6,7 @@ use axum::{ routing::{get, post}, Router, }; -use tower_http::cors::{AllowOrigin, Any, CorsLayer}; +use tower_http::cors::{AllowHeaders, AllowOrigin, CorsLayer}; use tower_http::trace::TraceLayer; use crate::{billing_limits::BillingLimiter, capture, redis::Client, sink, time::TimeSource}; @@ -47,7 +47,8 @@ pub fn router< // and reverse proxies might send funky headers. let cors = CorsLayer::new() .allow_methods([Method::GET, Method::POST, Method::OPTIONS]) - .allow_headers(Any) + .allow_headers(AllowHeaders::mirror_request()) + .allow_credentials(true) .allow_origin(AllowOrigin::mirror_request()); let router = Router::new() From bcc45497e2bbd2f1f9c2deba4602e40a9c8619d9 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 6 Nov 2023 14:21:29 +0100 Subject: [PATCH 045/247] add end2end testing harness to capture-server crate (#45) --- .github/workflows/rust.yml | 5 + Cargo.lock | 227 ++++++++++++++++++++++++++++++--- Cargo.toml | 2 + capture-server/Cargo.toml | 18 ++- capture-server/src/main.rs | 62 +-------- capture-server/tests/common.rs | 169 ++++++++++++++++++++++++ capture-server/tests/events.rs | 75 +++++++++++ capture/Cargo.toml | 3 +- capture/src/config.rs | 19 +++ capture/src/lib.rs | 2 + capture/src/router.rs | 1 - capture/src/server.rs | 52 ++++++++ docker-compose.yml | 51 ++++++++ 13 files changed, 605 insertions(+), 81 deletions(-) create mode 100644 capture-server/tests/common.rs create mode 100644 capture-server/tests/events.rs create mode 100644 capture/src/config.rs create mode 100644 capture/src/server.rs create mode 100644 docker-compose.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 478017af701f2..10636f11554e9 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -38,6 +38,11 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Setup end2end dependencies + run: | + docker compose up -d --wait + echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts + - name: Install rust uses: dtolnay/rust-toolchain@master with: diff --git a/Cargo.lock b/Cargo.lock index 3fca31c1399a7..685687f0932e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -205,6 +205,7 @@ dependencies = [ "axum-test-helper", "base64", "bytes", + "envconfig", "flate2", "governor", "metrics", @@ -231,9 +232,17 @@ dependencies = [ name = "capture-server" version = "0.1.0" dependencies = [ + "anyhow", + "assert-json-diff", "axum", "capture", "envconfig", + "futures", + "once_cell", + "rand", + "rdkafka", + "reqwest", + "serde_json", "time", "tokio", "tracing", @@ -406,6 +415,22 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + [[package]] name = "flate2" version = "1.0.27" @@ -431,6 +456,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -458,9 +498,9 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" dependencies = [ "futures-channel", "futures-core", @@ -473,9 +513,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -483,15 +523,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ "futures-core", "futures-task", @@ -500,15 +540,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", @@ -517,15 +557,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-timer" @@ -535,9 +575,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -696,6 +736,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.4.0" @@ -780,6 +833,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" + [[package]] name = "lock_api" version = "0.4.10" @@ -953,6 +1012,24 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "no-std-compat" version = "0.4.1" @@ -1042,6 +1119,38 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "openssl" +version = "0.10.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "openssl-sys" version = "0.9.93" @@ -1078,7 +1187,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.3.5", "smallvec", "windows-targets", ] @@ -1343,6 +1452,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "regex" version = "1.10.0" @@ -1387,11 +1505,13 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-tls", "ipnet", "js-sys", "log", "mime", "mime_guess", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -1400,6 +1520,7 @@ dependencies = [ "serde_urlencoded", "system-configuration", "tokio", + "tokio-native-tls", "tokio-util", "tower-service", "url", @@ -1416,6 +1537,19 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustix" +version = "0.38.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -1428,12 +1562,44 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.188" @@ -1601,6 +1767,19 @@ dependencies = [ "libc", ] +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall 0.4.1", + "rustix", + "windows-sys", +] + [[package]] name = "termtree" version = "0.4.1" @@ -1710,6 +1889,16 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.9" diff --git a/Cargo.toml b/Cargo.toml index 318b18098c46f..402243e69515c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ ] [workspace.dependencies] +assert-json-diff = "2.0.2" axum = "0.6.15" axum-client-ip = "0.4.1" tokio = { version = "1.0", features = ["full"] } @@ -28,3 +29,4 @@ rdkafka = { version = "0.34", features = ["cmake-build", "ssl"] } metrics = "0.21.1" metrics-exporter-prometheus = "0.12.1" thiserror = "1.0.48" +envconfig = "0.10.0" diff --git a/capture-server/Cargo.toml b/capture-server/Cargo.toml index fa7151ed497da..19200043f577f 100644 --- a/capture-server/Cargo.toml +++ b/capture-server/Cargo.toml @@ -4,10 +4,20 @@ version = "0.1.0" edition = "2021" [dependencies] -capture = { path = "../capture" } axum = { workspace = true } +capture = { path = "../capture" } +envconfig = { workspace = true } +time = { workspace = true } tokio = { workspace = true } -tracing-subscriber = { workspace = true } tracing = { workspace = true } -time = { workspace = true } -envconfig = "0.10.0" +tracing-subscriber = { workspace = true } + +[dev-dependencies] +anyhow = { workspace = true, features = [] } +assert-json-diff = { workspace = true } +futures = "0.3.29" +once_cell = "1.18.0" +rand = { workspace = true } +rdkafka = { workspace = true } +reqwest = "0.11.22" +serde_json = { workspace = true } diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index c2d342d2bb13d..4874a43a32aa7 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -1,24 +1,10 @@ -use envconfig::Envconfig; -use std::net::SocketAddr; -use std::sync::Arc; +use std::net::TcpListener; -use capture::{billing_limits::BillingLimiter, redis::RedisClient, router, sink}; -use time::Duration; +use envconfig::Envconfig; use tokio::signal; -#[derive(Envconfig)] -struct Config { - #[envconfig(default = "false")] - print_sink: bool, - #[envconfig(default = "127.0.0.1:3000")] - address: SocketAddr, - redis_url: String, - - kafka_hosts: String, - kafka_topic: String, - #[envconfig(default = "false")] - kafka_tls: bool, -} +use capture::config::Config; +use capture::server::serve; async fn shutdown() { let mut term = signal::unix::signal(signal::unix::SignalKind::terminate()) @@ -41,42 +27,6 @@ async fn main() { tracing_subscriber::fmt::init(); let config = Config::init_from_env().expect("Invalid configuration:"); - - let redis_client = - Arc::new(RedisClient::new(config.redis_url).expect("failed to create redis client")); - - let billing = BillingLimiter::new(Duration::seconds(5), redis_client.clone()) - .expect("failed to create billing limiter"); - - let app = if config.print_sink { - router::router( - capture::time::SystemTime {}, - sink::PrintSink {}, - redis_client, - billing, - true, - ) - } else { - let sink = - sink::KafkaSink::new(config.kafka_topic, config.kafka_hosts, config.kafka_tls).unwrap(); - - router::router( - capture::time::SystemTime {}, - sink, - redis_client, - billing, - true, - ) - }; - - // run our app with hyper - // `axum::Server` is a re-export of `hyper::Server` - - tracing::info!("listening on {}", config.address); - - axum::Server::bind(&config.address) - .serve(app.into_make_service_with_connect_info::()) - .with_graceful_shutdown(shutdown()) - .await - .unwrap(); + let listener = TcpListener::bind(config.address).unwrap(); + serve(config, listener, shutdown()).await } diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs new file mode 100644 index 0000000000000..40836ca7f6b73 --- /dev/null +++ b/capture-server/tests/common.rs @@ -0,0 +1,169 @@ +#![allow(dead_code)] + +use std::default::Default; +use std::net::{SocketAddr, TcpListener}; +use std::str::FromStr; +use std::string::ToString; +use std::sync::{Arc, Once}; +use std::time::Duration; + +use anyhow::bail; +use once_cell::sync::Lazy; +use rand::distributions::Alphanumeric; +use rand::Rng; +use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication}; +use rdkafka::config::{ClientConfig, FromClientConfig}; +use rdkafka::consumer::{BaseConsumer, Consumer}; +use rdkafka::util::Timeout; +use rdkafka::{Message, TopicPartitionList}; +use tokio::sync::Notify; +use tracing::debug; + +use capture::config::Config; +use capture::server::serve; + +pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { + print_sink: false, + address: SocketAddr::from_str("127.0.0.1:0").unwrap(), + export_prometheus: false, + redis_url: "redis://localhost:6379/".to_string(), + kafka_hosts: "kafka:9092".to_string(), + kafka_topic: "events_plugin_ingestion".to_string(), + kafka_tls: false, +}); + +static TRACING_INIT: Once = Once::new(); +pub fn setup_tracing() { + TRACING_INIT.call_once(|| { + tracing_subscriber::fmt() + .with_writer(tracing_subscriber::fmt::TestWriter::new()) + .init() + }); +} +pub struct ServerHandle { + pub addr: SocketAddr, + shutdown: Arc, +} + +impl ServerHandle { + pub fn for_topic(topic: &EphemeralTopic) -> Self { + let mut config = DEFAULT_CONFIG.clone(); + config.kafka_topic = topic.topic_name().to_string(); + Self::for_config(config) + } + pub fn for_config(config: Config) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let notify = Arc::new(Notify::new()); + let shutdown = notify.clone(); + + tokio::spawn( + async move { serve(config, listener, async { notify.notified().await }).await }, + ); + Self { addr, shutdown } + } + + pub async fn capture_events>(&self, body: T) -> reqwest::Response { + let client = reqwest::Client::new(); + client + .post(format!("http://{:?}/i/v0/e", self.addr)) + .body(body) + .send() + .await + .expect("failed to send request") + } +} + +impl Drop for ServerHandle { + fn drop(&mut self) { + self.shutdown.notify_one() + } +} + +pub struct EphemeralTopic { + consumer: BaseConsumer, + read_timeout: Timeout, + topic_name: String, +} + +impl EphemeralTopic { + pub async fn new() -> Self { + let mut config = ClientConfig::new(); + config.set("group.id", "capture_integration_tests"); + config.set("bootstrap.servers", DEFAULT_CONFIG.kafka_hosts.clone()); + config.set("debug", "all"); + + // TODO: check for name collision? + let topic_name = random_string("events_", 16); + let admin = AdminClient::from_config(&config).expect("failed to create admin client"); + admin + .create_topics( + &[NewTopic { + name: &topic_name, + num_partitions: 1, + replication: TopicReplication::Fixed(1), + config: vec![], + }], + &AdminOptions::default(), + ) + .await + .expect("failed to create topic"); + + let consumer: BaseConsumer = config.create().expect("failed to create consumer"); + let mut assignment = TopicPartitionList::new(); + assignment.add_partition(&topic_name, 0); + consumer + .assign(&assignment) + .expect("failed to assign topic"); + + Self { + consumer, + read_timeout: Timeout::After(Duration::from_secs(5)), + topic_name, + } + } + + pub fn next_event(&self) -> anyhow::Result { + match self.consumer.poll(self.read_timeout) { + Some(Ok(message)) => { + let body = message.payload().expect("empty kafka message"); + let event = serde_json::from_slice(body)?; + Ok(event) + } + Some(Err(err)) => bail!("kafka read error: {}", err), + None => bail!("kafka read timeout"), + } + } + + pub fn topic_name(&self) -> &str { + &self.topic_name + } +} + +impl Drop for EphemeralTopic { + fn drop(&mut self) { + debug!("dropping EphemeralTopic {}...", self.topic_name); + _ = self.consumer.unassign(); + futures::executor::block_on(delete_topic(self.topic_name.clone())); + debug!("dropped topic"); + } +} + +async fn delete_topic(topic: String) { + let mut config = ClientConfig::new(); + config.set("bootstrap.servers", DEFAULT_CONFIG.kafka_hosts.clone()); + let admin = AdminClient::from_config(&config).expect("failed to create admin client"); + admin + .delete_topics(&[&topic], &AdminOptions::default()) + .await + .expect("failed to delete topic"); +} + +pub fn random_string(prefix: &str, length: usize) -> String { + let suffix: String = rand::thread_rng() + .sample_iter(Alphanumeric) + .take(length) + .map(char::from) + .collect(); + format!("{}_{}", prefix, suffix) +} diff --git a/capture-server/tests/events.rs b/capture-server/tests/events.rs new file mode 100644 index 0000000000000..42facd86554ec --- /dev/null +++ b/capture-server/tests/events.rs @@ -0,0 +1,75 @@ +use anyhow::Result; +use assert_json_diff::assert_json_include; +use reqwest::StatusCode; +use serde_json::json; + +use crate::common::*; +mod common; + +#[tokio::test] +async fn it_captures_one_event() -> Result<()> { + setup_tracing(); + let token = random_string("token", 16); + let distinct_id = random_string("id", 16); + let topic = EphemeralTopic::new().await; + let server = ServerHandle::for_topic(&topic); + + let event = json!({ + "token": token, + "event": "testing", + "distinct_id": distinct_id + }); + let res = server.capture_events(event.to_string()).await; + assert_eq!(StatusCode::OK, res.status()); + + let event = topic.next_event()?; + assert_json_include!( + actual: event, + expected: json!({ + "token": token, + "distinct_id": distinct_id + }) + ); + + Ok(()) +} + +#[tokio::test] +async fn it_captures_a_batch() -> Result<()> { + setup_tracing(); + let token = random_string("token", 16); + let distinct_id1 = random_string("id", 16); + let distinct_id2 = random_string("id", 16); + + let topic = EphemeralTopic::new().await; + let server = ServerHandle::for_topic(&topic); + + let event = json!([{ + "token": token, + "event": "event1", + "distinct_id": distinct_id1 + },{ + "token": token, + "event": "event2", + "distinct_id": distinct_id2 + }]); + let res = server.capture_events(event.to_string()).await; + assert_eq!(StatusCode::OK, res.status()); + + assert_json_include!( + actual: topic.next_event()?, + expected: json!({ + "token": token, + "distinct_id": distinct_id1 + }) + ); + assert_json_include!( + actual: topic.next_event()?, + expected: json!({ + "token": token, + "distinct_id": distinct_id2 + }) + ); + + Ok(()) +} diff --git a/capture/Cargo.toml b/capture/Cargo.toml index 7c84c6ce111ef..6bdfc39b20216 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -30,9 +30,10 @@ metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } thiserror = { workspace = true } redis = { version="0.23.3", features=["tokio-comp", "cluster", "cluster-async"] } +envconfig = { workspace = true } [dev-dependencies] -assert-json-diff = "2.0.2" +assert-json-diff = { workspace = true } axum-test-helper = "0.2.0" mockall = "0.11.2" redis-test = "0.2.3" diff --git a/capture/src/config.rs b/capture/src/config.rs new file mode 100644 index 0000000000000..6edf438264f13 --- /dev/null +++ b/capture/src/config.rs @@ -0,0 +1,19 @@ +use std::net::SocketAddr; + +use envconfig::Envconfig; + +#[derive(Envconfig, Clone)] +pub struct Config { + #[envconfig(default = "false")] + pub print_sink: bool, + #[envconfig(default = "127.0.0.1:3000")] + pub address: SocketAddr, + pub redis_url: String, + #[envconfig(default = "true")] + pub export_prometheus: bool, + + pub kafka_hosts: String, + pub kafka_topic: String, + #[envconfig(default = "false")] + pub kafka_tls: bool, +} diff --git a/capture/src/lib.rs b/capture/src/lib.rs index fcd802b43f7cf..70f754807bc13 100644 --- a/capture/src/lib.rs +++ b/capture/src/lib.rs @@ -1,10 +1,12 @@ pub mod api; pub mod billing_limits; pub mod capture; +pub mod config; pub mod event; pub mod prometheus; pub mod redis; pub mod router; +pub mod server; pub mod sink; pub mod time; pub mod token; diff --git a/capture/src/router.rs b/capture/src/router.rs index 58fef22a61e3f..9acc7f4f6f920 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -66,7 +66,6 @@ pub fn router< // does not work well. if metrics { let recorder_handle = setup_metrics_recorder(); - router.route("/metrics", get(move || ready(recorder_handle.render()))) } else { router diff --git a/capture/src/server.rs b/capture/src/server.rs new file mode 100644 index 0000000000000..bee579ef3774e --- /dev/null +++ b/capture/src/server.rs @@ -0,0 +1,52 @@ +use std::future::Future; +use std::net::{SocketAddr, TcpListener}; +use std::sync::Arc; + +use time::Duration; + +use crate::billing_limits::BillingLimiter; +use crate::config::Config; +use crate::redis::RedisClient; +use crate::{router, sink}; + +pub async fn serve(config: Config, listener: TcpListener, shutdown: F) +where + F: Future, +{ + let redis_client = + Arc::new(RedisClient::new(config.redis_url).expect("failed to create redis client")); + + let billing = BillingLimiter::new(Duration::seconds(5), redis_client.clone()) + .expect("failed to create billing limiter"); + + let app = if config.print_sink { + router::router( + crate::time::SystemTime {}, + sink::PrintSink {}, + redis_client, + billing, + config.export_prometheus, + ) + } else { + let sink = + sink::KafkaSink::new(config.kafka_topic, config.kafka_hosts, config.kafka_tls).unwrap(); + + router::router( + crate::time::SystemTime {}, + sink, + redis_client, + billing, + config.export_prometheus, + ) + }; + + // run our app with hyper + // `axum::Server` is a re-export of `hyper::Server` + tracing::info!("listening on {:?}", listener.local_addr().unwrap()); + axum::Server::from_tcp(listener) + .unwrap() + .serve(app.into_make_service_with_connect_info::()) + .with_graceful_shutdown(shutdown) + .await + .unwrap() +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000..804ae78ec7512 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,51 @@ +version: "3" + +services: + zookeeper: + image: zookeeper:3.7.0 + restart: on-failure + + kafka: + image: ghcr.io/posthog/kafka-container:v2.8.2 + restart: on-failure + depends_on: + - zookeeper + environment: + KAFKA_BROKER_ID: 1001 + KAFKA_CFG_RESERVED_BROKER_MAX_ID: 1001 + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: 'true' + ports: + - '9092:9092' + healthcheck: + test: kafka-cluster.sh cluster-id --bootstrap-server localhost:9092 || exit 1 + interval: 3s + timeout: 10s + retries: 10 + + redis: + image: redis:6.2.7-alpine + restart: on-failure + command: redis-server --maxmemory-policy allkeys-lru --maxmemory 200mb + ports: + - '6379:6379' + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 10s + retries: 10 + + kafka-ui: + image: provectuslabs/kafka-ui:latest + profiles: ["ui"] + ports: + - '8080:8080' + depends_on: + - zookeeper + - kafka + environment: + KAFKA_CLUSTERS_0_NAME: local + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092 + KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181 From 8f6003267357b7ca1eb497b184adbbae7cce8057 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 6 Nov 2023 16:46:44 +0100 Subject: [PATCH 046/247] kafka sink: expose more rdkafka settings (#46) --- capture-server/tests/common.rs | 25 ++++++++++++----- capture/src/config.rs | 11 ++++++++ capture/src/server.rs | 4 +-- capture/src/sink.rs | 49 +++++++++++++++++++++------------- 4 files changed, 61 insertions(+), 28 deletions(-) diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index 40836ca7f6b73..d4665cfedc748 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -19,7 +19,7 @@ use rdkafka::{Message, TopicPartitionList}; use tokio::sync::Notify; use tracing::debug; -use capture::config::Config; +use capture::config::{Config, KafkaConfig}; use capture::server::serve; pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { @@ -27,9 +27,14 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { address: SocketAddr::from_str("127.0.0.1:0").unwrap(), export_prometheus: false, redis_url: "redis://localhost:6379/".to_string(), - kafka_hosts: "kafka:9092".to_string(), - kafka_topic: "events_plugin_ingestion".to_string(), - kafka_tls: false, + kafka: KafkaConfig { + kafka_producer_linger_ms: 0, // Send messages as soon as possible + kafka_producer_queue_mib: 10, + kafka_compression_codec: "none".to_string(), + kafka_hosts: "kafka:9092".to_string(), + kafka_topic: "events_plugin_ingestion".to_string(), + kafka_tls: false, + }, }); static TRACING_INIT: Once = Once::new(); @@ -48,7 +53,7 @@ pub struct ServerHandle { impl ServerHandle { pub fn for_topic(topic: &EphemeralTopic) -> Self { let mut config = DEFAULT_CONFIG.clone(); - config.kafka_topic = topic.topic_name().to_string(); + config.kafka.kafka_topic = topic.topic_name().to_string(); Self::for_config(config) } pub fn for_config(config: Config) -> Self { @@ -90,7 +95,10 @@ impl EphemeralTopic { pub async fn new() -> Self { let mut config = ClientConfig::new(); config.set("group.id", "capture_integration_tests"); - config.set("bootstrap.servers", DEFAULT_CONFIG.kafka_hosts.clone()); + config.set( + "bootstrap.servers", + DEFAULT_CONFIG.kafka.kafka_hosts.clone(), + ); config.set("debug", "all"); // TODO: check for name collision? @@ -151,7 +159,10 @@ impl Drop for EphemeralTopic { async fn delete_topic(topic: String) { let mut config = ClientConfig::new(); - config.set("bootstrap.servers", DEFAULT_CONFIG.kafka_hosts.clone()); + config.set( + "bootstrap.servers", + DEFAULT_CONFIG.kafka.kafka_hosts.clone(), + ); let admin = AdminClient::from_config(&config).expect("failed to create admin client"); admin .delete_topics(&[&topic], &AdminOptions::default()) diff --git a/capture/src/config.rs b/capture/src/config.rs index 6edf438264f13..e3ea1e8461a27 100644 --- a/capture/src/config.rs +++ b/capture/src/config.rs @@ -11,7 +11,18 @@ pub struct Config { pub redis_url: String, #[envconfig(default = "true")] pub export_prometheus: bool, + #[envconfig(nested = true)] + pub kafka: KafkaConfig, +} +#[derive(Envconfig, Clone)] +pub struct KafkaConfig { + #[envconfig(default = "20")] + pub kafka_producer_linger_ms: u32, // Maximum time between producer batches during low traffic + #[envconfig(default = "400")] + pub kafka_producer_queue_mib: u32, // Size of the in-memory producer queue in mebibytes + #[envconfig(default = "none")] + pub kafka_compression_codec: String, // none, gzip, snappy, lz4, zstd pub kafka_hosts: String, pub kafka_topic: String, #[envconfig(default = "false")] diff --git a/capture/src/server.rs b/capture/src/server.rs index bee579ef3774e..e4372ae0b584a 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -28,9 +28,7 @@ where config.export_prometheus, ) } else { - let sink = - sink::KafkaSink::new(config.kafka_topic, config.kafka_hosts, config.kafka_tls).unwrap(); - + let sink = sink::KafkaSink::new(config.kafka).unwrap(); router::router( crate::time::SystemTime {}, sink, diff --git a/capture/src/sink.rs b/capture/src/sink.rs index cc686eee44cf4..d6b60d82024b4 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -1,16 +1,17 @@ -use async_trait::async_trait; -use metrics::{absolute_counter, counter, gauge, histogram}; use std::time::Duration; -use tokio::task::JoinSet; -use crate::api::CaptureError; -use rdkafka::config::{ClientConfig, FromClientConfigAndContext}; +use async_trait::async_trait; +use metrics::{absolute_counter, counter, gauge, histogram}; +use rdkafka::config::ClientConfig; use rdkafka::error::RDKafkaErrorCode; use rdkafka::producer::future_producer::{FutureProducer, FutureRecord}; use rdkafka::producer::Producer; use rdkafka::util::Timeout; -use tracing::info; +use tokio::task::JoinSet; +use tracing::{debug, info}; +use crate::api::CaptureError; +use crate::config::KafkaConfig; use crate::event::ProcessedEvent; use crate::prometheus::report_dropped_events; @@ -106,29 +107,41 @@ pub struct KafkaSink { } impl KafkaSink { - pub fn new(topic: String, brokers: String, tls: bool) -> anyhow::Result { - info!("connecting to Kafka brokers at {}...", brokers); - let mut config = ClientConfig::new(); - config - .set("bootstrap.servers", &brokers) - .set("statistics.interval.ms", "10000"); - - if tls { - config + pub fn new(config: KafkaConfig) -> anyhow::Result { + info!("connecting to Kafka brokers at {}...", config.kafka_hosts); + + let mut client_config = ClientConfig::new(); + client_config + .set("bootstrap.servers", &config.kafka_hosts) + .set("statistics.interval.ms", "10000") + .set("linger.ms", config.kafka_producer_linger_ms.to_string()) + .set("compression.codec", config.kafka_compression_codec) + .set( + "queue.buffering.max.kbytes", + (config.kafka_producer_queue_mib * 1024).to_string(), + ); + + if config.kafka_tls { + client_config .set("security.protocol", "ssl") .set("enable.ssl.certificate.verification", "false"); }; - let producer = FutureProducer::from_config_and_context(&config, KafkaContext)?; + debug!("rdkafka configuration: {:?}", client_config); + let producer: FutureProducer = + client_config.create_with_context(KafkaContext)?; - // Ping the cluster to make sure we can reach brokers + // Ping the cluster to make sure we can reach brokers, fail after 10 seconds _ = producer.client().fetch_metadata( Some("__consumer_offsets"), Timeout::After(Duration::new(10, 0)), )?; info!("connected to Kafka brokers"); - Ok(KafkaSink { producer, topic }) + Ok(KafkaSink { + producer, + topic: config.kafka_topic, + }) } } From 780f3909c08662a0be5d08a0274cd9efe0ff7f59 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 7 Nov 2023 13:53:26 +0100 Subject: [PATCH 047/247] implement liveness checks based on rdkafka health (#41) --- capture/src/health.rs | 344 +++++++++++++++++++++++++++++++++ capture/src/lib.rs | 1 + capture/src/router.rs | 4 + capture/src/server.rs | 17 +- capture/src/sink.rs | 13 +- capture/tests/django_compat.rs | 11 +- 6 files changed, 384 insertions(+), 6 deletions(-) create mode 100644 capture/src/health.rs diff --git a/capture/src/health.rs b/capture/src/health.rs new file mode 100644 index 0000000000000..dcddbe477e7cc --- /dev/null +++ b/capture/src/health.rs @@ -0,0 +1,344 @@ +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use std::collections::HashMap; +use std::ops::Add; +use std::sync::{Arc, RwLock}; + +use time::Duration; +use tokio::sync::mpsc; +use tracing::{info, warn}; + +/// Health reporting for components of the service. +/// +/// The capture server contains several asynchronous loops, and +/// the process can only be trusted with user data if all the +/// loops are properly running and reporting. +/// +/// HealthRegistry allows an arbitrary number of components to +/// be registered and report their health. The process' health +/// status is the combination of these individual health status: +/// - if any component is unhealthy, the process is unhealthy +/// - if all components recently reported healthy, the process is healthy +/// - if a component failed to report healthy for its defined deadline, +/// it is considered unhealthy, and the check fails. +/// +/// Trying to merge the k8s concepts of liveness and readiness in +/// a single state is full of foot-guns, so HealthRegistry does not +/// try to do it. Each probe should have its separate instance of +/// the registry to avoid confusions. + +#[derive(Default, Debug)] +pub struct HealthStatus { + /// The overall status: true of all components are healthy + pub healthy: bool, + /// Current status of each registered component, for display + pub components: HashMap, +} +impl IntoResponse for HealthStatus { + /// Computes the axum status code based on the overall health status, + /// and prints each component status in the body for debugging. + fn into_response(self) -> Response { + let body = format!("{:?}", self); + match self.healthy { + true => (StatusCode::OK, body), + false => (StatusCode::INTERNAL_SERVER_ERROR, body), + } + .into_response() + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum ComponentStatus { + /// Automatically set when a component is newly registered + Starting, + /// Recently reported healthy, will need to report again before the date + HealthyUntil(time::OffsetDateTime), + /// Reported unhealthy + Unhealthy, + /// Automatically set when the HealthyUntil deadline is reached + Stalled, +} +struct HealthMessage { + component: String, + status: ComponentStatus, +} + +pub struct HealthHandle { + component: String, + deadline: Duration, + sender: mpsc::Sender, +} + +impl HealthHandle { + /// Asynchronously report healthy, returns when the message is queued. + /// Must be called more frequently than the configured deadline. + pub async fn report_healthy(&self) { + self.report_status(ComponentStatus::HealthyUntil( + time::OffsetDateTime::now_utc().add(self.deadline), + )) + .await + } + + /// Asynchronously report component status, returns when the message is queued. + pub async fn report_status(&self, status: ComponentStatus) { + let message = HealthMessage { + component: self.component.clone(), + status, + }; + if let Err(err) = self.sender.send(message).await { + warn!("failed to report heath status: {}", err) + } + } + + /// Synchronously report as healthy, returns when the message is queued. + /// Must be called more frequently than the configured deadline. + pub fn report_healthy_blocking(&self) { + self.report_status_blocking(ComponentStatus::HealthyUntil( + time::OffsetDateTime::now_utc().add(self.deadline), + )) + } + + /// Asynchronously report component status, returns when the message is queued. + pub fn report_status_blocking(&self, status: ComponentStatus) { + let message = HealthMessage { + component: self.component.clone(), + status, + }; + if let Err(err) = self.sender.blocking_send(message) { + warn!("failed to report heath status: {}", err) + } + } +} + +#[derive(Clone)] +pub struct HealthRegistry { + name: String, + components: Arc>>, + sender: mpsc::Sender, +} + +impl HealthRegistry { + pub fn new(name: &str) -> Self { + let (tx, mut rx) = mpsc::channel::(16); + let registry = Self { + name: name.to_owned(), + components: Default::default(), + sender: tx, + }; + + let components = registry.components.clone(); + tokio::spawn(async move { + while let Some(message) = rx.recv().await { + if let Ok(mut map) = components.write() { + _ = map.insert(message.component, message.status); + } else { + // Poisoned mutex: Just warn, the probes will fail and the process restart + warn!("poisoned HeathRegistry mutex") + } + } + }); + + registry + } + + /// Registers a new component in the registry. The returned handle should be passed + /// to the component, to allow it to frequently report its health status. + pub async fn register(&self, component: String, deadline: Duration) -> HealthHandle { + let handle = HealthHandle { + component, + deadline, + sender: self.sender.clone(), + }; + handle.report_status(ComponentStatus::Starting).await; + handle + } + + /// Returns the overall process status, computed from the status of all the components + /// currently registered. Can be used as an axum handler. + pub fn get_status(&self) -> HealthStatus { + let components = self + .components + .read() + .expect("poisoned HeathRegistry mutex"); + + let result = HealthStatus { + healthy: !components.is_empty(), // unhealthy if no component has registered yet + components: Default::default(), + }; + let now = time::OffsetDateTime::now_utc(); + + let result = components + .iter() + .fold(result, |mut result, (name, status)| { + match status { + ComponentStatus::HealthyUntil(until) => { + if until.gt(&now) { + _ = result.components.insert(name.clone(), status.clone()) + } else { + result.healthy = false; + _ = result + .components + .insert(name.clone(), ComponentStatus::Stalled) + } + } + _ => { + result.healthy = false; + _ = result.components.insert(name.clone(), status.clone()) + } + } + result + }); + match result.healthy { + true => info!("{} health check ok", self.name), + false => warn!("{} health check failed: {:?}", self.name, result.components), + } + result + } +} + +#[cfg(test)] +mod tests { + use crate::health::{ComponentStatus, HealthRegistry, HealthStatus}; + use axum::http::StatusCode; + use axum::response::IntoResponse; + use std::ops::{Add, Sub}; + use time::{Duration, OffsetDateTime}; + + async fn assert_or_retry(check: F) + where + F: Fn() -> bool, + { + assert_or_retry_for_duration(check, Duration::seconds(5)).await + } + + async fn assert_or_retry_for_duration(check: F, timeout: Duration) + where + F: Fn() -> bool, + { + let deadline = OffsetDateTime::now_utc().add(timeout); + while !check() && OffsetDateTime::now_utc().lt(&deadline) { + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert!(check()) + } + #[tokio::test] + async fn defaults_to_unhealthy() { + let registry = HealthRegistry::new("liveness"); + assert!(!registry.get_status().healthy); + } + + #[tokio::test] + async fn one_component() { + let registry = HealthRegistry::new("liveness"); + + // New components are registered in Starting + let handle = registry + .register("one".to_string(), Duration::seconds(30)) + .await; + assert_or_retry(|| registry.get_status().components.len() == 1).await; + let mut status = registry.get_status(); + assert!(!status.healthy); + assert_eq!( + status.components.get("one"), + Some(&ComponentStatus::Starting) + ); + + // Status goes healthy once the component reports + handle.report_healthy().await; + assert_or_retry(|| registry.get_status().healthy).await; + status = registry.get_status(); + assert_eq!(status.components.len(), 1); + + // Status goes unhealthy if the components says so + handle.report_status(ComponentStatus::Unhealthy).await; + assert_or_retry(|| !registry.get_status().healthy).await; + status = registry.get_status(); + assert_eq!(status.components.len(), 1); + assert_eq!( + status.components.get("one"), + Some(&ComponentStatus::Unhealthy) + ); + } + + #[tokio::test] + async fn staleness_check() { + let registry = HealthRegistry::new("liveness"); + let handle = registry + .register("one".to_string(), Duration::seconds(30)) + .await; + + // Status goes healthy once the component reports + handle.report_healthy().await; + assert_or_retry(|| registry.get_status().healthy).await; + let mut status = registry.get_status(); + assert_eq!(status.components.len(), 1); + + // If the component's ping is too old, it is considered stalled and the healthcheck fails + // FIXME: we should mock the time instead + handle + .report_status(ComponentStatus::HealthyUntil( + OffsetDateTime::now_utc().sub(Duration::seconds(1)), + )) + .await; + assert_or_retry(|| !registry.get_status().healthy).await; + status = registry.get_status(); + assert_eq!(status.components.len(), 1); + assert_eq!( + status.components.get("one"), + Some(&ComponentStatus::Stalled) + ); + } + + #[tokio::test] + async fn several_components() { + let registry = HealthRegistry::new("liveness"); + let handle1 = registry + .register("one".to_string(), Duration::seconds(30)) + .await; + let handle2 = registry + .register("two".to_string(), Duration::seconds(30)) + .await; + assert_or_retry(|| registry.get_status().components.len() == 2).await; + + // First component going healthy is not enough + handle1.report_healthy().await; + assert_or_retry(|| { + registry.get_status().components.get("one").unwrap() != &ComponentStatus::Starting + }) + .await; + assert!(!registry.get_status().healthy); + + // Second component going healthy brings the health to green + handle2.report_healthy().await; + assert_or_retry(|| { + registry.get_status().components.get("two").unwrap() != &ComponentStatus::Starting + }) + .await; + assert!(registry.get_status().healthy); + + // First component going unhealthy takes down the health to red + handle1.report_status(ComponentStatus::Unhealthy).await; + assert_or_retry(|| !registry.get_status().healthy).await; + + // First component recovering returns the health to green + handle1.report_healthy().await; + assert_or_retry(|| registry.get_status().healthy).await; + + // Second component going unhealthy takes down the health to red + handle2.report_status(ComponentStatus::Unhealthy).await; + assert_or_retry(|| !registry.get_status().healthy).await; + } + + #[tokio::test] + async fn into_response() { + let nok = HealthStatus::default().into_response(); + assert_eq!(nok.status(), StatusCode::INTERNAL_SERVER_ERROR); + + let ok = HealthStatus { + healthy: true, + components: Default::default(), + } + .into_response(); + assert_eq!(ok.status(), StatusCode::OK); + } +} diff --git a/capture/src/lib.rs b/capture/src/lib.rs index 70f754807bc13..50f670567c33b 100644 --- a/capture/src/lib.rs +++ b/capture/src/lib.rs @@ -3,6 +3,7 @@ pub mod billing_limits; pub mod capture; pub mod config; pub mod event; +pub mod health; pub mod prometheus; pub mod redis; pub mod router; diff --git a/capture/src/router.rs b/capture/src/router.rs index 9acc7f4f6f920..bae787c113853 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -9,6 +9,7 @@ use axum::{ use tower_http::cors::{AllowHeaders, AllowOrigin, CorsLayer}; use tower_http::trace::TraceLayer; +use crate::health::HealthRegistry; use crate::{billing_limits::BillingLimiter, capture, redis::Client, sink, time::TimeSource}; use crate::prometheus::{setup_metrics_recorder, track_metrics}; @@ -31,6 +32,7 @@ pub fn router< R: Client + Send + Sync + 'static, >( timesource: TZ, + liveness: HealthRegistry, sink: S, redis: Arc, billing: BillingLimiter, @@ -54,6 +56,8 @@ pub fn router< let router = Router::new() // TODO: use NormalizePathLayer::trim_trailing_slash .route("/", get(index)) + .route("/_readiness", get(index)) + .route("/_liveness", get(move || ready(liveness.get_status()))) .route("/i/v0/e", post(capture::event).options(capture::options)) .route("/i/v0/e/", post(capture::event).options(capture::options)) .layer(TraceLayer::new_for_http()) diff --git a/capture/src/server.rs b/capture/src/server.rs index e4372ae0b584a..8c40fd3fa4112 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -6,13 +6,15 @@ use time::Duration; use crate::billing_limits::BillingLimiter; use crate::config::Config; +use crate::health::{ComponentStatus, HealthRegistry}; use crate::redis::RedisClient; use crate::{router, sink}; - pub async fn serve(config: Config, listener: TcpListener, shutdown: F) where F: Future, { + let liveness = HealthRegistry::new("liveness"); + let redis_client = Arc::new(RedisClient::new(config.redis_url).expect("failed to create redis client")); @@ -20,17 +22,28 @@ where .expect("failed to create billing limiter"); let app = if config.print_sink { + // Print sink is only used for local debug, don't allow a container with it to run on prod + liveness + .register("print_sink".to_string(), Duration::seconds(30)) + .await + .report_status(ComponentStatus::Unhealthy) + .await; router::router( crate::time::SystemTime {}, + liveness, sink::PrintSink {}, redis_client, billing, config.export_prometheus, ) } else { - let sink = sink::KafkaSink::new(config.kafka).unwrap(); + let sink_liveness = liveness + .register("rdkafka".to_string(), Duration::seconds(30)) + .await; + let sink = sink::KafkaSink::new(config.kafka, sink_liveness).unwrap(); router::router( crate::time::SystemTime {}, + liveness, sink, redis_client, billing, diff --git a/capture/src/sink.rs b/capture/src/sink.rs index d6b60d82024b4..f044df0047c7f 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -13,6 +13,7 @@ use tracing::{debug, info}; use crate::api::CaptureError; use crate::config::KafkaConfig; use crate::event::ProcessedEvent; +use crate::health::HealthHandle; use crate::prometheus::report_dropped_events; #[async_trait] @@ -45,10 +46,16 @@ impl EventSink for PrintSink { } } -struct KafkaContext; +struct KafkaContext { + liveness: HealthHandle, +} impl rdkafka::ClientContext for KafkaContext { fn stats(&self, stats: rdkafka::Statistics) { + // Signal liveness, as the main rdkafka loop is running and calling us + self.liveness.report_healthy_blocking(); + + // Update exported metrics gauge!("capture_kafka_callback_queue_depth", stats.replyq as f64); gauge!("capture_kafka_producer_queue_depth", stats.msg_cnt as f64); gauge!( @@ -107,7 +114,7 @@ pub struct KafkaSink { } impl KafkaSink { - pub fn new(config: KafkaConfig) -> anyhow::Result { + pub fn new(config: KafkaConfig, liveness: HealthHandle) -> anyhow::Result { info!("connecting to Kafka brokers at {}...", config.kafka_hosts); let mut client_config = ClientConfig::new(); @@ -129,7 +136,7 @@ impl KafkaSink { debug!("rdkafka configuration: {:?}", client_config); let producer: FutureProducer = - client_config.create_with_context(KafkaContext)?; + client_config.create_with_context(KafkaContext { liveness })?; // Ping the cluster to make sure we can reach brokers, fail after 10 seconds _ = producer.client().fetch_metadata( diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index d418996d6b06a..b95c78e48501f 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -7,6 +7,7 @@ use base64::Engine; use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode}; use capture::billing_limits::BillingLimiter; use capture::event::ProcessedEvent; +use capture::health::HealthRegistry; use capture::redis::MockRedisClient; use capture::router::router; use capture::sink::EventSink; @@ -76,6 +77,7 @@ impl EventSink for MemorySink { async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let file = File::open(REQUESTS_DUMP_FILE_NAME)?; let reader = BufReader::new(file); + let liveness = HealthRegistry::new("dummy"); let mut mismatches = 0; @@ -100,7 +102,14 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let billing = BillingLimiter::new(Duration::weeks(1), redis.clone()) .expect("failed to create billing limiter"); - let app = router(timesource, sink.clone(), redis, billing, false); + let app = router( + timesource, + liveness.clone(), + sink.clone(), + redis, + billing, + false, + ); let client = TestClient::new(app); let mut req = client.post(&format!("/i/v0{}", case.path)).body(raw_body); From e56e1f1dd7d7bd3c730165869b39b27618efb34f Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 13 Nov 2023 14:20:01 +0000 Subject: [PATCH 048/247] Add tracing setup (#47) * Add tracing setup * fix --------- Co-authored-by: Xavier Vello --- Cargo.lock | 276 ++++++++++++++++++++++++++++++++- capture-server/Cargo.toml | 8 +- capture-server/src/main.rs | 55 ++++++- capture-server/tests/common.rs | 4 +- capture/src/billing_limits.rs | 4 + capture/src/capture.rs | 5 + capture/src/config.rs | 9 +- capture/src/sink.rs | 4 +- 8 files changed, 352 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 685687f0932e3..fab963c8fe5b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -239,6 +239,9 @@ dependencies = [ "envconfig", "futures", "once_cell", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "rand", "rdkafka", "reqwest", @@ -246,6 +249,7 @@ dependencies = [ "time", "tokio", "tracing", + "tracing-opentelemetry", "tracing-subscriber", ] @@ -318,6 +322,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.15" @@ -608,6 +622,12 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "governor" version = "0.5.1" @@ -736,6 +756,18 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -873,6 +905,15 @@ dependencies = [ "libc", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "matchit" version = "0.7.3" @@ -1163,6 +1204,93 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +dependencies = [ + "futures-core", + "futures-sink", + "indexmap 2.0.2", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930" +dependencies = [ + "async-trait", + "futures-core", + "http", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "prost", + "thiserror", + "tokio", + "tonic", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5b3ce3f5705e2ae493be467a0b23be4bc563c193cdb7713e55372c89a906b34" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "ordered-float", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "ordered-float" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8" +dependencies = [ + "num-traits", +] + [[package]] name = "overload" version = "0.1.1" @@ -1297,6 +1425,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "quanta" version = "0.9.3" @@ -1469,8 +1620,17 @@ checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.0", + "regex-syntax 0.8.0", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -1481,9 +1641,15 @@ checksum = "5d58da636bd923eae52b7e9120271cbefb16f399069ee566ca5ebf9c30e32238" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.0", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.0" @@ -1878,6 +2044,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.1.0" @@ -1899,6 +2075,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.9" @@ -1930,6 +2117,34 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -1938,9 +2153,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -2042,18 +2261,51 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log 0.2.0", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-subscriber" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.1.3", ] [[package]] @@ -2103,6 +2355,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "uuid" version = "1.4.1" @@ -2240,6 +2498,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57099a701fb3a8043f993e8228dc24229c7b942e2b009a1b962e54489ba1d3bf" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/capture-server/Cargo.toml b/capture-server/Cargo.toml index 19200043f577f..e8aa5595486a6 100644 --- a/capture-server/Cargo.toml +++ b/capture-server/Cargo.toml @@ -7,10 +7,14 @@ edition = "2021" axum = { workspace = true } capture = { path = "../capture" } envconfig = { workspace = true } +opentelemetry = { version = "0.21.0", features = ["trace"]} +opentelemetry-otlp = "0.14.0" +opentelemetry_sdk = { version = "0.21.0", features = ["trace", "rt-tokio"] } time = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } +tracing-opentelemetry = "0.22.0" +tracing-subscriber = { workspace = true, features = ["env-filter"] } [dev-dependencies] anyhow = { workspace = true, features = [] } @@ -20,4 +24,4 @@ once_cell = "1.18.0" rand = { workspace = true } rdkafka = { workspace = true } reqwest = "0.11.22" -serde_json = { workspace = true } +serde_json = { workspace = true } \ No newline at end of file diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index 4874a43a32aa7..402fc3245d883 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -1,7 +1,18 @@ use std::net::TcpListener; +use std::time::Duration; use envconfig::Envconfig; +use opentelemetry::KeyValue; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::trace::{BatchConfig, RandomIdGenerator, Sampler, Tracer}; +use opentelemetry_sdk::{runtime, Resource}; use tokio::signal; +use tracing::level_filters::LevelFilter; +use tracing::Level; +use tracing_opentelemetry::OpenTelemetryLayer; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{EnvFilter, Layer}; use capture::config::Config; use capture::server::serve; @@ -21,12 +32,50 @@ async fn shutdown() { tracing::info!("Shutting down gracefully..."); } +fn init_tracer(sink_url: &str, sampling_rate: f64) -> Tracer { + opentelemetry_otlp::new_pipeline() + .tracing() + .with_trace_config( + opentelemetry_sdk::trace::Config::default() + .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased( + sampling_rate, + )))) + .with_id_generator(RandomIdGenerator::default()) + .with_resource(Resource::new(vec![KeyValue::new( + "service.name", + "capture", + )])), + ) + .with_batch_config(BatchConfig::default()) + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint(sink_url) + .with_timeout(Duration::from_secs(3)), + ) + .install_batch(runtime::Tokio) + .unwrap() +} + #[tokio::main] async fn main() { - // initialize tracing - tracing_subscriber::fmt::init(); - let config = Config::init_from_env().expect("Invalid configuration:"); + + // Instantiate tracing outputs: + // - stdout with a level configured by the RUST_LOG envvar (default=ERROR) + // - OpenTelemetry if enabled, for levels INFO and higher + let log_layer = tracing_subscriber::fmt::layer().with_filter(EnvFilter::from_default_env()); + let otel_layer = config + .otel_url + .clone() + .map(|url| OpenTelemetryLayer::new(init_tracer(&url, config.otel_sampling_rate))) + .with_filter(LevelFilter::from_level(Level::INFO)); + tracing_subscriber::registry() + .with(log_layer) + .with(otel_layer) + .init(); + + // Open the TCP port and start the server let listener = TcpListener::bind(config.address).unwrap(); serve(config, listener, shutdown()).await } diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index d4665cfedc748..e9329cf857f6d 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -25,7 +25,6 @@ use capture::server::serve; pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { print_sink: false, address: SocketAddr::from_str("127.0.0.1:0").unwrap(), - export_prometheus: false, redis_url: "redis://localhost:6379/".to_string(), kafka: KafkaConfig { kafka_producer_linger_ms: 0, // Send messages as soon as possible @@ -35,6 +34,9 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { kafka_topic: "events_plugin_ingestion".to_string(), kafka_tls: false, }, + otel_url: None, + otel_sampling_rate: 0.0, + export_prometheus: false, }); static TRACING_INIT: Once = Once::new(); diff --git a/capture/src/billing_limits.rs b/capture/src/billing_limits.rs index 5f1540009d0d7..9fa0fdd0e953e 100644 --- a/capture/src/billing_limits.rs +++ b/capture/src/billing_limits.rs @@ -22,10 +22,12 @@ use crate::redis::Client; use thiserror::Error; use time::{Duration, OffsetDateTime}; use tokio::sync::RwLock; +use tracing::instrument; // todo: fetch from env const QUOTA_LIMITER_CACHE_KEY: &str = "@posthog/quota-limits/"; +#[derive(Debug)] pub enum QuotaResource { Events, Recordings, @@ -81,6 +83,7 @@ impl BillingLimiter { }) } + #[instrument(skip_all)] async fn fetch_limited( client: &Arc, resource: QuotaResource, @@ -96,6 +99,7 @@ impl BillingLimiter { .await } + #[instrument(skip_all, fields(key = key))] pub async fn is_limited(&self, key: &str, resource: QuotaResource) -> bool { // hold the read lock to clone it, very briefly. clone is ok because it's very small 🤏 // rwlock can have many readers, but one writer. the writer will wait in a queue with all diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 45fc5f70f889a..dbff970feffc0 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -13,6 +13,7 @@ use base64::Engine; use metrics::counter; use time::OffsetDateTime; +use tracing::instrument; use crate::billing_limits::QuotaResource; use crate::event::ProcessingContext; @@ -25,6 +26,7 @@ use crate::{ utils::uuid_v7, }; +#[instrument(skip_all)] pub async fn event( state: State, InsecureClientIp(ip): InsecureClientIp, @@ -110,6 +112,7 @@ pub async fn options() -> Result, CaptureError> { })) } +#[instrument(skip_all)] pub fn process_single_event( event: &RawEvent, context: &ProcessingContext, @@ -141,6 +144,7 @@ pub fn process_single_event( }) } +#[instrument(skip_all, fields(events = events.len()))] pub fn extract_and_verify_token(events: &[RawEvent]) -> Result { let distinct_tokens: HashSet> = HashSet::from_iter( events @@ -162,6 +166,7 @@ pub fn extract_and_verify_token(events: &[RawEvent]) -> Result( sink: Arc, events: &'a [RawEvent], diff --git a/capture/src/config.rs b/capture/src/config.rs index e3ea1e8461a27..da8377afa3a14 100644 --- a/capture/src/config.rs +++ b/capture/src/config.rs @@ -9,10 +9,15 @@ pub struct Config { #[envconfig(default = "127.0.0.1:3000")] pub address: SocketAddr, pub redis_url: String, - #[envconfig(default = "true")] - pub export_prometheus: bool, + #[envconfig(nested = true)] pub kafka: KafkaConfig, + + pub otel_url: Option, + #[envconfig(default = "1.0")] + pub otel_sampling_rate: f64, + #[envconfig(default = "true")] + pub export_prometheus: bool, } #[derive(Envconfig, Clone)] diff --git a/capture/src/sink.rs b/capture/src/sink.rs index f044df0047c7f..a7be4588e0a20 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -8,7 +8,7 @@ use rdkafka::producer::future_producer::{FutureProducer, FutureRecord}; use rdkafka::producer::Producer; use rdkafka::util::Timeout; use tokio::task::JoinSet; -use tracing::{debug, info}; +use tracing::{debug, info, instrument}; use crate::api::CaptureError; use crate::config::KafkaConfig; @@ -192,6 +192,7 @@ impl KafkaSink { #[async_trait] impl EventSink for KafkaSink { + #[instrument(skip_all)] async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { Self::kafka_send(self.producer.clone(), self.topic.clone(), event).await?; @@ -200,6 +201,7 @@ impl EventSink for KafkaSink { Ok(()) } + #[instrument(skip_all)] async fn send_batch(&self, events: Vec) -> Result<(), CaptureError> { let mut set = JoinSet::new(); let batch_size = events.len(); From 014456dbaafa6e21d579dcabef83546825fd1160 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 13 Nov 2023 15:01:19 +0000 Subject: [PATCH 049/247] feat: add partition limiter + overflow (#48) * Partition limiter * add basic limiting * add tests * fmt * oops * test it more * not needed * Update capture/src/partition_limits.rs Co-authored-by: Xavier Vello * fmt * Update events.rs --------- Co-authored-by: Xavier Vello --- Cargo.lock | 1 + Cargo.toml | 2 +- capture-server/tests/common.rs | 21 ++++++++ capture-server/tests/events.rs | 93 +++++++++++++++++++++++++++++++++ capture/Cargo.toml | 1 + capture/src/capture.rs | 4 +- capture/src/config.rs | 13 ++++- capture/src/lib.rs | 1 + capture/src/partition_limits.rs | 58 ++++++++++++++++++++ capture/src/prometheus.rs | 5 ++ capture/src/server.rs | 8 ++- capture/src/sink.rs | 20 +++++-- 12 files changed, 217 insertions(+), 10 deletions(-) create mode 100644 capture/src/partition_limits.rs diff --git a/Cargo.lock b/Cargo.lock index fab963c8fe5b9..8e3b8737e2daa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -205,6 +205,7 @@ dependencies = [ "axum-test-helper", "base64", "bytes", + "dashmap", "envconfig", "flate2", "governor", diff --git a/Cargo.toml b/Cargo.toml index 402243e69515c..5a0d5015ee8b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ tracing = "0.1" tracing-subscriber = "0.3" serde = { version = "1.0.160", features = ["derive"] } serde_json = "1.0.96" -governor = "0.5.1" +governor = {version = "0.5.1", features=["dashmap"]} tower_governor = "0.0.4" time = { version = "0.3.20", features = ["formatting", "macros", "parsing", "serde"] } tower-http = { version = "0.4.0", features = ["cors", "trace"] } diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index e9329cf857f6d..c0ee9ba9f843a 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -2,6 +2,7 @@ use std::default::Default; use std::net::{SocketAddr, TcpListener}; +use std::num::NonZeroU32; use std::str::FromStr; use std::string::ToString; use std::sync::{Arc, Once}; @@ -26,6 +27,8 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { print_sink: false, address: SocketAddr::from_str("127.0.0.1:0").unwrap(), redis_url: "redis://localhost:6379/".to_string(), + burst_limit: NonZeroU32::new(5).unwrap(), + per_second_limit: NonZeroU32::new(10).unwrap(), kafka: KafkaConfig { kafka_producer_linger_ms: 0, // Send messages as soon as possible kafka_producer_queue_mib: 10, @@ -144,6 +147,24 @@ impl EphemeralTopic { None => bail!("kafka read timeout"), } } + pub fn next_message_key(&self) -> anyhow::Result> { + match self.consumer.poll(self.read_timeout) { + Some(Ok(message)) => { + let key = message.key(); + + if let Some(key) = key { + let key = std::str::from_utf8(key)?; + let key = String::from_str(key)?; + + Ok(Some(key)) + } else { + Ok(None) + } + } + Some(Err(err)) => bail!("kafka read error: {}", err), + None => bail!("kafka read timeout"), + } + } pub fn topic_name(&self) -> &str { &self.topic_name diff --git a/capture-server/tests/events.rs b/capture-server/tests/events.rs index 42facd86554ec..27db9a7c897de 100644 --- a/capture-server/tests/events.rs +++ b/capture-server/tests/events.rs @@ -1,3 +1,5 @@ +use std::num::NonZeroU32; + use anyhow::Result; use assert_json_diff::assert_json_include; use reqwest::StatusCode; @@ -73,3 +75,94 @@ async fn it_captures_a_batch() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn it_is_limited_with_burst() -> Result<()> { + setup_tracing(); + + let token = random_string("token", 16); + let distinct_id = random_string("id", 16); + + let topic = EphemeralTopic::new().await; + + let mut config = DEFAULT_CONFIG.clone(); + config.kafka.kafka_topic = topic.topic_name().to_string(); + config.burst_limit = NonZeroU32::new(2).unwrap(); + config.per_second_limit = NonZeroU32::new(1).unwrap(); + + let server = ServerHandle::for_config(config); + + let event = json!([{ + "token": token, + "event": "event1", + "distinct_id": distinct_id + },{ + "token": token, + "event": "event2", + "distinct_id": distinct_id + },{ + "token": token, + "event": "event3", + "distinct_id": distinct_id + }]); + + let res = server.capture_events(event.to_string()).await; + assert_eq!(StatusCode::OK, res.status()); + + assert_eq!( + topic.next_message_key()?.unwrap(), + format!("{}:{}", token, distinct_id) + ); + + assert_eq!( + topic.next_message_key()?.unwrap(), + format!("{}:{}", token, distinct_id) + ); + + assert_eq!(topic.next_message_key()?, None); + + Ok(()) +} + +#[tokio::test] +async fn it_does_not_partition_limit_different_ids() -> Result<()> { + setup_tracing(); + + let token = random_string("token", 16); + let distinct_id = random_string("id", 16); + let distinct_id2 = random_string("id", 16); + + let topic = EphemeralTopic::new().await; + + let mut config = DEFAULT_CONFIG.clone(); + config.kafka.kafka_topic = topic.topic_name().to_string(); + config.burst_limit = NonZeroU32::new(1).unwrap(); + config.per_second_limit = NonZeroU32::new(1).unwrap(); + + let server = ServerHandle::for_config(config); + + let event = json!([{ + "token": token, + "event": "event1", + "distinct_id": distinct_id + },{ + "token": token, + "event": "event2", + "distinct_id": distinct_id2 + }]); + + let res = server.capture_events(event.to_string()).await; + assert_eq!(StatusCode::OK, res.status()); + + assert_eq!( + topic.next_message_key()?.unwrap(), + format!("{}:{}", token, distinct_id) + ); + + assert_eq!( + topic.next_message_key()?.unwrap(), + format!("{}:{}", token, distinct_id2) + ); + + Ok(()) +} diff --git a/capture/Cargo.toml b/capture/Cargo.toml index 6bdfc39b20216..3b2b100074a60 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -31,6 +31,7 @@ metrics-exporter-prometheus = { workspace = true } thiserror = { workspace = true } redis = { version="0.23.3", features=["tokio-comp", "cluster", "cluster-async"] } envconfig = { workspace = true } +dashmap = "5.5.3" [dev-dependencies] assert-json-diff = { workspace = true } diff --git a/capture/src/capture.rs b/capture/src/capture.rs index dbff970feffc0..83466eb7653a5 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -78,12 +78,12 @@ pub async fn event( client_ip: ip.to_string(), }; - let limited = state + let billing_limited = state .billing .is_limited(context.token.as_str(), QuotaResource::Events) .await; - if limited { + if billing_limited { report_dropped_events("over_quota", 1); // for v0 we want to just return ok 🙃 diff --git a/capture/src/config.rs b/capture/src/config.rs index da8377afa3a14..8a471b3d288d6 100644 --- a/capture/src/config.rs +++ b/capture/src/config.rs @@ -1,4 +1,4 @@ -use std::net::SocketAddr; +use std::{net::SocketAddr, num::NonZeroU32}; use envconfig::Envconfig; @@ -6,16 +6,25 @@ use envconfig::Envconfig; pub struct Config { #[envconfig(default = "false")] pub print_sink: bool, + #[envconfig(default = "127.0.0.1:3000")] pub address: SocketAddr, + pub redis_url: String, + pub otel_url: Option, + + #[envconfig(default = "100")] + pub per_second_limit: NonZeroU32, + + #[envconfig(default = "1000")] + pub burst_limit: NonZeroU32, #[envconfig(nested = true)] pub kafka: KafkaConfig, - pub otel_url: Option, #[envconfig(default = "1.0")] pub otel_sampling_rate: f64, + #[envconfig(default = "true")] pub export_prometheus: bool, } diff --git a/capture/src/lib.rs b/capture/src/lib.rs index 50f670567c33b..eea915c307f71 100644 --- a/capture/src/lib.rs +++ b/capture/src/lib.rs @@ -4,6 +4,7 @@ pub mod capture; pub mod config; pub mod event; pub mod health; +pub mod partition_limits; pub mod prometheus; pub mod redis; pub mod router; diff --git a/capture/src/partition_limits.rs b/capture/src/partition_limits.rs new file mode 100644 index 0000000000000..fe63ec157141c --- /dev/null +++ b/capture/src/partition_limits.rs @@ -0,0 +1,58 @@ +/// When a customer is writing too often to the same key, we get hot partitions. This negatively +/// affects our write latency and cluster health. We try to provide ordering guarantees wherever +/// possible, but this does require that we map key -> partition. +/// +/// If the write-rate reaches a certain amount, we need to be able to handle the hot partition +/// before it causes a negative impact. In this case, instead of passing the error to the customer +/// with a 429, we relax our ordering constraints and temporarily override the key, meaning the +/// customers data will be spread across all partitions. +use std::{num::NonZeroU32, sync::Arc}; + +use governor::{clock, state::keyed::DefaultKeyedStateStore, Quota, RateLimiter}; + +// See: https://docs.rs/governor/latest/governor/_guide/index.html#usage-in-multiple-threads +#[derive(Clone)] +pub struct PartitionLimiter { + limiter: Arc, clock::DefaultClock>>, +} + +impl PartitionLimiter { + pub fn new(per_second: NonZeroU32, burst: NonZeroU32) -> Self { + let quota = Quota::per_second(per_second).allow_burst(burst); + let limiter = Arc::new(governor::RateLimiter::dashmap(quota)); + + PartitionLimiter { limiter } + } + + pub fn is_limited(&self, key: &String) -> bool { + self.limiter.check_key(key).is_err() + } +} + +#[cfg(test)] +mod tests { + use crate::partition_limits::PartitionLimiter; + use std::num::NonZeroU32; + + #[tokio::test] + async fn low_limits() { + let limiter = + PartitionLimiter::new(NonZeroU32::new(1).unwrap(), NonZeroU32::new(1).unwrap()); + let token = String::from("test"); + + assert!(!limiter.is_limited(&token)); + assert!(limiter.is_limited(&token)); + } + + #[tokio::test] + async fn bursting() { + let limiter = + PartitionLimiter::new(NonZeroU32::new(1).unwrap(), NonZeroU32::new(3).unwrap()); + let token = String::from("test"); + + assert!(!limiter.is_limited(&token)); + assert!(!limiter.is_limited(&token)); + assert!(!limiter.is_limited(&token)); + assert!(limiter.is_limited(&token)); + } +} diff --git a/capture/src/prometheus.rs b/capture/src/prometheus.rs index d9dbea8831703..f2255200f1c2f 100644 --- a/capture/src/prometheus.rs +++ b/capture/src/prometheus.rs @@ -9,6 +9,11 @@ use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; pub fn report_dropped_events(cause: &'static str, quantity: u64) { counter!("capture_events_dropped_total", quantity, "cause" => cause); } + +pub fn report_overflow_partition(quantity: u64) { + counter!("capture_partition_key_capacity_exceeded_total", quantity); +} + pub fn setup_metrics_recorder() -> PrometheusHandle { // Ok I broke it at the end, but the limit on our ingress is 60 and that's a nicer way of reaching it const EXPONENTIAL_SECONDS: &[f64] = &[ diff --git a/capture/src/server.rs b/capture/src/server.rs index 8c40fd3fa4112..c84bd20c94ca6 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -7,8 +7,10 @@ use time::Duration; use crate::billing_limits::BillingLimiter; use crate::config::Config; use crate::health::{ComponentStatus, HealthRegistry}; +use crate::partition_limits::PartitionLimiter; use crate::redis::RedisClient; use crate::{router, sink}; + pub async fn serve(config: Config, listener: TcpListener, shutdown: F) where F: Future, @@ -28,6 +30,7 @@ where .await .report_status(ComponentStatus::Unhealthy) .await; + router::router( crate::time::SystemTime {}, liveness, @@ -40,7 +43,10 @@ where let sink_liveness = liveness .register("rdkafka".to_string(), Duration::seconds(30)) .await; - let sink = sink::KafkaSink::new(config.kafka, sink_liveness).unwrap(); + + let partition = PartitionLimiter::new(config.per_second_limit, config.burst_limit); + let sink = sink::KafkaSink::new(config.kafka, sink_liveness, partition).unwrap(); + router::router( crate::time::SystemTime {}, liveness, diff --git a/capture/src/sink.rs b/capture/src/sink.rs index a7be4588e0a20..9d915b0e15b2b 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -14,6 +14,7 @@ use crate::api::CaptureError; use crate::config::KafkaConfig; use crate::event::ProcessedEvent; use crate::health::HealthHandle; +use crate::partition_limits::PartitionLimiter; use crate::prometheus::report_dropped_events; #[async_trait] @@ -111,10 +112,15 @@ impl rdkafka::ClientContext for KafkaContext { pub struct KafkaSink { producer: FutureProducer, topic: String, + partition: PartitionLimiter, } impl KafkaSink { - pub fn new(config: KafkaConfig, liveness: HealthHandle) -> anyhow::Result { + pub fn new( + config: KafkaConfig, + liveness: HealthHandle, + partition: PartitionLimiter, + ) -> anyhow::Result { info!("connecting to Kafka brokers at {}...", config.kafka_hosts); let mut client_config = ClientConfig::new(); @@ -147,6 +153,7 @@ impl KafkaSink { Ok(KafkaSink { producer, + partition, topic: config.kafka_topic, }) } @@ -157,6 +164,7 @@ impl KafkaSink { producer: FutureProducer, topic: String, event: ProcessedEvent, + limited: bool, ) -> Result<(), CaptureError> { let payload = serde_json::to_string(&event).map_err(|e| { tracing::error!("failed to serialize event: {}", e); @@ -164,12 +172,13 @@ impl KafkaSink { })?; let key = event.key(); + let partition_key = if limited { None } else { Some(key.as_str()) }; match producer.send_result(FutureRecord { topic: topic.as_str(), payload: Some(&payload), partition: None, - key: Some(&key), + key: partition_key, timestamp: None, headers: None, }) { @@ -194,10 +203,12 @@ impl KafkaSink { impl EventSink for KafkaSink { #[instrument(skip_all)] async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { - Self::kafka_send(self.producer.clone(), self.topic.clone(), event).await?; + let limited = self.partition.is_limited(&event.key()); + Self::kafka_send(self.producer.clone(), self.topic.clone(), event, limited).await?; histogram!("capture_event_batch_size", 1.0); counter!("capture_events_ingested_total", 1); + Ok(()) } @@ -209,7 +220,8 @@ impl EventSink for KafkaSink { let producer = self.producer.clone(); let topic = self.topic.clone(); - set.spawn(Self::kafka_send(producer, topic, event)); + let limited = self.partition.is_limited(&event.key()); + set.spawn(Self::kafka_send(producer, topic, event, limited)); } // Await on all the produce promises From 0b1b0a2ab534a20532581df794c300d9755fb801 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 14 Nov 2023 14:55:49 +0100 Subject: [PATCH 050/247] tracing: add batch_size and token to root span (#49) --- capture/src/capture.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 83466eb7653a5..4177ad13f35be 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -26,7 +26,7 @@ use crate::{ utils::uuid_v7, }; -#[instrument(skip_all)] +#[instrument(skip_all, fields(token, batch_size))] pub async fn event( state: State, InsecureClientIp(ip): InsecureClientIp, @@ -48,6 +48,8 @@ pub async fn event( _ => RawEvent::from_bytes(&meta, body), }?; + tracing::Span::current().record("batch_size", events.len()); + if events.is_empty() { return Err(CaptureError::EmptyBatch); } @@ -57,6 +59,8 @@ pub async fn event( err })?; + tracing::Span::current().record("token", &token); + counter!("capture_events_received_total", events.len() as u64); let sent_at = meta.sent_at.and_then(|value| { From 3e70e7106b10e6eff94c842e0f4f3ba0dde1034f Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 20 Nov 2023 12:54:03 +0100 Subject: [PATCH 051/247] route GETs to the legacy endpoint too (#53) --- capture/src/router.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/capture/src/router.rs b/capture/src/router.rs index bae787c113853..6f2f044f88c67 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -58,8 +58,18 @@ pub fn router< .route("/", get(index)) .route("/_readiness", get(index)) .route("/_liveness", get(move || ready(liveness.get_status()))) - .route("/i/v0/e", post(capture::event).options(capture::options)) - .route("/i/v0/e/", post(capture::event).options(capture::options)) + .route( + "/i/v0/e", + post(capture::event) + .get(capture::event) + .options(capture::options), + ) + .route( + "/i/v0/e/", + post(capture::event) + .get(capture::event) + .options(capture::options), + ) .layer(TraceLayer::new_for_http()) .layer(cors) .layer(axum::middleware::from_fn(track_metrics)) From b68bd7e338c25ce0e6ff373e3edc41892cbd75bd Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 21 Nov 2023 15:08:32 +0100 Subject: [PATCH 052/247] fix: increment over_quota by batch size instead of 1 (#56) --- capture/src/capture.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 4177ad13f35be..39596da57a891 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -88,7 +88,7 @@ pub async fn event( .await; if billing_limited { - report_dropped_events("over_quota", 1); + report_dropped_events("over_quota", events.len() as u64); // for v0 we want to just return ok 🙃 // this is because the clients are pretty dumb and will just retry over and over and From f9fe9f11d83ca154fd87281cb9ec7a26c19acd78 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 21 Nov 2023 14:12:48 +0000 Subject: [PATCH 053/247] add more span attributes (#55) * add more spans * adjust * adjust again * fmt * bleh --- capture/src/capture.rs | 44 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 39596da57a891..d1d302ece64e5 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -16,7 +16,7 @@ use time::OffsetDateTime; use tracing::instrument; use crate::billing_limits::QuotaResource; -use crate::event::ProcessingContext; +use crate::event::{Compression, ProcessingContext}; use crate::prometheus::report_dropped_events; use crate::token::validate_token; use crate::{ @@ -26,7 +26,18 @@ use crate::{ utils::uuid_v7, }; -#[instrument(skip_all, fields(token, batch_size))] +#[instrument( + skip_all, + fields( + token, + batch_size, + user_agent, + content_encoding, + content_type, + version, + compression + ) +)] pub async fn event( state: State, InsecureClientIp(ip): InsecureClientIp, @@ -34,21 +45,48 @@ pub async fn event( headers: HeaderMap, body: Bytes, ) -> Result, CaptureError> { + // content-type + // user-agent + + let user_agent = headers + .get("user_agent") + .map_or("unknown", |v| v.to_str().unwrap_or("unknown")); + let content_encoding = headers + .get("content_encoding") + .map_or("unknown", |v| v.to_str().unwrap_or("unknown")); + + tracing::Span::current().record("user_agent", user_agent); + tracing::Span::current().record("content_encoding", content_encoding); + let events = match headers .get("content-type") .map_or("", |v| v.to_str().unwrap_or("")) { "application/x-www-form-urlencoded" => { + tracing::Span::current().record("content_type", "application/x-www-form-urlencoded"); + let input: EventFormData = serde_urlencoded::from_bytes(body.deref()).unwrap(); let payload = base64::engine::general_purpose::STANDARD .decode(input.data) .unwrap(); RawEvent::from_bytes(&meta, payload.into()) } - _ => RawEvent::from_bytes(&meta, body), + ct => { + tracing::Span::current().record("content_type", ct); + + RawEvent::from_bytes(&meta, body) + } }?; + let comp = match meta.compression { + None => String::from("unknown"), + Some(Compression::Gzip) => String::from("gzip"), + Some(Compression::Unsupported) => String::from("unsupported"), + }; + tracing::Span::current().record("batch_size", events.len()); + tracing::Span::current().record("version", meta.lib_version.clone()); + tracing::Span::current().record("compression", comp.as_str()); if events.is_empty() { return Err(CaptureError::EmptyBatch); From 6cee0a57c1522f0b2dc0e1dd3d83208a957ea3dc Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 21 Nov 2023 14:31:41 +0000 Subject: [PATCH 054/247] fix new span setup (#57) * fix spans * fmt * add method --- capture/src/capture.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/capture/src/capture.rs b/capture/src/capture.rs index d1d302ece64e5..1d889d371b3ae 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -7,7 +7,7 @@ use bytes::Bytes; use axum::Json; // TODO: stream this instead use axum::extract::{Query, State}; -use axum::http::HeaderMap; +use axum::http::{HeaderMap, Method}; use axum_client_ip::InsecureClientIp; use base64::Engine; use metrics::counter; @@ -43,20 +43,30 @@ pub async fn event( InsecureClientIp(ip): InsecureClientIp, meta: Query, headers: HeaderMap, + method: Method, body: Bytes, ) -> Result, CaptureError> { // content-type // user-agent let user_agent = headers - .get("user_agent") + .get("user-agent") .map_or("unknown", |v| v.to_str().unwrap_or("unknown")); let content_encoding = headers - .get("content_encoding") + .get("content-encoding") .map_or("unknown", |v| v.to_str().unwrap_or("unknown")); + let comp = match meta.compression { + None => String::from("unknown"), + Some(Compression::Gzip) => String::from("gzip"), + Some(Compression::Unsupported) => String::from("unsupported"), + }; + tracing::Span::current().record("user_agent", user_agent); tracing::Span::current().record("content_encoding", content_encoding); + tracing::Span::current().record("version", meta.lib_version.clone()); + tracing::Span::current().record("compression", comp.as_str()); + tracing::Span::current().record("method", method.as_str()); let events = match headers .get("content-type") @@ -78,15 +88,7 @@ pub async fn event( } }?; - let comp = match meta.compression { - None => String::from("unknown"), - Some(Compression::Gzip) => String::from("gzip"), - Some(Compression::Unsupported) => String::from("unsupported"), - }; - tracing::Span::current().record("batch_size", events.len()); - tracing::Span::current().record("version", meta.lib_version.clone()); - tracing::Span::current().record("compression", comp.as_str()); if events.is_empty() { return Err(CaptureError::EmptyBatch); From f8a07fbfa3dead0b73cad250ffa4b2db0440ae1d Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 21 Nov 2023 16:36:02 +0100 Subject: [PATCH 055/247] account for process_events errors in capture_events_dropped_total and log them (#58) --- capture/src/capture.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 1d889d371b3ae..6935ee3ff6e19 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -143,7 +143,11 @@ pub async fn event( tracing::debug!(context=?context, events=?events, "decoded request"); - process_events(state.sink.clone(), &events, &context).await?; + if let Err(err) = process_events(state.sink.clone(), &events, &context).await { + report_dropped_events("process_events_error", events.len() as u64); + tracing::log::warn!("rejected invalid payload: {}", err); + return Err(err); + } Ok(Json(CaptureResponse { status: CaptureResponseCode::Ok, From 7228307d2319d6c8d63362510795313d1c71463a Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 27 Nov 2023 13:55:08 +0100 Subject: [PATCH 056/247] properly process Kafka produce ACKs to propagate errors (#59) --- Cargo.lock | 8 +- Cargo.toml | 4 +- capture-server/tests/common.rs | 15 ++- capture/src/capture.rs | 5 +- capture/src/config.rs | 2 + capture/src/server.rs | 3 +- capture/src/sink.rs | 202 +++++++++++++++++++++++++++++---- 7 files changed, 207 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e3b8737e2daa..6bf02dfc43650 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1531,9 +1531,9 @@ dependencies = [ [[package]] name = "rdkafka" -version = "0.34.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053adfa02fab06e86c01d586cc68aa47ee0ff4489a59469081dc12cbcde578bf" +checksum = "d54f02a5a40220f8a2dfa47ddb38ba9064475a5807a69504b6f91711df2eea63" dependencies = [ "futures-channel", "futures-util", @@ -1549,9 +1549,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.6.0+2.2.0" +version = "4.7.0+2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad63c279fca41a27c231c450a2d2ad18288032e9cbb159ad16c9d96eba35aaaf" +checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615" dependencies = [ "cmake", "libc", diff --git a/Cargo.toml b/Cargo.toml index 5a0d5015ee8b5..983cbf2b7d696 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,6 @@ [workspace] +resolver = "2" + members = [ "capture", "capture-server" @@ -25,7 +27,7 @@ uuid = { version = "1.3.3", features = ["serde"] } async-trait = "0.1.68" serde_urlencoded = "0.7.1" rand = "0.8.5" -rdkafka = { version = "0.34", features = ["cmake-build", "ssl"] } +rdkafka = { version = "0.36.0", features = ["cmake-build", "ssl"] } metrics = "0.21.1" metrics-exporter-prometheus = "0.12.1" thiserror = "1.0.48" diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index c0ee9ba9f843a..d74b5bf9eed2b 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -18,7 +18,8 @@ use rdkafka::consumer::{BaseConsumer, Consumer}; use rdkafka::util::Timeout; use rdkafka::{Message, TopicPartitionList}; use tokio::sync::Notify; -use tracing::debug; +use tokio::time::timeout; +use tracing::{debug, warn}; use capture::config::{Config, KafkaConfig}; use capture::server::serve; @@ -32,6 +33,7 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { kafka: KafkaConfig { kafka_producer_linger_ms: 0, // Send messages as soon as possible kafka_producer_queue_mib: 10, + kafka_message_timeout_ms: 10000, // 10s, ACKs can be slow on low volumes, should be tuned kafka_compression_codec: "none".to_string(), kafka_hosts: "kafka:9092".to_string(), kafka_topic: "events_plugin_ingestion".to_string(), @@ -174,9 +176,14 @@ impl EphemeralTopic { impl Drop for EphemeralTopic { fn drop(&mut self) { debug!("dropping EphemeralTopic {}...", self.topic_name); - _ = self.consumer.unassign(); - futures::executor::block_on(delete_topic(self.topic_name.clone())); - debug!("dropped topic"); + self.consumer.unsubscribe(); + match futures::executor::block_on(timeout( + Duration::from_secs(10), + delete_topic(self.topic_name.clone()), + )) { + Ok(_) => debug!("dropped topic"), + Err(err) => warn!("failed to drop topic: {}", err), + } } } diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 6935ee3ff6e19..da4a72693f51f 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -228,11 +228,10 @@ pub async fn process_events<'a>( tracing::debug!(events=?events, "processed {} events", events.len()); if events.len() == 1 { - sink.send(events[0].clone()).await?; + sink.send(events[0].clone()).await } else { - sink.send_batch(events).await?; + sink.send_batch(events).await } - Ok(()) } #[cfg(test)] diff --git a/capture/src/config.rs b/capture/src/config.rs index 8a471b3d288d6..d69d4a96262f9 100644 --- a/capture/src/config.rs +++ b/capture/src/config.rs @@ -35,6 +35,8 @@ pub struct KafkaConfig { pub kafka_producer_linger_ms: u32, // Maximum time between producer batches during low traffic #[envconfig(default = "400")] pub kafka_producer_queue_mib: u32, // Size of the in-memory producer queue in mebibytes + #[envconfig(default = "20000")] + pub kafka_message_timeout_ms: u32, // Time before we stop retrying producing a message: 20 seconds #[envconfig(default = "none")] pub kafka_compression_codec: String, // none, gzip, snappy, lz4, zstd pub kafka_hosts: String, diff --git a/capture/src/server.rs b/capture/src/server.rs index c84bd20c94ca6..ad9150907df98 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -45,7 +45,8 @@ where .await; let partition = PartitionLimiter::new(config.per_second_limit, config.burst_limit); - let sink = sink::KafkaSink::new(config.kafka, sink_liveness, partition).unwrap(); + let sink = sink::KafkaSink::new(config.kafka, sink_liveness, partition) + .expect("failed to start Kafka sink"); router::router( crate::time::SystemTime {}, diff --git a/capture/src/sink.rs b/capture/src/sink.rs index 9d915b0e15b2b..c1d291a275628 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -3,12 +3,13 @@ use std::time::Duration; use async_trait::async_trait; use metrics::{absolute_counter, counter, gauge, histogram}; use rdkafka::config::ClientConfig; -use rdkafka::error::RDKafkaErrorCode; +use rdkafka::error::{KafkaError, RDKafkaErrorCode}; use rdkafka::producer::future_producer::{FutureProducer, FutureRecord}; -use rdkafka::producer::Producer; +use rdkafka::producer::{DeliveryFuture, Producer}; use rdkafka::util::Timeout; use tokio::task::JoinSet; -use tracing::{debug, info, instrument}; +use tracing::instrument; +use tracing::log::{debug, error, info}; use crate::api::CaptureError; use crate::config::KafkaConfig; @@ -128,6 +129,10 @@ impl KafkaSink { .set("bootstrap.servers", &config.kafka_hosts) .set("statistics.interval.ms", "10000") .set("linger.ms", config.kafka_producer_linger_ms.to_string()) + .set( + "message.timeout.ms", + config.kafka_message_timeout_ms.to_string(), + ) .set("compression.codec", config.kafka_compression_codec) .set( "queue.buffering.max.kbytes", @@ -157,17 +162,20 @@ impl KafkaSink { topic: config.kafka_topic, }) } -} -impl KafkaSink { + pub fn flush(&self) -> Result<(), KafkaError> { + // TODO: hook it up on shutdown + self.producer.flush(Duration::new(30, 0)) + } + async fn kafka_send( producer: FutureProducer, topic: String, event: ProcessedEvent, limited: bool, - ) -> Result<(), CaptureError> { + ) -> Result { let payload = serde_json::to_string(&event).map_err(|e| { - tracing::error!("failed to serialize event: {}", e); + error!("failed to serialize event: {}", e); CaptureError::NonRetryableSinkError })?; @@ -182,7 +190,7 @@ impl KafkaSink { timestamp: None, headers: None, }) { - Ok(_) => Ok(()), + Ok(ack) => Ok(ack), Err((e, _)) => match e.rdkafka_error_code() { Some(RDKafkaErrorCode::InvalidMessageSize) => { report_dropped_events("kafka_message_size", 1); @@ -191,12 +199,38 @@ impl KafkaSink { _ => { // TODO(maybe someday): Don't drop them but write them somewhere and try again report_dropped_events("kafka_write_error", 1); - tracing::error!("failed to produce event: {}", e); + error!("failed to produce event: {}", e); Err(CaptureError::RetryableSinkError) } }, } } + + async fn process_ack(delivery: DeliveryFuture) -> Result<(), CaptureError> { + match delivery.await { + Err(_) => { + // Cancelled due to timeout while retrying + counter!("capture_kafka_produce_errors_total", 1); + error!("failed to produce to Kafka before write timeout"); + Err(CaptureError::RetryableSinkError) + } + Ok(Err((KafkaError::MessageProduction(RDKafkaErrorCode::MessageSizeTooLarge), _))) => { + // Rejected by broker due to message size + report_dropped_events("kafka_message_size", 1); + Err(CaptureError::EventTooBig) + } + Ok(Err((err, _))) => { + // Unretriable produce error + counter!("capture_kafka_produce_errors_total", 1); + error!("failed to produce to Kafka: {}", err); + Err(CaptureError::RetryableSinkError) + } + Ok(Ok(_)) => { + counter!("capture_events_ingested_total", 1); + Ok(()) + } + } + } } #[async_trait] @@ -204,12 +238,10 @@ impl EventSink for KafkaSink { #[instrument(skip_all)] async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { let limited = self.partition.is_limited(&event.key()); - Self::kafka_send(self.producer.clone(), self.topic.clone(), event, limited).await?; - + let ack = + Self::kafka_send(self.producer.clone(), self.topic.clone(), event, limited).await?; histogram!("capture_event_batch_size", 1.0); - counter!("capture_events_ingested_total", 1); - - Ok(()) + Self::process_ack(ack).await } #[instrument(skip_all)] @@ -219,16 +251,148 @@ impl EventSink for KafkaSink { for event in events { let producer = self.producer.clone(); let topic = self.topic.clone(); - let limited = self.partition.is_limited(&event.key()); - set.spawn(Self::kafka_send(producer, topic, event, limited)); + + // We await kafka_send to get events in the producer queue sequentially + let ack = Self::kafka_send(producer, topic, event, limited).await?; + + // Then stash the returned DeliveryFuture, waiting concurrently for the write ACKs from brokers. + set.spawn(Self::process_ack(ack)); } - // Await on all the produce promises - while (set.join_next().await).is_some() {} + // Await on all the produce promises, fail batch on first failure + while let Some(res) = set.join_next().await { + match res { + Ok(Ok(_)) => {} + Ok(Err(err)) => { + set.abort_all(); + return Err(err); + } + Err(err) => { + set.abort_all(); + error!("join error while waiting on Kafka ACK: {:?}", err); + return Err(CaptureError::RetryableSinkError); + } + } + } histogram!("capture_event_batch_size", batch_size as f64); - counter!("capture_events_ingested_total", batch_size as u64); Ok(()) } } + +#[cfg(test)] +mod tests { + use crate::api::CaptureError; + use crate::config; + use crate::event::ProcessedEvent; + use crate::health::HealthRegistry; + use crate::partition_limits::PartitionLimiter; + use crate::sink::{EventSink, KafkaSink}; + use crate::utils::uuid_v7; + use rdkafka::mocking::MockCluster; + use rdkafka::producer::DefaultProducerContext; + use rdkafka::types::{RDKafkaApiKey, RDKafkaRespErr}; + use std::num::NonZeroU32; + use time::Duration; + + async fn start_on_mocked_sink() -> (MockCluster<'static, DefaultProducerContext>, KafkaSink) { + let registry = HealthRegistry::new("liveness"); + let handle = registry + .register("one".to_string(), Duration::seconds(30)) + .await; + let limiter = + PartitionLimiter::new(NonZeroU32::new(10).unwrap(), NonZeroU32::new(10).unwrap()); + let cluster = MockCluster::new(1).expect("failed to create mock brokers"); + let config = config::KafkaConfig { + kafka_producer_linger_ms: 0, + kafka_producer_queue_mib: 50, + kafka_message_timeout_ms: 500, + kafka_compression_codec: "none".to_string(), + kafka_hosts: cluster.bootstrap_servers(), + kafka_topic: "events_plugin_ingestion".to_string(), + kafka_tls: false, + }; + let sink = KafkaSink::new(config, handle, limiter).expect("failed to create sink"); + (cluster, sink) + } + + #[tokio::test] + async fn kafka_sink_error_handling() { + // Uses a mocked Kafka broker that allows injecting write errors, to check error handling. + // We test different cases in a single test to amortize the startup cost of the producer. + + let (cluster, sink) = start_on_mocked_sink().await; + let event: ProcessedEvent = ProcessedEvent { + uuid: uuid_v7(), + distinct_id: "id1".to_string(), + ip: "".to_string(), + data: "".to_string(), + now: "".to_string(), + sent_at: None, + token: "token1".to_string(), + }; + + // Wait for producer to be healthy, to keep kafka_message_timeout_ms short and tests faster + for _ in 0..20 { + if sink.send(event.clone()).await.is_ok() { + break; + } + } + + // Send events to confirm happy path + sink.send(event.clone()) + .await + .expect("failed to send one initial event"); + sink.send_batch(vec![event.clone(), event.clone()]) + .await + .expect("failed to send initial event batch"); + + // Simulate unretriable errors + cluster.clear_request_errors(RDKafkaApiKey::Produce); + let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE; 1]; + cluster.request_errors(RDKafkaApiKey::Produce, &err); + match sink.send(event.clone()).await { + Err(CaptureError::EventTooBig) => {} // Expected + Err(err) => panic!("wrong error code {}", err), + Ok(()) => panic!("should have errored"), + }; + cluster.clear_request_errors(RDKafkaApiKey::Produce); + let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_INVALID_PARTITIONS; 1]; + cluster.request_errors(RDKafkaApiKey::Produce, &err); + match sink.send_batch(vec![event.clone(), event.clone()]).await { + Err(CaptureError::RetryableSinkError) => {} // Expected + Err(err) => panic!("wrong error code {}", err), + Ok(()) => panic!("should have errored"), + }; + + // Simulate transient errors, messages should go through OK + cluster.clear_request_errors(RDKafkaApiKey::Produce); + let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE; 2]; + cluster.request_errors(RDKafkaApiKey::Produce, &err); + sink.send(event.clone()) + .await + .expect("failed to send one event after recovery"); + cluster.clear_request_errors(RDKafkaApiKey::Produce); + let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE; 2]; + cluster.request_errors(RDKafkaApiKey::Produce, &err); + sink.send_batch(vec![event.clone(), event.clone()]) + .await + .expect("failed to send event batch after recovery"); + + // Timeout on a sustained transient error + cluster.clear_request_errors(RDKafkaApiKey::Produce); + let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE; 50]; + cluster.request_errors(RDKafkaApiKey::Produce, &err); + match sink.send(event.clone()).await { + Err(CaptureError::RetryableSinkError) => {} // Expected + Err(err) => panic!("wrong error code {}", err), + Ok(()) => panic!("should have errored"), + }; + match sink.send_batch(vec![event.clone(), event.clone()]).await { + Err(CaptureError::RetryableSinkError) => {} // Expected + Err(err) => panic!("wrong error code {}", err), + Ok(()) => panic!("should have errored"), + }; + } +} From 7056e808d91747029cc28d0f097cb7ed523ff817 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 27 Nov 2023 14:05:12 +0100 Subject: [PATCH 057/247] feat(overflow): add overflow_forced_keys envvar (#54) --- capture-server/tests/common.rs | 1 + capture/src/config.rs | 2 ++ capture/src/partition_limits.rs | 55 ++++++++++++++++++++++++++++----- capture/src/server.rs | 6 +++- capture/src/sink.rs | 7 +++-- 5 files changed, 60 insertions(+), 11 deletions(-) diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index d74b5bf9eed2b..b71fdf62c9a35 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -30,6 +30,7 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { redis_url: "redis://localhost:6379/".to_string(), burst_limit: NonZeroU32::new(5).unwrap(), per_second_limit: NonZeroU32::new(10).unwrap(), + overflow_forced_keys: None, kafka: KafkaConfig { kafka_producer_linger_ms: 0, // Send messages as soon as possible kafka_producer_queue_mib: 10, diff --git a/capture/src/config.rs b/capture/src/config.rs index d69d4a96262f9..69a085dd927b0 100644 --- a/capture/src/config.rs +++ b/capture/src/config.rs @@ -19,6 +19,8 @@ pub struct Config { #[envconfig(default = "1000")] pub burst_limit: NonZeroU32, + pub overflow_forced_keys: Option, // Coma-delimited keys + #[envconfig(nested = true)] pub kafka: KafkaConfig, diff --git a/capture/src/partition_limits.rs b/capture/src/partition_limits.rs index fe63ec157141c..386665780ad1c 100644 --- a/capture/src/partition_limits.rs +++ b/capture/src/partition_limits.rs @@ -6,7 +6,9 @@ /// before it causes a negative impact. In this case, instead of passing the error to the customer /// with a 429, we relax our ordering constraints and temporarily override the key, meaning the /// customers data will be spread across all partitions. -use std::{num::NonZeroU32, sync::Arc}; +use std::collections::HashSet; +use std::num::NonZeroU32; +use std::sync::Arc; use governor::{clock, state::keyed::DefaultKeyedStateStore, Quota, RateLimiter}; @@ -14,18 +16,27 @@ use governor::{clock, state::keyed::DefaultKeyedStateStore, Quota, RateLimiter}; #[derive(Clone)] pub struct PartitionLimiter { limiter: Arc, clock::DefaultClock>>, + forced_keys: HashSet, } impl PartitionLimiter { - pub fn new(per_second: NonZeroU32, burst: NonZeroU32) -> Self { + pub fn new(per_second: NonZeroU32, burst: NonZeroU32, forced_keys: Option) -> Self { let quota = Quota::per_second(per_second).allow_burst(burst); let limiter = Arc::new(governor::RateLimiter::dashmap(quota)); - PartitionLimiter { limiter } + let forced_keys: HashSet = match forced_keys { + None => HashSet::new(), + Some(values) => values.split(',').map(String::from).collect(), + }; + + PartitionLimiter { + limiter, + forced_keys, + } } pub fn is_limited(&self, key: &String) -> bool { - self.limiter.check_key(key).is_err() + self.forced_keys.contains(key) || self.limiter.check_key(key).is_err() } } @@ -36,8 +47,11 @@ mod tests { #[tokio::test] async fn low_limits() { - let limiter = - PartitionLimiter::new(NonZeroU32::new(1).unwrap(), NonZeroU32::new(1).unwrap()); + let limiter = PartitionLimiter::new( + NonZeroU32::new(1).unwrap(), + NonZeroU32::new(1).unwrap(), + None, + ); let token = String::from("test"); assert!(!limiter.is_limited(&token)); @@ -46,8 +60,11 @@ mod tests { #[tokio::test] async fn bursting() { - let limiter = - PartitionLimiter::new(NonZeroU32::new(1).unwrap(), NonZeroU32::new(3).unwrap()); + let limiter = PartitionLimiter::new( + NonZeroU32::new(1).unwrap(), + NonZeroU32::new(3).unwrap(), + None, + ); let token = String::from("test"); assert!(!limiter.is_limited(&token)); @@ -55,4 +72,26 @@ mod tests { assert!(!limiter.is_limited(&token)); assert!(limiter.is_limited(&token)); } + + #[tokio::test] + async fn forced_key() { + let key_one = String::from("one"); + let key_two = String::from("two"); + let key_three = String::from("three"); + let forced_keys = Some(String::from("one,three")); + + let limiter = PartitionLimiter::new( + NonZeroU32::new(1).unwrap(), + NonZeroU32::new(1).unwrap(), + forced_keys, + ); + + // One and three are limited from the start, two is not + assert!(limiter.is_limited(&key_one)); + assert!(!limiter.is_limited(&key_two)); + assert!(limiter.is_limited(&key_three)); + + // Two is limited on the second event + assert!(limiter.is_limited(&key_two)); + } } diff --git a/capture/src/server.rs b/capture/src/server.rs index ad9150907df98..32bafa83b252c 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -44,7 +44,11 @@ where .register("rdkafka".to_string(), Duration::seconds(30)) .await; - let partition = PartitionLimiter::new(config.per_second_limit, config.burst_limit); + let partition = PartitionLimiter::new( + config.per_second_limit, + config.burst_limit, + config.overflow_forced_keys, + ); let sink = sink::KafkaSink::new(config.kafka, sink_liveness, partition) .expect("failed to start Kafka sink"); diff --git a/capture/src/sink.rs b/capture/src/sink.rs index c1d291a275628..93ba6c57e0989 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -301,8 +301,11 @@ mod tests { let handle = registry .register("one".to_string(), Duration::seconds(30)) .await; - let limiter = - PartitionLimiter::new(NonZeroU32::new(10).unwrap(), NonZeroU32::new(10).unwrap()); + let limiter = PartitionLimiter::new( + NonZeroU32::new(10).unwrap(), + NonZeroU32::new(10).unwrap(), + None, + ); let cluster = MockCluster::new(1).expect("failed to create mock brokers"); let config = config::KafkaConfig { kafka_producer_linger_ms: 0, From 03e47f497063fddef50a48db5b7857ecd2695753 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 27 Nov 2023 13:10:15 +0000 Subject: [PATCH 058/247] Initial commit --- LICENSE | 21 +++++++++++++++++++++ README.md | 2 ++ 2 files changed, 23 insertions(+) create mode 100644 LICENSE create mode 100644 README.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000..d1e439cba370e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 PostHog + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000..cf48fa8902586 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# rusty-hook +A reliable and performant webhook system for PostHog From 80f5270df14f1da01382ddf9bbeef5bb74f1e3a9 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 27 Nov 2023 13:20:47 +0000 Subject: [PATCH 059/247] Structure --- .gitignore | 1 + Cargo.lock | 15 +++++++++++++++ Cargo.toml | 11 +++++++++++ hook-common/Cargo.toml | 8 ++++++++ hook-common/src/lib.rs | 14 ++++++++++++++ hook-consumer/Cargo.toml | 8 ++++++++ hook-consumer/src/main.rs | 3 +++ hook-producer/Cargo.toml | 8 ++++++++ hook-producer/src/main.rs | 3 +++ 9 files changed, 71 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 hook-common/Cargo.toml create mode 100644 hook-common/src/lib.rs create mode 100644 hook-consumer/Cargo.toml create mode 100644 hook-consumer/src/main.rs create mode 100644 hook-producer/Cargo.toml create mode 100644 hook-producer/src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000..ea8c4bf7f35f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000000..456c3857724f0 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,15 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "hook-common" +version = "0.1.0" + +[[package]] +name = "hook-consumer" +version = "0.1.0" + +[[package]] +name = "hook-producer" +version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000000..d880e2cd85940 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,11 @@ +[workspace] +resolver = "2" + +members = [ + "hook-common", + "hook-producer", + "hook-consumer", +] + +[workspace.dependencies] +sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", "postgres", "uuid", "json" ] } diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml new file mode 100644 index 0000000000000..1d1418563ca5e --- /dev/null +++ b/hook-common/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "hook-common" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/hook-common/src/lib.rs b/hook-common/src/lib.rs new file mode 100644 index 0000000000000..7d12d9af8195b --- /dev/null +++ b/hook-common/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: usize, right: usize) -> usize { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} diff --git a/hook-consumer/Cargo.toml b/hook-consumer/Cargo.toml new file mode 100644 index 0000000000000..49c2d9f84b17d --- /dev/null +++ b/hook-consumer/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "hook-consumer" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs new file mode 100644 index 0000000000000..e7a11a969c037 --- /dev/null +++ b/hook-consumer/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/hook-producer/Cargo.toml b/hook-producer/Cargo.toml new file mode 100644 index 0000000000000..96fbb4d7528fe --- /dev/null +++ b/hook-producer/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "hook-producer" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/hook-producer/src/main.rs b/hook-producer/src/main.rs new file mode 100644 index 0000000000000..e7a11a969c037 --- /dev/null +++ b/hook-producer/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} From 2ca64c086c96c997688dc9abca8f7e2fbb39cf31 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 27 Nov 2023 14:44:09 +0000 Subject: [PATCH 060/247] Add rust workflow --- .github/workflows/rust.yml | 93 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000000000..6fc44a98c29ea --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,93 @@ +name: Rust + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + runs-on: buildjet-4vcpu-ubuntu-2204 + + steps: + - uses: actions/checkout@v3 + + - name: Install rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }} + + - name: Run cargo build + run: cargo build --all --locked --release + + test: + runs-on: buildjet-4vcpu-ubuntu-2204 + + steps: + - uses: actions/checkout@v3 + + - name: Install rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${ runner.os }-cargo-debug-${{ hashFiles('**/Cargo.lock') }} + + - name: Run cargo test + run: cargo test --all-features + + clippy: + runs-on: buildjet-4vcpu-ubuntu-2204 + + steps: + - uses: actions/checkout@v3 + + - name: Install latest rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy + + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }} + + - name: Run clippy + run: cargo clippy -- -D warnings + + format: + runs-on: buildjet-4vcpu-ubuntu-2204 + + steps: + - uses: actions/checkout@v3 + + - name: Install latest rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: rustfmt + + - name: Format + run: cargo fmt -- --check From cd4d639faddf3b05b82fc79c2fbd765dc4ee2ecc Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 28 Nov 2023 15:24:07 +0100 Subject: [PATCH 061/247] tracing: add ack_wait spans (#60) --- capture/src/sink.rs | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/capture/src/sink.rs b/capture/src/sink.rs index 93ba6c57e0989..13397dcf83465 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -8,8 +8,8 @@ use rdkafka::producer::future_producer::{FutureProducer, FutureRecord}; use rdkafka::producer::{DeliveryFuture, Producer}; use rdkafka::util::Timeout; use tokio::task::JoinSet; -use tracing::instrument; use tracing::log::{debug, error, info}; +use tracing::{info_span, instrument, Instrument}; use crate::api::CaptureError; use crate::config::KafkaConfig; @@ -241,7 +241,9 @@ impl EventSink for KafkaSink { let ack = Self::kafka_send(self.producer.clone(), self.topic.clone(), event, limited).await?; histogram!("capture_event_batch_size", 1.0); - Self::process_ack(ack).await + Self::process_ack(ack) + .instrument(info_span!("ack_wait_one")) + .await } #[instrument(skip_all)] @@ -261,20 +263,25 @@ impl EventSink for KafkaSink { } // Await on all the produce promises, fail batch on first failure - while let Some(res) = set.join_next().await { - match res { - Ok(Ok(_)) => {} - Ok(Err(err)) => { - set.abort_all(); - return Err(err); - } - Err(err) => { - set.abort_all(); - error!("join error while waiting on Kafka ACK: {:?}", err); - return Err(CaptureError::RetryableSinkError); + async move { + while let Some(res) = set.join_next().await { + match res { + Ok(Ok(_)) => {} + Ok(Err(err)) => { + set.abort_all(); + return Err(err); + } + Err(err) => { + set.abort_all(); + error!("join error while waiting on Kafka ACK: {:?}", err); + return Err(CaptureError::RetryableSinkError); + } } } + Ok(()) } + .instrument(info_span!("ack_wait_many")) + .await?; histogram!("capture_event_batch_size", batch_size as f64); Ok(()) From cc43e214729cefe5d4b58482d506bbef1ccf8691 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 28 Nov 2023 17:44:08 +0100 Subject: [PATCH 062/247] feat: trim distinct_id to 200 chars (#61) --- capture-server/tests/events.rs | 41 ++++++++++++++++++++++++++++++++++ capture/src/capture.rs | 8 ++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/capture-server/tests/events.rs b/capture-server/tests/events.rs index 27db9a7c897de..b38ac5a1a63fb 100644 --- a/capture-server/tests/events.rs +++ b/capture-server/tests/events.rs @@ -166,3 +166,44 @@ async fn it_does_not_partition_limit_different_ids() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn it_trims_distinct_id() -> Result<()> { + setup_tracing(); + let token = random_string("token", 16); + let distinct_id1 = random_string("id", 200 - 3); + let distinct_id2 = random_string("id", 222); + let (trimmed_distinct_id2, _) = distinct_id2.split_at(200); // works because ascii chars + + let topic = EphemeralTopic::new().await; + let server = ServerHandle::for_topic(&topic); + + let event = json!([{ + "token": token, + "event": "event1", + "distinct_id": distinct_id1 + },{ + "token": token, + "event": "event2", + "distinct_id": distinct_id2 + }]); + let res = server.capture_events(event.to_string()).await; + assert_eq!(StatusCode::OK, res.status()); + + assert_json_include!( + actual: topic.next_event()?, + expected: json!({ + "token": token, + "distinct_id": distinct_id1 + }) + ); + assert_json_include!( + actual: topic.next_event()?, + expected: json!({ + "token": token, + "distinct_id": trimmed_distinct_id2 + }) + ); + + Ok(()) +} diff --git a/capture/src/capture.rs b/capture/src/capture.rs index da4a72693f51f..8cc37e0737a00 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -172,6 +172,12 @@ pub fn process_single_event( _ => return Err(CaptureError::MissingDistinctId), }, }; + // Limit the size of distinct_id to 200 chars + let distinct_id: String = match distinct_id.len() { + 0..=200 => distinct_id.to_owned(), + _ => distinct_id.chars().take(200).collect(), + }; + if event.event.is_empty() { return Err(CaptureError::MissingEventName); } @@ -183,7 +189,7 @@ pub fn process_single_event( Ok(ProcessedEvent { uuid: event.uuid.unwrap_or_else(uuid_v7), - distinct_id: distinct_id.to_string(), + distinct_id, ip: context.client_ip.clone(), data, now: context.now.clone(), From 681fcafe74d3ffebdd35b5b440f93cc5ca7b63b9 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 28 Nov 2023 17:06:36 +0000 Subject: [PATCH 063/247] chore: add dispatch trigger (#62) --- .github/workflows/docker.yml | 1 + .github/workflows/rust.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 98160d19edce4..1b5f29bd90da3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,7 @@ name: Build docker image on: + workflow_dispatch: push: branches: - 'main' diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 10636f11554e9..cadd2fb6ee3a6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,6 +1,7 @@ name: Rust on: + workflow_dispatch: push: branches: [ main ] pull_request: From 132ceeec9d7cffcafaeed46a1f12ad09374542c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 29 Nov 2023 18:52:12 +0100 Subject: [PATCH 064/247] feat: Initial PgQueue implementation --- .github/workflows/rust.yml | 22 +- Cargo.lock | 1865 +++++++++++++++++ README.md | 19 + docker-compose.yml | 15 + hook-common/Cargo.toml | 8 + hook-common/README.md | 2 + hook-common/src/lib.rs | 15 +- hook-common/src/pgqueue.rs | 215 ++ migrations/20231129172339_job_queue_table.sql | 16 + 9 files changed, 2161 insertions(+), 16 deletions(-) create mode 100644 docker-compose.yml create mode 100644 hook-common/README.md create mode 100644 hook-common/src/pgqueue.rs create mode 100644 migrations/20231129172339_job_queue_table.sql diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6fc44a98c29ea..a06e9ee211676 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -30,7 +30,7 @@ jobs: key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }} - name: Run cargo build - run: cargo build --all --locked --release + run: cargo build --all --locked --release test: runs-on: buildjet-4vcpu-ubuntu-2204 @@ -43,6 +43,24 @@ jobs: with: toolchain: stable + - name: Install rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - name: Stop/Start stack with Docker Compose + shell: bash + run: | + docker compose -f docker-compose.dev.yml down + docker compose -f docker-compose.dev.yml up -d + + - name: Run migrations + shell: bash + run: | + cargo install sqlx-cli --no-default-features --features native-tls,postgres + DATABASE_URL=postgres://posthog:posthog@localhost:15432/test_database sqlx database create + DATABASE_URL=postgres://posthog:posthog@localhost:15432/test_database sqlx migrate run + - uses: actions/cache@v3 with: path: | @@ -76,7 +94,7 @@ jobs: - name: Run clippy run: cargo clippy -- -D warnings - + format: runs-on: buildjet-4vcpu-ubuntu-2204 diff --git a/Cargo.lock b/Cargo.lock index 456c3857724f0..16de92647176f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,9 +2,542 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-write-file" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae364a6c1301604bbc6dfbf8c385c47ff82301dd01eef506195a029196d8d04" +dependencies = [ + "nix", + "rand", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +dependencies = [ + "serde", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.48.5", +] + +[[package]] +name = "const-oid" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "der" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +dependencies = [ + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "hook-common" version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "serde_derive", + "sqlx", + "thiserror", + "tokio", +] [[package]] name = "hook-consumer" @@ -13,3 +546,1335 @@ version = "0.1.0" [[package]] name = "hook-producer" version = "0.1.0" + +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] + +[[package]] +name = "libc" +version = "0.2.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "openssl" +version = "0.10.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.48.5", +] + +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "rsa" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af6c4b23d99685a1408194da11270ef8e9809aff951cc70ec9b17350b087e474" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.38.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" +dependencies = [ + "itertools", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dba03c279da73694ef99763320dea58b51095dfe87d001b1d4b5fe78ba8763cf" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" +dependencies = [ + "ahash", + "atoi", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "dotenvy", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap", + "log", + "memchr", + "native-tls", + "once_cell", + "paste", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0bd4519486723648186a08785143599760f7cc81c52334a55d6a83ea1e20841" +dependencies = [ + "atomic-write-file", + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" +dependencies = [ + "atoi", + "base64", + "bitflags 2.4.1", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" +dependencies = [ + "atoi", + "base64", + "bitflags 2.4.1", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "210976b7d948c7ba9fced8ca835b11cbb2d677c59c79de41ac0d397e14547490" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", + "uuid", +] + +[[package]] +name = "stringprep" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +dependencies = [ + "finl_unicode", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "uuid" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.39", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" + +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "zerocopy" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/README.md b/README.md index cf48fa8902586..2ce36b3908cc8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,21 @@ # rusty-hook A reliable and performant webhook system for PostHog + +## Testing + +1. Start a PostgreSQL instance: +```bash +docker compose -f docker-compose.yml up -d +``` + +2. Prepare test database: +```bash +export DATABASE_URL=postgres://posthog:posthog@localhost:15432/test_database +sqlx database create +sqlx migrate run +``` + +3. Test: +```bash +cargo test +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000..35b7a498d44b4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +services: + db: + image: docker.io/library/postgres:16-alpine + restart: on-failure + environment: + POSTGRES_USER: posthog + POSTGRES_DB: posthog + POSTGRES_PASSWORD: posthog + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U posthog'] + interval: 5s + timeout: 5s + ports: + - '15432:5432' + command: postgres -c max_connections=1000 -c idle_in_transaction_session_timeout=300000 diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index 1d1418563ca5e..673d8877f726a 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -6,3 +6,11 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +chrono = { version = "0.4" } +serde = { version = "1.0" } +serde_derive = { version = "1.0" } +sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", "postgres", "uuid", "json", "chrono" ] } +thiserror = { version = "1.0" } + +[dev-dependencies] +tokio = { version = "1.34", features = ["macros"] } # We need a runtime for async tests diff --git a/hook-common/README.md b/hook-common/README.md new file mode 100644 index 0000000000000..d277a6c8600c9 --- /dev/null +++ b/hook-common/README.md @@ -0,0 +1,2 @@ +# hook-common +Library of common utilities used by rusty-hook. diff --git a/hook-common/src/lib.rs b/hook-common/src/lib.rs index 7d12d9af8195b..d1dadf32ba63e 100644 --- a/hook-common/src/lib.rs +++ b/hook-common/src/lib.rs @@ -1,14 +1 @@ -pub fn add(left: usize, right: usize) -> usize { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} +pub mod pgqueue; diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs new file mode 100644 index 0000000000000..f06f50d10335d --- /dev/null +++ b/hook-common/src/pgqueue.rs @@ -0,0 +1,215 @@ +use std::str::FromStr; + +use chrono::prelude::*; +use serde::{de::DeserializeOwned, Serialize}; +use sqlx::postgres::{PgPool, PgPoolOptions}; +use thiserror::Error; + +/// Enumeration of errors for operations with PgQueue. +/// Errors can originate from sqlx and are wrapped by us to provide additional context. +#[derive(Error, Debug)] +pub enum PgQueueError { + #[error("connection failed with: {error}")] + ConnectionError { + error: sqlx::Error + }, + #[error("{command} query failed with: {error}")] + QueryError { + command: String, + error: sqlx::Error + }, + #[error("{0} is not a valid JobStatus")] + ParseJobStatusError(String), +} + +/// Enumeration of possible statuses for a Job. +/// Available: A job that is waiting in the queue to be picked up by a worker. +/// Completed: A job that was successfully completed by a worker. +/// Failed: A job that was unsuccessfully completed by a worker. +/// Running: A job that was picked up by a worker and it's currentlly being run. +#[derive(Debug, PartialEq, sqlx::Type)] +#[sqlx(type_name = "job_status")] +#[sqlx(rename_all = "lowercase")] +pub enum JobStatus { + Available, + Completed, + Failed, + Running, +} + +/// Allow casting JobStatus from strings. +impl FromStr for JobStatus { + type Err = PgQueueError; + + fn from_str(s: &str) -> Result { + match s { + "available" => Ok(JobStatus::Available), + "completed" => Ok(JobStatus::Completed), + "failed" => Ok(JobStatus::Failed), + "running" => Ok(JobStatus::Running), + invalid => Err(PgQueueError::ParseJobStatusError(invalid.to_owned())), + } + } +} + +/// JobParameters are stored and read to and from a JSONB field, so we accept anything that fits `sqlx::types::Json`. +pub type JobParameters = sqlx::types::Json; + +/// A Job to be executed by a worker dequeueing a PgQueue. +#[derive(sqlx::FromRow)] +pub struct Job { + pub id: i64, + pub attempt: i32, + pub finished_at: Option>, + pub created_at: DateTime, + pub started_at: Option>, + pub status: JobStatus, + pub parameters: sqlx::types::Json, +} + +/// A NewJob to be enqueued into a PgQueue. +pub struct NewJob { + pub attempt: i32, + pub finished_at: Option>, + pub started_at: Option>, + pub status: JobStatus, + pub parameters: sqlx::types::Json, +} + +impl NewJob { + pub fn new(parameters: J) -> Self { + Self { + attempt: 0, + parameters: sqlx::types::Json(parameters), + finished_at: None, + started_at: None, + status: JobStatus::Available, + } + } +} + +/// A queue implemented on top of a PostgreSQL table. +pub struct PgQueue { + table: String, + pool: PgPool, +} + +pub type PgQueueResult = std::result::Result; + +impl PgQueue { + /// Initialize a new PgQueue backed by table in PostgreSQL. + pub async fn new(table: &str, url: &str) -> PgQueueResult { + let table = table.to_owned(); + let pool = PgPoolOptions::new() + .connect(url) + .await + .map_err(|error| PgQueueError::ConnectionError {error})?; + + Ok(Self { + table, + pool, + }) + } + + /// Dequeue a Job from this PgQueue. + pub async fn dequeue(&self) -> PgQueueResult> { + let base_query = format!( + r#" +WITH available_in_queue AS ( + SELECT + id + FROM + "{0}" + WHERE + status = 'available' + ORDER BY + id + LIMIT 1 + FOR UPDATE SKIP LOCKED +) +UPDATE + "{0}" +SET + started_at = NOW(), + status = 'running'::job_status, + attempt = "{0}".attempt + 1 +FROM + available_in_queue +WHERE + "{0}".id = available_in_queue.id +RETURNING + "{0}".* + "#, &self.table); + + let item: Job = sqlx::query_as(&base_query) + .bind(&self.table) + .bind(&self.table) + .bind(&self.table) + .fetch_one(&self.pool) + .await + .map_err(|error| PgQueueError::QueryError { command: "UPDATE".to_owned(), error})?; + + Ok(item) + } + + /// Enqueue a Job into this PgQueue. + /// We take ownership of NewJob to enforce a specific NewJob is only enqueued once. + pub async fn enqueue(&self, job: NewJob) -> PgQueueResult<()> { + // TODO: Escaping. I think sqlx doesn't support identifiers. + let base_query = format!( + r#" +INSERT INTO {} + (attempt, created_at, finished_at, started_at, status, parameters) +VALUES + ($1, NOW(), $2, $3, $4::job_status, $5) + "#, &self.table); + + sqlx::query(&base_query) + .bind(job.attempt) + .bind(job.finished_at) + .bind(job.started_at) + .bind(job.status) + .bind(&job.parameters) + .execute(&self.pool) + .await + .map_err(|error| PgQueueError::QueryError { command: "INSERT".to_owned(), error})?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + + #[derive(Serialize, Deserialize)] + struct JobParameters { + method: String, + body: String, + url: String, + } + + #[tokio::test] + async fn test_can_enqueue_and_dequeue_job() { + let job_parameters = JobParameters { + method: "POST".to_string(), + body: "{\"event\":\"event-name\"}".to_string(), + url: "https://localhost".to_string(), + }; + let new_job = NewJob::new(job_parameters); + + let queue = PgQueue::new("job_queue", "postgres://posthog:posthog@localhost:15432/test_database").await.unwrap(); + + queue.enqueue(new_job).await.unwrap(); + + let job: Job = queue.dequeue().await.unwrap(); + + assert_eq!(job.attempt, 1); + assert_eq!(job.parameters.method, "POST".to_string()); + assert_eq!(job.parameters.body, "{\"event\":\"event-name\"}".to_string()); + assert_eq!(job.parameters.url, "https://localhost".to_string()); + assert_eq!(job.finished_at, None); + assert_eq!(job.status, JobStatus::Running); + } +} diff --git a/migrations/20231129172339_job_queue_table.sql b/migrations/20231129172339_job_queue_table.sql new file mode 100644 index 0000000000000..078fdd9ef58b6 --- /dev/null +++ b/migrations/20231129172339_job_queue_table.sql @@ -0,0 +1,16 @@ +CREATE TYPE job_status AS ENUM( + 'available', + 'completed', + 'failed', + 'running' +); + +CREATE TABLE job_queue( + id BIGSERIAL PRIMARY KEY, + attempt INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + finished_at TIMESTAMPTZ DEFAULT NULL, + started_at TIMESTAMPTZ DEFAULT NULL, + status job_status NOT NULL DEFAULT 'available'::job_status, + parameters JSONB +); From 388b5444afd67d18b8cb66a84278f7725c072d42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 30 Nov 2023 12:18:17 +0100 Subject: [PATCH 065/247] feat: Support for attempted_by --- hook-common/src/pgqueue.rs | 52 ++++++++++++++----- migrations/20231129172339_job_queue_table.sql | 2 + 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index f06f50d10335d..e2c206ed6457d 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -1,3 +1,7 @@ +//! # PgQueue +//! +//! A job queue implementation backed by a PostgreSQL table. + use std::str::FromStr; use chrono::prelude::*; @@ -6,7 +10,7 @@ use sqlx::postgres::{PgPool, PgPoolOptions}; use thiserror::Error; /// Enumeration of errors for operations with PgQueue. -/// Errors can originate from sqlx and are wrapped by us to provide additional context. +/// Errors that can originate from sqlx and are wrapped by us to provide additional context. #[derive(Error, Debug)] pub enum PgQueueError { #[error("connection failed with: {error}")] @@ -23,17 +27,17 @@ pub enum PgQueueError { } /// Enumeration of possible statuses for a Job. -/// Available: A job that is waiting in the queue to be picked up by a worker. -/// Completed: A job that was successfully completed by a worker. -/// Failed: A job that was unsuccessfully completed by a worker. -/// Running: A job that was picked up by a worker and it's currentlly being run. #[derive(Debug, PartialEq, sqlx::Type)] #[sqlx(type_name = "job_status")] #[sqlx(rename_all = "lowercase")] pub enum JobStatus { + /// A job that is waiting in the queue to be picked up by a worker. Available, + /// A job that was successfully completed by a worker. Completed, + /// A job that was unsuccessfully completed by a worker. Failed, + /// A job that was picked up by a worker and it's currentlly being run. Running, } @@ -58,12 +62,23 @@ pub type JobParameters = sqlx::types::Json; /// A Job to be executed by a worker dequeueing a PgQueue. #[derive(sqlx::FromRow)] pub struct Job { + /// A unique id identifying a job. pub id: i64, + /// A number corresponding to the current job attempt. pub attempt: i32, + /// A datetime corresponding to when the current job attempt started. + pub attempted_at: Option>, + /// A vector of identifiers that have attempted this job. E.g. thread ids, pod names, etc... + pub attempted_by: Vec, + /// A datetime corresponding to when the job was finished (either successfully or unsuccessfully). pub finished_at: Option>, + /// A datetime corresponding to when the job was created. pub created_at: DateTime, + /// A datetime corresponding to when the first job attempt was started. pub started_at: Option>, + /// The current status of the job. pub status: JobStatus, + /// Arbitrary job parameters stored as JSON. pub parameters: sqlx::types::Json, } @@ -90,16 +105,21 @@ impl NewJob { /// A queue implemented on top of a PostgreSQL table. pub struct PgQueue { + /// The identifier of the PostgreSQL table this queue runs on. table: String, + /// A connection pool used to connect to the PostgreSQL database. pool: PgPool, + /// The identifier of the worker listening on this queue. + worker: String, } pub type PgQueueResult = std::result::Result; impl PgQueue { /// Initialize a new PgQueue backed by table in PostgreSQL. - pub async fn new(table: &str, url: &str) -> PgQueueResult { + pub async fn new(table: &str, url: &str, worker: &str) -> PgQueueResult { let table = table.to_owned(); + let worker = worker.to_owned(); let pool = PgPoolOptions::new() .connect(url) .await @@ -108,6 +128,7 @@ impl PgQueue { Ok(Self { table, pool, + worker, }) } @@ -132,7 +153,8 @@ UPDATE SET started_at = NOW(), status = 'running'::job_status, - attempt = "{0}".attempt + 1 + attempt = "{0}".attempt + 1, + attempted_by = array_append("{0}".attempted_by, $1::text) FROM available_in_queue WHERE @@ -142,9 +164,7 @@ RETURNING "#, &self.table); let item: Job = sqlx::query_as(&base_query) - .bind(&self.table) - .bind(&self.table) - .bind(&self.table) + .bind(&self.worker) .fetch_one(&self.pool) .await .map_err(|error| PgQueueError::QueryError { command: "UPDATE".to_owned(), error})?; @@ -199,17 +219,21 @@ mod tests { }; let new_job = NewJob::new(job_parameters); - let queue = PgQueue::new("job_queue", "postgres://posthog:posthog@localhost:15432/test_database").await.unwrap(); + let worker_id = std::process::id().to_string(); + let queue = PgQueue::new("job_queue", "postgres://posthog:posthog@localhost:15432/test_database", &worker_id) + .await + .expect("failed to connect to local test postgresql database"); - queue.enqueue(new_job).await.unwrap(); + queue.enqueue(new_job).await.expect("failed to enqueue job"); - let job: Job = queue.dequeue().await.unwrap(); + let job: Job = queue.dequeue().await.expect("failed to dequeue job"); assert_eq!(job.attempt, 1); assert_eq!(job.parameters.method, "POST".to_string()); assert_eq!(job.parameters.body, "{\"event\":\"event-name\"}".to_string()); assert_eq!(job.parameters.url, "https://localhost".to_string()); - assert_eq!(job.finished_at, None); + assert!(job.finished_at.is_none()); assert_eq!(job.status, JobStatus::Running); + assert!(job.attempted_by.contains(&worker_id)); } } diff --git a/migrations/20231129172339_job_queue_table.sql b/migrations/20231129172339_job_queue_table.sql index 078fdd9ef58b6..f2682eea2fef4 100644 --- a/migrations/20231129172339_job_queue_table.sql +++ b/migrations/20231129172339_job_queue_table.sql @@ -8,6 +8,8 @@ CREATE TYPE job_status AS ENUM( CREATE TABLE job_queue( id BIGSERIAL PRIMARY KEY, attempt INT NOT NULL DEFAULT 0, + attempted_at TIMESTAMPTZ DEFAULT NULL, + attempted_by TEXT[] DEFAULT ARRAY[]::TEXT[], created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), finished_at TIMESTAMPTZ DEFAULT NULL, started_at TIMESTAMPTZ DEFAULT NULL, From bfeab3642dab184f4c6a717071c75d85d61ab790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 30 Nov 2023 12:20:38 +0100 Subject: [PATCH 066/247] fix: Wait for Postgres to be up in CI --- .github/workflows/rust.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a06e9ee211676..ee0e5b3ebc785 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -38,11 +38,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - name: Install rust uses: dtolnay/rust-toolchain@master with: @@ -51,8 +46,8 @@ jobs: - name: Stop/Start stack with Docker Compose shell: bash run: | - docker compose -f docker-compose.dev.yml down - docker compose -f docker-compose.dev.yml up -d + docker compose -f docker-compose.yml down + docker compose -f docker-compose.yml up -d --wait - name: Run migrations shell: bash From 8b2325bddd2d907dc0e81bdf11c1c7dcf94a56c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 30 Nov 2023 12:23:05 +0100 Subject: [PATCH 067/247] fix: Formatting --- hook-common/src/pgqueue.rs | 55 +++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index e2c206ed6457d..18d324b733a57 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -14,14 +14,9 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum PgQueueError { #[error("connection failed with: {error}")] - ConnectionError { - error: sqlx::Error - }, + ConnectionError { error: sqlx::Error }, #[error("{command} query failed with: {error}")] - QueryError { - command: String, - error: sqlx::Error - }, + QueryError { command: String, error: sqlx::Error }, #[error("{0} is not a valid JobStatus")] ParseJobStatusError(String), } @@ -95,7 +90,7 @@ impl NewJob { pub fn new(parameters: J) -> Self { Self { attempt: 0, - parameters: sqlx::types::Json(parameters), + parameters: sqlx::types::Json(parameters), finished_at: None, started_at: None, status: JobStatus::Available, @@ -123,7 +118,7 @@ impl PgQueue { let pool = PgPoolOptions::new() .connect(url) .await - .map_err(|error| PgQueueError::ConnectionError {error})?; + .map_err(|error| PgQueueError::ConnectionError { error })?; Ok(Self { table, @@ -133,7 +128,9 @@ impl PgQueue { } /// Dequeue a Job from this PgQueue. - pub async fn dequeue(&self) -> PgQueueResult> { + pub async fn dequeue( + &self, + ) -> PgQueueResult> { let base_query = format!( r#" WITH available_in_queue AS ( @@ -161,20 +158,28 @@ WHERE "{0}".id = available_in_queue.id RETURNING "{0}".* - "#, &self.table); + "#, + &self.table + ); let item: Job = sqlx::query_as(&base_query) .bind(&self.worker) .fetch_one(&self.pool) .await - .map_err(|error| PgQueueError::QueryError { command: "UPDATE".to_owned(), error})?; + .map_err(|error| PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; Ok(item) } /// Enqueue a Job into this PgQueue. /// We take ownership of NewJob to enforce a specific NewJob is only enqueued once. - pub async fn enqueue(&self, job: NewJob) -> PgQueueResult<()> { + pub async fn enqueue( + &self, + job: NewJob, + ) -> PgQueueResult<()> { // TODO: Escaping. I think sqlx doesn't support identifiers. let base_query = format!( r#" @@ -182,7 +187,9 @@ INSERT INTO {} (attempt, created_at, finished_at, started_at, status, parameters) VALUES ($1, NOW(), $2, $3, $4::job_status, $5) - "#, &self.table); + "#, + &self.table + ); sqlx::query(&base_query) .bind(job.attempt) @@ -192,7 +199,10 @@ VALUES .bind(&job.parameters) .execute(&self.pool) .await - .map_err(|error| PgQueueError::QueryError { command: "INSERT".to_owned(), error})?; + .map_err(|error| PgQueueError::QueryError { + command: "INSERT".to_owned(), + error, + })?; Ok(()) } @@ -220,9 +230,13 @@ mod tests { let new_job = NewJob::new(job_parameters); let worker_id = std::process::id().to_string(); - let queue = PgQueue::new("job_queue", "postgres://posthog:posthog@localhost:15432/test_database", &worker_id) - .await - .expect("failed to connect to local test postgresql database"); + let queue = PgQueue::new( + "job_queue", + "postgres://posthog:posthog@localhost:15432/test_database", + &worker_id, + ) + .await + .expect("failed to connect to local test postgresql database"); queue.enqueue(new_job).await.expect("failed to enqueue job"); @@ -230,7 +244,10 @@ mod tests { assert_eq!(job.attempt, 1); assert_eq!(job.parameters.method, "POST".to_string()); - assert_eq!(job.parameters.body, "{\"event\":\"event-name\"}".to_string()); + assert_eq!( + job.parameters.body, + "{\"event\":\"event-name\"}".to_string() + ); assert_eq!(job.parameters.url, "https://localhost".to_string()); assert!(job.finished_at.is_none()); assert_eq!(job.status, JobStatus::Running); From ec6ba09b820a9b6515acfd719694a5dad696ce3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 30 Nov 2023 12:27:47 +0100 Subject: [PATCH 068/247] fix: README recommends --waiting for docker compose to be up --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2ce36b3908cc8..8eb2e2c347ca0 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A reliable and performant webhook system for PostHog 1. Start a PostgreSQL instance: ```bash -docker compose -f docker-compose.yml up -d +docker compose -f docker-compose.yml up -d --wait ``` 2. Prepare test database: From c6ab642c284d26c504fc66dbd5ec58d30eae5de2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 30 Nov 2023 12:31:02 +0100 Subject: [PATCH 069/247] chore: Add comment on SKIP LOCKED clause --- hook-common/src/pgqueue.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 18d324b733a57..8bb40e18af717 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -131,6 +131,8 @@ impl PgQueue { pub async fn dequeue( &self, ) -> PgQueueResult> { + // The query that follows uses a FOR UPDATE SKIP LOCKED clause. + // For more details on this see: 2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5. let base_query = format!( r#" WITH available_in_queue AS ( From da952a662956493b27380f01e2f8bde6ee487811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 1 Dec 2023 00:43:31 +0100 Subject: [PATCH 070/247] fix: Use the type alias I defined --- hook-common/src/pgqueue.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 8bb40e18af717..53398bfe27944 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -74,7 +74,7 @@ pub struct Job { /// The current status of the job. pub status: JobStatus, /// Arbitrary job parameters stored as JSON. - pub parameters: sqlx::types::Json, + pub parameters: JobParameters, } /// A NewJob to be enqueued into a PgQueue. @@ -83,7 +83,7 @@ pub struct NewJob { pub finished_at: Option>, pub started_at: Option>, pub status: JobStatus, - pub parameters: sqlx::types::Json, + pub parameters: JobParameters, } impl NewJob { From 5d0838f175bca4e4e6208dbce1d1150a220b3212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 1 Dec 2023 12:47:20 +0100 Subject: [PATCH 071/247] feat: Support for retrying --- hook-common/src/pgqueue.rs | 365 ++++++++++++++++-- migrations/20231129172339_job_queue_table.sql | 9 +- 2 files changed, 338 insertions(+), 36 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 53398bfe27944..a85b323521346 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -2,9 +2,10 @@ //! //! A job queue implementation backed by a PostgreSQL table. +use std::default::Default; use std::str::FromStr; -use chrono::prelude::*; +use chrono::{prelude::*, Duration}; use serde::{de::DeserializeOwned, Serialize}; use sqlx::postgres::{PgPool, PgPoolOptions}; use thiserror::Error; @@ -19,6 +20,8 @@ pub enum PgQueueError { QueryError { command: String, error: sqlx::Error }, #[error("{0} is not a valid JobStatus")] ParseJobStatusError(String), + #[error("{0} Job has reached max attempts and cannot be retried further")] + MaxAttemptsReachedError(String), } /// Enumeration of possible statuses for a Job. @@ -28,8 +31,12 @@ pub enum PgQueueError { pub enum JobStatus { /// A job that is waiting in the queue to be picked up by a worker. Available, + /// A job that was cancelled by a worker. + Cancelled, /// A job that was successfully completed by a worker. Completed, + /// A job that has + Discarded, /// A job that was unsuccessfully completed by a worker. Failed, /// A job that was picked up by a worker and it's currentlly being run. @@ -61,49 +68,123 @@ pub struct Job { pub id: i64, /// A number corresponding to the current job attempt. pub attempt: i32, - /// A datetime corresponding to when the current job attempt started. - pub attempted_at: Option>, + /// A datetime corresponding to when the job was attempted. + pub attempted_at: DateTime, /// A vector of identifiers that have attempted this job. E.g. thread ids, pod names, etc... pub attempted_by: Vec, - /// A datetime corresponding to when the job was finished (either successfully or unsuccessfully). - pub finished_at: Option>, /// A datetime corresponding to when the job was created. pub created_at: DateTime, - /// A datetime corresponding to when the first job attempt was started. - pub started_at: Option>, - /// The current status of the job. - pub status: JobStatus, + /// The current job's number of max attempts. + pub max_attempts: i32, /// Arbitrary job parameters stored as JSON. pub parameters: JobParameters, + /// The current status of the job. + pub status: JobStatus, + /// The target of the job. E.g. an endpoint or service we are trying to reach. + pub target: String, +} + +impl Job { + pub fn retry(self, error: E) -> Result, PgQueueError> { + if self.attempt == self.max_attempts { + Err(PgQueueError::MaxAttemptsReachedError(self.target)) + } else { + Ok(RetryableJob { + id: self.id, + attempt: self.attempt, + max_attempts: self.max_attempts, + error: sqlx::types::Json(error), + }) + } + } + + pub fn complete(self) -> CompletedJob { + CompletedJob { id: self.id } + } + + pub fn fail(self, error: E) -> FailedJob { + FailedJob { + id: self.id, + error: sqlx::types::Json(error), + } + } +} + +pub struct RetryPolicy { + backoff_coefficient: i32, + initial_interval: Duration, + maximum_interval: Option, +} + +impl RetryPolicy { + pub fn time_until_next_retry(&self, job: &RetryableJob) -> Duration { + let candidate_interval = + self.initial_interval * self.backoff_coefficient.pow(job.attempt as u32); + + if let Some(max_interval) = self.maximum_interval { + std::cmp::min(candidate_interval, max_interval) + } else { + candidate_interval + } + } +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + backoff_coefficient: 2, + initial_interval: Duration::seconds(1), + maximum_interval: None, + } + } +} + +pub struct RetryableJob { + pub id: i64, + pub attempt: i32, + pub max_attempts: i32, + pub error: sqlx::types::Json, +} + +pub struct CompletedJob { + pub id: i64, +} + +pub struct FailedJob { + pub id: i64, + pub error: sqlx::types::Json, } /// A NewJob to be enqueued into a PgQueue. pub struct NewJob { - pub attempt: i32, - pub finished_at: Option>, - pub started_at: Option>, - pub status: JobStatus, + /// The maximum amount of attempts this NewJob has to complete. + pub max_attempts: i32, + /// The JSON-deserializable parameters for this NewJob. pub parameters: JobParameters, + /// The target of the NewJob. E.g. an endpoint or service we are trying to reach. + pub target: String, } impl NewJob { - pub fn new(parameters: J) -> Self { + pub fn new(max_attempts: i32, parameters: J, target: &str) -> Self { Self { - attempt: 0, + max_attempts, parameters: sqlx::types::Json(parameters), - finished_at: None, - started_at: None, - status: JobStatus::Available, + target: target.to_owned(), } } } /// A queue implemented on top of a PostgreSQL table. pub struct PgQueue { - /// The identifier of the PostgreSQL table this queue runs on. - table: String, + /// A name to identify this PgQueue as multiple may share a table. + name: String, /// A connection pool used to connect to the PostgreSQL database. pool: PgPool, + /// The retry policy to use to enqueue any retryable jobs. + retry_policy: RetryPolicy, + /// The identifier of the PostgreSQL table this queue runs on. + table: String, /// The identifier of the worker listening on this queue. worker: String, } @@ -112,7 +193,14 @@ pub type PgQueueResult = std::result::Result; impl PgQueue { /// Initialize a new PgQueue backed by table in PostgreSQL. - pub async fn new(table: &str, url: &str, worker: &str) -> PgQueueResult { + pub async fn new( + name: &str, + table: &str, + retry_policy: RetryPolicy, + url: &str, + worker: &str, + ) -> PgQueueResult { + let name = name.to_owned(); let table = table.to_owned(); let worker = worker.to_owned(); let pool = PgPoolOptions::new() @@ -121,13 +209,15 @@ impl PgQueue { .map_err(|error| PgQueueError::ConnectionError { error })?; Ok(Self { + name, table, pool, worker, + retry_policy, }) } - /// Dequeue a Job from this PgQueue. + /// Dequeue a Job from this PgQueue to work on it. pub async fn dequeue( &self, ) -> PgQueueResult> { @@ -142,6 +232,8 @@ WITH available_in_queue AS ( "{0}" WHERE status = 'available' + AND scheduled_at <= NOW() + AND queue = $1 ORDER BY id LIMIT 1 @@ -150,10 +242,10 @@ WITH available_in_queue AS ( UPDATE "{0}" SET - started_at = NOW(), + attempted_at = NOW(), status = 'running'::job_status, attempt = "{0}".attempt + 1, - attempted_by = array_append("{0}".attempted_by, $1::text) + attempted_by = array_append("{0}".attempted_by, $2::text) FROM available_in_queue WHERE @@ -165,6 +257,7 @@ RETURNING ); let item: Job = sqlx::query_as(&base_query) + .bind(&self.name) .bind(&self.worker) .fetch_one(&self.pool) .await @@ -186,19 +279,18 @@ RETURNING let base_query = format!( r#" INSERT INTO {} - (attempt, created_at, finished_at, started_at, status, parameters) + (attempt, created_at, scheduled_at, max_attempts, parameters, queue, status, target) VALUES - ($1, NOW(), $2, $3, $4::job_status, $5) + (0, NOW(), NOW(), $1, $2, $3, 'available'::job_status, $4) "#, &self.table ); sqlx::query(&base_query) - .bind(job.attempt) - .bind(job.finished_at) - .bind(job.started_at) - .bind(job.status) + .bind(job.max_attempts) .bind(&job.parameters) + .bind(&self.name) + .bind(&job.target) .execute(&self.pool) .await .map_err(|error| PgQueueError::QueryError { @@ -208,6 +300,120 @@ VALUES Ok(()) } + + /// Enqueue a Job back into this PgQueue marked as completed. + /// We take ownership of Job to enforce a specific Job is only enqueued once. + pub async fn enqueue_completed(&self, job: CompletedJob) -> PgQueueResult<()> { + // TODO: Escaping. I think sqlx doesn't support identifiers. + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + completed_at = NOW(), + status = 'completed'::job_status +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + "#, + &self.table + ); + + sqlx::query(&base_query) + .bind(&self.name) + .bind(job.id) + .execute(&self.pool) + .await + .map_err(|error| PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + Ok(()) + } + + /// Enqueue a Job back into this PgQueue to be retried at a later time. + /// We take ownership of Job to enforce a specific Job is only enqueued once. + pub async fn enqueue_retryable( + &self, + job: RetryableJob, + ) -> PgQueueResult<()> { + // TODO: Escaping. I think sqlx doesn't support identifiers. + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + status = 'available'::job_status, + scheduled_at = NOW() + $3, + errors = array_append("{0}".errors, $4) +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + "#, + &self.table + ); + + sqlx::query(&base_query) + .bind(&self.name) + .bind(job.id) + .bind(self.retry_policy.time_until_next_retry(&job)) + .bind(&job.error) + .execute(&self.pool) + .await + .map_err(|error| PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + Ok(()) + } + + /// Enqueue a Job back into this PgQueue marked as failed. + /// Jobs marked as failed will remain in the queue for tracking purposes but will not be dequeued. + /// We take ownership of FailedJob to enforce a specific FailedJob is only enqueued once. + pub async fn enqueue_failed( + &self, + job: FailedJob, + ) -> PgQueueResult<()> { + // TODO: Escaping. I think sqlx doesn't support identifiers. + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + completed_at = NOW(), + status = 'failed'::job_status + errors = array_append("{0}".errors, $3) +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + "#, + &self.table + ); + + sqlx::query(&base_query) + .bind(&self.name) + .bind(job.id) + .bind(&job.error) + .execute(&self.pool) + .await + .map_err(|error| PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + Ok(()) + } } #[cfg(test)] @@ -223,17 +429,21 @@ mod tests { } #[tokio::test] - async fn test_can_enqueue_and_dequeue_job() { + async fn test_can_dequeue_job() { let job_parameters = JobParameters { method: "POST".to_string(), body: "{\"event\":\"event-name\"}".to_string(), url: "https://localhost".to_string(), }; - let new_job = NewJob::new(job_parameters); + let job_target = "https://myhost/endpoint"; + let new_job = NewJob::new(1, job_parameters, job_target); let worker_id = std::process::id().to_string(); + let retry_policy = RetryPolicy::default(); let queue = PgQueue::new( + "test_queue_1", "job_queue", + retry_policy, "postgres://posthog:posthog@localhost:15432/test_database", &worker_id, ) @@ -245,14 +455,101 @@ mod tests { let job: Job = queue.dequeue().await.expect("failed to dequeue job"); assert_eq!(job.attempt, 1); + assert!(job.attempted_by.contains(&worker_id)); + assert_eq!(job.attempted_by.len(), 1); + assert_eq!(job.max_attempts, 1); assert_eq!(job.parameters.method, "POST".to_string()); assert_eq!( job.parameters.body, "{\"event\":\"event-name\"}".to_string() ); assert_eq!(job.parameters.url, "https://localhost".to_string()); - assert!(job.finished_at.is_none()); assert_eq!(job.status, JobStatus::Running); - assert!(job.attempted_by.contains(&worker_id)); + assert_eq!(job.target, job_target.to_string()); + } + + #[tokio::test] + async fn test_can_retry_job_with_remaining_attempts() { + let job_parameters = JobParameters { + method: "POST".to_string(), + body: "{\"event\":\"event-name\"}".to_string(), + url: "https://localhost".to_string(), + }; + let job_target = "https://myhost/endpoint"; + let new_job = NewJob::new(2, job_parameters, job_target); + + let worker_id = std::process::id().to_string(); + let retry_policy = RetryPolicy { + backoff_coefficient: 0, + initial_interval: Duration::seconds(0), + maximum_interval: None, + }; + let queue = PgQueue::new( + "test_queue_2", + "job_queue", + retry_policy, + "postgres://posthog:posthog@localhost:15432/test_database", + &worker_id, + ) + .await + .expect("failed to connect to local test postgresql database"); + + queue.enqueue(new_job).await.expect("failed to enqueue job"); + let job: Job = queue.dequeue().await.expect("failed to dequeue job"); + let retryable_job = job + .retry("a very reasonable failure reason") + .expect("failed to retry job"); + + queue + .enqueue_retryable(retryable_job) + .await + .expect("failed to enqueue retryable job"); + let retried_job: Job = queue.dequeue().await.expect("failed to dequeue job"); + + assert_eq!(retried_job.attempt, 2); + assert!(retried_job.attempted_by.contains(&worker_id)); + assert_eq!(retried_job.attempted_by.len(), 2); + assert_eq!(retried_job.max_attempts, 2); + assert_eq!(retried_job.parameters.method, "POST".to_string()); + assert_eq!( + retried_job.parameters.body, + "{\"event\":\"event-name\"}".to_string() + ); + assert_eq!(retried_job.parameters.url, "https://localhost".to_string()); + assert_eq!(retried_job.status, JobStatus::Running); + assert_eq!(retried_job.target, job_target.to_string()); + } + + #[tokio::test] + #[should_panic(expected = "failed to retry job")] + async fn test_cannot_retry_job_without_remaining_attempts() { + let job_parameters = JobParameters { + method: "POST".to_string(), + body: "{\"event\":\"event-name\"}".to_string(), + url: "https://localhost".to_string(), + }; + let job_target = "https://myhost/endpoint"; + let new_job = NewJob::new(1, job_parameters, job_target); + + let worker_id = std::process::id().to_string(); + let retry_policy = RetryPolicy { + backoff_coefficient: 0, + initial_interval: Duration::seconds(0), + maximum_interval: None, + }; + let queue = PgQueue::new( + "test_queue_3", + "job_queue", + retry_policy, + "postgres://posthog:posthog@localhost:15432/test_database", + &worker_id, + ) + .await + .expect("failed to connect to local test postgresql database"); + + queue.enqueue(new_job).await.expect("failed to enqueue job"); + let job: Job = queue.dequeue().await.expect("failed to dequeue job"); + job.retry("a very reasonable failure reason") + .expect("failed to retry job"); } } diff --git a/migrations/20231129172339_job_queue_table.sql b/migrations/20231129172339_job_queue_table.sql index f2682eea2fef4..4631f0b0a5d3f 100644 --- a/migrations/20231129172339_job_queue_table.sql +++ b/migrations/20231129172339_job_queue_table.sql @@ -10,9 +10,14 @@ CREATE TABLE job_queue( attempt INT NOT NULL DEFAULT 0, attempted_at TIMESTAMPTZ DEFAULT NULL, attempted_by TEXT[] DEFAULT ARRAY[]::TEXT[], + completed_at TIMESTAMPTZ DEFAULT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + errors jsonb[], + max_attempts INT NOT NULL DEFAULT 1, finished_at TIMESTAMPTZ DEFAULT NULL, - started_at TIMESTAMPTZ DEFAULT NULL, + parameters JSONB, + queue TEXT NOT NULL DEFAULT 'default'::text, + scheduled_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), status job_status NOT NULL DEFAULT 'available'::job_status, - parameters JSONB + target TEXT NOT NULL ); From 67c28b0c6aa574198a3a7adddf02d4a373186cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 1 Dec 2023 14:49:58 +0100 Subject: [PATCH 072/247] chore: Add docstrings --- hook-common/src/pgqueue.rs | 71 ++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index a85b323521346..1aa2a8f080843 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -85,23 +85,36 @@ pub struct Job { } impl Job { + /// Consume Job to retry it. + /// This returns a RetryableJob that can be enqueued by PgQueue. + /// + /// # Arguments + /// + /// * `error`: Any JSON-serializable value to be stored as an error. pub fn retry(self, error: E) -> Result, PgQueueError> { - if self.attempt == self.max_attempts { + if self.attempt >= self.max_attempts { Err(PgQueueError::MaxAttemptsReachedError(self.target)) } else { Ok(RetryableJob { id: self.id, attempt: self.attempt, - max_attempts: self.max_attempts, error: sqlx::types::Json(error), }) } } + /// Consume Job to complete it. + /// This returns a CompletedJob that can be enqueued by PgQueue. pub fn complete(self) -> CompletedJob { CompletedJob { id: self.id } } + /// Consume Job to fail it. + /// This returns a FailedJob that can be enqueued by PgQueue. + /// + /// # Arguments + /// + /// * `error`: Any JSON-serializable value to be stored as an error. pub fn fail(self, error: E) -> FailedJob { FailedJob { id: self.id, @@ -110,25 +123,6 @@ impl Job { } } -pub struct RetryPolicy { - backoff_coefficient: i32, - initial_interval: Duration, - maximum_interval: Option, -} - -impl RetryPolicy { - pub fn time_until_next_retry(&self, job: &RetryableJob) -> Duration { - let candidate_interval = - self.initial_interval * self.backoff_coefficient.pow(job.attempt as u32); - - if let Some(max_interval) = self.maximum_interval { - std::cmp::min(candidate_interval, max_interval) - } else { - candidate_interval - } - } -} - impl Default for RetryPolicy { fn default() -> Self { Self { @@ -139,19 +133,28 @@ impl Default for RetryPolicy { } } +/// A Job that has failed but can still be enqueued into a PgQueue to be retried at a later point. +/// The time until retry will depend on the PgQueue's RetryPolicy. pub struct RetryableJob { + /// A unique id identifying a job. pub id: i64, + /// A number corresponding to the current job attempt. pub attempt: i32, - pub max_attempts: i32, + /// Any JSON-serializable value to be stored as an error. pub error: sqlx::types::Json, } +/// A Job that has completed to be enqueued into a PgQueue and marked as completed. pub struct CompletedJob { + /// A unique id identifying a job. pub id: i64, } +/// A Job that has failed to be enqueued into a PgQueue and marked as failed. pub struct FailedJob { + /// A unique id identifying a job. pub id: i64, + /// Any JSON-serializable value to be stored as an error. pub error: sqlx::types::Json, } @@ -175,6 +178,30 @@ impl NewJob { } } +/// The retry policy that PgQueue will use to determine how to set scheduled_at when enqueuing a retry. +pub struct RetryPolicy { + /// Coeficient to multiply initial_interval with for every past attempt. + backoff_coefficient: i32, + /// The backoff interval for the first retry. + initial_interval: Duration, + /// The maximum possible backoff between retries. + maximum_interval: Option, +} + +impl RetryPolicy { + /// Calculate the time until the next retry for a given RetryableJob. + pub fn time_until_next_retry(&self, job: &RetryableJob) -> Duration { + let candidate_interval = + self.initial_interval * self.backoff_coefficient.pow(job.attempt as u32); + + if let Some(max_interval) = self.maximum_interval { + std::cmp::min(candidate_interval, max_interval) + } else { + candidate_interval + } + } +} + /// A queue implemented on top of a PostgreSQL table. pub struct PgQueue { /// A name to identify this PgQueue as multiple may share a table. From 8c5c7ec19d4f984a23fb539626761f483759717f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 1 Dec 2023 15:01:22 +0100 Subject: [PATCH 073/247] chore: Add basic requirements --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 8eb2e2c347ca0..b17e7ae6ffbe7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,12 @@ # rusty-hook A reliable and performant webhook system for PostHog +## Requirements + +1. [Rust](https://www.rust-lang.org/tools/install). +2. [sqlx-cli](https://crates.io/crates/sqlx-cli): To setup database and run migrations. +3. [Docker](https://docs.docker.com/engine/install/) or [podman](https://podman.io/docs/installation) (and [podman-compose](https://github.com/containers/podman-compose#installation)): To setup testing services. + ## Testing 1. Start a PostgreSQL instance: From d05f6ebccbfdf5e6c8837d0e468513c893664740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 1 Dec 2023 16:48:03 +0100 Subject: [PATCH 074/247] feat: Implement PgTransactionJob to hold a transaction open for the job --- hook-common/src/pgqueue.rs | 290 ++++++++++++++++++++++++++++++++++++- 1 file changed, 282 insertions(+), 8 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 1aa2a8f080843..3d1b233b6d804 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -18,6 +18,8 @@ pub enum PgQueueError { ConnectionError { error: sqlx::Error }, #[error("{command} query failed with: {error}")] QueryError { command: String, error: sqlx::Error }, + #[error("transaction {command} failed with: {error}")] + TransactionError { command: String, error: sqlx::Error }, #[error("{0} is not a valid JobStatus")] ParseJobStatusError(String), #[error("{0} Job has reached max attempts and cannot be retried further")] @@ -78,6 +80,8 @@ pub struct Job { pub max_attempts: i32, /// Arbitrary job parameters stored as JSON. pub parameters: JobParameters, + /// The queue this job belongs to. + pub queue: String, /// The current status of the job. pub status: JobStatus, /// The target of the job. E.g. an endpoint or service we are trying to reach. @@ -99,6 +103,7 @@ impl Job { id: self.id, attempt: self.attempt, error: sqlx::types::Json(error), + queue: self.queue, }) } } @@ -106,7 +111,10 @@ impl Job { /// Consume Job to complete it. /// This returns a CompletedJob that can be enqueued by PgQueue. pub fn complete(self) -> CompletedJob { - CompletedJob { id: self.id } + CompletedJob { + id: self.id, + queue: self.queue, + } } /// Consume Job to fail it. @@ -119,17 +127,152 @@ impl Job { FailedJob { id: self.id, error: sqlx::types::Json(error), + queue: self.queue, } } } -impl Default for RetryPolicy { - fn default() -> Self { - Self { - backoff_coefficient: 2, - initial_interval: Duration::seconds(1), - maximum_interval: None, - } +/// A Job within an open PostgreSQL transaction. +/// This implementation allows 'hiding' the job from any other workers running SKIP LOCKED queries. +pub struct PgTransactionJob<'c, J> { + pub job: Job, + pub table: String, + pub transaction: sqlx::Transaction<'c, sqlx::postgres::Postgres>, +} + +impl<'c, J> PgTransactionJob<'c, J> { + pub async fn retry( + mut self, + error: E, + retry_policy: &RetryPolicy, + ) -> Result, PgQueueError> { + let retryable_job = self.job.retry(error)?; + + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + status = 'available'::job_status, + scheduled_at = NOW() + $3, + errors = array_append("{0}".errors, $4) +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + + "#, + &self.table + ); + + sqlx::query(&base_query) + .bind(&retryable_job.queue) + .bind(retryable_job.id) + .bind(retry_policy.time_until_next_retry(&retryable_job)) + .bind(&retryable_job.error) + .execute(&mut *self.transaction) + .await + .map_err(|error| PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + self.transaction + .commit() + .await + .map_err(|error| PgQueueError::TransactionError { + command: "COMMIT".to_owned(), + error, + })?; + + Ok(retryable_job) + } + + pub async fn complete(mut self) -> Result { + let completed_job = self.job.complete(); + + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + status = 'completed'::job_status, +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + + "#, + &self.table + ); + + sqlx::query(&base_query) + .bind(&completed_job.queue) + .bind(completed_job.id) + .execute(&mut *self.transaction) + .await + .map_err(|error| PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + self.transaction + .commit() + .await + .map_err(|error| PgQueueError::TransactionError { + command: "COMMIT".to_owned(), + error, + })?; + + Ok(completed_job) + } + + pub async fn fail( + mut self, + error: E, + ) -> Result, PgQueueError> { + let failed_job = self.job.fail(error); + + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + status = 'failed'::job_status, +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + + "#, + &self.table + ); + + sqlx::query(&base_query) + .bind(&failed_job.queue) + .bind(failed_job.id) + .execute(&mut *self.transaction) + .await + .map_err(|error| PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + self.transaction + .commit() + .await + .map_err(|error| PgQueueError::TransactionError { + command: "COMMIT".to_owned(), + error, + })?; + + Ok(failed_job) } } @@ -142,12 +285,16 @@ pub struct RetryableJob { pub attempt: i32, /// Any JSON-serializable value to be stored as an error. pub error: sqlx::types::Json, + /// A unique id identifying a job queue. + pub queue: String, } /// A Job that has completed to be enqueued into a PgQueue and marked as completed. pub struct CompletedJob { /// A unique id identifying a job. pub id: i64, + /// A unique id identifying a job queue. + pub queue: String, } /// A Job that has failed to be enqueued into a PgQueue and marked as failed. @@ -156,6 +303,8 @@ pub struct FailedJob { pub id: i64, /// Any JSON-serializable value to be stored as an error. pub error: sqlx::types::Json, + /// A unique id identifying a job queue. + pub queue: String, } /// A NewJob to be enqueued into a PgQueue. @@ -202,6 +351,16 @@ impl RetryPolicy { } } +impl Default for RetryPolicy { + fn default() -> Self { + Self { + backoff_coefficient: 2, + initial_interval: Duration::seconds(1), + maximum_interval: None, + } + } +} + /// A queue implemented on top of a PostgreSQL table. pub struct PgQueue { /// A name to identify this PgQueue as multiple may share a table. @@ -296,6 +455,73 @@ RETURNING Ok(item) } + /// Dequeue a Job from this PgQueue to work on it. + pub async fn dequeue_tx< + J: DeserializeOwned + std::marker::Send + std::marker::Unpin + 'static, + >( + &self, + ) -> PgQueueResult>> { + // The query that follows uses a FOR UPDATE SKIP LOCKED clause. + // For more details on this see: 2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5. + let mut tx = self.pool.begin().await.unwrap(); + + let base_query = format!( + r#" +WITH available_in_queue AS ( + SELECT + id + FROM + "{0}" + WHERE + status = 'available' + AND scheduled_at <= NOW() + AND queue = $1 + ORDER BY + id + LIMIT 1 + FOR UPDATE SKIP LOCKED +) +UPDATE + "{0}" +SET + attempted_at = NOW(), + status = 'running'::job_status, + attempt = "{0}".attempt + 1, + attempted_by = array_append("{0}".attempted_by, $2::text) +FROM + available_in_queue +WHERE + "{0}".id = available_in_queue.id +RETURNING + "{0}".* + "#, + &self.table + ); + + let query_result: Result, sqlx::Error> = sqlx::query_as(&base_query) + .bind(&self.name) + .bind(&self.worker) + .fetch_one(&mut *tx) + .await; + + let job: Job = match query_result { + Ok(j) => j, + Err(sqlx::Error::RowNotFound) => return Ok(None), + Err(e) => { + return Err(PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error: e, + }) + } + }; + + Ok(Some(PgTransactionJob { + job, + table: self.table.to_owned(), + transaction: tx, + })) + } + /// Enqueue a Job into this PgQueue. /// We take ownership of NewJob to enforce a specific NewJob is only enqueued once. pub async fn enqueue( @@ -495,6 +721,54 @@ mod tests { assert_eq!(job.target, job_target.to_string()); } + #[tokio::test] + async fn test_can_dequeue_tx_job() { + let job_parameters = JobParameters { + method: "POST".to_string(), + body: "{\"event\":\"event-name\"}".to_string(), + url: "https://localhost".to_string(), + }; + let job_target = "https://myhost/endpoint"; + let new_job = NewJob::new(1, job_parameters, job_target); + + let worker_id = std::process::id().to_string(); + let retry_policy = RetryPolicy::default(); + let queue = PgQueue::new( + "test_queue_tx_1", + "job_queue", + retry_policy, + "postgres://posthog:posthog@localhost:15432/test_database", + &worker_id, + ) + .await + .expect("failed to connect to local test postgresql database"); + + queue.enqueue(new_job).await.expect("failed to enqueue job"); + + let tx_job: PgTransactionJob<'_, JobParameters> = queue + .dequeue_tx() + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + let another_job: Option> = + queue.dequeue_tx().await.expect("failed to dequeue job"); + + assert!(another_job.is_none()); + + assert_eq!(tx_job.job.attempt, 1); + assert!(tx_job.job.attempted_by.contains(&worker_id)); + assert_eq!(tx_job.job.attempted_by.len(), 1); + assert_eq!(tx_job.job.max_attempts, 1); + assert_eq!(tx_job.job.parameters.method, "POST".to_string()); + assert_eq!( + tx_job.job.parameters.body, + "{\"event\":\"event-name\"}".to_string() + ); + assert_eq!(tx_job.job.parameters.url, "https://localhost".to_string()); + assert_eq!(tx_job.job.status, JobStatus::Running); + assert_eq!(tx_job.job.target, job_target.to_string()); + } + #[tokio::test] async fn test_can_retry_job_with_remaining_attempts() { let job_parameters = JobParameters { From 4bf65168952211982ec543646d54006a215fc1c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 1 Dec 2023 18:09:52 +0100 Subject: [PATCH 075/247] refactor: Move queries to state handling methods in Job --- hook-common/src/pgqueue.rs | 505 +++++++++++++++++++++---------------- 1 file changed, 286 insertions(+), 219 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 3d1b233b6d804..451724debd364 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -132,19 +132,138 @@ impl Job { } } +/// A Job that can be updated in PostgreSQL. +pub struct PgJob { + pub job: Job, + pub table: String, + pub connection: sqlx::pool::PoolConnection, + pub retry_policy: RetryPolicy, +} + +impl PgJob { + pub async fn retry( + mut self, + error: E, + ) -> Result, PgQueueError> { + let retryable_job = self.job.retry(error)?; + + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + status = 'available'::job_status, + scheduled_at = NOW() + $3, + errors = array_append("{0}".errors, $4) +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + + "#, + &self.table + ); + + sqlx::query(&base_query) + .bind(&retryable_job.queue) + .bind(retryable_job.id) + .bind(self.retry_policy.time_until_next_retry(&retryable_job)) + .bind(&retryable_job.error) + .execute(&mut *self.connection) + .await + .map_err(|error| PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + Ok(retryable_job) + } + + pub async fn complete(mut self) -> Result { + let completed_job = self.job.complete(); + + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + status = 'completed'::job_status, +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + + "#, + &self.table + ); + + sqlx::query(&base_query) + .bind(&completed_job.queue) + .bind(completed_job.id) + .execute(&mut *self.connection) + .await + .map_err(|error| PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + Ok(completed_job) + } + + pub async fn fail( + mut self, + error: E, + ) -> Result, PgQueueError> { + let failed_job = self.job.fail(error); + + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + status = 'failed'::job_status, +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + + "#, + &self.table + ); + + sqlx::query(&base_query) + .bind(&failed_job.queue) + .bind(failed_job.id) + .execute(&mut *self.connection) + .await + .map_err(|error| PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; + + Ok(failed_job) + } +} + /// A Job within an open PostgreSQL transaction. /// This implementation allows 'hiding' the job from any other workers running SKIP LOCKED queries. pub struct PgTransactionJob<'c, J> { pub job: Job, pub table: String, pub transaction: sqlx::Transaction<'c, sqlx::postgres::Postgres>, + pub retry_policy: RetryPolicy, } impl<'c, J> PgTransactionJob<'c, J> { pub async fn retry( mut self, error: E, - retry_policy: &RetryPolicy, ) -> Result, PgQueueError> { let retryable_job = self.job.retry(error)?; @@ -170,7 +289,7 @@ RETURNING sqlx::query(&base_query) .bind(&retryable_job.queue) .bind(retryable_job.id) - .bind(retry_policy.time_until_next_retry(&retryable_job)) + .bind(self.retry_policy.time_until_next_retry(&retryable_job)) .bind(&retryable_job.error) .execute(&mut *self.transaction) .await @@ -327,6 +446,7 @@ impl NewJob { } } +#[derive(Copy, Clone)] /// The retry policy that PgQueue will use to determine how to set scheduled_at when enqueuing a retry. pub struct RetryPolicy { /// Coeficient to multiply initial_interval with for every past attempt. @@ -367,7 +487,7 @@ pub struct PgQueue { name: String, /// A connection pool used to connect to the PostgreSQL database. pool: PgPool, - /// The retry policy to use to enqueue any retryable jobs. + /// The retry policy to be assigned to Jobs in this PgQueue. retry_policy: RetryPolicy, /// The identifier of the PostgreSQL table this queue runs on. table: String, @@ -382,9 +502,9 @@ impl PgQueue { pub async fn new( name: &str, table: &str, - retry_policy: RetryPolicy, url: &str, worker: &str, + retry_policy: RetryPolicy, ) -> PgQueueResult { let name = name.to_owned(); let table = table.to_owned(); @@ -396,17 +516,23 @@ impl PgQueue { Ok(Self { name, - table, pool, - worker, retry_policy, + table, + worker, }) } /// Dequeue a Job from this PgQueue to work on it. pub async fn dequeue( &self, - ) -> PgQueueResult> { + ) -> PgQueueResult>> { + let mut connection = self + .pool + .acquire() + .await + .map_err(|error| PgQueueError::ConnectionError { error })?; + // The query that follows uses a FOR UPDATE SKIP LOCKED clause. // For more details on this see: 2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5. let base_query = format!( @@ -442,17 +568,29 @@ RETURNING &self.table ); - let item: Job = sqlx::query_as(&base_query) + let query_result: Result, sqlx::Error> = sqlx::query_as(&base_query) .bind(&self.name) .bind(&self.worker) - .fetch_one(&self.pool) - .await - .map_err(|error| PgQueueError::QueryError { - command: "UPDATE".to_owned(), - error, - })?; + .fetch_one(&mut *connection) + .await; + + let job: Job = match query_result { + Ok(j) => j, + Err(sqlx::Error::RowNotFound) => return Ok(None), + Err(e) => { + return Err(PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error: e, + }) + } + }; - Ok(item) + Ok(Some(PgJob { + job, + table: self.table.to_owned(), + connection, + retry_policy: self.retry_policy, + })) } /// Dequeue a Job from this PgQueue to work on it. @@ -461,10 +599,14 @@ RETURNING >( &self, ) -> PgQueueResult>> { + let mut tx = self + .pool + .begin() + .await + .map_err(|error| PgQueueError::ConnectionError { error })?; + // The query that follows uses a FOR UPDATE SKIP LOCKED clause. // For more details on this see: 2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5. - let mut tx = self.pool.begin().await.unwrap(); - let base_query = format!( r#" WITH available_in_queue AS ( @@ -519,6 +661,7 @@ RETURNING job, table: self.table.to_owned(), transaction: tx, + retry_policy: self.retry_policy, })) } @@ -553,120 +696,6 @@ VALUES Ok(()) } - - /// Enqueue a Job back into this PgQueue marked as completed. - /// We take ownership of Job to enforce a specific Job is only enqueued once. - pub async fn enqueue_completed(&self, job: CompletedJob) -> PgQueueResult<()> { - // TODO: Escaping. I think sqlx doesn't support identifiers. - let base_query = format!( - r#" -UPDATE - "{0}" -SET - finished_at = NOW(), - completed_at = NOW(), - status = 'completed'::job_status -WHERE - "{0}".id = $2 - AND queue = $1 -RETURNING - "{0}".* - "#, - &self.table - ); - - sqlx::query(&base_query) - .bind(&self.name) - .bind(job.id) - .execute(&self.pool) - .await - .map_err(|error| PgQueueError::QueryError { - command: "UPDATE".to_owned(), - error, - })?; - - Ok(()) - } - - /// Enqueue a Job back into this PgQueue to be retried at a later time. - /// We take ownership of Job to enforce a specific Job is only enqueued once. - pub async fn enqueue_retryable( - &self, - job: RetryableJob, - ) -> PgQueueResult<()> { - // TODO: Escaping. I think sqlx doesn't support identifiers. - let base_query = format!( - r#" -UPDATE - "{0}" -SET - finished_at = NOW(), - status = 'available'::job_status, - scheduled_at = NOW() + $3, - errors = array_append("{0}".errors, $4) -WHERE - "{0}".id = $2 - AND queue = $1 -RETURNING - "{0}".* - "#, - &self.table - ); - - sqlx::query(&base_query) - .bind(&self.name) - .bind(job.id) - .bind(self.retry_policy.time_until_next_retry(&job)) - .bind(&job.error) - .execute(&self.pool) - .await - .map_err(|error| PgQueueError::QueryError { - command: "UPDATE".to_owned(), - error, - })?; - - Ok(()) - } - - /// Enqueue a Job back into this PgQueue marked as failed. - /// Jobs marked as failed will remain in the queue for tracking purposes but will not be dequeued. - /// We take ownership of FailedJob to enforce a specific FailedJob is only enqueued once. - pub async fn enqueue_failed( - &self, - job: FailedJob, - ) -> PgQueueResult<()> { - // TODO: Escaping. I think sqlx doesn't support identifiers. - let base_query = format!( - r#" -UPDATE - "{0}" -SET - finished_at = NOW(), - completed_at = NOW(), - status = 'failed'::job_status - errors = array_append("{0}".errors, $3) -WHERE - "{0}".id = $2 - AND queue = $1 -RETURNING - "{0}".* - "#, - &self.table - ); - - sqlx::query(&base_query) - .bind(&self.name) - .bind(job.id) - .bind(&job.error) - .execute(&self.pool) - .await - .map_err(|error| PgQueueError::QueryError { - command: "UPDATE".to_owned(), - error, - })?; - - Ok(()) - } } #[cfg(test)] @@ -674,71 +703,99 @@ mod tests { use super::*; use serde::Deserialize; - #[derive(Serialize, Deserialize)] + #[derive(Serialize, Deserialize, PartialEq, Debug)] struct JobParameters { method: String, body: String, url: String, } + impl Default for JobParameters { + fn default() -> Self { + Self { + method: "POST".to_string(), + body: "{\"event\":\"event-name\"}".to_string(), + url: "https://localhost".to_string(), + } + } + } + + /// Use process id as a worker id for tests. + fn worker_id() -> String { + std::process::id().to_string() + } + + /// Hardcoded test value for job target. + fn job_target() -> String { + "https://myhost/endpoint".to_owned() + } + #[tokio::test] async fn test_can_dequeue_job() { - let job_parameters = JobParameters { - method: "POST".to_string(), - body: "{\"event\":\"event-name\"}".to_string(), - url: "https://localhost".to_string(), - }; - let job_target = "https://myhost/endpoint"; - let new_job = NewJob::new(1, job_parameters, job_target); + let job_target = job_target(); + let job_parameters = JobParameters::default(); + let worker_id = worker_id(); + let new_job = NewJob::new(1, job_parameters, &job_target); - let worker_id = std::process::id().to_string(); - let retry_policy = RetryPolicy::default(); let queue = PgQueue::new( - "test_queue_1", + "test_can_dequeue_job", "job_queue", - retry_policy, "postgres://posthog:posthog@localhost:15432/test_database", &worker_id, + RetryPolicy::default(), ) .await .expect("failed to connect to local test postgresql database"); queue.enqueue(new_job).await.expect("failed to enqueue job"); - let job: Job = queue.dequeue().await.expect("failed to dequeue job"); + let pg_job: PgJob = queue + .dequeue() + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + + assert_eq!(pg_job.job.attempt, 1); + assert!(pg_job.job.attempted_by.contains(&worker_id)); + assert_eq!(pg_job.job.attempted_by.len(), 1); + assert_eq!(pg_job.job.max_attempts, 1); + assert_eq!(*pg_job.job.parameters.as_ref(), JobParameters::default()); + assert_eq!(pg_job.job.status, JobStatus::Running); + assert_eq!(pg_job.job.target, job_target); + } - assert_eq!(job.attempt, 1); - assert!(job.attempted_by.contains(&worker_id)); - assert_eq!(job.attempted_by.len(), 1); - assert_eq!(job.max_attempts, 1); - assert_eq!(job.parameters.method, "POST".to_string()); - assert_eq!( - job.parameters.body, - "{\"event\":\"event-name\"}".to_string() - ); - assert_eq!(job.parameters.url, "https://localhost".to_string()); - assert_eq!(job.status, JobStatus::Running); - assert_eq!(job.target, job_target.to_string()); + #[tokio::test] + async fn test_dequeue_returns_none_on_no_jobs() { + let worker_id = worker_id(); + let queue = PgQueue::new( + "test_dequeue_returns_none_on_no_jobs", + "job_queue", + "postgres://posthog:posthog@localhost:15432/test_database", + &worker_id, + RetryPolicy::default(), + ) + .await + .expect("failed to connect to local test postgresql database"); + + let pg_job: Option> = + queue.dequeue().await.expect("failed to dequeue job"); + + assert!(pg_job.is_none()); } #[tokio::test] async fn test_can_dequeue_tx_job() { - let job_parameters = JobParameters { - method: "POST".to_string(), - body: "{\"event\":\"event-name\"}".to_string(), - url: "https://localhost".to_string(), - }; - let job_target = "https://myhost/endpoint"; - let new_job = NewJob::new(1, job_parameters, job_target); + let job_target = job_target(); + let job_parameters = JobParameters::default(); + let worker_id = worker_id(); + let new_job = NewJob::new(1, job_parameters, &job_target); - let worker_id = std::process::id().to_string(); - let retry_policy = RetryPolicy::default(); let queue = PgQueue::new( - "test_queue_tx_1", + "test_can_dequeue_tx_job", "job_queue", - retry_policy, "postgres://posthog:posthog@localhost:15432/test_database", &worker_id, + RetryPolicy::default(), ) .await .expect("failed to connect to local test postgresql database"); @@ -750,107 +807,117 @@ mod tests { .await .expect("failed to dequeue job") .expect("didn't find a job to dequeue"); - let another_job: Option> = - queue.dequeue_tx().await.expect("failed to dequeue job"); - - assert!(another_job.is_none()); assert_eq!(tx_job.job.attempt, 1); assert!(tx_job.job.attempted_by.contains(&worker_id)); assert_eq!(tx_job.job.attempted_by.len(), 1); assert_eq!(tx_job.job.max_attempts, 1); - assert_eq!(tx_job.job.parameters.method, "POST".to_string()); - assert_eq!( - tx_job.job.parameters.body, - "{\"event\":\"event-name\"}".to_string() - ); - assert_eq!(tx_job.job.parameters.url, "https://localhost".to_string()); + assert_eq!(*tx_job.job.parameters.as_ref(), JobParameters::default()); assert_eq!(tx_job.job.status, JobStatus::Running); - assert_eq!(tx_job.job.target, job_target.to_string()); + assert_eq!(tx_job.job.target, job_target); } #[tokio::test] - async fn test_can_retry_job_with_remaining_attempts() { - let job_parameters = JobParameters { - method: "POST".to_string(), - body: "{\"event\":\"event-name\"}".to_string(), - url: "https://localhost".to_string(), - }; - let job_target = "https://myhost/endpoint"; - let new_job = NewJob::new(2, job_parameters, job_target); + async fn test_dequeue_tx_returns_none_on_no_jobs() { + let worker_id = worker_id(); + let queue = PgQueue::new( + "test_dequeue_tx_returns_none_on_no_jobs", + "job_queue", + "postgres://posthog:posthog@localhost:15432/test_database", + &worker_id, + RetryPolicy::default(), + ) + .await + .expect("failed to connect to local test postgresql database"); + + let tx_job: Option> = + queue.dequeue_tx().await.expect("failed to dequeue job"); + + assert!(tx_job.is_none()); + } - let worker_id = std::process::id().to_string(); + #[tokio::test] + async fn test_can_retry_job_with_remaining_attempts() { + let job_target = job_target(); + let job_parameters = JobParameters::default(); + let worker_id = worker_id(); + let new_job = NewJob::new(2, job_parameters, &job_target); let retry_policy = RetryPolicy { backoff_coefficient: 0, initial_interval: Duration::seconds(0), maximum_interval: None, }; + let queue = PgQueue::new( - "test_queue_2", + "test_can_retry_job_with_remaining_attempts", "job_queue", - retry_policy, "postgres://posthog:posthog@localhost:15432/test_database", &worker_id, + retry_policy, ) .await .expect("failed to connect to local test postgresql database"); queue.enqueue(new_job).await.expect("failed to enqueue job"); - let job: Job = queue.dequeue().await.expect("failed to dequeue job"); - let retryable_job = job + let job: PgJob = queue + .dequeue() + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + let _ = job .retry("a very reasonable failure reason") + .await .expect("failed to retry job"); - - queue - .enqueue_retryable(retryable_job) + let retried_job: PgJob = queue + .dequeue() .await - .expect("failed to enqueue retryable job"); - let retried_job: Job = queue.dequeue().await.expect("failed to dequeue job"); - - assert_eq!(retried_job.attempt, 2); - assert!(retried_job.attempted_by.contains(&worker_id)); - assert_eq!(retried_job.attempted_by.len(), 2); - assert_eq!(retried_job.max_attempts, 2); - assert_eq!(retried_job.parameters.method, "POST".to_string()); + .expect("failed to dequeue job") + .expect("didn't find retried job to dequeue"); + + assert_eq!(retried_job.job.attempt, 2); + assert!(retried_job.job.attempted_by.contains(&worker_id)); + assert_eq!(retried_job.job.attempted_by.len(), 2); + assert_eq!(retried_job.job.max_attempts, 2); assert_eq!( - retried_job.parameters.body, - "{\"event\":\"event-name\"}".to_string() + *retried_job.job.parameters.as_ref(), + JobParameters::default() ); - assert_eq!(retried_job.parameters.url, "https://localhost".to_string()); - assert_eq!(retried_job.status, JobStatus::Running); - assert_eq!(retried_job.target, job_target.to_string()); + assert_eq!(retried_job.job.status, JobStatus::Running); + assert_eq!(retried_job.job.target, job_target); } #[tokio::test] #[should_panic(expected = "failed to retry job")] async fn test_cannot_retry_job_without_remaining_attempts() { - let job_parameters = JobParameters { - method: "POST".to_string(), - body: "{\"event\":\"event-name\"}".to_string(), - url: "https://localhost".to_string(), - }; - let job_target = "https://myhost/endpoint"; - let new_job = NewJob::new(1, job_parameters, job_target); - - let worker_id = std::process::id().to_string(); + let job_target = job_target(); + let job_parameters = JobParameters::default(); + let worker_id = worker_id(); + let new_job = NewJob::new(1, job_parameters, &job_target); let retry_policy = RetryPolicy { backoff_coefficient: 0, initial_interval: Duration::seconds(0), maximum_interval: None, }; + let queue = PgQueue::new( - "test_queue_3", + "test_cannot_retry_job_without_remaining_attempts", "job_queue", - retry_policy, "postgres://posthog:posthog@localhost:15432/test_database", &worker_id, + retry_policy, ) .await .expect("failed to connect to local test postgresql database"); queue.enqueue(new_job).await.expect("failed to enqueue job"); - let job: Job = queue.dequeue().await.expect("failed to dequeue job"); + + let job: PgJob = queue + .dequeue() + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); job.retry("a very reasonable failure reason") + .await .expect("failed to retry job"); } } From 401336a853a1c5a8eec0505c00fc5ff811a7d994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 1 Dec 2023 18:27:51 +0100 Subject: [PATCH 076/247] refactor: Return from match and close connection as recommended --- hook-common/src/pgqueue.rs | 61 ++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 451724debd364..599785cf3f914 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -574,23 +574,28 @@ RETURNING .fetch_one(&mut *connection) .await; - let job: Job = match query_result { - Ok(j) => j, - Err(sqlx::Error::RowNotFound) => return Ok(None), + match query_result { + Ok(job) => Ok(Some(PgJob { + job, + table: self.table.to_owned(), + connection, + retry_policy: self.retry_policy, + })), + + // Although connection would be closed once it goes out of scope, sqlx recommends explicitly calling close(). + // See: https://docs.rs/sqlx/latest/sqlx/postgres/any/trait.AnyConnectionBackend.html#tymethod.close. + Err(sqlx::Error::RowNotFound) => { + let _ = connection.close().await; + Ok(None) + } Err(e) => { - return Err(PgQueueError::QueryError { + let _ = connection.close().await; + Err(PgQueueError::QueryError { command: "UPDATE".to_owned(), error: e, }) } - }; - - Ok(Some(PgJob { - job, - table: self.table.to_owned(), - connection, - retry_policy: self.retry_policy, - })) + } } /// Dequeue a Job from this PgQueue to work on it. @@ -646,23 +651,21 @@ RETURNING .fetch_one(&mut *tx) .await; - let job: Job = match query_result { - Ok(j) => j, - Err(sqlx::Error::RowNotFound) => return Ok(None), - Err(e) => { - return Err(PgQueueError::QueryError { - command: "UPDATE".to_owned(), - error: e, - }) - } - }; - - Ok(Some(PgTransactionJob { - job, - table: self.table.to_owned(), - transaction: tx, - retry_policy: self.retry_policy, - })) + match query_result { + Ok(job) => Ok(Some(PgTransactionJob { + job, + table: self.table.to_owned(), + transaction: tx, + retry_policy: self.retry_policy, + })), + + // Transaction is rolledback on drop. + Err(sqlx::Error::RowNotFound) => Ok(None), + Err(e) => Err(PgQueueError::QueryError { + command: "UPDATE".to_owned(), + error: e, + }), + } } /// Enqueue a Job into this PgQueue. From 2ab6250078d782b7f90d08089ed7304fecf4b2cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Mon, 4 Dec 2023 10:24:28 +0100 Subject: [PATCH 077/247] fix: Update docstring Co-authored-by: Brett Hoerner --- hook-common/src/pgqueue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 599785cf3f914..dd38fa24525ee 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -109,7 +109,7 @@ impl Job { } /// Consume Job to complete it. - /// This returns a CompletedJob that can be enqueued by PgQueue. + /// This returns a CompletedJob that can be marked as completed by PgQueue. pub fn complete(self) -> CompletedJob { CompletedJob { id: self.id, From 521fb51264c1e0a7202f7d8aa74199bf4e27ee67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Mon, 4 Dec 2023 10:24:43 +0100 Subject: [PATCH 078/247] fix: More docstring updates. Co-authored-by: Brett Hoerner --- hook-common/src/pgqueue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index dd38fa24525ee..6575be8a0b6c8 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -118,7 +118,7 @@ impl Job { } /// Consume Job to fail it. - /// This returns a FailedJob that can be enqueued by PgQueue. + /// This returns a FailedJob that can be marked as failed by PgQueue. /// /// # Arguments /// From 746120b4cd8e843c8ee53b500bc07322f79fabfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Mon, 4 Dec 2023 10:25:03 +0100 Subject: [PATCH 079/247] fix: Typo Co-authored-by: Brett Hoerner --- hook-common/src/pgqueue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 6575be8a0b6c8..7204f976cd1eb 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -449,7 +449,7 @@ impl NewJob { #[derive(Copy, Clone)] /// The retry policy that PgQueue will use to determine how to set scheduled_at when enqueuing a retry. pub struct RetryPolicy { - /// Coeficient to multiply initial_interval with for every past attempt. + /// Coefficient to multiply initial_interval with for every past attempt. backoff_coefficient: i32, /// The backoff interval for the first retry. initial_interval: Duration, From d5f345dadd6f19734b2d9c195550de1a46c24d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Mon, 4 Dec 2023 10:31:59 +0100 Subject: [PATCH 080/247] fix: Update documentation and names --- hook-common/src/pgqueue.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 7204f976cd1eb..89aab7ee45ab9 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -499,16 +499,24 @@ pub type PgQueueResult = std::result::Result; impl PgQueue { /// Initialize a new PgQueue backed by table in PostgreSQL. + /// + /// # Arguments + /// + /// * `queue_name`: A name for the queue we are going to initialize. + /// * `table_name`: The name for the table the queue will use in PostgreSQL. + /// * `url`: A URL pointing to where the PostgreSQL database is hosted. + /// * `worker_name`: The name of the worker that is operating with this queue. + /// * `retry_policy`: A retry policy to pass to jobs from this queue. pub async fn new( - name: &str, - table: &str, + queue_name: &str, + table_name: &str, url: &str, - worker: &str, + worker_name: &str, retry_policy: RetryPolicy, ) -> PgQueueResult { - let name = name.to_owned(); - let table = table.to_owned(); - let worker = worker.to_owned(); + let name = queue_name.to_owned(); + let table = table_name.to_owned(); + let worker = worker_name.to_owned(); let pool = PgPoolOptions::new() .connect(url) .await From e4554641d0afdf7c20cb195ffcc5aded4f17c03f Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 27 Nov 2023 15:13:04 +0000 Subject: [PATCH 081/247] add deps --- Cargo.lock | 348 +++++++++++++++++++++++++++++++++++++-- Cargo.toml | 1 + hook-producer/Cargo.toml | 2 + 3 files changed, 334 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16de92647176f..aa7f4880c709b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,6 +51,17 @@ dependencies = [ "libc", ] +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "atoi" version = "2.0.0" @@ -62,9 +73,9 @@ dependencies = [ [[package]] name = "atomic-write-file" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae364a6c1301604bbc6dfbf8c385c47ff82301dd01eef506195a029196d8d04" +checksum = "edcdbedc2236483ab103a53415653d6b4442ea6141baf1ffa85df29635e88436" dependencies = [ "nix", "rand", @@ -76,6 +87,59 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202651474fe73c62d9e0a56c6133f7a0ff1dc1c8cf7a5b03381af2a26553ac9d" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77cb22c689c44d4c07b0ab44ebc25d69d8ae601a2f28fb8d672d344178fa17aa" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -182,9 +246,9 @@ checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -192,9 +256,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" @@ -343,6 +407,12 @@ dependencies = [ "spin 0.9.8", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foreign-types" version = "0.3.2" @@ -466,6 +536,25 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "h2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d308f63daf4181410c242d34c11f928dcb3aa105852019e043c9d1f4e4368a" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.14.3" @@ -494,6 +583,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + [[package]] name = "hex" version = "0.4.3" @@ -546,6 +641,95 @@ version = "0.1.0" [[package]] name = "hook-producer" version = "0.1.0" +dependencies = [ + "axum", + "tokio", +] + +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", +] [[package]] name = "iana-time-zone" @@ -648,9 +832,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lock_api" @@ -668,6 +852,12 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -684,6 +874,12 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -797,6 +993,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "object" version = "0.32.1" @@ -900,6 +1106,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -1004,9 +1230,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6c4b23d99685a1408194da11270ef8e9809aff951cc70ec9b17350b087e474" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" dependencies = [ "const-oid", "digest", @@ -1030,17 +1256,23 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + [[package]] name = "ryu" version = "1.0.15" @@ -1116,6 +1348,28 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +dependencies = [ + "itoa", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -1138,6 +1392,15 @@ dependencies = [ "digest", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + [[package]] name = "signature" version = "2.2.0" @@ -1454,6 +1717,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "tempfile" version = "3.8.1" @@ -1512,7 +1781,10 @@ dependencies = [ "bytes", "libc", "mio", + "num_cpus", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.48.0", @@ -1540,6 +1812,48 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.40" @@ -1855,18 +2169,18 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "zerocopy" -version = "0.7.26" +version = "0.7.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.26" +version = "0.7.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index d880e2cd85940..ea188390ae387 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,3 +9,4 @@ members = [ [workspace.dependencies] sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", "postgres", "uuid", "json" ] } +tokio = { version = "1.34.0", features = ["full"] } diff --git a/hook-producer/Cargo.toml b/hook-producer/Cargo.toml index 96fbb4d7528fe..7626cc58033d5 100644 --- a/hook-producer/Cargo.toml +++ b/hook-producer/Cargo.toml @@ -6,3 +6,5 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +tokio = { workspace = true } +axum = { version="0.7.1", features=["http2"] } From 749dd1798c7f44b4dbddd615a4c4f82ab1c0e36d Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 4 Dec 2023 15:32:38 +0100 Subject: [PATCH 082/247] feat: auto-detect gzip compression on input payloads (#63) --- capture/src/event.rs | 52 +++++++++++++++---------------- capture/tests/django_compat.rs | 7 ++++- capture/tests/requests_dump.jsonl | 3 ++ 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/capture/src/event.rs b/capture/src/event.rs index 81eb754fd5bd3..92b0f0bc9e444 100644 --- a/capture/src/event.rs +++ b/capture/src/event.rs @@ -1,15 +1,16 @@ use std::collections::HashMap; use std::io::prelude::*; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use crate::api::CaptureError; use bytes::{Buf, Bytes}; use flate2::read::GzDecoder; +use serde::{Deserialize, Serialize}; +use serde_json::Value; use time::OffsetDateTime; +use tracing::instrument; use uuid::Uuid; +use crate::api::CaptureError; + #[derive(Deserialize, Default)] pub enum Compression { #[default] @@ -59,6 +60,8 @@ pub struct RawEvent { pub set_once: Option>, } +static GZIP_MAGIC_NUMBERS: [u8; 3] = [0x1f, 0x8b, 8]; + #[derive(Deserialize)] #[serde(untagged)] enum RawRequest { @@ -78,33 +81,30 @@ impl RawRequest { } impl RawEvent { - /// We post up _at least one_ event, so when decompressiong and deserializing there - /// could be more than one. Hence this function has to return a Vec. - /// TODO: Use an axum extractor for this - pub fn from_bytes(query: &EventQuery, bytes: Bytes) -> Result, CaptureError> { + /// Takes a request payload and tries to decompress and unmarshall it into events. + /// While posthog-js sends a compression query param, a sizable portion of requests + /// fail due to it being missing when the body is compressed. + /// Instead of trusting the parameter, we peek at the payload's first three bytes to + /// detect gzip, fallback to uncompressed utf8 otherwise. + #[instrument(skip_all)] + pub fn from_bytes(_query: &EventQuery, bytes: Bytes) -> Result, CaptureError> { tracing::debug!(len = bytes.len(), "decoding new event"); - let payload = match query.compression { - Some(Compression::Gzip) => { - let mut d = GzDecoder::new(bytes.reader()); - let mut s = String::new(); - d.read_to_string(&mut s).map_err(|e| { - tracing::error!("failed to decode gzip: {}", e); - CaptureError::RequestDecodingError(String::from("invalid gzip data")) - })?; - s - } - Some(_) => { - return Err(CaptureError::RequestDecodingError(String::from( - "unsupported compression format", - ))) - } - - None => String::from_utf8(bytes.into()).map_err(|e| { + let payload = if bytes.starts_with(&GZIP_MAGIC_NUMBERS) { + let mut d = GzDecoder::new(bytes.reader()); + let mut s = String::new(); + d.read_to_string(&mut s).map_err(|e| { + tracing::error!("failed to decode gzip: {}", e); + CaptureError::RequestDecodingError(String::from("invalid gzip data")) + })?; + s + } else { + String::from_utf8(bytes.into()).map_err(|e| { tracing::error!("failed to decode body: {}", e); CaptureError::RequestDecodingError(String::from("invalid body encoding")) - })?, + })? }; + tracing::debug!(json = payload, "decoded event data"); Ok(serde_json::from_str::(&payload)?.events()) } diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index b95c78e48501f..d1d075bdab97c 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -82,7 +82,12 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { let mut mismatches = 0; for (line_number, line_contents) in reader.lines().enumerate() { - let case: RequestDump = serde_json::from_str(&line_contents?)?; + let line_contents = line_contents?; + if line_contents.starts_with('#') { + // Skip comment lines + continue; + } + let case: RequestDump = serde_json::from_str(&line_contents)?; if !case.path.starts_with("/e/") { println!("Skipping {} test case", &case.path); continue; diff --git a/capture/tests/requests_dump.jsonl b/capture/tests/requests_dump.jsonl index 88d30102039ee..ec0f4df482afb 100644 --- a/capture/tests/requests_dump.jsonl +++ b/capture/tests/requests_dump.jsonl @@ -10,3 +10,6 @@ {"path":"/e/?compression=gzip-js&ip=1&_=1694769323385&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:23.391445+00:00","body":"H4sIAAAAAAAAA+0a7W7bNvBVDCPYrzLV90f+temSdli7ZF3bDUUhUCRlMZZFRaLiOEWAPcsebU+yOzm2ZdlOlq7YklV/bOnueHe8b5P++HlY15IPD4aGGdAwMBMSuokgvkfhKTE54dQVruXFjhGYwydDcSFyDeR7tNaK0ULXpQBwUapClFqKanjwebin4Gv4mrLBT28HvwIaANGFKCupckCYxr7p7hsIj0s1rUQJwCNZikRdIpCLC8lEpGeFAMQLUY21KhDB6rIE8VFdZoBItS4Onj7NQI0sVZU+CAzDeApqVCqvkByBQLdOgIiC6jSnE+Tepr9RZqWpafotcEbzUU1HuErk5N1bXFKxUog8SoUcpSDLNFxrBZ1KrlMAepYBwAsppoUq9ZI4NM02eEHtOgGAMxmDnKmIUQq8tM237wf7LsJlDnrpaO6/sjqvrMC6HBX2lDsK8VriHk0vdHwvtC1z34H9cFlpmbObZcWHZ+NXv8xO7cM3vm1b785PL/OXXmEfvQ/e/8D94jfP8b/nb8aXH1jLNesR48QWJ35ABQFBMYkFjyk1LGokDq6p0XhfIExWERcTFUFsnQkGBktoVglgOCpVXTSBtkQtlBEkCWKLgJ8Nwl3BSWgHhhAupdxBU6pyRHN5RfXclKtVrsvmq2jgcmK5zISwd13uctQkrzTNGXp+a9ANr0GrVkJEYGMaZ4JHsHVwXFRJDosX+lOm5YWIEkEb4iSjI9jNx0+AasOigs4yRTnuFASUgmYT1ACkwsZYJtk4VWBczMoJlVkjDL1DL+AN5S9FVhll41vwKa2iWGaZzEEqxPkSAdnBIOMg6qO6gs99sDivIXhoTrOZlgyUg+BeJuYtNBusKlFhREelAJGzLXy2EGww6Rhxg8cmfg/qjAASLDp7HKoOxM8SigbgCqyJ0dHCNmVvUZEa0zf1SLRqBVJlYgJ06MzPQ1AyuikyVQEmBc0yWsGeADvU4lITLhJaZ8gfIno6BP9TrcsoauhgWZto0JA8GeY6jVgqM0gnkIhvKrnRa65BhKtg8aFKoapgXZszTWGDWO/YDfz6yR0q/gi5lz+vtQYPREzlGgv/ho5bqdbUhIK4pqbVkUwb+27Ru62LzNHgLWHrb4Ro7D60nHXhkLi6ruALsgfKaweb1FlG5oW3g4GMIBL2s2XHoMugRdt+XumxDl3XYh230mEdvtRglwcXcE41JfgISAi/mkgtJmRFdVvIdHyRyQ693V1gdFbU2f0kcHmx7tq3UBqfU9hGlcET2nlXqO2k/KoK3Cn364jbKWadvdNhb9/FPiXz4QOgSSZwnMIviIZsU+KCdoAkzUdD9wX7m/NtenypVMcjGzHUYRErjmk756FTKKFNJOKM1NX49spiXsMWtRrD7mHWSFl0Pnp3VLjTK+v52aHzc3V09vq4OLVPjl9AzfZOjcNj4/DEefOMMuNVM8zd9JuN6Qbm4ZjbJI5tg3ie6ceJEwSBF+Kiqcy5mt65JjCNRHDPTXBWUElSCRwXQ7Nx6MYI7nkO8allkjBwLeJYTJghCwKDee0RfG1YgHkEBo4HNYqvStDfG8Vb9I96FJ9NEq8Yu06dz/K0DrC3bI7irg950c/i/Szez+L3nMXbbHByTdWU3GyTyFyXiojLJq9GC3BTUtbcWYqqgF/9K6s/qrYB0B1tIyY+Z4zQWEAeOWYchB71HYYpvGgbYEZVcrDOAOud4IOphNEzVwN0LJapQVVPJvNRuu8l/3Uvqa9c93xmqTgurSvbG23vJcC6byV9K+lbyf1aybwARphP0QTY2FYAmYRarWDAelkxI16XTYwtMSWkcjSlZY5YpmqssAAHkaqcv+MaRK6YIP8YHDenzEewHzw1iiogAN8B8HE1I2PXbxhYbvkWCeKQkzhJOHNZ4Bjcav+GKWD36Ia+2TyAZuNObR1YM31GZ0Z+LtEp25oNqNd3m77b9N3mft3mwVd1UFFqdOjiEH7w5+9/DE4gVF6qUafmN4d4GzWfWwnUfBGTIAxDyGEWJhYNE9dZq/kP9er4Wy37emzqfKbklTOK87HVRHKn7Nv7Norsy35f9vuy/6/eHXeuR0/mf1cZfDc4nkdyf0+665509c+e2+5JV1T/5J60exUV9tek/TVpf036dY4Y/B3Dpu0T3wQOsQEHDJ7J7CRgie1ZODk87AOGb/VPip4T5GcTd6Snl5ZIzs52TJphP2n2k2Y/af6PDxi6Y+yukwbPuf70FzjrKd2kLgAA","output":[{"uuid":"018a981f-95fe-76af-9f1d-da5e526b4081","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-95fe-76af-9f1d-da5e526b4081\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0rsqs282xgp3wd4o\", \"$time\": 1694769321.47, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"classes\": [\"text-default\", \"grow\"], \"attr__class\": \"text-default grow\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Cohorts\", \"attr__href\": \"/cohorts\"}, {\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"a\", \"$el_text\": \"Cohorts\", \"classes\": [\"Link\", \"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-stealth\", \"LemonButton--full-width\", \"LemonButton--has-icon\"], \"attr__class\": \"Link LemonButton LemonButton--tertiary LemonButton--status-stealth LemonButton--full-width LemonButton--has-icon\", \"attr__href\": \"/cohorts\", \"attr__data-attr\": \"menu-item-cohorts\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"nth_child\": 13, \"nth_of_type\": 10}, {\"tag_name\": \"ul\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider__content\"], \"attr__class\": \"SideBar__slider__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider\"], \"attr__class\": \"SideBar__slider\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1913}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-9664-7a21-9852-42ce19c880c6","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9664-7a21-9852-42ce19c880c6\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ymf6pk54unynhu8h\", \"$time\": 1694769321.573, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"show-product-intro-existing-products\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1810}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-966b-7dcc-abee-f41b896a74cc","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-966b-7dcc-abee-f41b896a74cc\", \"event\": \"recording viewed with no playtime summary\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"uz55qy2obbr2z36g\", \"$time\": 1694769321.58, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"viewed_time_ms\": 3288, \"play_time_ms\": 0, \"recording_duration_ms\": 0, \"rrweb_warning_count\": 0, \"error_count_during_recording_playback\": 0, \"engagement_score\": 0, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1803}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-966e-7272-8b9d-bffdc5c840d2","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-966e-7272-8b9d-bffdc5c840d2\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"5w3t82ytjay0nqiw\", \"$time\": 1694769321.582, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Cohorts \\u2022 PostHog\"}, \"offset\": 1801}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-9d2f-72eb-8999-bec9f2a9f542","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9d2f-72eb-8999-bec9f2a9f542\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"tk1tnyoiz4gbnk2t\", \"$time\": 1694769323.311, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"a\", \"$el_text\": \"Persons & Groups\", \"classes\": [\"Link\", \"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-stealth\", \"LemonButton--full-width\", \"LemonButton--has-icon\"], \"attr__class\": \"Link LemonButton LemonButton--tertiary LemonButton--status-stealth LemonButton--full-width LemonButton--has-icon\", \"attr__href\": \"/persons\", \"attr__data-attr\": \"menu-item-persons\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"nth_child\": 12, \"nth_of_type\": 9}, {\"tag_name\": \"ul\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider__content\"], \"attr__class\": \"SideBar__slider__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider\"], \"attr__class\": \"SideBar__slider\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 71}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-9d37-712e-b09d-61c3f8cf3624","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9d37-712e-b09d-61c3f8cf3624\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"648njm5gtwx2efjj\", \"$time\": 1694769323.319, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Persons & Groups \\u2022 PostHog\"}, \"offset\": 64}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} {"path":"/e/?compression=gzip-js&ip=1&_=1694769326396&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:26.539412+00:00","body":"H4sIAAAAAAAAA+1b6Y7cNhJ+lUavf5pt3ccsFtjEWTtZbLx2HCe7CAKBIqmWptWiRqT6GMPAPss+Wp4kH9Xn9JV08scz0PyYaRWLVcW6PrKp+enjsG0LPrwZWnZE48jOSMxTn4R2FJHIDmwS2yLOeGA5GQ+Hz4diJioN9rtWNMsBk9O6FFpwjNSNrEWjC6GGNx+HzyT+DL+lbPDv94P/YBiEZCYaVcgKA7Y1sv2RZehpI+dKNCC+KhqRyYUhcjErmEj0shYY+EqoiZa1GWBt08CCpG1KDORa1zcvXpSS0TKXSt9ElmW9gBlKVsqwGyL4HjKYgZrqvKJTI32ff23MzlLbDvfIJa3GLR2bWaIiH96bKYo1QlRJLopxDl225Ts76rzgOgcxcCwQZ4WY17LRW+bYtvfJG27fi0AuixR65iI1WvCw775RGI18Qy8q2KWTLoTMX97Hd9mcRsEsX9Buni7MGu0g9sIgdh135Hn+8yEvlC4qtp5X//jF5Jvvl+/cl29C13U+3L1bVF8Htfvqh+iHf/Kw/m/ghf/gbyaLH9lebB5mjZc6nIQRFQSaUpIKnlJqOdTKPDOnNd77A8oKlXAxlQmS61YweCyjpRIQOG5kW3eZth3aGCNIFqUOQaAtwn3BSexGlhA+pdwzPpHNmFbFPdUrX+5m+T5bzaKRz4njM9sXvu9z36Q3HK1pxUzoT2bd8BOsoq0GsdZtIxL4mKal4AmWjsglquCYvLGfMl3MRJIJ2jFnJR1jNT/9jKF9WlLTZSkpNyuFgkbQcmosgFYsjJUFm+QSzjWVOaVF2Skz0aEzPBn9W5WqpGxyYTynKkmLsiwqaEWibwdQHgwlh7RPWoXfI3ict0geWtFyqQsG45Dd28q8wHMkSgllUjppBFQuT8g5wXAk5MCJRzKOx5+h0QiwmK7zjKPtIH+2VOMALuFNkx270a7jmYSbFJVJ5LerrvFGIqwIDW+bdUbZFhqAlhNh5tc5S+7GH17V/vze+fL2pfedenX77ev6nfv29Vco5+Cd9fK19fKt9+YLyqxvuoayXvJRgQkSptwlaepaJAjsMM28KIqC2Eyawyg5/805kW1lggd+ZkyWWaYECseJPf/T82MkoA7KIPRcfIpslwQ8tmI3tjKfGZUbJHiYrygJ5PxnCAcvfm+rugIyfrfMRw0rhW1Zd7RN/ZyJtuTdgg5gxRsFLhbUw0oPKz2sXAcrD+JiNnGKcKryVNIG4TmMWyNUDdDZufcx4Ywd+u5pnAk8ElKHosjdmNhxFIcuY2EURns4A3/JhsOLA9PYBB/MC50PKjkwETT9aKDa6ZQCo3vweTrgM79vZpbj5WFqTUMadUhxDD79maYHnx58rgefVSdNTEElU4ixnRiVZKza0SB623qTzSlnO9KglpM5bSozymRrWjXoUCmb1bOZYwZ3Qoz8FIFbcVZjrGdqVqbAgNiB+LhQ7dzpKQhI6HKH0CDIiJ2ljLkOjzPL2T891Vi9CUOPWk8ItSZK6ftFfNuWY1nPQ3oGtfojU49aPWpdjVqfPTrAxEKbgG6+JBz88r//D94iVb6W4wPsOH0i8lIXwj2LpB6Ex17scztFKcfuHnasOur6ONTjxxPCD+Vk97f3AQu1JSZjPT2BH/7IcYMeP3r86PHjWvzY9cnNicVGlRqvdZ4f3uimXbtx1XzW7mOt0nJqMvhgugeZiGMu4e9j2Rj97BFrH5PQtE5iUhDiPBM6DloCE8TJ4tTlQRp5qKS988xe4fSQ9IQgqaqXE2RZE+Tav7tP85OQFEQ9JPWQ1EPS9bdAXf/ctLbO9V1jE3vNwnCV3TdlJpgfhzByjU9DVcOlz4dU6yZJONWUmI/bIwJZL5Rsv4NTRHfvB1U6T1helCgeyDdPMltbsdKXaLEw6f/ddiZy8YFuXswgiJVUQQsMG/4LNVV9T1OVJFBCmKy0wQZk38q+jhUTz/BdNupAeVlc0v1bOrcua2R3Wtsn0aag8FuJ+OBwdTPs8nU7Ck5ArFiYEn9osXtgcXe+27O4Lc9anNLmosVm/NjiEu32Oqedj9gF9VvNSi871YQgGZFTMEERVaIBNaTDmJtBEI3sGD8AhsjxHDuuF38dnOJf7TluBhYYLuRvl63Ggr1FYlf1YJHdnuXSIk3NEVrX5xPyiOPPefU9lvglNfLPKTzi+HMrXIs7q+ey+MNEPRKfk9W+A9SsROqv/sBZ5bHGDe/AsHS/Or4/4NCV3A7dGykPXHRYbIciUsnNdfC6ZnM0z65nmO3RocUPBWOXdSAYS3xMu/jQOreJT3HVjvuIGG/skJRRygRzvDgwG6Incynxl9UOCH3rb2vP7+FXv7s/3t1nuWqDOfazy1DPZq0pmhO7+xgq+919v7vvd/dP9sJiQdFSmr+vv84a4R88VncXl+8xgjg+iTZhhGuM2PFIikIiLp7jLLSF45sb0eMXu8xWdpAJzfLHe5/RQ8+10MNEOrsNdDtO51447869R9AT+/0bXj309NBzNfSYMHTvcpmLCNSQabGGdVeV5kb5sweoPazxXP/Tz78Ca5NvR7g4AAA=","output":[{"uuid":"018a981f-9db5-7188-8161-91e9fd602fd7","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9db5-7188-8161-91e9fd602fd7\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"c5yz9qfwa86vhxab\", \"$time\": 1694769323.445, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"PersonsNode\"}, \"duration\": 102, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2945}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a25d-743f-a813-6d909390f5c9","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a25d-743f-a813-6d909390f5c9\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"i100qaub5hceuld4\", \"$time\": 1694769324.637, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"cs-dashboards\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1753}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a264-7a2a-9439-198973cc7878","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a264-7a2a-9439-198973cc7878\", \"event\": \"recording viewed with no playtime summary\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"wzrv024h7b0m7a8c\", \"$time\": 1694769324.645, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"viewed_time_ms\": 1295, \"play_time_ms\": 0, \"recording_duration_ms\": 0, \"rrweb_warning_count\": 0, \"error_count_during_recording_playback\": 0, \"engagement_score\": 0, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1745}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a266-73d2-a66f-1fbcc32d9f02","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a266-73d2-a66f-1fbcc32d9f02\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ksstzx9julgopw7a\", \"$time\": 1694769324.647, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Persons \\u2022 PostHog\"}, \"offset\": 1743}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a4b3-7b40-b430-9495d1bbed93","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a4b3-7b40-b430-9495d1bbed93\", \"event\": \"person viewed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"s2fzjz6c7t0ekgtm\", \"$time\": 1694769325.236, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"properties_count\": 18, \"has_email\": true, \"has_name\": false, \"custom_properties_count\": 4, \"posthog_properties_count\": 14, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1154}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a676-7722-bece-2f9b3d6b84ce","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a676-7722-bece-2f9b3d6b84ce\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"npyk617r6ht5qzbh\", \"$time\": 1694769325.686, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"attr__data-attr\": \"person-session-recordings-tab\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Recordings\"}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 68.19999694824219px; --lemon-tabs-slider-offset: 0px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 704}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a67b-7a6f-9637-bcaacec2496c","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a67b-7a6f-9637-bcaacec2496c\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"fhsu6w5edy7tvvuy\", \"$time\": 1694769325.691, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 699}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a783-7924-b938-3a789f71e25a","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a783-7924-b938-3a789f71e25a\", \"event\": \"recording list fetched\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"fcebvj6tugbw47wk\", \"$time\": 1694769325.955, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"load_time\": 145, \"listing_version\": \"3\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 435}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} {"path":"/e/?compression=gzip-js&ip=1&_=1694769329412&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:29.500667+00:00","body":"H4sIAAAAAAAAA+0Ya2/bNvCvGF4/bEPo6EVSyjBgbdq0HdauaZp2Q1EIFElZqmlRkehXiwL7Lftp+yU7yo7jV2KkHYoikz/Y1t3xXryX7u3H7miUi+5R13FDFoVuihhLMaJp6qAoTQUKUyzdNMTUpWn3oCvHsjBAfo+NjOasNKNKArisdCkrk8u6e/Sxe0/DT/cZ453fzzp/ABoA8VhWda4LQLhOz8U9x8KTSk9qWQHwJK9kqqcWKOQ45zI2s1IC4qGsB0aXFsFHVQXi41GlAJEZUx4dHipQQ2W6Nkeh4ziHoEati8OlOUHiCURDJlEU0AQlUiSMOR5z0uA7xk0+lq9Y8nMta6vcS8l1JfKiX1txlinIWRdgESUzWcGGVrvbyFsx+MobrktXwIoV/RHrW86yQOdn9kjNKymLOJN5PwN9XAd7V9BJLkwGQOI5ABznclLqyiyJI9ddBV9S4yAEsMoTkDORiZUCD6tX1KNhD1t4XoBeJp7HiB6LQTRVxThKs5QNLN7k1g8uAWtJ5HukF/pgkMhrkxd8ca58c3/w9NXs1D9+Tn3fO784nRZPSOmfvA5f/ypo+ScJ6CPxfDB9w1fufz0s9/h1ZL33GcLyOhZyqGMI4PeSg8dSpmoJDPuVHpVNNC9Rl8pIlIaJhyAYHCSwFCjyQ0dKzJgIrC911WdF/oGZuS+vTmHM56dYiAXyMHexxBgLLKwmRW1Ywe3V74zs7ifQaiXrYvAxS5QUMZgONxfXuYDDl/rPYztOJWuIU8UgqI/evgPUKiwu2UxpJqylIKCSTA2tBiAVDOMq54NMg3Nt6g9Zrhph9nbYGJ6s/KXIWjE+uAGfsTpOcqUgveISAn2JgBTikNYQ9vGohu8eeFyMIHhYwdTM5ByUg+heZv8NNFusFokdVxJEznbw2UGwxWTDiVs8tvH3oJhJILGV7Z6A0gbxs4RaBwgN3rTRsYJtautl2Wtc3xQ9uVIsLJWSQ6Czl/mxC0rGi0pUl+DSgy4zpopjwQxD9q9NiKZC1aiSitlLbbREhtlQLUwW8yxXkDjA2z7pdKHBXFZs5NSG/sncxM7cxE8Ha6JFPgZeXLEavAl6dX+DlCqgrtZxDHIQ14WxbQOCb65eQwoHr6G7Wa8N4Sq/SfY+mUuPVdqGKjC+ArEqZ6gGf3NwG+CacF1igTIvhJzaDF/XGG9ojDc0HqlrNU5YdaPGFr+tsYJqezunXX9jN4hfSq7NrBGNEMSiLmws1ahWUH8q1LSYow4Neh6Bj4cjSjD8lNOfOrvodZrW0hx1XEp6XhRFIaWRF5GQ2hM3xrPlsm51sGF1sM9qm4OIleX1EbpF8WVuPgObHzDL/zqBWxRfZuGC3bVybmbv72OfofkcAtBUQS7Mf8BZalviJS3UEDltvhq6z3DonG/T7SutN1zk72GRaDG7SuIMimlTROy4tKnxOmOYujYYg4lGD8B6CMqMxxf985MSTz54D94fBy/rk/fPHpen/ovHD6F6k1Pn+LFz/CJ4fp9x52kz1y06z9acIxFNhI+SxHcQIS5N0iAMQxLZQxOoOXqy90zoOqkUBKd2apinGBiAqWedsWPiZ4hGMkQMRKFAklAmJOC+szbxl9Dq7CR5B8b9RcM+aZpZO+lvT/ozf0qJZGR4cZF7Fw7bPekHoF476beTfjvp327S/+abBqiYm2bEmzIoKdUvJURJpvs9roedf/76u/NiPoPN/wPuie6vdxpCd3aa1IfUjkQAWR0wJKgXcQqFxhG2wtzR3VLbbPY0G2UmtFJBmuVB0r8g/o5mQ3uRD8Np22zaZtM2m6+6Vvq6u521ndPmmunb2/RsvhI2r1f/s01PCHsbHyqi5wY+xgGJ9m16fEJ7Dmx6KMYhjvx20dMuetpFz3+66HED2uyct8fvgMCw43soYiRCHBNCscvtlvaOLnoam9qpe9fUTWbKxxNhxnjsupo0I/L21N2ueNqpu5262xXP1orHDXaveBLHCRH1KQOZAchkhIVuwjwvsKl92WMuRrKadUBUqaQdudtOc4c7TdgfTcbD0Jtlk1mWarKj04Q91ydtp2k7TdtpbttpmlJqA24ALQAwj5pCdNpAwbvNUsO+wf24LL+NqlB97BueZkrWXH5/VX57q0XyoLOKWBQPW8N+gFf8znmlOoeds8sXw1VSWyRsixlKCM9haV8S50Kfrqcnj9g8tkmYwqNLAwBTjhNbRVhqGlcgL8hsHItRtcgOl0KB+uZ77Gq79Kj/6d2/ToySO28rAAA=","output":[{"uuid":"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0ovdk9xlnv9fhfak\", \"$time\": 1694769326.837, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"attr__data-attr\": \"persons-related-flags-tab\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Feature flags\"}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 74.26666259765625px; --lemon-tabs-slider-offset: 176.29998779296875px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2572}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-aafa-79e8-abf4-4e68eb64c30f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aafa-79e8-abf4-4e68eb64c30f\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"y3x76ea6mqqi2q0a\", \"$time\": 1694769326.842, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 2567}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af3d-79d4-be4a-d729c776e0da","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af3d-79d4-be4a-d729c776e0da\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ltw7rl4fhi4bgq63\", \"$time\": 1694769327.934, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"\"}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 86.23332214355469px; --lemon-tabs-slider-offset: 367.0999755859375px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1475}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af46-7832-9a69-c566751c6625","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af46-7832-9a69-c566751c6625\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"6yl35wdtv5v11o6c\", \"$time\": 1694769327.942, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 1467}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-b008-737a-bb40-6a6a81ba2244","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-b008-737a-bb40-6a6a81ba2244\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"8guwvm82yhwyhfo6\", \"$time\": 1694769328.136, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"EventsQuery\", \"select\": [\"*\", \"event\", \"person\", \"coalesce(properties.$current_url, properties.$screen_name) -- Url / Screen\", \"properties.$lib\", \"timestamp\"], \"personId\": \"018a981f-4c9a-0000-68ff-417481f7c5b5\", \"after\": \"-24h\"}, \"duration\": 171, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1273}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +### Compression query param mismatch, to confirm gzip autodetection +{"path":"/e/?compression=gzip-js&ip=1&_=1694769302319&ver=1.78.5","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-09-15T09:15:02.321230+00:00","body":"ZGF0YT1leUoxZFdsa0lqb2lNREU0WVRrNE1XWXROR0l5WlMwM1ltUXpMV0ppTXpBdE5qWXhOalkxTjJNek9HWmhJaXdpWlhabGJuUWlPaUlrYjNCMFgybHVJaXdpY0hKdmNHVnlkR2xsY3lJNmV5SWtiM01pT2lKTllXTWdUMU1nV0NJc0lpUnZjMTkyWlhKemFXOXVJam9pTVRBdU1UVXVNQ0lzSWlSaWNtOTNjMlZ5SWpvaVJtbHlaV1p2ZUNJc0lpUmtaWFpwWTJWZmRIbHdaU0k2SWtSbGMydDBiM0FpTENJa1kzVnljbVZ1ZEY5MWNtd2lPaUpvZEhSd09pOHZiRzlqWVd4b2IzTjBPamd3TURBdklpd2lKR2h2YzNRaU9pSnNiMk5oYkdodmMzUTZPREF3TUNJc0lpUndZWFJvYm1GdFpTSTZJaThpTENJa1luSnZkM05sY2w5MlpYSnphVzl1SWpveE1UY3NJaVJpY205M2MyVnlYMnhoYm1kMVlXZGxJam9pWlc0dFZWTWlMQ0lrYzJOeVpXVnVYMmhsYVdkb2RDSTZNVEExTWl3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRZeU1Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPVEV4TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFMU5EZ3NJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpjNExqVWlMQ0lrYVc1elpYSjBYMmxrSWpvaU1XVTRaV1p5WkdSbE1HSTBNV2RtYkNJc0lpUjBhVzFsSWpveE5qazBOelk1TXpBeUxqTXhPQ3dpWkdsemRHbHVZM1JmYVdRaU9pSXdNVGhoT1RneFppMDBZakprTFRjNFlXVXRPVFEzWWkxaVpXUmlZV0V3TW1Fd1pqUWlMQ0lrWkdWMmFXTmxYMmxrSWpvaU1ERTRZVGs0TVdZdE5HSXlaQzAzT0dGbExUazBOMkl0WW1Wa1ltRmhNREpoTUdZMElpd2lKSEpsWm1WeWNtVnlJam9pSkdScGNtVmpkQ0lzSWlSeVpXWmxjbkpwYm1kZlpHOXRZV2x1SWpvaUpHUnBjbVZqZENJc0luUnZhMlZ1SWpvaWNHaGpYM0ZuVlVad05YZDZNa0pxUXpSU2MwWnFUVWR3VVROUVIwUnljMmsyVVRCRFJ6QkRVRFJPUVdGak1Fa2lMQ0lrYzJWemMybHZibDlwWkNJNklqQXhPR0U1T0RGbUxUUmlNbVV0TjJKa015MWlZak13TFRZMk1UZGlaalE0T0RnMk9TSXNJaVIzYVc1a2IzZGZhV1FpT2lJd01UaGhPVGd4WmkwMFlqSmxMVGRpWkRNdFltSXpNQzAyTmpFNE1UQm1aV1EyTldZaWZTd2lkR2x0WlhOMFlXMXdJam9pTWpBeU15MHdPUzB4TlZRd09Ub3hOVG93TWk0ek1UaGFJbjAlM0Q=","output":[{"uuid":"018a981f-4b2e-7bd3-bb30-6616657c38fa","distinct_id":"018a981f-4b2d-78ae-947b-bedbaa02a0f4","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b2e-7bd3-bb30-6616657c38fa\", \"event\": \"$opt_in\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"1e8efrdde0b41gfl\", \"$time\": 1694769302.318, \"distinct_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"timestamp\": \"2023-09-15T09:15:02.318Z\"}","now":"2023-09-15T09:15:02.321230+00:00","sent_at":"2023-09-15T09:15:02.319000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +{"path":"/e/?ip=1&_=1694769329412&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:29.500667+00:00","body":"H4sIAAAAAAAAA+0Ya2/bNvCvGF4/bEPo6EVSyjBgbdq0HdauaZp2Q1EIFElZqmlRkehXiwL7Lftp+yU7yo7jV2KkHYoikz/Y1t3xXryX7u3H7miUi+5R13FDFoVuihhLMaJp6qAoTQUKUyzdNMTUpWn3oCvHsjBAfo+NjOasNKNKArisdCkrk8u6e/Sxe0/DT/cZ453fzzp/ABoA8VhWda4LQLhOz8U9x8KTSk9qWQHwJK9kqqcWKOQ45zI2s1IC4qGsB0aXFsFHVQXi41GlAJEZUx4dHipQQ2W6Nkeh4ziHoEati8OlOUHiCURDJlEU0AQlUiSMOR5z0uA7xk0+lq9Y8nMta6vcS8l1JfKiX1txlinIWRdgESUzWcGGVrvbyFsx+MobrktXwIoV/RHrW86yQOdn9kjNKymLOJN5PwN9XAd7V9BJLkwGQOI5ABznclLqyiyJI9ddBV9S4yAEsMoTkDORiZUCD6tX1KNhD1t4XoBeJp7HiB6LQTRVxThKs5QNLN7k1g8uAWtJ5HukF/pgkMhrkxd8ca58c3/w9NXs1D9+Tn3fO784nRZPSOmfvA5f/ypo+ScJ6CPxfDB9w1fufz0s9/h1ZL33GcLyOhZyqGMI4PeSg8dSpmoJDPuVHpVNNC9Rl8pIlIaJhyAYHCSwFCjyQ0dKzJgIrC911WdF/oGZuS+vTmHM56dYiAXyMHexxBgLLKwmRW1Ywe3V74zs7ifQaiXrYvAxS5QUMZgONxfXuYDDl/rPYztOJWuIU8UgqI/evgPUKiwu2UxpJqylIKCSTA2tBiAVDOMq54NMg3Nt6g9Zrhph9nbYGJ6s/KXIWjE+uAGfsTpOcqUgveISAn2JgBTikNYQ9vGohu8eeFyMIHhYwdTM5ByUg+heZv8NNFusFokdVxJEznbw2UGwxWTDiVs8tvH3oJhJILGV7Z6A0gbxs4RaBwgN3rTRsYJtautl2Wtc3xQ9uVIsLJWSQ6Czl/mxC0rGi0pUl+DSgy4zpopjwQxD9q9NiKZC1aiSitlLbbREhtlQLUwW8yxXkDjA2z7pdKHBXFZs5NSG/sncxM7cxE8Ha6JFPgZeXLEavAl6dX+DlCqgrtZxDHIQ14WxbQOCb65eQwoHr6G7Wa8N4Sq/SfY+mUuPVdqGKjC+ArEqZ6gGf3NwG+CacF1igTIvhJzaDF/XGG9ojDc0HqlrNU5YdaPGFr+tsYJqezunXX9jN4hfSq7NrBGNEMSiLmws1ahWUH8q1LSYow4Neh6Bj4cjSjD8lNOfOrvodZrW0hx1XEp6XhRFIaWRF5GQ2hM3xrPlsm51sGF1sM9qm4OIleX1EbpF8WVuPgObHzDL/zqBWxRfZuGC3bVybmbv72OfofkcAtBUQS7Mf8BZalviJS3UEDltvhq6z3DonG/T7SutN1zk72GRaDG7SuIMimlTROy4tKnxOmOYujYYg4lGD8B6CMqMxxf985MSTz54D94fBy/rk/fPHpen/ovHD6F6k1Pn+LFz/CJ4fp9x52kz1y06z9acIxFNhI+SxHcQIS5N0iAMQxLZQxOoOXqy90zoOqkUBKd2apinGBiAqWedsWPiZ4hGMkQMRKFAklAmJOC+szbxl9Dq7CR5B8b9RcM+aZpZO+lvT/ozf0qJZGR4cZF7Fw7bPekHoF476beTfjvp327S/+abBqiYm2bEmzIoKdUvJURJpvs9roedf/76u/NiPoPN/wPuie6vdxpCd3aa1IfUjkQAWR0wJKgXcQqFxhG2wtzR3VLbbPY0G2UmtFJBmuVB0r8g/o5mQ3uRD8Np22zaZtM2m6+6Vvq6u521ndPmmunb2/RsvhI2r1f/s01PCHsbHyqi5wY+xgGJ9m16fEJ7Dmx6KMYhjvx20dMuetpFz3+66HED2uyct8fvgMCw43soYiRCHBNCscvtlvaOLnoam9qpe9fUTWbKxxNhxnjsupo0I/L21N2ueNqpu5262xXP1orHDXaveBLHCRH1KQOZAchkhIVuwjwvsKl92WMuRrKadUBUqaQdudtOc4c7TdgfTcbD0Jtlk1mWarKj04Q91ydtp2k7TdtpbttpmlJqA24ALQAwj5pCdNpAwbvNUsO+wf24LL+NqlB97BueZkrWXH5/VX57q0XyoLOKWBQPW8N+gFf8znmlOoeds8sXw1VSWyRsixlKCM9haV8S50Kfrqcnj9g8tkmYwqNLAwBTjhNbRVhqGlcgL8hsHItRtcgOl0KB+uZ77Gq79Kj/6d2/ToySO28rAAA=","output":[{"uuid":"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0ovdk9xlnv9fhfak\", \"$time\": 1694769326.837, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"attr__data-attr\": \"persons-related-flags-tab\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Feature flags\"}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 74.26666259765625px; --lemon-tabs-slider-offset: 176.29998779296875px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2572}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-aafa-79e8-abf4-4e68eb64c30f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aafa-79e8-abf4-4e68eb64c30f\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"y3x76ea6mqqi2q0a\", \"$time\": 1694769326.842, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 2567}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af3d-79d4-be4a-d729c776e0da","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af3d-79d4-be4a-d729c776e0da\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ltw7rl4fhi4bgq63\", \"$time\": 1694769327.934, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"\"}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 86.23332214355469px; --lemon-tabs-slider-offset: 367.0999755859375px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1475}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af46-7832-9a69-c566751c6625","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af46-7832-9a69-c566751c6625\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"6yl35wdtv5v11o6c\", \"$time\": 1694769327.942, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 1467}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-b008-737a-bb40-6a6a81ba2244","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-b008-737a-bb40-6a6a81ba2244\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"8guwvm82yhwyhfo6\", \"$time\": 1694769328.136, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"EventsQuery\", \"select\": [\"*\", \"event\", \"person\", \"coalesce(properties.$current_url, properties.$screen_name) -- Url / Screen\", \"properties.$lib\", \"timestamp\"], \"personId\": \"018a981f-4c9a-0000-68ff-417481f7c5b5\", \"after\": \"-24h\"}, \"duration\": 171, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1273}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} From efe67f8fef07926cab24fe6799d43b1458f4e3d8 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 27 Nov 2023 15:29:45 +0000 Subject: [PATCH 083/247] boot http server --- Cargo.lock | 108 ++++++++++++++++++++++++++++ hook-producer/Cargo.toml | 3 + hook-producer/src/handlers/index.rs | 3 + hook-producer/src/handlers/mod.rs | 9 +++ hook-producer/src/main.rs | 23 +++++- 5 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 hook-producer/src/handlers/index.rs create mode 100644 hook-producer/src/handlers/mod.rs diff --git a/Cargo.lock b/Cargo.lock index aa7f4880c709b..be15d2dc20319 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -384,6 +384,16 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "eyre" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80f656be11ddf91bd709454d15d5bd896fbaf4cc3314e69349e4d1569f5b46cd" +dependencies = [ + "indenter", + "once_cell", +] + [[package]] name = "fastrand" version = "2.0.1" @@ -643,7 +653,10 @@ name = "hook-producer" version = "0.1.0" dependencies = [ "axum", + "eyre", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -764,6 +777,12 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + [[package]] name = "indexmap" version = "2.1.0" @@ -945,6 +964,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -1062,6 +1091,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1392,6 +1427,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -1756,6 +1800,16 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1884,6 +1938,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", ] [[package]] @@ -1948,6 +2028,12 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vcpkg" version = "0.2.15" @@ -2026,6 +2112,28 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.51.1" diff --git a/hook-producer/Cargo.toml b/hook-producer/Cargo.toml index 7626cc58033d5..35af92d685ccb 100644 --- a/hook-producer/Cargo.toml +++ b/hook-producer/Cargo.toml @@ -8,3 +8,6 @@ edition = "2021" [dependencies] tokio = { workspace = true } axum = { version="0.7.1", features=["http2"] } +eyre = "0.6.9" +tracing = "0.1.40" +tracing-subscriber = "0.3.18" diff --git a/hook-producer/src/handlers/index.rs b/hook-producer/src/handlers/index.rs new file mode 100644 index 0000000000000..56896fa63a483 --- /dev/null +++ b/hook-producer/src/handlers/index.rs @@ -0,0 +1,3 @@ +pub async fn get() -> &'static str { + "rusty hook" +} diff --git a/hook-producer/src/handlers/mod.rs b/hook-producer/src/handlers/mod.rs new file mode 100644 index 0000000000000..a83e46ec58f42 --- /dev/null +++ b/hook-producer/src/handlers/mod.rs @@ -0,0 +1,9 @@ +use axum::{Router, routing}; + +mod index; + +pub fn router() -> Router { + let app = Router::new().route("/", routing::get(index::get)); + + app +} diff --git a/hook-producer/src/main.rs b/hook-producer/src/main.rs index e7a11a969c037..a8a1ce528d704 100644 --- a/hook-producer/src/main.rs +++ b/hook-producer/src/main.rs @@ -1,3 +1,22 @@ -fn main() { - println!("Hello, world!"); +use axum::Router; +use eyre::Result; +mod handlers; + +async fn listen(app: Router) -> Result<()> { + let listener = tokio::net::TcpListener::bind("0.0.0.0:8000").await?; + axum::serve(listener, app).await?; + + Ok(()) +} + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + + let app = handlers::router(); + + match listen(app).await { + Ok(_) => {}, + Err(e) => tracing::error!("failed to start hook-producer http server, {}", e) + } } From 9fb81ef3ca045d18f9163c70eda57c74cb6a7389 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 4 Dec 2023 14:51:19 +0000 Subject: [PATCH 084/247] Formatting, move shared deps into workspace --- Cargo.toml | 9 ++++++++- hook-common/Cargo.toml | 12 ++++++------ hook-producer/Cargo.toml | 8 ++++---- hook-producer/src/handlers/mod.rs | 2 +- hook-producer/src/main.rs | 4 ++-- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ea188390ae387..8f557673559dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,5 +8,12 @@ members = [ ] [workspace.dependencies] -sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", "postgres", "uuid", "json" ] } +chrono = { version = "0.4" } +serde = { version = "1.0" } +serde_derive = { version = "1.0" } +thiserror = { version = "1.0" } +sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", "postgres", "uuid", "json", "chrono" ] } tokio = { version = "1.34.0", features = ["full"] } +eyre = "0.6.9" +tracing = "0.1.40" +tracing-subscriber = "0.3.18" diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index 673d8877f726a..b55a9ecd84d4d 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -6,11 +6,11 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -chrono = { version = "0.4" } -serde = { version = "1.0" } -serde_derive = { version = "1.0" } -sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", "postgres", "uuid", "json", "chrono" ] } -thiserror = { version = "1.0" } +chrono = { workspace = true} +serde = { workspace = true } +serde_derive = { workspace = true} +thiserror = { workspace = true } +sqlx = { workspace = true } [dev-dependencies] -tokio = { version = "1.34", features = ["macros"] } # We need a runtime for async tests +tokio = { workspace = true } # We need a runtime for async tests diff --git a/hook-producer/Cargo.toml b/hook-producer/Cargo.toml index 35af92d685ccb..85100d099bf4e 100644 --- a/hook-producer/Cargo.toml +++ b/hook-producer/Cargo.toml @@ -6,8 +6,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { workspace = true } axum = { version="0.7.1", features=["http2"] } -eyre = "0.6.9" -tracing = "0.1.40" -tracing-subscriber = "0.3.18" +tokio = { workspace = true } +eyre = {workspace = true } +tracing = {workspace = true} +tracing-subscriber = {workspace = true} diff --git a/hook-producer/src/handlers/mod.rs b/hook-producer/src/handlers/mod.rs index a83e46ec58f42..8b4f83c0f19fc 100644 --- a/hook-producer/src/handlers/mod.rs +++ b/hook-producer/src/handlers/mod.rs @@ -1,4 +1,4 @@ -use axum::{Router, routing}; +use axum::{routing, Router}; mod index; diff --git a/hook-producer/src/main.rs b/hook-producer/src/main.rs index a8a1ce528d704..f05edab10fee2 100644 --- a/hook-producer/src/main.rs +++ b/hook-producer/src/main.rs @@ -16,7 +16,7 @@ async fn main() { let app = handlers::router(); match listen(app).await { - Ok(_) => {}, - Err(e) => tracing::error!("failed to start hook-producer http server, {}", e) + Ok(_) => {} + Err(e) => tracing::error!("failed to start hook-producer http server, {}", e), } } From 731ae143e7a58586973b441b6ec863bfe85a46a2 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Mon, 4 Dec 2023 15:08:00 +0000 Subject: [PATCH 085/247] add metrics --- Cargo.lock | 297 ++++++++++++++++++++++++++++-- Cargo.toml | 3 + hook-producer/Cargo.toml | 3 + hook-producer/src/config.rs | 16 ++ hook-producer/src/handlers/mod.rs | 10 +- hook-producer/src/main.rs | 16 +- hook-producer/src/metrics.rs | 53 ++++++ 7 files changed, 373 insertions(+), 25 deletions(-) create mode 100644 hook-producer/src/config.rs create mode 100644 hook-producer/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index be15d2dc20319..451d42483e301 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,10 +97,10 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http", - "http-body", + "http 1.0.0", + "http-body 1.0.0", "http-body-util", - "hyper", + "hyper 1.0.1", "hyper-util", "itoa", "matchit", @@ -129,8 +129,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 1.0.0", + "http-body 1.0.0", "http-body-util", "mime", "pin-project-lite", @@ -284,6 +284,19 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-queue" version = "0.3.8" @@ -351,6 +364,26 @@ dependencies = [ "serde", ] +[[package]] +name = "envconfig" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" +dependencies = [ + "envconfig_derive", +] + +[[package]] +name = "envconfig_derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -557,14 +590,29 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", - "indexmap", + "http 1.0.0", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.14.3" @@ -581,7 +629,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown", + "hashbrown 0.14.3", ] [[package]] @@ -653,12 +701,26 @@ name = "hook-producer" version = "0.1.0" dependencies = [ "axum", + "envconfig", "eyre", + "metrics", + "metrics-exporter-prometheus", "tokio", "tracing", "tracing-subscriber", ] +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.0.0" @@ -670,6 +732,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http 0.2.11", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.0" @@ -677,7 +750,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http", + "http 1.0.0", ] [[package]] @@ -688,8 +761,8 @@ checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -705,6 +778,29 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 0.2.11", + "http-body 0.4.5", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.0.1" @@ -715,8 +811,8 @@ dependencies = [ "futures-channel", "futures-util", "h2", - "http", - "http-body", + "http 1.0.0", + "http-body 1.0.0", "httparse", "httpdate", "itoa", @@ -733,11 +829,11 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.0.1", "pin-project-lite", - "socket2", + "socket2 0.5.5", "tokio", "tower", "tower-service", @@ -783,6 +879,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.1.0" @@ -790,9 +896,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.3", ] +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "itertools" version = "0.11.0" @@ -871,6 +983,15 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + [[package]] name = "matchit" version = "0.7.3" @@ -893,6 +1014,70 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "metrics" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +dependencies = [ + "ahash", + "metrics-macros", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" +dependencies = [ + "base64", + "hyper 0.14.27", + "indexmap 1.9.3", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "metrics-util" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.13.1", + "metrics", + "num_cpus", + "quanta", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -1200,6 +1385,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "portable-atomic" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1215,6 +1406,22 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quanta" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.33" @@ -1254,6 +1461,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -1455,6 +1671,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "sketches-ddsketch" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" + [[package]] name = "slab" version = "0.4.9" @@ -1470,6 +1692,16 @@ version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "socket2" version = "0.5.5" @@ -1552,7 +1784,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap", + "indexmap 2.1.0", "log", "memchr", "native-tls", @@ -1839,7 +2071,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", ] @@ -1966,6 +2198,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "typenum" version = "1.17.0" @@ -2046,6 +2284,15 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2106,6 +2353,16 @@ version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +[[package]] +name = "web-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "whoami" version = "1.4.1" diff --git a/Cargo.toml b/Cargo.toml index 8f557673559dc..e92db695f324a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,3 +17,6 @@ tokio = { version = "1.34.0", features = ["full"] } eyre = "0.6.9" tracing = "0.1.40" tracing-subscriber = "0.3.18" +envconfig = "0.10.0" +metrics = "0.21.1" +metrics-exporter-prometheus = "0.12.1" diff --git a/hook-producer/Cargo.toml b/hook-producer/Cargo.toml index 85100d099bf4e..47ef532891436 100644 --- a/hook-producer/Cargo.toml +++ b/hook-producer/Cargo.toml @@ -11,3 +11,6 @@ tokio = { workspace = true } eyre = {workspace = true } tracing = {workspace = true} tracing-subscriber = {workspace = true} +envconfig = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } diff --git a/hook-producer/src/config.rs b/hook-producer/src/config.rs new file mode 100644 index 0000000000000..9d093c652efbb --- /dev/null +++ b/hook-producer/src/config.rs @@ -0,0 +1,16 @@ +use envconfig::Envconfig; + +#[derive(Envconfig)] +pub struct Config { + #[envconfig(from = "BIND_HOST", default = "0.0.0.0")] + pub host: String, + + #[envconfig(from = "BIND_PORT", default = "8000")] + pub port: u16, +} + +impl Config { + pub fn bind(&self) -> String { + format!("{}:{}", self.host, self.port) + } +} diff --git a/hook-producer/src/handlers/mod.rs b/hook-producer/src/handlers/mod.rs index 8b4f83c0f19fc..25040731d5688 100644 --- a/hook-producer/src/handlers/mod.rs +++ b/hook-producer/src/handlers/mod.rs @@ -3,7 +3,13 @@ use axum::{routing, Router}; mod index; pub fn router() -> Router { - let app = Router::new().route("/", routing::get(index::get)); + let recorder_handle = crate::metrics::setup_metrics_recorder(); - app + Router::new() + .route("/", routing::get(index::get)) + .route( + "/metrics", + routing::get(move || std::future::ready(recorder_handle.render())), + ) + .layer(axum::middleware::from_fn(crate::metrics::track_metrics)) } diff --git a/hook-producer/src/main.rs b/hook-producer/src/main.rs index f05edab10fee2..118829b00d78b 100644 --- a/hook-producer/src/main.rs +++ b/hook-producer/src/main.rs @@ -1,9 +1,17 @@ use axum::Router; + +use config::Config; +use envconfig::Envconfig; + use eyre::Result; + +mod config; mod handlers; +mod metrics; + +async fn listen(app: Router, bind: String) -> Result<()> { + let listener = tokio::net::TcpListener::bind(bind).await?; -async fn listen(app: Router) -> Result<()> { - let listener = tokio::net::TcpListener::bind("0.0.0.0:8000").await?; axum::serve(listener, app).await?; Ok(()) @@ -15,7 +23,9 @@ async fn main() { let app = handlers::router(); - match listen(app).await { + let config = Config::init_from_env().expect("failed to load configuration from env"); + + match listen(app, config.bind()).await { Ok(_) => {} Err(e) => tracing::error!("failed to start hook-producer http server, {}", e), } diff --git a/hook-producer/src/metrics.rs b/hook-producer/src/metrics.rs new file mode 100644 index 0000000000000..dbdc7b1fa1107 --- /dev/null +++ b/hook-producer/src/metrics.rs @@ -0,0 +1,53 @@ +use std::time::Instant; + +use axum::{ + body::Body, extract::MatchedPath, http::Request, middleware::Next, response::IntoResponse, +}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; + +pub fn setup_metrics_recorder() -> PrometheusHandle { + const EXPONENTIAL_SECONDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]; + + PrometheusBuilder::new() + .set_buckets_for_metric( + Matcher::Full("http_requests_duration_seconds".to_string()), + EXPONENTIAL_SECONDS, + ) + .unwrap() + .install_recorder() + .unwrap() +} + +/// Middleware to record some common HTTP metrics +/// Someday tower-http might provide a metrics middleware: https://github.com/tower-rs/tower-http/issues/57 +pub async fn track_metrics(req: Request, next: Next) -> impl IntoResponse { + let start = Instant::now(); + + let path = if let Some(matched_path) = req.extensions().get::() { + matched_path.as_str().to_owned() + } else { + req.uri().path().to_owned() + }; + + let method = req.method().clone(); + + // Run the rest of the request handling first, so we can measure it and get response + // codes. + let response = next.run(req).await; + + let latency = start.elapsed().as_secs_f64(); + let status = response.status().as_u16().to_string(); + + let labels = [ + ("method", method.to_string()), + ("path", path), + ("status", status), + ]; + + metrics::increment_counter!("http_requests_total", &labels); + metrics::histogram!("http_requests_duration_seconds", latency, &labels); + + response +} From 3524d26ecedbd4b9cd59016669b93dbb4be8a4e6 Mon Sep 17 00:00:00 2001 From: Pranav Rajveer Date: Tue, 5 Dec 2023 15:25:48 +0530 Subject: [PATCH 086/247] refactor:typo (#64) --- capture/src/api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capture/src/api.rs b/capture/src/api.rs index ff245b5072ac1..b27b1a9630c45 100644 --- a/capture/src/api.rs +++ b/capture/src/api.rs @@ -29,7 +29,7 @@ pub struct CaptureResponse { pub enum CaptureError { #[error("failed to decode request: {0}")] RequestDecodingError(String), - #[error("failed to decode request: {0}")] + #[error("failed to parse request: {0}")] RequestParsingError(#[from] serde_json::Error), #[error("request holds no event")] From 7ff5b80c28d7a11de6b92e42c18041a9e0b9b9b4 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 5 Dec 2023 14:17:59 +0100 Subject: [PATCH 087/247] feat: accept other JSON types as distinct_id, stringify them (#65) --- capture/src/capture.rs | 27 ++----- capture/src/event.rs | 159 +++++++++++++++++++++++++++++++++++------ 2 files changed, 146 insertions(+), 40 deletions(-) diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 8cc37e0737a00..37e2872a9f9b4 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -79,12 +79,12 @@ pub async fn event( let payload = base64::engine::general_purpose::STANDARD .decode(input.data) .unwrap(); - RawEvent::from_bytes(&meta, payload.into()) + RawEvent::from_bytes(payload.into()) } ct => { tracing::Span::current().record("content_type", ct); - RawEvent::from_bytes(&meta, body) + RawEvent::from_bytes(body) } }?; @@ -165,19 +165,6 @@ pub fn process_single_event( event: &RawEvent, context: &ProcessingContext, ) -> Result { - let distinct_id = match &event.distinct_id { - Some(id) => id, - None => match event.properties.get("distinct_id").map(|v| v.as_str()) { - Some(Some(id)) => id, - _ => return Err(CaptureError::MissingDistinctId), - }, - }; - // Limit the size of distinct_id to 200 chars - let distinct_id: String = match distinct_id.len() { - 0..=200 => distinct_id.to_owned(), - _ => distinct_id.chars().take(200).collect(), - }; - if event.event.is_empty() { return Err(CaptureError::MissingEventName); } @@ -189,7 +176,7 @@ pub fn process_single_event( Ok(ProcessedEvent { uuid: event.uuid.unwrap_or_else(uuid_v7), - distinct_id, + distinct_id: event.extract_distinct_id()?, ip: context.client_ip.clone(), data, now: context.now.clone(), @@ -252,7 +239,7 @@ mod tests { let events = vec![ RawEvent { token: Some(String::from("hello")), - distinct_id: Some("testing".to_string()), + distinct_id: Some(json!("testing")), uuid: None, event: String::new(), properties: HashMap::new(), @@ -263,7 +250,7 @@ mod tests { }, RawEvent { token: None, - distinct_id: Some("testing".to_string()), + distinct_id: Some(json!("testing")), uuid: None, event: String::new(), properties: HashMap::from([(String::from("token"), json!("hello"))]), @@ -283,7 +270,7 @@ mod tests { let events = vec![ RawEvent { token: Some(String::from("hello")), - distinct_id: Some("testing".to_string()), + distinct_id: Some(json!("testing")), uuid: None, event: String::new(), properties: HashMap::new(), @@ -294,7 +281,7 @@ mod tests { }, RawEvent { token: None, - distinct_id: Some("testing".to_string()), + distinct_id: Some(json!("testing")), uuid: None, event: String::new(), properties: HashMap::from([(String::from("token"), json!("goodbye"))]), diff --git a/capture/src/event.rs b/capture/src/event.rs index 92b0f0bc9e444..ea71a3f276704 100644 --- a/capture/src/event.rs +++ b/capture/src/event.rs @@ -44,8 +44,8 @@ pub struct RawEvent { skip_serializing_if = "Option::is_none" )] pub token: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub distinct_id: Option, + #[serde(alias = "$distinct_id", skip_serializing_if = "Option::is_none")] + pub distinct_id: Option, // posthog-js accepts arbitrary values as distinct_id pub uuid: Option, pub event: String, #[serde(default)] @@ -87,7 +87,7 @@ impl RawEvent { /// Instead of trusting the parameter, we peek at the payload's first three bytes to /// detect gzip, fallback to uncompressed utf8 otherwise. #[instrument(skip_all)] - pub fn from_bytes(_query: &EventQuery, bytes: Bytes) -> Result, CaptureError> { + pub fn from_bytes(bytes: Bytes) -> Result, CaptureError> { tracing::debug!(len = bytes.len(), "decoding new event"); let payload = if bytes.starts_with(&GZIP_MAGIC_NUMBERS) { @@ -119,6 +119,30 @@ impl RawEvent { .map(String::from), } } + + /// Extracts, stringifies and trims the distinct_id to a 200 chars String. + /// SDKs send the distinct_id either in the root field or as a property, + /// and can send string, number, array, or map values. We try to best-effort + /// stringify complex values, and make sure it's not longer than 200 chars. + pub fn extract_distinct_id(&self) -> Result { + // Breaking change compared to capture-py: None / Null is not allowed. + let value = match &self.distinct_id { + None | Some(Value::Null) => match self.properties.get("distinct_id") { + None | Some(Value::Null) => return Err(CaptureError::MissingDistinctId), + Some(id) => id, + }, + Some(id) => id, + }; + + let distinct_id = value + .as_str() + .map(|s| s.to_owned()) + .unwrap_or_else(|| value.to_string()); + Ok(match distinct_id.len() { + 0..=200 => distinct_id, + _ => distinct_id.chars().take(200).collect(), + }) + } } #[derive(Debug)] @@ -150,29 +174,124 @@ impl ProcessedEvent { #[cfg(test)] mod tests { - use super::Compression; use base64::Engine as _; use bytes::Bytes; + use rand::distributions::Alphanumeric; + use rand::Rng; + use serde_json::json; - use super::{EventQuery, RawEvent}; + use super::CaptureError; + use super::RawEvent; #[test] - fn decode_bytes() { - let horrible_blob = "H4sIAAAAAAAAA31T207cMBD9lSrikSy+5bIrVX2g4oWWUlEqBEKRY08Sg4mD4+xCEf/e8XLZBSGeEp+ZOWOfmXPxkMAS+pAskp1BtmBBLiHZTQbvBvDBwJgsHpIdh5/kp1Rffp18OcMwAtUS/GhcjwFKZjSbkYjX3q1G8AgeGA+Nu4ughqVRUIX7ATDwHcbr4IYYUJP32LyavMVAF8Kw2NuzTknbuTEsSkIIHlvTf+vhLnzdizUxgslvs2JgkKHr5U1s8VS0dZ/NZSnlW7CVfTvhs7EG+vT0JJaMygP0VQem7bDTvBAbcGV06JAkIwTBpYHV4Hx4zS1FJH+FX7IFj7A1NbZZQR2b4GFbwFlWzFjETY/XCpXRiN538yt/S9mdnm7bSa+lDCY+kOalKDJGs/msZMVuos0YTK+e62hZciHqes7LnDcpoVmTg+TAaqnKMhWUaaa4TllBoCDpJn2uYK3k87xeyFjZFHWdzxmdq5Q0IstBzRXlDMiHbM/5kgnerKfs+tFZqHAolQflvDZ9W0Evawu6wveiENVoND4s+Ami2jBGZbayn/42g3xblizX4skp4FYMYfJQoSQf8DfSjrGBVMEsoWpArpMbK1vc8ItLDG1j1SDvrZM6muBxN/Eg7U1cVFw70KmyRl13bhqjYeBGGrtuFqWTSzzF/q8tRyvV9SfxHXQLoBuidXY0ekeF+KQnNCqgHXaIy7KJBncNERk6VUFhhB33j8zv5uhQ/rCTvbq9/9seH5Pj3Bf/TsuzYf9g2j+3h9N6yZ8Vfpmx4KSguSY5S0lOqc5LmgmhidoMmOaixoFvktFKOo9kK9Nrt3rPxViWk5RwIhtJykZzXohP2DjmZ08+bnH/4B1fkUnGSp2SMmNlIYTguS5ga//eERZZTSVeD8cWPTMGeTMgHSOMpyRLGftDyUKwBV9b6Dx5vPwPzQHjFwsFAAA="; - let decoded_horrible_blob = base64::engine::general_purpose::STANDARD - .decode(horrible_blob) - .unwrap(); - - let bytes = Bytes::from(decoded_horrible_blob); - let events = RawEvent::from_bytes( - &EventQuery { - compression: Some(Compression::Gzip), - lib_version: None, - sent_at: None, - }, - bytes, + fn decode_uncompressed_raw_event() { + let base64_payload = "ewogICAgImRpc3RpbmN0X2lkIjogIm15X2lkMSIsCiAgICAiZXZlbnQiOiAibXlfZXZlbnQxIiwKICAgICJwcm9wZXJ0aWVzIjogewogICAgICAgICIkZGV2aWNlX3R5cGUiOiAiRGVza3RvcCIKICAgIH0sCiAgICAiYXBpX2tleSI6ICJteV90b2tlbjEiCn0K"; + let compressed_bytes = Bytes::from( + base64::engine::general_purpose::STANDARD + .decode(base64_payload) + .expect("payload is not base64"), + ); + + let events = RawEvent::from_bytes(compressed_bytes).expect("failed to parse"); + assert_eq!(1, events.len()); + assert_eq!(Some("my_token1".to_string()), events[0].extract_token()); + assert_eq!("my_event1".to_string(), events[0].event); + assert_eq!( + "my_id1".to_string(), + events[0] + .extract_distinct_id() + .expect("cannot find distinct_id") + ); + } + #[test] + fn decode_gzipped_raw_event() { + let base64_payload = "H4sIADQSbmUCAz2MsQqAMAxE936FBEcnR2f/o4i9IRTb0AahiP9urcVMx3t3ucxQjxxn5bCrZUfLQEepYabpkzgRtOOWfyMpCpIyctVXY42PDifvsFoE73BF9hqFWuPu403YepT+WKNHmMnc5gENoFu2kwAAAA=="; + let compressed_bytes = Bytes::from( + base64::engine::general_purpose::STANDARD + .decode(base64_payload) + .expect("payload is not base64"), ); - assert!(events.is_ok()); + let events = RawEvent::from_bytes(compressed_bytes).expect("failed to parse"); + assert_eq!(1, events.len()); + assert_eq!(Some("my_token2".to_string()), events[0].extract_token()); + assert_eq!("my_event2".to_string(), events[0].event); + assert_eq!( + "my_id2".to_string(), + events[0] + .extract_distinct_id() + .expect("cannot find distinct_id") + ); + } + + #[test] + fn extract_distinct_id() { + let parse_and_extract = |input: &'static str| -> Result { + let parsed = RawEvent::from_bytes(input.into()).expect("failed to parse"); + parsed[0].extract_distinct_id() + }; + // Return MissingDistinctId if not found + assert!(matches!( + parse_and_extract(r#"{"event": "e"}"#), + Err(CaptureError::MissingDistinctId) + )); + // Return MissingDistinctId if null, breaking compat with capture-py + assert!(matches!( + parse_and_extract(r#"{"event": "e", "distinct_id": null}"#), + Err(CaptureError::MissingDistinctId) + )); + + let assert_extracted_id = |input: &'static str, expected: &str| { + let id = parse_and_extract(input).expect("failed to extract"); + assert_eq!(id, expected); + }; + // Happy path: toplevel field present + assert_extracted_id(r#"{"event": "e", "distinct_id": "myid"}"#, "myid"); + assert_extracted_id(r#"{"event": "e", "$distinct_id": "23"}"#, "23"); + + // Sourced from properties if not present in toplevel field, but toplevel wins if both present + assert_extracted_id( + r#"{"event": "e", "properties":{"distinct_id": "myid"}}"#, + "myid", + ); + assert_extracted_id( + r#"{"event": "e", "distinct_id": 23, "properties":{"distinct_id": "myid"}}"#, + "23", + ); + + // Numbers are stringified + assert_extracted_id(r#"{"event": "e", "distinct_id": 23}"#, "23"); + assert_extracted_id(r#"{"event": "e", "distinct_id": 23.4}"#, "23.4"); + + // Containers are stringified + assert_extracted_id( + r#"{"event": "e", "distinct_id": ["a", "b"]}"#, + r#"["a","b"]"#, + ); + assert_extracted_id( + r#"{"event": "e", "distinct_id": {"string": "a", "number": 3}}"#, + r#"{"number":3,"string":"a"}"#, + ); + } + + #[test] + fn extract_distinct_id_trims_to_200_chars() { + let distinct_id: String = rand::thread_rng() + .sample_iter(Alphanumeric) + .take(222) + .map(char::from) + .collect(); + let (expected_distinct_id, _) = distinct_id.split_at(200); // works because ascii chars only + let input = json!([{ + "token": "mytoken", + "event": "myevent", + "distinct_id": distinct_id + }]); + + let parsed = RawEvent::from_bytes(input.to_string().into()).expect("failed to parse"); + assert_eq!( + parsed[0].extract_distinct_id().expect("failed to extract"), + expected_distinct_id + ); } } From fa43c0a8362d2ab2f045747abc6106a420871bb3 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Thu, 7 Dec 2023 16:50:03 +0100 Subject: [PATCH 088/247] fix MessageSizeTooLarge handling (#66) --- capture/src/sink.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/capture/src/sink.rs b/capture/src/sink.rs index 13397dcf83465..af83e20c1a763 100644 --- a/capture/src/sink.rs +++ b/capture/src/sink.rs @@ -192,7 +192,7 @@ impl KafkaSink { }) { Ok(ack) => Ok(ack), Err((e, _)) => match e.rdkafka_error_code() { - Some(RDKafkaErrorCode::InvalidMessageSize) => { + Some(RDKafkaErrorCode::MessageSizeTooLarge) => { report_dropped_events("kafka_message_size", 1); Err(CaptureError::EventTooBig) } @@ -297,6 +297,8 @@ mod tests { use crate::partition_limits::PartitionLimiter; use crate::sink::{EventSink, KafkaSink}; use crate::utils::uuid_v7; + use rand::distributions::Alphanumeric; + use rand::Rng; use rdkafka::mocking::MockCluster; use rdkafka::producer::DefaultProducerContext; use rdkafka::types::{RDKafkaApiKey, RDKafkaRespErr}; @@ -358,6 +360,27 @@ mod tests { .await .expect("failed to send initial event batch"); + // Producer should reject a 2MB message, twice the default `message.max.bytes` + let big_data = rand::thread_rng() + .sample_iter(Alphanumeric) + .take(2_000_000) + .map(char::from) + .collect(); + let big_event: ProcessedEvent = ProcessedEvent { + uuid: uuid_v7(), + distinct_id: "id1".to_string(), + ip: "".to_string(), + data: big_data, + now: "".to_string(), + sent_at: None, + token: "token1".to_string(), + }; + match sink.send(big_event).await { + Err(CaptureError::EventTooBig) => {} // Expected + Err(err) => panic!("wrong error code {}", err), + Ok(()) => panic!("should have errored"), + }; + // Simulate unretriable errors cluster.clear_request_errors(RDKafkaApiKey::Produce); let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE; 1]; From a63a00ce667c99fb4a185da3dd4b20c431f5d3ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 6 Dec 2023 13:46:09 +0100 Subject: [PATCH 089/247] feat: Include migrations and echo-server in docker-compose stack --- Dockerfile.sqlx | 5 +++++ README.md | 14 +++----------- docker-compose.yml | 25 +++++++++++++++++++++++++ docker/echo-server/Caddyfile | 17 +++++++++++++++++ hook-consumer/README.md | 2 ++ 5 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 Dockerfile.sqlx create mode 100644 docker/echo-server/Caddyfile create mode 100644 hook-consumer/README.md diff --git a/Dockerfile.sqlx b/Dockerfile.sqlx new file mode 100644 index 0000000000000..c55dfaa8a960a --- /dev/null +++ b/Dockerfile.sqlx @@ -0,0 +1,5 @@ +FROM docker.io/library/rust:1.74.0 + +RUN cargo install sqlx-cli --no-default-features --features native-tls,postgres + +WORKDIR /sqlx diff --git a/README.md b/README.md index b17e7ae6ffbe7..a3a674c28ff38 100644 --- a/README.md +++ b/README.md @@ -4,24 +4,16 @@ A reliable and performant webhook system for PostHog ## Requirements 1. [Rust](https://www.rust-lang.org/tools/install). -2. [sqlx-cli](https://crates.io/crates/sqlx-cli): To setup database and run migrations. -3. [Docker](https://docs.docker.com/engine/install/) or [podman](https://podman.io/docs/installation) (and [podman-compose](https://github.com/containers/podman-compose#installation)): To setup testing services. +2. [Docker](https://docs.docker.com/engine/install/), or [podman](https://podman.io/docs/installation) and [podman-compose](https://github.com/containers/podman-compose#installation): To setup development stack. ## Testing -1. Start a PostgreSQL instance: +1. Start development stack: ```bash docker compose -f docker-compose.yml up -d --wait ``` -2. Prepare test database: -```bash -export DATABASE_URL=postgres://posthog:posthog@localhost:15432/test_database -sqlx database create -sqlx migrate run -``` - -3. Test: +2. Test: ```bash cargo test ``` diff --git a/docker-compose.yml b/docker-compose.yml index 35b7a498d44b4..6f62692df0743 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,6 @@ services: db: + container_name: db image: docker.io/library/postgres:16-alpine restart: on-failure environment: @@ -13,3 +14,27 @@ services: ports: - '15432:5432' command: postgres -c max_connections=1000 -c idle_in_transaction_session_timeout=300000 + + setup_test_db: + container_name: setup-test-db + build: + context: . + dockerfile: Dockerfile.sqlx + restart: on-failure + command: > + sh -c "sqlx database create && sqlx migrate run" + requires: + - db + environment: + DATABASE_URL: postgres://posthog:posthog@db:5432/test_database + volumes: + - ./migrations:/sqlx/migrations/ + + echo_server: + image: docker.io/library/caddy:2 + container_name: echo-server + restart: on-failure + ports: + - '18081:8081' + volumes: + - ./docker/echo-server/Caddyfile:/etc/caddy/Caddyfile diff --git a/docker/echo-server/Caddyfile b/docker/echo-server/Caddyfile new file mode 100644 index 0000000000000..a13ac68a24d6b --- /dev/null +++ b/docker/echo-server/Caddyfile @@ -0,0 +1,17 @@ +{ + auto_https off +} + +:8081 + +route /echo { + respond `{http.request.body}` 200 { + close + } +} + +route /fail { + respond `{http.request.body}` 400 { + close + } +} diff --git a/hook-consumer/README.md b/hook-consumer/README.md new file mode 100644 index 0000000000000..1adab6ea571f4 --- /dev/null +++ b/hook-consumer/README.md @@ -0,0 +1,2 @@ +# hook-consumer +Consume and process webhook jobs From 690888a250733c0f7036895fd64901f55b419380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 6 Dec 2023 13:55:06 +0100 Subject: [PATCH 090/247] refactor: Support in pgqueue for consumer --- hook-common/src/pgqueue.rs | 237 ++++++++++++++++++++++--------------- 1 file changed, 140 insertions(+), 97 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 89aab7ee45ab9..7ed46550bd8bd 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -4,9 +4,10 @@ use std::default::Default; use std::str::FromStr; +use std::time; -use chrono::{prelude::*, Duration}; -use serde::{de::DeserializeOwned, Serialize}; +use chrono; +use serde; use sqlx::postgres::{PgPool, PgPoolOptions}; use thiserror::Error; @@ -18,12 +19,18 @@ pub enum PgQueueError { ConnectionError { error: sqlx::Error }, #[error("{command} query failed with: {error}")] QueryError { command: String, error: sqlx::Error }, - #[error("transaction {command} failed with: {error}")] - TransactionError { command: String, error: sqlx::Error }, #[error("{0} is not a valid JobStatus")] ParseJobStatusError(String), - #[error("{0} Job has reached max attempts and cannot be retried further")] - MaxAttemptsReachedError(String), +} + +#[derive(Error, Debug)] +pub enum PgJobError { + #[error("retry is an invalid state for this PgJob: {error}")] + RetryInvalidError { job: T, error: String }, + #[error("{command} query failed with: {error}")] + QueryError { command: String, error: sqlx::Error }, + #[error("transaction {command} failed with: {error}")] + TransactionError { command: String, error: sqlx::Error }, } /// Enumeration of possible statuses for a Job. @@ -64,18 +71,18 @@ impl FromStr for JobStatus { pub type JobParameters = sqlx::types::Json; /// A Job to be executed by a worker dequeueing a PgQueue. -#[derive(sqlx::FromRow)] +#[derive(sqlx::FromRow, Debug)] pub struct Job { /// A unique id identifying a job. pub id: i64, /// A number corresponding to the current job attempt. pub attempt: i32, /// A datetime corresponding to when the job was attempted. - pub attempted_at: DateTime, + pub attempted_at: chrono::DateTime, /// A vector of identifiers that have attempted this job. E.g. thread ids, pod names, etc... pub attempted_by: Vec, /// A datetime corresponding to when the job was created. - pub created_at: DateTime, + pub created_at: chrono::DateTime, /// The current job's number of max attempts. pub max_attempts: i32, /// Arbitrary job parameters stored as JSON. @@ -89,28 +96,29 @@ pub struct Job { } impl Job { + /// Return true if this job attempt is greater or equal to the maximum number of possible attempts. + pub fn is_gte_max_attempts(&self) -> bool { + self.attempt >= self.max_attempts + } + /// Consume Job to retry it. /// This returns a RetryableJob that can be enqueued by PgQueue. /// /// # Arguments /// /// * `error`: Any JSON-serializable value to be stored as an error. - pub fn retry(self, error: E) -> Result, PgQueueError> { - if self.attempt >= self.max_attempts { - Err(PgQueueError::MaxAttemptsReachedError(self.target)) - } else { - Ok(RetryableJob { - id: self.id, - attempt: self.attempt, - error: sqlx::types::Json(error), - queue: self.queue, - }) + fn retry(self, error: E) -> RetryableJob { + RetryableJob { + id: self.id, + attempt: self.attempt, + error: sqlx::types::Json(error), + queue: self.queue, } } /// Consume Job to complete it. /// This returns a CompletedJob that can be marked as completed by PgQueue. - pub fn complete(self) -> CompletedJob { + fn complete(self) -> CompletedJob { CompletedJob { id: self.id, queue: self.queue, @@ -123,7 +131,7 @@ impl Job { /// # Arguments /// /// * `error`: Any JSON-serializable value to be stored as an error. - pub fn fail(self, error: E) -> FailedJob { + fn fail(self, error: E) -> FailedJob { FailedJob { id: self.id, error: sqlx::types::Json(error), @@ -133,6 +141,7 @@ impl Job { } /// A Job that can be updated in PostgreSQL. +#[derive(Debug)] pub struct PgJob { pub job: Job, pub table: String, @@ -141,11 +150,21 @@ pub struct PgJob { } impl PgJob { - pub async fn retry( + pub async fn retry( mut self, error: E, - ) -> Result, PgQueueError> { - let retryable_job = self.job.retry(error)?; + preferred_retry_interval: Option, + ) -> Result, PgJobError>> { + if self.job.is_gte_max_attempts() { + return Err(PgJobError::RetryInvalidError { + job: self, + error: "Maximum attempts reached".to_owned(), + }); + } + let retryable_job = self.job.retry(error); + let retry_interval = self + .retry_policy + .time_until_next_retry(&retryable_job, preferred_retry_interval); let base_query = format!( r#" @@ -161,7 +180,6 @@ WHERE AND queue = $1 RETURNING "{0}".* - "#, &self.table ); @@ -169,11 +187,11 @@ RETURNING sqlx::query(&base_query) .bind(&retryable_job.queue) .bind(retryable_job.id) - .bind(self.retry_policy.time_until_next_retry(&retryable_job)) + .bind(retry_interval) .bind(&retryable_job.error) .execute(&mut *self.connection) .await - .map_err(|error| PgQueueError::QueryError { + .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), error, })?; @@ -181,7 +199,7 @@ RETURNING Ok(retryable_job) } - pub async fn complete(mut self) -> Result { + pub async fn complete(mut self) -> Result>> { let completed_job = self.job.complete(); let base_query = format!( @@ -190,13 +208,12 @@ UPDATE "{0}" SET finished_at = NOW(), - status = 'completed'::job_status, + status = 'completed'::job_status WHERE "{0}".id = $2 AND queue = $1 RETURNING "{0}".* - "#, &self.table ); @@ -206,7 +223,7 @@ RETURNING .bind(completed_job.id) .execute(&mut *self.connection) .await - .map_err(|error| PgQueueError::QueryError { + .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), error, })?; @@ -214,10 +231,10 @@ RETURNING Ok(completed_job) } - pub async fn fail( + pub async fn fail( mut self, error: E, - ) -> Result, PgQueueError> { + ) -> Result, PgJobError>> { let failed_job = self.job.fail(error); let base_query = format!( @@ -226,7 +243,7 @@ UPDATE "{0}" SET finished_at = NOW(), - status = 'failed'::job_status, + status = 'failed'::job_status WHERE "{0}".id = $2 AND queue = $1 @@ -242,7 +259,7 @@ RETURNING .bind(failed_job.id) .execute(&mut *self.connection) .await - .map_err(|error| PgQueueError::QueryError { + .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), error, })?; @@ -253,6 +270,7 @@ RETURNING /// A Job within an open PostgreSQL transaction. /// This implementation allows 'hiding' the job from any other workers running SKIP LOCKED queries. +#[derive(Debug)] pub struct PgTransactionJob<'c, J> { pub job: Job, pub table: String, @@ -261,11 +279,21 @@ pub struct PgTransactionJob<'c, J> { } impl<'c, J> PgTransactionJob<'c, J> { - pub async fn retry( + pub async fn retry( mut self, error: E, - ) -> Result, PgQueueError> { - let retryable_job = self.job.retry(error)?; + preferred_retry_interval: Option, + ) -> Result, PgJobError>> { + if self.job.is_gte_max_attempts() { + return Err(PgJobError::RetryInvalidError { + job: self, + error: "Maximum attempts reached".to_owned(), + }); + } + let retryable_job = self.job.retry(error); + let retry_interval = self + .retry_policy + .time_until_next_retry(&retryable_job, preferred_retry_interval); let base_query = format!( r#" @@ -289,11 +317,11 @@ RETURNING sqlx::query(&base_query) .bind(&retryable_job.queue) .bind(retryable_job.id) - .bind(self.retry_policy.time_until_next_retry(&retryable_job)) + .bind(retry_interval) .bind(&retryable_job.error) .execute(&mut *self.transaction) .await - .map_err(|error| PgQueueError::QueryError { + .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), error, })?; @@ -301,7 +329,7 @@ RETURNING self.transaction .commit() .await - .map_err(|error| PgQueueError::TransactionError { + .map_err(|error| PgJobError::TransactionError { command: "COMMIT".to_owned(), error, })?; @@ -309,7 +337,7 @@ RETURNING Ok(retryable_job) } - pub async fn complete(mut self) -> Result { + pub async fn complete(mut self) -> Result>> { let completed_job = self.job.complete(); let base_query = format!( @@ -318,13 +346,12 @@ UPDATE "{0}" SET finished_at = NOW(), - status = 'completed'::job_status, + status = 'completed'::job_status WHERE "{0}".id = $2 AND queue = $1 RETURNING "{0}".* - "#, &self.table ); @@ -334,7 +361,7 @@ RETURNING .bind(completed_job.id) .execute(&mut *self.transaction) .await - .map_err(|error| PgQueueError::QueryError { + .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), error, })?; @@ -342,7 +369,7 @@ RETURNING self.transaction .commit() .await - .map_err(|error| PgQueueError::TransactionError { + .map_err(|error| PgJobError::TransactionError { command: "COMMIT".to_owned(), error, })?; @@ -350,10 +377,10 @@ RETURNING Ok(completed_job) } - pub async fn fail( + pub async fn fail( mut self, error: E, - ) -> Result, PgQueueError> { + ) -> Result, PgJobError>> { let failed_job = self.job.fail(error); let base_query = format!( @@ -362,13 +389,12 @@ UPDATE "{0}" SET finished_at = NOW(), - status = 'failed'::job_status, + status = 'failed'::job_status WHERE "{0}".id = $2 AND queue = $1 RETURNING "{0}".* - "#, &self.table ); @@ -378,7 +404,7 @@ RETURNING .bind(failed_job.id) .execute(&mut *self.transaction) .await - .map_err(|error| PgQueueError::QueryError { + .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), error, })?; @@ -386,7 +412,7 @@ RETURNING self.transaction .commit() .await - .map_err(|error| PgQueueError::TransactionError { + .map_err(|error| PgJobError::TransactionError { command: "COMMIT".to_owned(), error, })?; @@ -399,7 +425,7 @@ RETURNING /// The time until retry will depend on the PgQueue's RetryPolicy. pub struct RetryableJob { /// A unique id identifying a job. - pub id: i64, + id: i64, /// A number corresponding to the current job attempt. pub attempt: i32, /// Any JSON-serializable value to be stored as an error. @@ -411,7 +437,7 @@ pub struct RetryableJob { /// A Job that has completed to be enqueued into a PgQueue and marked as completed. pub struct CompletedJob { /// A unique id identifying a job. - pub id: i64, + id: i64, /// A unique id identifying a job queue. pub queue: String, } @@ -419,7 +445,7 @@ pub struct CompletedJob { /// A Job that has failed to be enqueued into a PgQueue and marked as failed. pub struct FailedJob { /// A unique id identifying a job. - pub id: i64, + id: i64, /// Any JSON-serializable value to be stored as an error. pub error: sqlx::types::Json, /// A unique id identifying a job queue. @@ -446,27 +472,47 @@ impl NewJob { } } -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Debug)] /// The retry policy that PgQueue will use to determine how to set scheduled_at when enqueuing a retry. pub struct RetryPolicy { /// Coefficient to multiply initial_interval with for every past attempt. - backoff_coefficient: i32, + backoff_coefficient: u32, /// The backoff interval for the first retry. - initial_interval: Duration, + initial_interval: time::Duration, /// The maximum possible backoff between retries. - maximum_interval: Option, + maximum_interval: Option, } impl RetryPolicy { + pub fn new( + backoff_coefficient: u32, + initial_interval: time::Duration, + maximum_interval: Option, + ) -> Self { + Self { + backoff_coefficient, + initial_interval, + maximum_interval, + } + } + /// Calculate the time until the next retry for a given RetryableJob. - pub fn time_until_next_retry(&self, job: &RetryableJob) -> Duration { + pub fn time_until_next_retry( + &self, + job: &RetryableJob, + preferred_retry_interval: Option, + ) -> time::Duration { let candidate_interval = self.initial_interval * self.backoff_coefficient.pow(job.attempt as u32); - if let Some(max_interval) = self.maximum_interval { - std::cmp::min(candidate_interval, max_interval) - } else { - candidate_interval + match (preferred_retry_interval, self.maximum_interval) { + (Some(duration), Some(max_interval)) => std::cmp::min( + std::cmp::max(std::cmp::min(candidate_interval, max_interval), duration), + max_interval, + ), + (Some(duration), None) => std::cmp::max(candidate_interval, duration), + (None, Some(max_interval)) => std::cmp::min(candidate_interval, max_interval), + (None, None) => candidate_interval, } } } @@ -475,7 +521,7 @@ impl Default for RetryPolicy { fn default() -> Self { Self { backoff_coefficient: 2, - initial_interval: Duration::seconds(1), + initial_interval: time::Duration::from_secs(1), maximum_interval: None, } } @@ -491,8 +537,6 @@ pub struct PgQueue { retry_policy: RetryPolicy, /// The identifier of the PostgreSQL table this queue runs on. table: String, - /// The identifier of the worker listening on this queue. - worker: String, } pub type PgQueueResult = std::result::Result; @@ -511,12 +555,10 @@ impl PgQueue { queue_name: &str, table_name: &str, url: &str, - worker_name: &str, retry_policy: RetryPolicy, ) -> PgQueueResult { let name = queue_name.to_owned(); let table = table_name.to_owned(); - let worker = worker_name.to_owned(); let pool = PgPoolOptions::new() .connect(url) .await @@ -527,13 +569,15 @@ impl PgQueue { pool, retry_policy, table, - worker, }) } /// Dequeue a Job from this PgQueue to work on it. - pub async fn dequeue( + pub async fn dequeue< + J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, + >( &self, + attempted_by: &str, ) -> PgQueueResult>> { let mut connection = self .pool @@ -578,7 +622,7 @@ RETURNING let query_result: Result, sqlx::Error> = sqlx::query_as(&base_query) .bind(&self.name) - .bind(&self.worker) + .bind(attempted_by) .fetch_one(&mut *connection) .await; @@ -608,10 +652,12 @@ RETURNING /// Dequeue a Job from this PgQueue to work on it. pub async fn dequeue_tx< - J: DeserializeOwned + std::marker::Send + std::marker::Unpin + 'static, + 'a, + J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, >( &self, - ) -> PgQueueResult>> { + attempted_by: &str, + ) -> PgQueueResult>> { let mut tx = self .pool .begin() @@ -655,7 +701,7 @@ RETURNING let query_result: Result, sqlx::Error> = sqlx::query_as(&base_query) .bind(&self.name) - .bind(&self.worker) + .bind(attempted_by) .fetch_one(&mut *tx) .await; @@ -678,7 +724,7 @@ RETURNING /// Enqueue a Job into this PgQueue. /// We take ownership of NewJob to enforce a specific NewJob is only enqueued once. - pub async fn enqueue( + pub async fn enqueue( &self, job: NewJob, ) -> PgQueueResult<()> { @@ -712,9 +758,8 @@ VALUES #[cfg(test)] mod tests { use super::*; - use serde::Deserialize; - #[derive(Serialize, Deserialize, PartialEq, Debug)] + #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] struct JobParameters { method: String, body: String, @@ -752,7 +797,6 @@ mod tests { "test_can_dequeue_job", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - &worker_id, RetryPolicy::default(), ) .await @@ -761,7 +805,7 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); let pg_job: PgJob = queue - .dequeue() + .dequeue(&worker_id) .await .expect("failed to dequeue job") .expect("didn't find a job to dequeue"); @@ -782,14 +826,15 @@ mod tests { "test_dequeue_returns_none_on_no_jobs", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - &worker_id, RetryPolicy::default(), ) .await .expect("failed to connect to local test postgresql database"); - let pg_job: Option> = - queue.dequeue().await.expect("failed to dequeue job"); + let pg_job: Option> = queue + .dequeue(&worker_id) + .await + .expect("failed to dequeue job"); assert!(pg_job.is_none()); } @@ -805,7 +850,6 @@ mod tests { "test_can_dequeue_tx_job", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - &worker_id, RetryPolicy::default(), ) .await @@ -814,7 +858,7 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); let tx_job: PgTransactionJob<'_, JobParameters> = queue - .dequeue_tx() + .dequeue_tx(&worker_id) .await .expect("failed to dequeue job") .expect("didn't find a job to dequeue"); @@ -835,14 +879,15 @@ mod tests { "test_dequeue_tx_returns_none_on_no_jobs", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - &worker_id, RetryPolicy::default(), ) .await .expect("failed to connect to local test postgresql database"); - let tx_job: Option> = - queue.dequeue_tx().await.expect("failed to dequeue job"); + let tx_job: Option> = queue + .dequeue_tx(&worker_id) + .await + .expect("failed to dequeue job"); assert!(tx_job.is_none()); } @@ -855,7 +900,7 @@ mod tests { let new_job = NewJob::new(2, job_parameters, &job_target); let retry_policy = RetryPolicy { backoff_coefficient: 0, - initial_interval: Duration::seconds(0), + initial_interval: time::Duration::from_secs(0), maximum_interval: None, }; @@ -863,7 +908,6 @@ mod tests { "test_can_retry_job_with_remaining_attempts", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - &worker_id, retry_policy, ) .await @@ -871,16 +915,16 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); let job: PgJob = queue - .dequeue() + .dequeue(&worker_id) .await .expect("failed to dequeue job") .expect("didn't find a job to dequeue"); let _ = job - .retry("a very reasonable failure reason") + .retry("a very reasonable failure reason", None) .await .expect("failed to retry job"); let retried_job: PgJob = queue - .dequeue() + .dequeue(&worker_id) .await .expect("failed to dequeue job") .expect("didn't find retried job to dequeue"); @@ -906,7 +950,7 @@ mod tests { let new_job = NewJob::new(1, job_parameters, &job_target); let retry_policy = RetryPolicy { backoff_coefficient: 0, - initial_interval: Duration::seconds(0), + initial_interval: time::Duration::from_secs(0), maximum_interval: None, }; @@ -914,7 +958,6 @@ mod tests { "test_cannot_retry_job_without_remaining_attempts", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - &worker_id, retry_policy, ) .await @@ -923,11 +966,11 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); let job: PgJob = queue - .dequeue() + .dequeue(&worker_id) .await .expect("failed to dequeue job") .expect("didn't find a job to dequeue"); - job.retry("a very reasonable failure reason") + job.retry("a very reasonable failure reason", None) .await .expect("failed to retry job"); } From 7824d69da6dd104650ef3f8fcc245be8b8b6a6cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 6 Dec 2023 13:55:21 +0100 Subject: [PATCH 091/247] feat: First consumer implementation --- hook-consumer/Cargo.toml | 13 + hook-consumer/src/config.rs | 56 ++++ hook-consumer/src/consumer.rs | 492 ++++++++++++++++++++++++++++++++++ hook-consumer/src/lib.rs | 2 + hook-consumer/src/main.rs | 34 ++- 5 files changed, 595 insertions(+), 2 deletions(-) create mode 100644 hook-consumer/src/config.rs create mode 100644 hook-consumer/src/consumer.rs create mode 100644 hook-consumer/src/lib.rs diff --git a/hook-consumer/Cargo.toml b/hook-consumer/Cargo.toml index 49c2d9f84b17d..2e95a6b071903 100644 --- a/hook-consumer/Cargo.toml +++ b/hook-consumer/Cargo.toml @@ -6,3 +6,16 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-std = { version = "1.12" } +chrono = { version = "0.4" } +envconfig = { version = "0.10" } +futures = "0.3" +hook-common = { path = "../hook-common" } +http = { version = "0.2" } +reqwest = { version = "0.11" } +serde = { version = "1.0" } +serde_derive = { version = "1.0" } +sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", "postgres", "uuid", "json", "chrono" ] } +thiserror = { version = "1.0" } +tokio = { version = "1.34", features = ["macros", "rt", "rt-multi-thread"] } +url = { version = "2.2" } diff --git a/hook-consumer/src/config.rs b/hook-consumer/src/config.rs new file mode 100644 index 0000000000000..fde137337e8f8 --- /dev/null +++ b/hook-consumer/src/config.rs @@ -0,0 +1,56 @@ +use std::str::FromStr; +use std::time; + +use envconfig::Envconfig; + +#[derive(Envconfig, Clone)] +pub struct Config { + #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] + pub database_url: String, + + #[envconfig(default = "consumer")] + pub consumer_name: String, + + #[envconfig(default = "default")] + pub queue_name: String, + + #[envconfig(default = "100")] + pub poll_interval: EnvMsDuration, + + #[envconfig(default = "5000")] + pub request_timeout: EnvMsDuration, + + #[envconfig(nested = true)] + pub retry_policy: RetryPolicyConfig, + + #[envconfig(default = "job_queue")] + pub table_name: String, +} + +#[derive(Debug, Clone, Copy)] +pub struct EnvMsDuration(pub time::Duration); + +#[derive(Debug, PartialEq, Eq)] +pub struct ParseEnvMsDurationError; + +impl FromStr for EnvMsDuration { + type Err = ParseEnvMsDurationError; + + fn from_str(s: &str) -> Result { + let ms = s.parse::().map_err(|_| ParseEnvMsDurationError)?; + + Ok(EnvMsDuration(time::Duration::from_millis(ms))) + } +} + +#[derive(Envconfig, Clone)] +pub struct RetryPolicyConfig { + #[envconfig(default = "2")] + pub backoff_coefficient: u32, + + #[envconfig(default = "1000")] + pub initial_interval: EnvMsDuration, + + #[envconfig(default = "100000")] + pub maximum_interval: EnvMsDuration, +} diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs new file mode 100644 index 0000000000000..33b09b0caacbe --- /dev/null +++ b/hook-consumer/src/consumer.rs @@ -0,0 +1,492 @@ +use std::collections; +use std::fmt; +use std::str::FromStr; +use std::time; + +use async_std::task; +use hook_common::pgqueue::{PgJobError, PgQueue, PgQueueError, PgTransactionJob}; +use http::StatusCode; +use serde::{de::Visitor, Deserialize, Serialize}; +use thiserror::Error; + +/// Enumeration of errors for operations with WebhookConsumer. +#[derive(Error, Debug)] +pub enum WebhookConsumerError { + #[error("timed out while waiting for jobs to be available")] + TimeoutError, + #[error("{0} is not a valid HttpMethod")] + ParseHttpMethodError(String), + #[error("error parsing webhook headers")] + ParseHeadersError(http::Error), + #[error("error parsing webhook url")] + ParseUrlError(url::ParseError), + #[error("an error occurred in the underlying queue")] + QueueError(#[from] PgQueueError), + #[error("an error occurred in the underlying job")] + PgJobError(String), + #[error("an error occurred when attempting to send a request")] + RequestError(#[from] reqwest::Error), + #[error("a webhook could not be delivered but it could be retried later: {reason}")] + RetryableWebhookError { + reason: String, + retry_after: Option, + }, + #[error("a webhook could not be delivered and it cannot be retried further: {0}")] + NonRetryableWebhookError(String), +} + +/// Supported HTTP methods for webhooks. +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum HttpMethod { + DELETE, + GET, + PATCH, + POST, + PUT, +} + +/// Allow casting `HttpMethod` from strings. +impl FromStr for HttpMethod { + type Err = WebhookConsumerError; + + fn from_str(s: &str) -> Result { + match s.to_ascii_uppercase().as_ref() { + "DELETE" => Ok(HttpMethod::DELETE), + "GET" => Ok(HttpMethod::GET), + "PATCH" => Ok(HttpMethod::PATCH), + "POST" => Ok(HttpMethod::POST), + "PUT" => Ok(HttpMethod::PUT), + invalid => Err(WebhookConsumerError::ParseHttpMethodError( + invalid.to_owned(), + )), + } + } +} + +/// Implement `std::fmt::Display` to convert HttpMethod to string. +impl fmt::Display for HttpMethod { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + HttpMethod::DELETE => write!(f, "DELETE"), + HttpMethod::GET => write!(f, "GET"), + HttpMethod::PATCH => write!(f, "PATCH"), + HttpMethod::POST => write!(f, "POST"), + HttpMethod::PUT => write!(f, "PUT"), + } + } +} + +struct HttpMethodVisitor; + +impl<'de> Visitor<'de> for HttpMethodVisitor { + type Value = HttpMethod; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "the string representation of HttpMethod") + } + + fn visit_str(self, s: &str) -> Result + where + E: serde::de::Error, + { + match HttpMethod::from_str(s) { + Ok(method) => Ok(method), + Err(_) => Err(serde::de::Error::invalid_value( + serde::de::Unexpected::Str(s), + &self, + )), + } + } +} + +/// Deserialize required to read `HttpMethod` from database. +impl<'de> Deserialize<'de> for HttpMethod { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_str(HttpMethodVisitor) + } +} + +/// Serialize required to write `HttpMethod` to database. +impl Serialize for HttpMethod { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +/// Convinience to cast `HttpMethod` to `http::Method`. +/// Not all `http::Method` variants are valid `HttpMethod` variants, hence why we +/// can't just use the former. +impl Into for HttpMethod { + fn into(self) -> http::Method { + match self { + HttpMethod::DELETE => http::Method::DELETE, + HttpMethod::GET => http::Method::GET, + HttpMethod::PATCH => http::Method::PATCH, + HttpMethod::POST => http::Method::POST, + HttpMethod::PUT => http::Method::PUT, + } + } +} + +impl Into for &HttpMethod { + fn into(self) -> http::Method { + match self { + HttpMethod::DELETE => http::Method::DELETE, + HttpMethod::GET => http::Method::GET, + HttpMethod::PATCH => http::Method::PATCH, + HttpMethod::POST => http::Method::POST, + HttpMethod::PUT => http::Method::PUT, + } + } +} + +/// `JobParameters` required for the `WebhookConsumer` to execute a webhook. +/// These parameters should match the exported Webhook interface that PostHog plugins. +/// implement. See: https://github.com/PostHog/plugin-scaffold/blob/main/src/types.ts#L15. +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct WebhookJobParameters { + body: String, + headers: collections::HashMap, + method: HttpMethod, + url: String, +} + +/// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. +pub struct WebhookConsumer<'p> { + /// An identifier for this consumer. Used to mark jobs we have consumed. + name: String, + /// The queue we will be dequeuing jobs from. + queue: &'p PgQueue, + /// The interval for polling the queue. + poll_interval: time::Duration, + /// A timeout for webhook requests. + request_timeout: time::Duration, +} + +impl<'p> WebhookConsumer<'p> { + pub fn new( + name: &str, + queue: &'p PgQueue, + poll_interval: time::Duration, + request_timeout: time::Duration, + ) -> Self { + Self { + name: name.to_owned(), + queue, + poll_interval, + request_timeout, + } + } + + /// Wait until a job becomes available in our queue. + async fn wait_for_job<'a>( + &self, + ) -> Result, WebhookConsumerError> { + loop { + if let Some(job) = self.queue.dequeue_tx(&self.name).await? { + return Ok(job); + } else { + task::sleep(self.poll_interval).await; + } + } + } + + /// Run this consumer to continuously process any jobs that become available. + pub async fn run(&self) -> Result<(), WebhookConsumerError> { + loop { + let webhook_job = self.wait_for_job().await?; + + let request_timeout = self.request_timeout; // Required to avoid capturing self in closure. + tokio::spawn(async move { process_webhook_job(webhook_job, request_timeout) }); + } + } +} + +/// Process a webhook job by transitioning it to its appropriate state after its request is sent. +/// After we finish, the webhook job will be set as completed (if the request was successful), retryable (if the request +/// was unsuccessful but we can still attempt a retry), or failed (if the request was unsuccessful and no more retries +/// may be attempted). +/// +/// A webhook job is considered retryable after a failing request if: +/// 1. The job has attempts remaining (i.e. hasn't reached `max_attempts`), and... +/// 2. The status code indicates retrying at a later point could resolve the issue. This means: 429 and any 5XX. +/// +/// # Arguments +/// +/// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. +/// * `request_timeout`: A timeout for the HTTP request. +async fn process_webhook_job( + webhook_job: PgTransactionJob<'_, WebhookJobParameters>, + request_timeout: std::time::Duration, +) -> Result<(), WebhookConsumerError> { + match send_webhook( + &webhook_job.job.parameters.method, + &webhook_job.job.parameters.url, + &webhook_job.job.parameters.headers, + webhook_job.job.parameters.body.clone(), + request_timeout, + ) + .await + { + Ok(_) => { + webhook_job + .complete() + .await + .map_err(|error| WebhookConsumerError::PgJobError(error.to_string()))?; + Ok(()) + } + Err(WebhookConsumerError::RetryableWebhookError { + reason, + retry_after, + }) => match webhook_job.retry(reason.to_string(), retry_after).await { + Ok(_) => Ok(()), + Err(PgJobError::RetryInvalidError { + job: webhook_job, + error: fail_error, + }) => { + webhook_job + .fail(fail_error.to_string()) + .await + .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; + Ok(()) + } + Err(job_error) => Err(WebhookConsumerError::PgJobError(job_error.to_string())), + }, + Err(error) => { + webhook_job + .fail(error.to_string()) + .await + .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; + Ok(()) + } + } +} + +/// Make an HTTP request to a webhook endpoint. +/// +/// # Arguments +/// +/// * `method`: The HTTP method to use in the HTTP request. +/// * `url`: The URL we are targetting with our request. Parsing this URL fail. +/// * `headers`: Key, value pairs of HTTP headers in a `std::collections::HashMap`. Can fail if headers are not valid. +/// * `body`: The body of the request. Ownership is required. +/// * `timeout`: A timeout for the HTTP request. +async fn send_webhook( + method: &HttpMethod, + url: &str, + headers: &collections::HashMap, + body: String, + timeout: std::time::Duration, +) -> Result { + let client = reqwest::Client::new(); + let method: http::Method = method.into(); + let url: reqwest::Url = (url) + .parse() + .map_err(|error| WebhookConsumerError::ParseUrlError(error))?; + let headers: reqwest::header::HeaderMap = (headers) + .try_into() + .map_err(|error| WebhookConsumerError::ParseHeadersError(error))?; + + let body = reqwest::Body::from(body); + let response = client + .request(method, url) + .headers(headers) + .timeout(timeout) + .body(body) + .send() + .await?; + + let status = response.status(); + + if status.is_success() { + Ok(response) + } else if is_retryable_status(status) { + let retry_after = parse_retry_after_header(response.headers()); + + Err(WebhookConsumerError::RetryableWebhookError { + reason: format!("retryable status code {}", status), + retry_after, + }) + } else { + Err(WebhookConsumerError::NonRetryableWebhookError(format!( + "non-retryable status code {}", + status + ))) + } +} + +fn is_retryable_status(status: StatusCode) -> bool { + status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error() +} + +/// Attempt to parse a chrono::Duration from a Retry-After header, returning None if not possible. +/// Retry-After header can specify a date in RFC2822 or a number of seconds; we try to parse both. +/// If a Retry-After header is not present in the provided `header_map`, `None` is returned. +/// +/// # Arguments +/// +/// * `header_map`: A `&reqwest::HeaderMap` of response headers that could contain Retry-After. +fn parse_retry_after_header(header_map: &reqwest::header::HeaderMap) -> Option { + let retry_after_header = header_map.get(reqwest::header::RETRY_AFTER); + + let retry_after = match retry_after_header { + Some(header_value) => match header_value.to_str() { + Ok(s) => s, + Err(_) => { + return None; + } + }, + None => { + return None; + } + }; + + if let Ok(u) = u64::from_str_radix(retry_after, 10) { + let duration = time::Duration::from_secs(u); + return Some(duration); + } + + if let Ok(dt) = chrono::DateTime::parse_from_rfc2822(retry_after) { + let duration = + chrono::DateTime::::from(dt) - chrono::offset::Utc::now(); + + // This can only fail when negative, in which case we return None. + return duration.to_std().ok(); + } + + None +} + +mod tests { + use super::*; + // Note we are ignoring some warnings in this module. + // This is due to a long-standing cargo bug that reports imports and helper functions as unused. + // See: https://github.com/rust-lang/rust/issues/46379. + #[allow(unused_imports)] + use hook_common::pgqueue::{JobStatus, NewJob, RetryPolicy}; + + /// Use process id as a worker id for tests. + #[allow(dead_code)] + fn worker_id() -> String { + std::process::id().to_string() + } + + #[allow(dead_code)] + async fn enqueue_job( + queue: &PgQueue, + max_attempts: i32, + job_parameters: WebhookJobParameters, + ) -> Result<(), PgQueueError> { + let job_target = job_parameters.url.to_owned(); + let new_job = NewJob::new(max_attempts, job_parameters, &job_target); + queue.enqueue(new_job).await?; + Ok(()) + } + + #[test] + fn test_is_retryable_status() { + assert!(!is_retryable_status(http::StatusCode::FORBIDDEN)); + assert!(!is_retryable_status(http::StatusCode::BAD_REQUEST)); + assert!(is_retryable_status(http::StatusCode::TOO_MANY_REQUESTS)); + assert!(is_retryable_status(http::StatusCode::INTERNAL_SERVER_ERROR)); + } + + #[test] + fn test_parse_retry_after_header() { + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert(reqwest::header::RETRY_AFTER, "120".parse().unwrap()); + + let duration = parse_retry_after_header(&headers).unwrap(); + assert_eq!(duration, time::Duration::from_secs(120)); + + headers.remove(reqwest::header::RETRY_AFTER); + + let duration = parse_retry_after_header(&headers); + assert_eq!(duration, None); + + headers.insert( + reqwest::header::RETRY_AFTER, + "Wed, 21 Oct 2015 07:28:00 GMT".parse().unwrap(), + ); + + let duration = parse_retry_after_header(&headers); + assert_eq!(duration, None); + } + + #[tokio::test] + async fn test_wait_for_job() { + let worker_id = worker_id(); + let queue_name = "test_wait_for_job".to_string(); + let table_name = "job_queue".to_string(); + let db_url = "postgres://posthog:posthog@localhost:15432/test_database".to_string(); + let queue = PgQueue::new(&queue_name, &table_name, &db_url, RetryPolicy::default()) + .await + .expect("failed to connect to PG"); + + let webhook_job = WebhookJobParameters { + body: "a webhook job body. much wow.".to_owned(), + headers: collections::HashMap::new(), + method: HttpMethod::POST, + url: "localhost".to_owned(), + }; + // enqueue takes ownership of the job enqueued to avoid bugs that can cause duplicate jobs. + // Normally, a separate application would be enqueueing jobs for us to consume, so no ownership + // conflicts would arise. However, in this test we need to do the enqueueing ourselves. + // So, we clone the job to keep it around and assert the values returned by wait_for_job. + enqueue_job(&queue, 1, webhook_job.clone()) + .await + .expect("failed to enqueue job"); + let consumer = WebhookConsumer::new( + &worker_id, + &queue, + time::Duration::from_millis(100), + time::Duration::from_millis(5000), + ); + let consumed_job = consumer + .wait_for_job() + .await + .expect("failed to wait and read job"); + + assert_eq!(consumed_job.job.attempt, 1); + assert!(consumed_job.job.attempted_by.contains(&worker_id)); + assert_eq!(consumed_job.job.attempted_by.len(), 1); + assert_eq!(consumed_job.job.max_attempts, 1); + assert_eq!(*consumed_job.job.parameters.as_ref(), webhook_job); + assert_eq!(consumed_job.job.status, JobStatus::Running); + assert_eq!(consumed_job.job.target, webhook_job.url); + + consumed_job + .complete() + .await + .expect("job not successfully completed"); + } + + #[tokio::test] + async fn test_send_webhook() { + let method = HttpMethod::POST; + let url = "http://localhost:18081/echo"; + let headers = collections::HashMap::new(); + let body = "a very relevant request body"; + let response = send_webhook( + &method, + url, + &headers, + body.to_owned(), + time::Duration::from_millis(5000), + ) + .await + .expect("send_webhook failed"); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.text().await.expect("failed to read response body"), + body.to_owned(), + ); + } +} diff --git a/hook-consumer/src/lib.rs b/hook-consumer/src/lib.rs new file mode 100644 index 0000000000000..cc746b0833b0c --- /dev/null +++ b/hook-consumer/src/lib.rs @@ -0,0 +1,2 @@ +pub mod config; +pub mod consumer; diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index e7a11a969c037..22acee1263ef8 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -1,3 +1,33 @@ -fn main() { - println!("Hello, world!"); +use envconfig::Envconfig; + +use hook_common::pgqueue::{PgQueue, RetryPolicy}; +use hook_consumer::config::Config; +use hook_consumer::consumer::WebhookConsumer; + +#[tokio::main] +async fn main() { + let config = Config::init_from_env().expect("Invalid configuration:"); + + let retry_policy = RetryPolicy::new( + config.retry_policy.backoff_coefficient, + config.retry_policy.initial_interval.0, + Some(config.retry_policy.maximum_interval.0), + ); + let queue = PgQueue::new( + &config.queue_name, + &config.table_name, + &config.database_url, + retry_policy, + ) + .await + .expect("failed to initialize queue"); + + let consumer = WebhookConsumer::new( + &config.consumer_name, + &queue, + config.poll_interval.0, + config.request_timeout.0, + ); + + let _ = consumer.run().await; } From d91c90a89af31c4895b7fc4109b20caff90a9bb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 6 Dec 2023 14:17:08 +0100 Subject: [PATCH 092/247] chore: Cargo lock update --- Cargo.lock | 731 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 559 insertions(+), 172 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 451d42483e301..07643585e3152 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,16 +52,149 @@ dependencies = [ ] [[package]] -name = "async-trait" -version = "0.1.74" +name = "async-channel" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" +dependencies = [ + "concurrent-queue", + "event-listener 4.0.0", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" +dependencies = [ + "async-lock 3.2.0", + "async-task", + "concurrent-queue", + "fastrand 2.0.1", + "futures-lite 2.1.0", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4353121d5644cdf2beb5726ab752e79a8db1ebb52031770ec47db31d245526" +dependencies = [ + "async-channel 2.1.1", + "async-executor", + "async-io 2.2.1", + "async-lock 3.2.0", + "blocking", + "futures-lite 2.1.0", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock 2.8.0", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-lite 1.13.0", + "log", + "parking", + "polling 2.8.0", + "rustix 0.37.27", + "slab", + "socket2 0.4.10", + "waker-fn", +] + +[[package]] +name = "async-io" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6d3b15875ba253d1110c740755e246537483f152fa334f91abd7fe84c88b3ff" +dependencies = [ + "async-lock 3.2.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.1.0", + "parking", + "polling 3.3.1", + "rustix 0.38.25", + "slab", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" +dependencies = [ + "event-listener 4.0.0", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-channel 1.9.0", + "async-global-executor", + "async-io 1.13.0", + "async-lock 2.8.0", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite 1.13.0", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", ] +[[package]] +name = "async-task" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" + [[package]] name = "atoi" version = "2.0.0" @@ -71,6 +204,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "atomic-write-file" version = "0.1.2" @@ -191,6 +330,22 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +dependencies = [ + "async-channel 2.1.1", + "async-lock 3.2.0", + "async-task", + "fastrand 2.0.1", + "futures-io", + "futures-lite 2.1.0", + "piper", + "tracing", +] + [[package]] name = "bumpalo" version = "3.14.0" @@ -238,6 +393,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "concurrent-queue" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "const-oid" version = "0.9.5" @@ -364,6 +528,15 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "envconfig" version = "0.10.0" @@ -418,13 +591,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] -name = "eyre" -version = "0.6.9" +name = "event-listener" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80f656be11ddf91bd709454d15d5bd896fbaf4cc3314e69349e4d1569f5b46cd" +checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" dependencies = [ - "indenter", - "once_cell", + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.0", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", ] [[package]] @@ -480,6 +673,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.29" @@ -524,6 +732,45 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-lite" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143" +dependencies = [ + "fastrand 2.0.1", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "futures-sink" version = "0.3.29" @@ -542,8 +789,10 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -579,40 +828,37 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" -version = "0.4.0" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d308f63daf4181410c242d34c11f928dcb3aa105852019e043c9d1f4e4368a" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", - "http 1.0.0", - "indexmap 2.1.0", + "http", + "indexmap", "slab", "tokio", "tokio-util", "tracing", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" -dependencies = [ - "ahash", -] - [[package]] name = "hashbrown" version = "0.14.3" @@ -695,6 +941,21 @@ dependencies = [ [[package]] name = "hook-consumer" version = "0.1.0" +dependencies = [ + "async-std", + "chrono", + "envconfig", + "futures", + "hook-common", + "http", + "reqwest", + "serde", + "serde_derive", + "sqlx", + "thiserror", + "tokio", + "url", +] [[package]] name = "hook-producer" @@ -840,6 +1101,77 @@ dependencies = [ "tracing", ] +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -899,6 +1231,26 @@ dependencies = [ "hashbrown 0.14.3", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "ipnet" version = "2.9.0" @@ -929,6 +1281,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -963,7 +1324,13 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "linux-raw-sys" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" @@ -982,6 +1349,9 @@ name = "log" version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +dependencies = [ + "value-bag", +] [[package]] name = "mach2" @@ -1014,70 +1384,6 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - -[[package]] -name = "metrics" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" -dependencies = [ - "ahash", - "metrics-macros", - "portable-atomic", -] - -[[package]] -name = "metrics-exporter-prometheus" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" -dependencies = [ - "base64", - "hyper 0.14.27", - "indexmap 1.9.3", - "ipnet", - "metrics", - "metrics-util", - "quanta", - "thiserror", - "tokio", - "tracing", -] - -[[package]] -name = "metrics-macros" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "metrics-util" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", - "hashbrown 0.13.1", - "metrics", - "num_cpus", - "quanta", - "sketches-ddsketch", -] - [[package]] name = "mime" version = "0.3.17" @@ -1277,10 +1583,10 @@ dependencies = [ ] [[package]] -name = "overload" -version = "0.1.1" +name = "parking" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" @@ -1358,6 +1664,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -1386,10 +1703,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] -name = "portable-atomic" -version = "1.5.1" +name = "polling" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] + +[[package]] +name = "polling" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf63fa624ab313c11656b4cda960bfc46c410187ad493c41f6ba2d8c1e991c9e" +dependencies = [ + "cfg-if", + "concurrent-queue", + "pin-project-lite", + "rustix 0.38.25", + "tracing", + "windows-sys 0.52.0", +] [[package]] name = "ppv-lite86" @@ -1479,6 +1820,44 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "reqwest" +version = "0.11.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rsa" version = "0.9.6" @@ -1507,15 +1886,29 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.26" +version = "0.37.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" dependencies = [ "bitflags 2.4.1", "errno", "libc", - "linux-raw-sys", - "windows-sys 0.52.0", + "linux-raw-sys 0.4.11", + "windows-sys 0.48.0", ] [[package]] @@ -1599,16 +1992,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_path_to_error" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" -dependencies = [ - "itoa", - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1776,7 +2159,7 @@ dependencies = [ "crossbeam-queue", "dotenvy", "either", - "event-listener", + "event-listener 2.5.3", "futures-channel", "futures-core", "futures-intrusive", @@ -1994,10 +2377,25 @@ dependencies = [ ] [[package]] -name = "sync_wrapper" -version = "0.1.2" +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] [[package]] name = "tempfile" @@ -2006,9 +2404,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", - "fastrand", + "fastrand 2.0.1", "redox_syscall", - "rustix", + "rustix 0.38.25", "windows-sys 0.48.0", ] @@ -2068,9 +2466,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot", "pin-project-lite", - "signal-hook-registry", "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", @@ -2087,6 +2483,16 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -2112,28 +2518,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - [[package]] name = "tower-service" version = "0.3.2" @@ -2173,31 +2557,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" -dependencies = [ - "nu-ansi-term", - "sharded-slab", - "smallvec", - "thread_local", - "tracing-core", - "tracing-log", -] - [[package]] name = "try-lock" version = "0.2.4" @@ -2267,10 +2626,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" [[package]] -name = "valuable" -version = "0.1.0" +name = "value-bag" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" [[package]] name = "vcpkg" @@ -2284,6 +2643,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "waker-fn" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" + [[package]] name = "want" version = "0.3.1" @@ -2324,6 +2689,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.89" @@ -2532,6 +2909,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "zerocopy" version = "0.7.28" From 3bdd1c05d736835eaa8454fd3348fac60430bf02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 6 Dec 2023 14:17:35 +0100 Subject: [PATCH 093/247] chore: Remove install of sqlx from CI --- .github/workflows/rust.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ee0e5b3ebc785..5ddac411a47bf 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -49,13 +49,6 @@ jobs: docker compose -f docker-compose.yml down docker compose -f docker-compose.yml up -d --wait - - name: Run migrations - shell: bash - run: | - cargo install sqlx-cli --no-default-features --features native-tls,postgres - DATABASE_URL=postgres://posthog:posthog@localhost:15432/test_database sqlx database create - DATABASE_URL=postgres://posthog:posthog@localhost:15432/test_database sqlx migrate run - - uses: actions/cache@v3 with: path: | From c25ac6fde1d322b1c6cdccab574ae503d4ae83da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 6 Dec 2023 14:17:46 +0100 Subject: [PATCH 094/247] fix: Use depends_on instead of requires --- docker-compose.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6f62692df0743..afaf48ef86d06 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,8 +23,10 @@ services: restart: on-failure command: > sh -c "sqlx database create && sqlx migrate run" - requires: - - db + depends_on: + db: + condition: service_healthy + restart: true environment: DATABASE_URL: postgres://posthog:posthog@db:5432/test_database volumes: From 8c7cb796bca701d7b2b520f15582d7c90bb8dd2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 6 Dec 2023 14:25:33 +0100 Subject: [PATCH 095/247] fix: Address clippy linting issues --- hook-consumer/src/consumer.rs | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 33b09b0caacbe..bf9f8187c7b97 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -121,10 +121,10 @@ impl Serialize for HttpMethod { /// Convinience to cast `HttpMethod` to `http::Method`. /// Not all `http::Method` variants are valid `HttpMethod` variants, hence why we -/// can't just use the former. -impl Into for HttpMethod { - fn into(self) -> http::Method { - match self { +/// can't just use the former or implement `From`. +impl From for http::Method { + fn from(val: HttpMethod) -> Self { + match val { HttpMethod::DELETE => http::Method::DELETE, HttpMethod::GET => http::Method::GET, HttpMethod::PATCH => http::Method::PATCH, @@ -134,9 +134,9 @@ impl Into for HttpMethod { } } -impl Into for &HttpMethod { - fn into(self) -> http::Method { - match self { +impl From<&HttpMethod> for http::Method { + fn from(val: &HttpMethod) -> Self { + match val { HttpMethod::DELETE => http::Method::DELETE, HttpMethod::GET => http::Method::GET, HttpMethod::PATCH => http::Method::PATCH, @@ -203,7 +203,7 @@ impl<'p> WebhookConsumer<'p> { let webhook_job = self.wait_for_job().await?; let request_timeout = self.request_timeout; // Required to avoid capturing self in closure. - tokio::spawn(async move { process_webhook_job(webhook_job, request_timeout) }); + tokio::spawn(async move { process_webhook_job(webhook_job, request_timeout).await }); } } } @@ -286,12 +286,10 @@ async fn send_webhook( ) -> Result { let client = reqwest::Client::new(); let method: http::Method = method.into(); - let url: reqwest::Url = (url) - .parse() - .map_err(|error| WebhookConsumerError::ParseUrlError(error))?; + let url: reqwest::Url = (url).parse().map_err(WebhookConsumerError::ParseUrlError)?; let headers: reqwest::header::HeaderMap = (headers) .try_into() - .map_err(|error| WebhookConsumerError::ParseHeadersError(error))?; + .map_err(WebhookConsumerError::ParseHeadersError)?; let body = reqwest::Body::from(body); let response = client @@ -347,7 +345,7 @@ fn parse_retry_after_header(header_map: &reqwest::header::HeaderMap) -> Option() { let duration = time::Duration::from_secs(u); return Some(duration); } From 2d99f77d80ffaae8fb1e754e85d4b8774b8dfc93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 6 Dec 2023 14:36:44 +0100 Subject: [PATCH 096/247] fix: Split up docker compose start up in CI --- .github/workflows/rust.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5ddac411a47bf..b811c1a52e668 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -47,7 +47,8 @@ jobs: shell: bash run: | docker compose -f docker-compose.yml down - docker compose -f docker-compose.yml up -d --wait + docker compose -f docker-compose.yml up db echo_server -d --wait + docker compose -f docker-compose.yml up setup_test_db - uses: actions/cache@v3 with: From 71d59299c2dc49608ae8807a57b0ecc738c73d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 7 Dec 2023 10:51:33 +0100 Subject: [PATCH 097/247] fix: Typo Co-authored-by: Brett Hoerner --- hook-consumer/src/consumer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index bf9f8187c7b97..a8f2fa61941c8 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -119,7 +119,7 @@ impl Serialize for HttpMethod { } } -/// Convinience to cast `HttpMethod` to `http::Method`. +/// Convenience to cast `HttpMethod` to `http::Method`. /// Not all `http::Method` variants are valid `HttpMethod` variants, hence why we /// can't just use the former or implement `From`. impl From for http::Method { From 09acfef888a5c1835ca9118a3320fe7995c6a1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 11:37:24 +0100 Subject: [PATCH 098/247] fix: Delete Cargo.lock --- Cargo.lock | 2946 ---------------------------------------------------- 1 file changed, 2946 deletions(-) delete mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 07643585e3152..0000000000000 --- a/Cargo.lock +++ /dev/null @@ -1,2946 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "addr2line" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "ahash" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "allocator-api2" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - -[[package]] -name = "async-channel" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" -dependencies = [ - "concurrent-queue", - "event-listener 4.0.0", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-executor" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" -dependencies = [ - "async-lock 3.2.0", - "async-task", - "concurrent-queue", - "fastrand 2.0.1", - "futures-lite 2.1.0", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4353121d5644cdf2beb5726ab752e79a8db1ebb52031770ec47db31d245526" -dependencies = [ - "async-channel 2.1.1", - "async-executor", - "async-io 2.2.1", - "async-lock 3.2.0", - "blocking", - "futures-lite 2.1.0", - "once_cell", -] - -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock 2.8.0", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite 1.13.0", - "log", - "parking", - "polling 2.8.0", - "rustix 0.37.27", - "slab", - "socket2 0.4.10", - "waker-fn", -] - -[[package]] -name = "async-io" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6d3b15875ba253d1110c740755e246537483f152fa334f91abd7fe84c88b3ff" -dependencies = [ - "async-lock 3.2.0", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite 2.1.0", - "parking", - "polling 3.3.1", - "rustix 0.38.25", - "slab", - "tracing", - "windows-sys 0.52.0", -] - -[[package]] -name = "async-lock" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" -dependencies = [ - "event-listener 2.5.3", -] - -[[package]] -name = "async-lock" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" -dependencies = [ - "event-listener 4.0.0", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-std" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" -dependencies = [ - "async-channel 1.9.0", - "async-global-executor", - "async-io 1.13.0", - "async-lock 2.8.0", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite 1.13.0", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - -[[package]] -name = "async-task" -version = "4.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" - -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "atomic-write-file" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edcdbedc2236483ab103a53415653d6b4442ea6141baf1ffa85df29635e88436" -dependencies = [ - "nix", - "rand", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "axum" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202651474fe73c62d9e0a56c6133f7a0ff1dc1c8cf7a5b03381af2a26553ac9d" -dependencies = [ - "async-trait", - "axum-core", - "bytes", - "futures-util", - "http 1.0.0", - "http-body 1.0.0", - "http-body-util", - "hyper 1.0.1", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77cb22c689c44d4c07b0ab44ebc25d69d8ae601a2f28fb8d672d344178fa17aa" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.0.0", - "http-body 1.0.0", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", -] - -[[package]] -name = "backtrace" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base64" -version = "0.21.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" - -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" -dependencies = [ - "serde", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "blocking" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" -dependencies = [ - "async-channel 2.1.1", - "async-lock 3.2.0", - "async-task", - "fastrand 2.0.1", - "futures-io", - "futures-lite 2.1.0", - "piper", - "tracing", -] - -[[package]] -name = "bumpalo" -version = "3.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" - -[[package]] -name = "cc" -version = "1.0.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "chrono" -version = "0.4.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-targets 0.48.5", -] - -[[package]] -name = "concurrent-queue" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "const-oid" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" - -[[package]] -name = "cpufeatures" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" -dependencies = [ - "libc", -] - -[[package]] -name = "crc" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - -[[package]] -name = "crossbeam-epoch" -version = "0.9.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" -dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "memoffset", - "scopeguard", -] - -[[package]] -name = "crossbeam-queue" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "der" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "subtle", -] - -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - -[[package]] -name = "either" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" -dependencies = [ - "serde", -] - -[[package]] -name = "encoding_rs" -version = "0.8.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "envconfig" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" -dependencies = [ - "envconfig_derive", -] - -[[package]] -name = "envconfig_derive" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "errno" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "etcetera" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" -dependencies = [ - "cfg-if", - "home", - "windows-sys 0.48.0", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "event-listener" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener 4.0.0", - "pin-project-lite", -] - -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "fastrand" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" - -[[package]] -name = "finl_unicode" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" - -[[package]] -name = "flume" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" -dependencies = [ - "futures-core", - "futures-sink", - "spin 0.9.8", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" - -[[package]] -name = "futures-executor" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-intrusive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot", -] - -[[package]] -name = "futures-io" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" - -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - -[[package]] -name = "futures-lite" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143" -dependencies = [ - "fastrand 2.0.1", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - -[[package]] -name = "futures-macro" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "futures-sink" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" - -[[package]] -name = "futures-task" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" - -[[package]] -name = "futures-util" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "gimli" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" - -[[package]] -name = "gloo-timers" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "h2" -version = "0.3.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" -dependencies = [ - "ahash", - "allocator-api2", -] - -[[package]] -name = "hashlink" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" -dependencies = [ - "hashbrown 0.14.3", -] - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "hermit-abi" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hkdf" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" -dependencies = [ - "hmac", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys 0.48.0", -] - -[[package]] -name = "hook-common" -version = "0.1.0" -dependencies = [ - "chrono", - "serde", - "serde_derive", - "sqlx", - "thiserror", - "tokio", -] - -[[package]] -name = "hook-consumer" -version = "0.1.0" -dependencies = [ - "async-std", - "chrono", - "envconfig", - "futures", - "hook-common", - "http", - "reqwest", - "serde", - "serde_derive", - "sqlx", - "thiserror", - "tokio", - "url", -] - -[[package]] -name = "hook-producer" -version = "0.1.0" -dependencies = [ - "axum", - "envconfig", - "eyre", - "metrics", - "metrics-exporter-prometheus", - "tokio", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "http" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" -dependencies = [ - "bytes", - "http 0.2.11", - "pin-project-lite", -] - -[[package]] -name = "http-body" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" -dependencies = [ - "bytes", - "http 1.0.0", -] - -[[package]] -name = "http-body-util" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" -dependencies = [ - "bytes", - "futures-util", - "http 1.0.0", - "http-body 1.0.0", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "0.14.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http 0.2.11", - "http-body 0.4.5", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.4.10", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2", - "http 1.0.0", - "http-body 1.0.0", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "hyper-util" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http 1.0.0", - "http-body 1.0.0", - "hyper 1.0.1", - "pin-project-lite", - "socket2 0.5.5", - "tokio", - "tower", - "tower-service", - "tracing", -] - -[[package]] -name = "http" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "0.14.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.4.10", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "idna" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "indenter" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indexmap" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" -dependencies = [ - "equivalent", - "hashbrown 0.14.3", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "ipnet" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" - -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" - -[[package]] -name = "js-sys" -version = "0.3.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -dependencies = [ - "spin 0.5.2", -] - -[[package]] -name = "libc" -version = "0.2.150" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" - -[[package]] -name = "libm" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" - -[[package]] -name = "libsqlite3-sys" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - -[[package]] -name = "linux-raw-sys" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" - -[[package]] -name = "lock_api" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -dependencies = [ - "value-bag", -] - -[[package]] -name = "mach2" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" -dependencies = [ - "libc", -] - -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - -[[package]] -name = "memchr" -version = "2.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.48.0", -] - -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "nix" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" -dependencies = [ - "bitflags 2.4.1", - "cfg-if", - "libc", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand", - "smallvec", - "zeroize", -] - -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "object" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" - -[[package]] -name = "openssl" -version = "0.10.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" -dependencies = [ - "bitflags 2.4.1", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-sys" -version = "0.9.96" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "parking" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.48.5", -] - -[[package]] -name = "paste" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pin-project" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "piper" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" -dependencies = [ - "atomic-waker", - "fastrand 2.0.1", - "futures-io", -] - -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - -[[package]] -name = "pkg-config" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" - -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags 1.3.2", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", -] - -[[package]] -name = "polling" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf63fa624ab313c11656b4cda960bfc46c410187ad493c41f6ba2d8c1e991c9e" -dependencies = [ - "cfg-if", - "concurrent-queue", - "pin-project-lite", - "rustix 0.38.25", - "tracing", - "windows-sys 0.52.0", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "proc-macro2" -version = "1.0.70" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quanta" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" -dependencies = [ - "crossbeam-utils", - "libc", - "mach2", - "once_cell", - "raw-cpuid", - "wasi", - "web-sys", - "winapi", -] - -[[package]] -name = "quote" -version = "1.0.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "raw-cpuid" -version = "10.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "reqwest" -version = "0.11.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" -dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "system-configuration", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - -[[package]] -name = "rsa" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" -dependencies = [ - "const-oid", - "digest", - "num-bigint-dig", - "num-integer", - "num-traits", - "pkcs1", - "pkcs8", - "rand_core", - "signature", - "spki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - -[[package]] -name = "rustix" -version = "0.37.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustix" -version = "0.38.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" -dependencies = [ - "bitflags 2.4.1", - "errno", - "libc", - "linux-raw-sys 0.4.11", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustversion" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" - -[[package]] -name = "ryu" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" - -[[package]] -name = "schannel" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" -dependencies = [ - "windows-sys 0.48.0", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "security-framework" -version = "2.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "serde" -version = "1.0.193" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.193" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "serde_json" -version = "1.0.108" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest", - "rand_core", -] - -[[package]] -name = "sketches-ddsketch" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "1.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" - -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" -dependencies = [ - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "sqlformat" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" -dependencies = [ - "itertools", - "nom", - "unicode_categories", -] - -[[package]] -name = "sqlx" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dba03c279da73694ef99763320dea58b51095dfe87d001b1d4b5fe78ba8763cf" -dependencies = [ - "sqlx-core", - "sqlx-macros", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", -] - -[[package]] -name = "sqlx-core" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" -dependencies = [ - "ahash", - "atoi", - "byteorder", - "bytes", - "chrono", - "crc", - "crossbeam-queue", - "dotenvy", - "either", - "event-listener 2.5.3", - "futures-channel", - "futures-core", - "futures-intrusive", - "futures-io", - "futures-util", - "hashlink", - "hex", - "indexmap 2.1.0", - "log", - "memchr", - "native-tls", - "once_cell", - "paste", - "percent-encoding", - "serde", - "serde_json", - "sha2", - "smallvec", - "sqlformat", - "thiserror", - "tokio", - "tokio-stream", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "sqlx-macros" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" -dependencies = [ - "proc-macro2", - "quote", - "sqlx-core", - "sqlx-macros-core", - "syn 1.0.109", -] - -[[package]] -name = "sqlx-macros-core" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0bd4519486723648186a08785143599760f7cc81c52334a55d6a83ea1e20841" -dependencies = [ - "atomic-write-file", - "dotenvy", - "either", - "heck", - "hex", - "once_cell", - "proc-macro2", - "quote", - "serde", - "serde_json", - "sha2", - "sqlx-core", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", - "syn 1.0.109", - "tempfile", - "tokio", - "url", -] - -[[package]] -name = "sqlx-mysql" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" -dependencies = [ - "atoi", - "base64", - "bitflags 2.4.1", - "byteorder", - "bytes", - "chrono", - "crc", - "digest", - "dotenvy", - "either", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "generic-array", - "hex", - "hkdf", - "hmac", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "percent-encoding", - "rand", - "rsa", - "serde", - "sha1", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "uuid", - "whoami", -] - -[[package]] -name = "sqlx-postgres" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" -dependencies = [ - "atoi", - "base64", - "bitflags 2.4.1", - "byteorder", - "chrono", - "crc", - "dotenvy", - "etcetera", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "hex", - "hkdf", - "hmac", - "home", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "rand", - "serde", - "serde_json", - "sha1", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "uuid", - "whoami", -] - -[[package]] -name = "sqlx-sqlite" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "210976b7d948c7ba9fced8ca835b11cbb2d677c59c79de41ac0d397e14547490" -dependencies = [ - "atoi", - "chrono", - "flume", - "futures-channel", - "futures-core", - "futures-executor", - "futures-intrusive", - "futures-util", - "libsqlite3-sys", - "log", - "percent-encoding", - "serde", - "sqlx-core", - "tracing", - "url", - "urlencoding", - "uuid", -] - -[[package]] -name = "stringprep" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" -dependencies = [ - "finl_unicode", - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "subtle" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" -dependencies = [ - "cfg-if", - "fastrand 2.0.1", - "redox_syscall", - "rustix 0.38.25", - "windows-sys 0.48.0", -] - -[[package]] -name = "thiserror" -version = "1.0.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "thread_local" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" -dependencies = [ - "cfg-if", - "once_cell", -] - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "num_cpus", - "pin-project-lite", - "socket2 0.5.5", - "tokio-macros", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-macros" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" -dependencies = [ - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "try-lock" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" - -[[package]] -name = "typenum" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - -[[package]] -name = "unicode-bidi" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - -[[package]] -name = "url" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - -[[package]] -name = "uuid" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" - -[[package]] -name = "value-bag" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "waker-fn" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.39", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" - -[[package]] -name = "web-sys" -version = "0.3.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "whoami" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" -version = "0.51.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.0", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" -dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" - -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - -[[package]] -name = "zerocopy" -version = "0.7.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "zeroize" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" From 6d3080de965279d6b0bb82ce9c331eee2b6a9ef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 11:39:00 +0100 Subject: [PATCH 099/247] fix: Re-add Cargo.lock --- Cargo.lock | 3091 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 3091 insertions(+) create mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000000..b24af98ae2169 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3091 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" +dependencies = [ + "concurrent-queue", + "event-listener 4.0.0", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" +dependencies = [ + "async-lock 3.2.0", + "async-task", + "concurrent-queue", + "fastrand 2.0.1", + "futures-lite 2.1.0", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4353121d5644cdf2beb5726ab752e79a8db1ebb52031770ec47db31d245526" +dependencies = [ + "async-channel 2.1.1", + "async-executor", + "async-io 2.2.1", + "async-lock 3.2.0", + "blocking", + "futures-lite 2.1.0", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock 2.8.0", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-lite 1.13.0", + "log", + "parking", + "polling 2.8.0", + "rustix 0.37.27", + "slab", + "socket2 0.4.10", + "waker-fn", +] + +[[package]] +name = "async-io" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6d3b15875ba253d1110c740755e246537483f152fa334f91abd7fe84c88b3ff" +dependencies = [ + "async-lock 3.2.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.1.0", + "parking", + "polling 3.3.1", + "rustix 0.38.27", + "slab", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" +dependencies = [ + "event-listener 4.0.0", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-channel 1.9.0", + "async-global-executor", + "async-io 1.13.0", + "async-lock 2.8.0", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite 1.13.0", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "atomic-write-file" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edcdbedc2236483ab103a53415653d6b4442ea6141baf1ffa85df29635e88436" +dependencies = [ + "nix", + "rand", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "axum" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202651474fe73c62d9e0a56c6133f7a0ff1dc1c8cf7a5b03381af2a26553ac9d" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.0.1", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77cb22c689c44d4c07b0ab44ebc25d69d8ae601a2f28fb8d672d344178fa17aa" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +dependencies = [ + "serde", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +dependencies = [ + "async-channel 2.1.1", + "async-lock 3.2.0", + "async-task", + "fastrand 2.0.1", + "futures-io", + "futures-lite 2.1.0", + "piper", + "tracing", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.48.5", +] + +[[package]] +name = "concurrent-queue" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "der" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +dependencies = [ + "serde", +] + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "envconfig" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" +dependencies = [ + "envconfig_derive", +] + +[[package]] +name = "envconfig_derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.0", + "pin-project-lite", +] + +[[package]] +name = "eyre" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bbb8258be8305fb0237d7b295f47bb24ff1b136a535f473baf40e70468515aa" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-lite" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143" +dependencies = [ + "fastrand 2.0.1", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.11", + "indexmap 2.1.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d308f63daf4181410c242d34c11f928dcb3aa105852019e043c9d1f4e4368a" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.0.0", + "indexmap 2.1.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.3", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "hook-common" +version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "serde_derive", + "sqlx", + "thiserror", + "tokio", +] + +[[package]] +name = "hook-consumer" +version = "0.1.0" +dependencies = [ + "async-std", + "chrono", + "envconfig", + "futures", + "hook-common", + "http 0.2.11", + "reqwest", + "serde", + "serde_derive", + "sqlx", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "hook-producer" +version = "0.1.0" +dependencies = [ + "axum", + "envconfig", + "eyre", + "metrics", + "metrics-exporter-prometheus", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.5", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.0", + "http 1.0.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.27", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "hyper-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.0.1", + "pin-project-lite", + "socket2 0.5.5", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] + +[[package]] +name = "libc" +version = "0.2.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "linux-raw-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +dependencies = [ + "value-bag", +] + +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "metrics" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +dependencies = [ + "ahash", + "metrics-macros", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" +dependencies = [ + "base64", + "hyper 0.14.27", + "indexmap 1.9.3", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "metrics-util" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.13.1", + "metrics", + "num_cpus", + "quanta", + "sketches-ddsketch", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl" +version = "0.10.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.48.5", +] + +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "polling" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] + +[[package]] +name = "polling" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf63fa624ab313c11656b4cda960bfc46c410187ad493c41f6ba2d8c1e991c9e" +dependencies = [ + "cfg-if", + "concurrent-queue", + "pin-project-lite", + "rustix 0.38.27", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quanta" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "reqwest" +version = "0.11.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.27", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.37.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfeae074e687625746172d639330f1de242a178bf3189b51e35a7a21573513ac" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys 0.4.12", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +dependencies = [ + "itoa", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "sketches-ddsketch" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +dependencies = [ + "itertools", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dba03c279da73694ef99763320dea58b51095dfe87d001b1d4b5fe78ba8763cf" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" +dependencies = [ + "ahash", + "atoi", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "dotenvy", + "either", + "event-listener 2.5.3", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap 2.1.0", + "log", + "memchr", + "native-tls", + "once_cell", + "paste", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0bd4519486723648186a08785143599760f7cc81c52334a55d6a83ea1e20841" +dependencies = [ + "atomic-write-file", + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" +dependencies = [ + "atoi", + "base64", + "bitflags 2.4.1", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" +dependencies = [ + "atoi", + "base64", + "bitflags 2.4.1", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "210976b7d948c7ba9fced8ca835b11cbb2d677c59c79de41ac0d397e14547490" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", + "uuid", +] + +[[package]] +name = "stringprep" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +dependencies = [ + "finl_unicode", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +dependencies = [ + "cfg-if", + "fastrand 2.0.1", + "redox_syscall", + "rustix 0.38.27", + "windows-sys 0.48.0", +] + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.5.5", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-bidi" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "uuid" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "value-bag" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "waker-fn" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.39", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" + +[[package]] +name = "web-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "zerocopy" +version = "0.7.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d075cf85bbb114e933343e087b92f2146bac0d55b534cbb8188becf0039948e" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86cd5ca076997b97ef09d3ad65efe811fa68c9e874cb636ccb211223a813b0c2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" From f76915e0b54eeb852910844c63f3afb433b1867f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 11:41:06 +0100 Subject: [PATCH 100/247] chore: Add comment referencing connection limit --- hook-consumer/src/consumer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index a8f2fa61941c8..cff54f68f21ef 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -200,6 +200,7 @@ impl<'p> WebhookConsumer<'p> { /// Run this consumer to continuously process any jobs that become available. pub async fn run(&self) -> Result<(), WebhookConsumerError> { loop { + // TODO: The number of jobs processed will be capped by the PG connection limit when running in transactional mode. let webhook_job = self.wait_for_job().await?; let request_timeout = self.request_timeout; // Required to avoid capturing self in closure. From 104717937ec5f1537f6b587e0b3fb1839b98435a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 12:23:41 +0100 Subject: [PATCH 101/247] refactor: Re-use client in consumer --- hook-consumer/src/consumer.rs | 83 ++++++++++++++--------------------- hook-consumer/src/error.rs | 30 +++++++++++++ hook-consumer/src/lib.rs | 1 + hook-consumer/src/main.rs | 7 ++- 4 files changed, 69 insertions(+), 52 deletions(-) create mode 100644 hook-consumer/src/error.rs diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index cff54f68f21ef..d18aed5eee368 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -6,34 +6,10 @@ use std::time; use async_std::task; use hook_common::pgqueue::{PgJobError, PgQueue, PgQueueError, PgTransactionJob}; use http::StatusCode; +use reqwest::header; use serde::{de::Visitor, Deserialize, Serialize}; -use thiserror::Error; - -/// Enumeration of errors for operations with WebhookConsumer. -#[derive(Error, Debug)] -pub enum WebhookConsumerError { - #[error("timed out while waiting for jobs to be available")] - TimeoutError, - #[error("{0} is not a valid HttpMethod")] - ParseHttpMethodError(String), - #[error("error parsing webhook headers")] - ParseHeadersError(http::Error), - #[error("error parsing webhook url")] - ParseUrlError(url::ParseError), - #[error("an error occurred in the underlying queue")] - QueueError(#[from] PgQueueError), - #[error("an error occurred in the underlying job")] - PgJobError(String), - #[error("an error occurred when attempting to send a request")] - RequestError(#[from] reqwest::Error), - #[error("a webhook could not be delivered but it could be retried later: {reason}")] - RetryableWebhookError { - reason: String, - retry_after: Option, - }, - #[error("a webhook could not be delivered and it cannot be retried further: {0}")] - NonRetryableWebhookError(String), -} + +use crate::error::WebhookConsumerError; /// Supported HTTP methods for webhooks. #[derive(Debug, PartialEq, Clone, Copy)] @@ -165,8 +141,8 @@ pub struct WebhookConsumer<'p> { queue: &'p PgQueue, /// The interval for polling the queue. poll_interval: time::Duration, - /// A timeout for webhook requests. - request_timeout: time::Duration, + /// The client used for HTTP requests. + client: reqwest::Client, } impl<'p> WebhookConsumer<'p> { @@ -175,13 +151,24 @@ impl<'p> WebhookConsumer<'p> { queue: &'p PgQueue, poll_interval: time::Duration, request_timeout: time::Duration, - ) -> Self { - Self { + ) -> Result { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ); + + let client = reqwest::Client::builder() + .default_headers(headers) + .timeout(request_timeout) + .build()?; + + Ok(Self { name: name.to_owned(), queue, poll_interval, - request_timeout, - } + client, + }) } /// Wait until a job becomes available in our queue. @@ -203,8 +190,9 @@ impl<'p> WebhookConsumer<'p> { // TODO: The number of jobs processed will be capped by the PG connection limit when running in transactional mode. let webhook_job = self.wait_for_job().await?; - let request_timeout = self.request_timeout; // Required to avoid capturing self in closure. - tokio::spawn(async move { process_webhook_job(webhook_job, request_timeout).await }); + // reqwest::Client internally wraps with Arc, so this allocation is cheap. + let client = self.client.clone(); + tokio::spawn(async move { process_webhook_job(client, webhook_job).await }); } } } @@ -223,15 +211,15 @@ impl<'p> WebhookConsumer<'p> { /// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. /// * `request_timeout`: A timeout for the HTTP request. async fn process_webhook_job( + client: reqwest::Client, webhook_job: PgTransactionJob<'_, WebhookJobParameters>, - request_timeout: std::time::Duration, ) -> Result<(), WebhookConsumerError> { match send_webhook( + client, &webhook_job.job.parameters.method, &webhook_job.job.parameters.url, &webhook_job.job.parameters.headers, webhook_job.job.parameters.body.clone(), - request_timeout, ) .await { @@ -279,13 +267,12 @@ async fn process_webhook_job( /// * `body`: The body of the request. Ownership is required. /// * `timeout`: A timeout for the HTTP request. async fn send_webhook( + client: reqwest::Client, method: &HttpMethod, url: &str, headers: &collections::HashMap, body: String, - timeout: std::time::Duration, ) -> Result { - let client = reqwest::Client::new(); let method: http::Method = method.into(); let url: reqwest::Url = (url).parse().map_err(WebhookConsumerError::ParseUrlError)?; let headers: reqwest::header::HeaderMap = (headers) @@ -296,7 +283,6 @@ async fn send_webhook( let response = client .request(method, url) .headers(headers) - .timeout(timeout) .body(body) .send() .await?; @@ -446,7 +432,8 @@ mod tests { &queue, time::Duration::from_millis(100), time::Duration::from_millis(5000), - ); + ) + .expect("consumer failed to initialize"); let consumed_job = consumer .wait_for_job() .await @@ -472,15 +459,11 @@ mod tests { let url = "http://localhost:18081/echo"; let headers = collections::HashMap::new(); let body = "a very relevant request body"; - let response = send_webhook( - &method, - url, - &headers, - body.to_owned(), - time::Duration::from_millis(5000), - ) - .await - .expect("send_webhook failed"); + let client = reqwest::Client::new(); + + let response = send_webhook(client, &method, url, &headers, body.to_owned()) + .await + .expect("send_webhook failed"); assert_eq!(response.status(), StatusCode::OK); assert_eq!( diff --git a/hook-consumer/src/error.rs b/hook-consumer/src/error.rs new file mode 100644 index 0000000000000..34f0619ada4f0 --- /dev/null +++ b/hook-consumer/src/error.rs @@ -0,0 +1,30 @@ +use std::time; + +use hook_common::pgqueue; +use thiserror::Error; + +/// Enumeration of errors for operations with WebhookConsumer. +#[derive(Error, Debug)] +pub enum WebhookConsumerError { + #[error("timed out while waiting for jobs to be available")] + TimeoutError, + #[error("{0} is not a valid HttpMethod")] + ParseHttpMethodError(String), + #[error("error parsing webhook headers")] + ParseHeadersError(http::Error), + #[error("error parsing webhook url")] + ParseUrlError(url::ParseError), + #[error("an error occurred in the underlying queue")] + QueueError(#[from] pgqueue::PgQueueError), + #[error("an error occurred in the underlying job")] + PgJobError(String), + #[error("an error occurred when attempting to send a request")] + RequestError(#[from] reqwest::Error), + #[error("a webhook could not be delivered but it could be retried later: {reason}")] + RetryableWebhookError { + reason: String, + retry_after: Option, + }, + #[error("a webhook could not be delivered and it cannot be retried further: {0}")] + NonRetryableWebhookError(String), +} diff --git a/hook-consumer/src/lib.rs b/hook-consumer/src/lib.rs index cc746b0833b0c..b99481bbc3b58 100644 --- a/hook-consumer/src/lib.rs +++ b/hook-consumer/src/lib.rs @@ -1,2 +1,3 @@ pub mod config; pub mod consumer; +pub mod error; diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index 22acee1263ef8..16515640914d8 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -3,9 +3,10 @@ use envconfig::Envconfig; use hook_common::pgqueue::{PgQueue, RetryPolicy}; use hook_consumer::config::Config; use hook_consumer::consumer::WebhookConsumer; +use hook_consumer::error::WebhookConsumerError; #[tokio::main] -async fn main() { +async fn main() -> Result<(), WebhookConsumerError> { let config = Config::init_from_env().expect("Invalid configuration:"); let retry_policy = RetryPolicy::new( @@ -27,7 +28,9 @@ async fn main() { &queue, config.poll_interval.0, config.request_timeout.0, - ); + )?; let _ = consumer.run().await; + + Ok(()) } From 0525d7b71728fdfbd7915c6c49308325a921460f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 14:59:42 +0100 Subject: [PATCH 102/247] fix: Limit number of concurrent HTTP requests with Semaphore --- hook-consumer/src/config.rs | 3 +++ hook-consumer/src/consumer.rs | 19 +++++++++++++++++-- hook-consumer/src/main.rs | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/hook-consumer/src/config.rs b/hook-consumer/src/config.rs index fde137337e8f8..50d0f1cd4ec7d 100644 --- a/hook-consumer/src/config.rs +++ b/hook-consumer/src/config.rs @@ -20,6 +20,9 @@ pub struct Config { #[envconfig(default = "5000")] pub request_timeout: EnvMsDuration, + #[envconfig(default = "1024")] + pub max_requests: usize, + #[envconfig(nested = true)] pub retry_policy: RetryPolicyConfig, diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index d18aed5eee368..fba6cde74f301 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -1,6 +1,7 @@ use std::collections; use std::fmt; use std::str::FromStr; +use std::sync::Arc; use std::time; use async_std::task; @@ -8,6 +9,7 @@ use hook_common::pgqueue::{PgJobError, PgQueue, PgQueueError, PgTransactionJob}; use http::StatusCode; use reqwest::header; use serde::{de::Visitor, Deserialize, Serialize}; +use tokio::sync; use crate::error::WebhookConsumerError; @@ -143,6 +145,8 @@ pub struct WebhookConsumer<'p> { poll_interval: time::Duration, /// The client used for HTTP requests. client: reqwest::Client, + /// Maximum number of concurrent HTTP requests. + max_requests: usize, } impl<'p> WebhookConsumer<'p> { @@ -151,6 +155,7 @@ impl<'p> WebhookConsumer<'p> { queue: &'p PgQueue, poll_interval: time::Duration, request_timeout: time::Duration, + max_requests: usize, ) -> Result { let mut headers = header::HeaderMap::new(); headers.insert( @@ -168,6 +173,7 @@ impl<'p> WebhookConsumer<'p> { queue, poll_interval, client, + max_requests, }) } @@ -186,13 +192,21 @@ impl<'p> WebhookConsumer<'p> { /// Run this consumer to continuously process any jobs that become available. pub async fn run(&self) -> Result<(), WebhookConsumerError> { + let semaphore = Arc::new(sync::Semaphore::new(self.max_requests)); + loop { // TODO: The number of jobs processed will be capped by the PG connection limit when running in transactional mode. let webhook_job = self.wait_for_job().await?; // reqwest::Client internally wraps with Arc, so this allocation is cheap. let client = self.client.clone(); - tokio::spawn(async move { process_webhook_job(client, webhook_job).await }); + let permit = semaphore.clone().acquire_owned().await.unwrap(); + + tokio::spawn(async move { + let result = process_webhook_job(client, webhook_job).await; + drop(permit); + result.expect("webhook processing failed"); + }); } } } @@ -278,8 +292,8 @@ async fn send_webhook( let headers: reqwest::header::HeaderMap = (headers) .try_into() .map_err(WebhookConsumerError::ParseHeadersError)?; - let body = reqwest::Body::from(body); + let response = client .request(method, url) .headers(headers) @@ -432,6 +446,7 @@ mod tests { &queue, time::Duration::from_millis(100), time::Duration::from_millis(5000), + 10, ) .expect("consumer failed to initialize"); let consumed_job = consumer diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index 16515640914d8..5f0d46dce133c 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -28,6 +28,7 @@ async fn main() -> Result<(), WebhookConsumerError> { &queue, config.poll_interval.0, config.request_timeout.0, + config.max_requests, )?; let _ = consumer.run().await; From 35825378b0fa68a71bcb98403e57d33c4e602858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 16:31:06 +0100 Subject: [PATCH 103/247] chore: Use workspace dependencies --- hook-consumer/Cargo.toml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/hook-consumer/Cargo.toml b/hook-consumer/Cargo.toml index 2e95a6b071903..5ff1eb08eb660 100644 --- a/hook-consumer/Cargo.toml +++ b/hook-consumer/Cargo.toml @@ -3,19 +3,17 @@ name = "hook-consumer" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] async-std = { version = "1.12" } -chrono = { version = "0.4" } -envconfig = { version = "0.10" } +chrono = { workspace = true } +envconfig = { workspace = true } futures = "0.3" hook-common = { path = "../hook-common" } http = { version = "0.2" } reqwest = { version = "0.11" } -serde = { version = "1.0" } -serde_derive = { version = "1.0" } -sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", "postgres", "uuid", "json", "chrono" ] } -thiserror = { version = "1.0" } -tokio = { version = "1.34", features = ["macros", "rt", "rt-multi-thread"] } +serde = { workspace = true } +serde_derive = { workspace = true } +sqlx = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } url = { version = "2.2" } From 31b1e7ae0e6f19fcf9cf40fd0e2a87e779667f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 16:37:43 +0100 Subject: [PATCH 104/247] refactor: Rename to max_concurrent_jobs --- hook-consumer/src/config.rs | 2 +- hook-consumer/src/consumer.rs | 10 +++++----- hook-consumer/src/main.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hook-consumer/src/config.rs b/hook-consumer/src/config.rs index 50d0f1cd4ec7d..8e4bde9d956be 100644 --- a/hook-consumer/src/config.rs +++ b/hook-consumer/src/config.rs @@ -21,7 +21,7 @@ pub struct Config { pub request_timeout: EnvMsDuration, #[envconfig(default = "1024")] - pub max_requests: usize, + pub max_concurrent_jobs: usize, #[envconfig(nested = true)] pub retry_policy: RetryPolicyConfig, diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index fba6cde74f301..9abe9bddd09f6 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -145,8 +145,8 @@ pub struct WebhookConsumer<'p> { poll_interval: time::Duration, /// The client used for HTTP requests. client: reqwest::Client, - /// Maximum number of concurrent HTTP requests. - max_requests: usize, + /// Maximum number of concurrent jobs being processed. + max_concurrent_jobs: usize, } impl<'p> WebhookConsumer<'p> { @@ -155,7 +155,7 @@ impl<'p> WebhookConsumer<'p> { queue: &'p PgQueue, poll_interval: time::Duration, request_timeout: time::Duration, - max_requests: usize, + max_concurrent_jobs: usize, ) -> Result { let mut headers = header::HeaderMap::new(); headers.insert( @@ -173,7 +173,7 @@ impl<'p> WebhookConsumer<'p> { queue, poll_interval, client, - max_requests, + max_concurrent_jobs, }) } @@ -192,7 +192,7 @@ impl<'p> WebhookConsumer<'p> { /// Run this consumer to continuously process any jobs that become available. pub async fn run(&self) -> Result<(), WebhookConsumerError> { - let semaphore = Arc::new(sync::Semaphore::new(self.max_requests)); + let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); loop { // TODO: The number of jobs processed will be capped by the PG connection limit when running in transactional mode. diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index 5f0d46dce133c..f165b32409de1 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -28,7 +28,7 @@ async fn main() -> Result<(), WebhookConsumerError> { &queue, config.poll_interval.0, config.request_timeout.0, - config.max_requests, + config.max_concurrent_jobs, )?; let _ = consumer.run().await; From 5cdff6a884d6ed2ecaae8b880d2bd70ccfcf6d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 16:38:39 +0100 Subject: [PATCH 105/247] chore: Remove deprecated comment --- hook-consumer/src/consumer.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 9abe9bddd09f6..653a8907df984 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -195,7 +195,6 @@ impl<'p> WebhookConsumer<'p> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); loop { - // TODO: The number of jobs processed will be capped by the PG connection limit when running in transactional mode. let webhook_job = self.wait_for_job().await?; // reqwest::Client internally wraps with Arc, so this allocation is cheap. From 1358e3b62260dab524751a1d4a4538c99b6ddcce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 16:58:30 +0100 Subject: [PATCH 106/247] refactor: Return result from processing task --- hook-consumer/src/consumer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 653a8907df984..8da2df690c2fd 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -204,7 +204,7 @@ impl<'p> WebhookConsumer<'p> { tokio::spawn(async move { let result = process_webhook_job(client, webhook_job).await; drop(permit); - result.expect("webhook processing failed"); + result }); } } From 332c3d569afce1702edd0fe90714b457ccbcc5a3 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 11 Dec 2023 18:00:05 +0100 Subject: [PATCH 107/247] release: ship debug symbols in image for continuous profiling (#67) --- .dockerignore | 1 + Cargo.toml | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000..eb5a316cbd195 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +target diff --git a/Cargo.toml b/Cargo.toml index 983cbf2b7d696..faf9d2a0088be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,9 @@ members = [ "capture-server" ] +[profile.release] +debug = 2 # https://www.polarsignals.com/docs/rust + [workspace.dependencies] assert-json-diff = "2.0.2" axum = "0.6.15" From fb078fcc80cb275c7cb59ee16d3ebbe827e8eec2 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 12 Dec 2023 11:29:58 +0100 Subject: [PATCH 108/247] report partition_limits_key_count metric (#69) --- capture/src/partition_limits.rs | 11 +++++++++++ capture/src/server.rs | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/capture/src/partition_limits.rs b/capture/src/partition_limits.rs index 386665780ad1c..cd0148f10e1f4 100644 --- a/capture/src/partition_limits.rs +++ b/capture/src/partition_limits.rs @@ -11,6 +11,7 @@ use std::num::NonZeroU32; use std::sync::Arc; use governor::{clock, state::keyed::DefaultKeyedStateStore, Quota, RateLimiter}; +use metrics::gauge; // See: https://docs.rs/governor/latest/governor/_guide/index.html#usage-in-multiple-threads #[derive(Clone)] @@ -38,6 +39,16 @@ impl PartitionLimiter { pub fn is_limited(&self, key: &String) -> bool { self.forced_keys.contains(key) || self.limiter.check_key(key).is_err() } + + /// Reports the number of tracked keys to prometheus every 10 seconds, + /// needs to be spawned in a separate task. + pub async fn report_metrics(&self) { + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(10)); + loop { + interval.tick().await; + gauge!("partition_limits_key_count", self.limiter.len() as f64); + } + } } #[cfg(test)] diff --git a/capture/src/server.rs b/capture/src/server.rs index 32bafa83b252c..3eca676f3763f 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -49,6 +49,12 @@ where config.burst_limit, config.overflow_forced_keys, ); + if config.export_prometheus { + let partition = partition.clone(); + tokio::spawn(async move { + partition.report_metrics().await; + }); + } let sink = sink::KafkaSink::new(config.kafka, sink_liveness, partition) .expect("failed to start Kafka sink"); From f714f32ddfde6b0793180a09fa3a81b35d5abaec Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 12 Dec 2023 11:30:25 +0100 Subject: [PATCH 109/247] revert rdkafka to 0.34 to evaluate impact (#68) --- .github/workflows/rust.yml | 1 + Cargo.lock | 8 ++++---- Cargo.toml | 2 +- capture-server/tests/common.rs | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index cadd2fb6ee3a6..91cf2018ee7e4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -35,6 +35,7 @@ jobs: test: runs-on: buildjet-4vcpu-ubuntu-2204 + timeout-minutes: 10 steps: - uses: actions/checkout@v3 diff --git a/Cargo.lock b/Cargo.lock index 6bf02dfc43650..8e3b8737e2daa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1531,9 +1531,9 @@ dependencies = [ [[package]] name = "rdkafka" -version = "0.36.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54f02a5a40220f8a2dfa47ddb38ba9064475a5807a69504b6f91711df2eea63" +checksum = "053adfa02fab06e86c01d586cc68aa47ee0ff4489a59469081dc12cbcde578bf" dependencies = [ "futures-channel", "futures-util", @@ -1549,9 +1549,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.7.0+2.3.0" +version = "4.6.0+2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615" +checksum = "ad63c279fca41a27c231c450a2d2ad18288032e9cbb159ad16c9d96eba35aaaf" dependencies = [ "cmake", "libc", diff --git a/Cargo.toml b/Cargo.toml index faf9d2a0088be..ecd9b6a0ad78b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ uuid = { version = "1.3.3", features = ["serde"] } async-trait = "0.1.68" serde_urlencoded = "0.7.1" rand = "0.8.5" -rdkafka = { version = "0.36.0", features = ["cmake-build", "ssl"] } +rdkafka = { version = "0.34.0", features = ["cmake-build", "ssl"] } metrics = "0.21.1" metrics-exporter-prometheus = "0.12.1" thiserror = "1.0.48" diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index b71fdf62c9a35..ce31897583f16 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -177,6 +177,7 @@ impl EphemeralTopic { impl Drop for EphemeralTopic { fn drop(&mut self) { debug!("dropping EphemeralTopic {}...", self.topic_name); + _ = self.consumer.unassign(); self.consumer.unsubscribe(); match futures::executor::block_on(timeout( Duration::from_secs(10), From 3c7479a51a59cb5a512c84a0a76b9447965f28e6 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Tue, 12 Dec 2023 16:16:27 +0000 Subject: [PATCH 110/247] chore(ci): add short sha to docker ci image (#71) --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1b5f29bd90da3..77e731ecebcf2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,6 +31,7 @@ jobs: type=ref,event=branch type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} + type=sha - name: Set up Docker Buildx id: buildx From eae1cb13b6c534f73aba37e21d211b71e6b1eb36 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 7 Dec 2023 15:34:39 -0700 Subject: [PATCH 111/247] Add basic webhook produce endpoint --- Cargo.lock | 9 + Cargo.toml | 20 +- hook-common/Cargo.toml | 7 +- hook-common/src/lib.rs | 1 + hook-common/src/pgqueue.rs | 10 +- hook-common/src/webhook.rs | 139 +++++++++++++ hook-consumer/src/consumer.rs | 132 +----------- hook-producer/Cargo.toml | 18 +- hook-producer/src/config.rs | 9 + hook-producer/src/handlers/app.rs | 60 ++++++ hook-producer/src/handlers/index.rs | 3 - hook-producer/src/handlers/mod.rs | 17 +- hook-producer/src/handlers/webhook.rs | 278 ++++++++++++++++++++++++++ hook-producer/src/main.rs | 22 +- 14 files changed, 562 insertions(+), 163 deletions(-) create mode 100644 hook-common/src/webhook.rs create mode 100644 hook-producer/src/handlers/app.rs delete mode 100644 hook-producer/src/handlers/index.rs create mode 100644 hook-producer/src/handlers/webhook.rs diff --git a/Cargo.lock b/Cargo.lock index b24af98ae2169..73fd7cb4b7f75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -986,6 +986,7 @@ name = "hook-common" version = "0.1.0" dependencies = [ "chrono", + "http 0.2.11", "serde", "serde_derive", "sqlx", @@ -1019,11 +1020,19 @@ dependencies = [ "axum", "envconfig", "eyre", + "hook-common", + "http-body-util", "metrics", "metrics-exporter-prometheus", + "serde", + "serde_derive", + "serde_json", + "sqlx", "tokio", + "tower", "tracing", "tracing-subscriber", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e92db695f324a..60a7219d66630 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,18 +1,22 @@ [workspace] resolver = "2" -members = [ - "hook-common", - "hook-producer", - "hook-consumer", -] +members = ["hook-common", "hook-producer", "hook-consumer"] [workspace.dependencies] chrono = { version = "0.4" } serde = { version = "1.0" } serde_derive = { version = "1.0" } +serde_json = { version = "1.0" } thiserror = { version = "1.0" } -sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", "postgres", "uuid", "json", "chrono" ] } +sqlx = { version = "0.7", features = [ + "runtime-tokio", + "tls-native-tls", + "postgres", + "uuid", + "json", + "chrono", +] } tokio = { version = "1.34.0", features = ["full"] } eyre = "0.6.9" tracing = "0.1.40" @@ -20,3 +24,7 @@ tracing-subscriber = "0.3.18" envconfig = "0.10.0" metrics = "0.21.1" metrics-exporter-prometheus = "0.12.1" +http = { version = "0.2" } +url = { version = "2.5.0 " } +tower = "0.4.13" +http-body-util = "0.1.0" diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index b55a9ecd84d4d..24d1a0d663a5f 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -6,11 +6,12 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -chrono = { workspace = true} +chrono = { workspace = true } +http = { workspace = true } serde = { workspace = true } -serde_derive = { workspace = true} -thiserror = { workspace = true } +serde_derive = { workspace = true } sqlx = { workspace = true } +thiserror = { workspace = true } [dev-dependencies] tokio = { workspace = true } # We need a runtime for async tests diff --git a/hook-common/src/lib.rs b/hook-common/src/lib.rs index d1dadf32ba63e..3138f087a1ebc 100644 --- a/hook-common/src/lib.rs +++ b/hook-common/src/lib.rs @@ -1 +1,2 @@ pub mod pgqueue; +pub mod webhook; diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 7ed46550bd8bd..fb2211b33b02f 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -15,12 +15,16 @@ use thiserror::Error; /// Errors that can originate from sqlx and are wrapped by us to provide additional context. #[derive(Error, Debug)] pub enum PgQueueError { + #[error("pool creation failed with: {error}")] + PoolCreationError { error: sqlx::Error }, #[error("connection failed with: {error}")] ConnectionError { error: sqlx::Error }, #[error("{command} query failed with: {error}")] QueryError { command: String, error: sqlx::Error }, #[error("{0} is not a valid JobStatus")] ParseJobStatusError(String), + #[error("{0} is not a valid HttpMethod")] + ParseHttpMethodError(String), } #[derive(Error, Debug)] @@ -528,6 +532,7 @@ impl Default for RetryPolicy { } /// A queue implemented on top of a PostgreSQL table. +#[derive(Clone)] pub struct PgQueue { /// A name to identify this PgQueue as multiple may share a table. name: String, @@ -560,9 +565,8 @@ impl PgQueue { let name = queue_name.to_owned(); let table = table_name.to_owned(); let pool = PgPoolOptions::new() - .connect(url) - .await - .map_err(|error| PgQueueError::ConnectionError { error })?; + .connect_lazy(url) + .map_err(|error| PgQueueError::PoolCreationError { error })?; Ok(Self { name, diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs new file mode 100644 index 0000000000000..b17959ce0f6da --- /dev/null +++ b/hook-common/src/webhook.rs @@ -0,0 +1,139 @@ +use std::collections; +use std::fmt; +use std::str::FromStr; + +use serde::{de::Visitor, Deserialize, Serialize}; + +use crate::pgqueue::PgQueueError; + +/// Supported HTTP methods for webhooks. +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum HttpMethod { + DELETE, + GET, + PATCH, + POST, + PUT, +} + +/// Allow casting `HttpMethod` from strings. +impl FromStr for HttpMethod { + type Err = PgQueueError; + + fn from_str(s: &str) -> Result { + match s.to_ascii_uppercase().as_ref() { + "DELETE" => Ok(HttpMethod::DELETE), + "GET" => Ok(HttpMethod::GET), + "PATCH" => Ok(HttpMethod::PATCH), + "POST" => Ok(HttpMethod::POST), + "PUT" => Ok(HttpMethod::PUT), + invalid => Err(PgQueueError::ParseHttpMethodError(invalid.to_owned())), + } + } +} + +/// Implement `std::fmt::Display` to convert HttpMethod to string. +impl fmt::Display for HttpMethod { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + HttpMethod::DELETE => write!(f, "DELETE"), + HttpMethod::GET => write!(f, "GET"), + HttpMethod::PATCH => write!(f, "PATCH"), + HttpMethod::POST => write!(f, "POST"), + HttpMethod::PUT => write!(f, "PUT"), + } + } +} + +struct HttpMethodVisitor; + +impl<'de> Visitor<'de> for HttpMethodVisitor { + type Value = HttpMethod; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "the string representation of HttpMethod") + } + + fn visit_str(self, s: &str) -> Result + where + E: serde::de::Error, + { + match HttpMethod::from_str(s) { + Ok(method) => Ok(method), + Err(_) => Err(serde::de::Error::invalid_value( + serde::de::Unexpected::Str(s), + &self, + )), + } + } +} + +/// Deserialize required to read `HttpMethod` from database. +impl<'de> Deserialize<'de> for HttpMethod { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_str(HttpMethodVisitor) + } +} + +/// Serialize required to write `HttpMethod` to database. +impl Serialize for HttpMethod { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +/// Convenience to cast `HttpMethod` to `http::Method`. +/// Not all `http::Method` variants are valid `HttpMethod` variants, hence why we +/// can't just use the former or implement `From`. +impl From for http::Method { + fn from(val: HttpMethod) -> Self { + match val { + HttpMethod::DELETE => http::Method::DELETE, + HttpMethod::GET => http::Method::GET, + HttpMethod::PATCH => http::Method::PATCH, + HttpMethod::POST => http::Method::POST, + HttpMethod::PUT => http::Method::PUT, + } + } +} + +impl From<&HttpMethod> for http::Method { + fn from(val: &HttpMethod) -> Self { + match val { + HttpMethod::DELETE => http::Method::DELETE, + HttpMethod::GET => http::Method::GET, + HttpMethod::PATCH => http::Method::PATCH, + HttpMethod::POST => http::Method::POST, + HttpMethod::PUT => http::Method::PUT, + } + } +} + +/// `JobParameters` required for the `WebhookConsumer` to execute a webhook. +/// These parameters should match the exported Webhook interface that PostHog plugins. +/// implement. See: https://github.com/PostHog/plugin-scaffold/blob/main/src/types.ts#L15. +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct WebhookJobParameters { + pub body: String, + pub headers: collections::HashMap, + pub method: HttpMethod, + pub url: String, + + // These should be set if the Webhook is associated with a plugin `composeWebhook` invocation. + pub team_id: Option, + pub plugin_id: Option, + pub plugin_config_id: Option, + + #[serde(default = "default_max_attempts")] + pub max_attempts: i32, +} + +fn default_max_attempts() -> i32 { + 3 +} diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 8da2df690c2fd..406dd732ae797 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -1,140 +1,16 @@ use std::collections; -use std::fmt; -use std::str::FromStr; use std::sync::Arc; use std::time; use async_std::task; use hook_common::pgqueue::{PgJobError, PgQueue, PgQueueError, PgTransactionJob}; +use hook_common::webhook::{HttpMethod, WebhookJobParameters}; use http::StatusCode; use reqwest::header; -use serde::{de::Visitor, Deserialize, Serialize}; use tokio::sync; use crate::error::WebhookConsumerError; -/// Supported HTTP methods for webhooks. -#[derive(Debug, PartialEq, Clone, Copy)] -pub enum HttpMethod { - DELETE, - GET, - PATCH, - POST, - PUT, -} - -/// Allow casting `HttpMethod` from strings. -impl FromStr for HttpMethod { - type Err = WebhookConsumerError; - - fn from_str(s: &str) -> Result { - match s.to_ascii_uppercase().as_ref() { - "DELETE" => Ok(HttpMethod::DELETE), - "GET" => Ok(HttpMethod::GET), - "PATCH" => Ok(HttpMethod::PATCH), - "POST" => Ok(HttpMethod::POST), - "PUT" => Ok(HttpMethod::PUT), - invalid => Err(WebhookConsumerError::ParseHttpMethodError( - invalid.to_owned(), - )), - } - } -} - -/// Implement `std::fmt::Display` to convert HttpMethod to string. -impl fmt::Display for HttpMethod { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - HttpMethod::DELETE => write!(f, "DELETE"), - HttpMethod::GET => write!(f, "GET"), - HttpMethod::PATCH => write!(f, "PATCH"), - HttpMethod::POST => write!(f, "POST"), - HttpMethod::PUT => write!(f, "PUT"), - } - } -} - -struct HttpMethodVisitor; - -impl<'de> Visitor<'de> for HttpMethodVisitor { - type Value = HttpMethod; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - write!(formatter, "the string representation of HttpMethod") - } - - fn visit_str(self, s: &str) -> Result - where - E: serde::de::Error, - { - match HttpMethod::from_str(s) { - Ok(method) => Ok(method), - Err(_) => Err(serde::de::Error::invalid_value( - serde::de::Unexpected::Str(s), - &self, - )), - } - } -} - -/// Deserialize required to read `HttpMethod` from database. -impl<'de> Deserialize<'de> for HttpMethod { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_str(HttpMethodVisitor) - } -} - -/// Serialize required to write `HttpMethod` to database. -impl Serialize for HttpMethod { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -/// Convenience to cast `HttpMethod` to `http::Method`. -/// Not all `http::Method` variants are valid `HttpMethod` variants, hence why we -/// can't just use the former or implement `From`. -impl From for http::Method { - fn from(val: HttpMethod) -> Self { - match val { - HttpMethod::DELETE => http::Method::DELETE, - HttpMethod::GET => http::Method::GET, - HttpMethod::PATCH => http::Method::PATCH, - HttpMethod::POST => http::Method::POST, - HttpMethod::PUT => http::Method::PUT, - } - } -} - -impl From<&HttpMethod> for http::Method { - fn from(val: &HttpMethod) -> Self { - match val { - HttpMethod::DELETE => http::Method::DELETE, - HttpMethod::GET => http::Method::GET, - HttpMethod::PATCH => http::Method::PATCH, - HttpMethod::POST => http::Method::POST, - HttpMethod::PUT => http::Method::PUT, - } - } -} - -/// `JobParameters` required for the `WebhookConsumer` to execute a webhook. -/// These parameters should match the exported Webhook interface that PostHog plugins. -/// implement. See: https://github.com/PostHog/plugin-scaffold/blob/main/src/types.ts#L15. -#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] -pub struct WebhookJobParameters { - body: String, - headers: collections::HashMap, - method: HttpMethod, - url: String, -} - /// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. pub struct WebhookConsumer<'p> { /// An identifier for this consumer. Used to mark jobs we have consumed. @@ -432,6 +308,12 @@ mod tests { headers: collections::HashMap::new(), method: HttpMethod::POST, url: "localhost".to_owned(), + + team_id: Some(1), + plugin_id: Some(2), + plugin_config_id: Some(3), + + max_attempts: 1, }; // enqueue takes ownership of the job enqueued to avoid bugs that can cause duplicate jobs. // Normally, a separate application would be enqueueing jobs for us to consume, so no ownership diff --git a/hook-producer/Cargo.toml b/hook-producer/Cargo.toml index 47ef532891436..ef1a24b01820a 100644 --- a/hook-producer/Cargo.toml +++ b/hook-producer/Cargo.toml @@ -6,11 +6,19 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axum = { version="0.7.1", features=["http2"] } -tokio = { workspace = true } -eyre = {workspace = true } -tracing = {workspace = true} -tracing-subscriber = {workspace = true} +axum = { version = "0.7.1", features = ["http2"] } envconfig = { workspace = true } +eyre = { workspace = true } +hook-common = { path = "../hook-common" } +http-body-util = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } +serde = { workspace = true } +serde_derive = { workspace = true } +serde_json = { workspace = true } +sqlx = { workspace = true } +tokio = { workspace = true } +tower = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = { workspace = true } diff --git a/hook-producer/src/config.rs b/hook-producer/src/config.rs index 9d093c652efbb..87fad5d07dbb1 100644 --- a/hook-producer/src/config.rs +++ b/hook-producer/src/config.rs @@ -7,6 +7,15 @@ pub struct Config { #[envconfig(from = "BIND_PORT", default = "8000")] pub port: u16, + + #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] + pub database_url: String, + + #[envconfig(default = "job_queue")] + pub table_name: String, + + #[envconfig(default = "default")] + pub queue_name: String, } impl Config { diff --git a/hook-producer/src/handlers/app.rs b/hook-producer/src/handlers/app.rs new file mode 100644 index 0000000000000..911a04d397416 --- /dev/null +++ b/hook-producer/src/handlers/app.rs @@ -0,0 +1,60 @@ +use axum::{routing, Router}; +use metrics_exporter_prometheus::PrometheusHandle; + +use hook_common::pgqueue::PgQueue; + +use super::webhook; + +pub fn app(pg_pool: PgQueue, metrics: Option) -> Router { + Router::new() + .route("/", routing::get(index)) + .route( + "/metrics", + routing::get(move || match metrics { + Some(ref recorder_handle) => std::future::ready(recorder_handle.render()), + None => std::future::ready("no metrics recorder installed".to_owned()), + }), + ) + .route("/webhook", routing::post(webhook::post).with_state(pg_pool)) + .layer(axum::middleware::from_fn(crate::metrics::track_metrics)) +} + +pub async fn index() -> &'static str { + "rusty hook" +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{ + body::Body, + http::{Request, StatusCode}, + }; + use hook_common::pgqueue::{PgQueue, RetryPolicy}; + use http_body_util::BodyExt; // for `collect` + use tower::ServiceExt; // for `call`, `oneshot`, and `ready` + + #[tokio::test] + async fn index() { + let pg_queue = PgQueue::new( + "test_index", + "job_queue", + "postgres://posthog:posthog@localhost:15432/test_database", + RetryPolicy::default(), + ) + .await + .expect("failed to construct pg_queue"); + + let app = app(pg_queue, None); + + let response = app + .oneshot(Request::builder().uri("/").body(Body::empty()).unwrap()) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&body[..], b"rusty hook"); + } +} diff --git a/hook-producer/src/handlers/index.rs b/hook-producer/src/handlers/index.rs deleted file mode 100644 index 56896fa63a483..0000000000000 --- a/hook-producer/src/handlers/index.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub async fn get() -> &'static str { - "rusty hook" -} diff --git a/hook-producer/src/handlers/mod.rs b/hook-producer/src/handlers/mod.rs index 25040731d5688..88f96717c130a 100644 --- a/hook-producer/src/handlers/mod.rs +++ b/hook-producer/src/handlers/mod.rs @@ -1,15 +1,4 @@ -use axum::{routing, Router}; +mod app; +mod webhook; -mod index; - -pub fn router() -> Router { - let recorder_handle = crate::metrics::setup_metrics_recorder(); - - Router::new() - .route("/", routing::get(index::get)) - .route( - "/metrics", - routing::get(move || std::future::ready(recorder_handle.render())), - ) - .layer(axum::middleware::from_fn(crate::metrics::track_metrics)) -} +pub use app::app; diff --git a/hook-producer/src/handlers/webhook.rs b/hook-producer/src/handlers/webhook.rs new file mode 100644 index 0000000000000..7de1126fe4db8 --- /dev/null +++ b/hook-producer/src/handlers/webhook.rs @@ -0,0 +1,278 @@ +use axum::{extract::State, http::StatusCode, Json}; +use hook_common::webhook::WebhookJobParameters; +use serde_derive::Deserialize; +use url::Url; + +use hook_common::pgqueue::{NewJob, PgQueue}; +use serde::Serialize; +use tracing::{debug, error}; + +const MAX_BODY_SIZE: usize = 1_000_000; + +#[derive(Serialize, Deserialize)] +pub struct WebhookPostResponse { + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +pub async fn post( + State(pg_queue): State, + Json(payload): Json, +) -> Result, (StatusCode, Json)> { + debug!("received payload: {:?}", payload); + + if payload.body.len() > MAX_BODY_SIZE { + return Err(( + StatusCode::BAD_REQUEST, + Json(WebhookPostResponse { + error: Some("body too large".to_owned()), + }), + )); + } + + let url_hostname = get_hostname(&payload.url)?; + let job = NewJob::new(payload.max_attempts, payload, url_hostname.as_str()); + + pg_queue.enqueue(job).await.map_err(internal_error)?; + + Ok(Json(WebhookPostResponse { error: None })) +} + +fn internal_error(err: E) -> (StatusCode, Json) +where + E: std::error::Error, +{ + error!("internal error: {}", err); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(WebhookPostResponse { + error: Some(err.to_string()), + }), + ) +} + +fn get_hostname(url_str: &str) -> Result)> { + let url = Url::parse(url_str).map_err(|_| { + ( + StatusCode::BAD_REQUEST, + Json(WebhookPostResponse { + error: Some("could not parse url".to_owned()), + }), + ) + })?; + + match url.host_str() { + Some(hostname) => Ok(hostname.to_owned()), + None => Err(( + StatusCode::BAD_REQUEST, + Json(WebhookPostResponse { + error: Some("couldn't extract hostname from url".to_owned()), + }), + )), + } +} + +#[cfg(test)] +mod tests { + use axum::{ + body::Body, + http::{self, Request, StatusCode}, + }; + use hook_common::pgqueue::{PgQueue, RetryPolicy}; + use hook_common::webhook::{HttpMethod, WebhookJobParameters}; + use http_body_util::BodyExt; // for `collect` + use std::collections; + use tower::ServiceExt; // for `call`, `oneshot`, and `ready` + + use crate::handlers::app; + + #[tokio::test] + async fn webhook_success() { + let pg_queue = PgQueue::new( + "test_index", + "job_queue", + "postgres://posthog:posthog@localhost:15432/test_database", + RetryPolicy::default(), + ) + .await + .expect("failed to construct pg_queue"); + + let app = app(pg_queue, None); + + let mut headers = collections::HashMap::new(); + headers.insert("Content-Type".to_owned(), "application/json".to_owned()); + let response = app + .oneshot( + Request::builder() + .method(http::Method::POST) + .uri("/webhook") + .header(http::header::CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::to_string(&WebhookJobParameters { + headers, + method: HttpMethod::POST, + url: "http://example.com/".to_owned(), + body: r#"{"a": "b"}"#.to_owned(), + + team_id: Some(1), + plugin_id: Some(2), + plugin_config_id: Some(3), + + max_attempts: 1, + }) + .unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&body[..], b"{}"); + } + + #[tokio::test] + async fn webhook_bad_url() { + let pg_queue = PgQueue::new( + "test_index", + "job_queue", + "postgres://posthog:posthog@localhost:15432/test_database", + RetryPolicy::default(), + ) + .await + .expect("failed to construct pg_queue"); + + let app = app(pg_queue, None); + + let response = app + .oneshot( + Request::builder() + .method(http::Method::POST) + .uri("/webhook") + .header(http::header::CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::to_string(&WebhookJobParameters { + headers: collections::HashMap::new(), + method: HttpMethod::POST, + url: "invalid".to_owned(), + body: r#"{"a": "b"}"#.to_owned(), + + team_id: Some(1), + plugin_id: Some(2), + plugin_config_id: Some(3), + + max_attempts: 1, + }) + .unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn webhook_payload_missing_fields() { + let pg_queue = PgQueue::new( + "test_index", + "job_queue", + "postgres://posthog:posthog@localhost:15432/test_database", + RetryPolicy::default(), + ) + .await + .expect("failed to construct pg_queue"); + + let app = app(pg_queue, None); + + let response = app + .oneshot( + Request::builder() + .method(http::Method::POST) + .uri("/webhook") + .header(http::header::CONTENT_TYPE, "application/json") + .body("{}".to_owned()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); + } + + #[tokio::test] + async fn webhook_payload_not_json() { + let pg_queue = PgQueue::new( + "test_index", + "job_queue", + "postgres://posthog:posthog@localhost:15432/test_database", + RetryPolicy::default(), + ) + .await + .expect("failed to construct pg_queue"); + + let app = app(pg_queue, None); + + let response = app + .oneshot( + Request::builder() + .method(http::Method::POST) + .uri("/webhook") + .header(http::header::CONTENT_TYPE, "application/json") + .body("x".to_owned()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn webhook_payload_body_too_large() { + let pg_queue = PgQueue::new( + "test_index", + "job_queue", + "postgres://posthog:posthog@localhost:15432/test_database", + RetryPolicy::default(), + ) + .await + .expect("failed to construct pg_queue"); + + let app = app(pg_queue, None); + + let bytes: Vec = vec![b'a'; 1_000_000 * 2]; + let long_string = String::from_utf8_lossy(&bytes); + + let response = app + .oneshot( + Request::builder() + .method(http::Method::POST) + .uri("/webhook") + .header(http::header::CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::to_string(&WebhookJobParameters { + headers: collections::HashMap::new(), + method: HttpMethod::POST, + url: "http://example.com".to_owned(), + body: long_string.to_string(), + + team_id: Some(1), + plugin_id: Some(2), + plugin_config_id: Some(3), + + max_attempts: 1, + }) + .unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } +} diff --git a/hook-producer/src/main.rs b/hook-producer/src/main.rs index 118829b00d78b..29da8dd6efe0e 100644 --- a/hook-producer/src/main.rs +++ b/hook-producer/src/main.rs @@ -1,10 +1,10 @@ use axum::Router; - use config::Config; use envconfig::Envconfig; - use eyre::Result; +use hook_common::pgqueue::{PgQueue, RetryPolicy}; + mod config; mod handlers; mod metrics; @@ -21,10 +21,24 @@ async fn listen(app: Router, bind: String) -> Result<()> { async fn main() { tracing_subscriber::fmt::init(); - let app = handlers::router(); - let config = Config::init_from_env().expect("failed to load configuration from env"); + let pg_queue = PgQueue::new( + // TODO: Coupling the queue name to the PgQueue object doesn't seem ideal from the producer + // side, but we don't need more than one queue for now. + &config.queue_name, + &config.table_name, + &config.database_url, + // TODO: It seems unnecessary that the producer side needs to know about the retry policy. + RetryPolicy::default(), + ) + .await + .expect("failed to initialize queue"); + + let recorder_handle = crate::metrics::setup_metrics_recorder(); + + let app = handlers::app(pg_queue, Some(recorder_handle)); + match listen(app, config.bind()).await { Ok(_) => {} Err(e) => tracing::error!("failed to start hook-producer http server, {}", e), From 454b96d75c144b4dd428fc7862565d5a447ec4d9 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Wed, 13 Dec 2023 13:34:45 +0000 Subject: [PATCH 112/247] feat: clean up partition limiter state on schedule (#72) * feat: clean up partition limiter state on schedule * add randomness --- capture/src/partition_limits.rs | 18 ++++++++++++++++++ capture/src/server.rs | 11 +++++++++++ 2 files changed, 29 insertions(+) diff --git a/capture/src/partition_limits.rs b/capture/src/partition_limits.rs index cd0148f10e1f4..7059b452ce51f 100644 --- a/capture/src/partition_limits.rs +++ b/capture/src/partition_limits.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use governor::{clock, state::keyed::DefaultKeyedStateStore, Quota, RateLimiter}; use metrics::gauge; +use rand::Rng; // See: https://docs.rs/governor/latest/governor/_guide/index.html#usage-in-multiple-threads #[derive(Clone)] @@ -49,6 +50,23 @@ impl PartitionLimiter { gauge!("partition_limits_key_count", self.limiter.len() as f64); } } + + /// Clean up the rate limiter state, once per minute. Ensure we don't use more memory than + /// necessary. + pub async fn clean_state(&self) { + // Give a small amount of randomness to the interval to ensure we don't have all replicas + // locking at the same time. The lock isn't going to be held for long, but this will reduce + // impact regardless. + let interval_secs = rand::thread_rng().gen_range(60..70); + + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(interval_secs)); + loop { + interval.tick().await; + + self.limiter.retain_recent(); + self.limiter.shrink_to_fit(); + } + } } #[cfg(test)] diff --git a/capture/src/server.rs b/capture/src/server.rs index 3eca676f3763f..9b8d60cbc23c4 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -55,6 +55,17 @@ where partition.report_metrics().await; }); } + + { + // Ensure that the rate limiter state does not grow unbounded + + let partition = partition.clone(); + + tokio::spawn(async move { + partition.clean_state().await; + }); + } + let sink = sink::KafkaSink::new(config.kafka, sink_liveness, partition) .expect("failed to start Kafka sink"); From 1a7e87aa267aa36d6988a186a8cb4f7851582e15 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 13 Dec 2023 15:33:45 +0100 Subject: [PATCH 113/247] carve out the limiters and sinks sub-modules (#70) --- capture-server/tests/common.rs | 4 +- capture-server/tests/events.rs | 12 ++--- capture/src/capture.rs | 6 +-- capture/src/config.rs | 4 +- capture/src/lib.rs | 5 +- .../billing.rs} | 2 +- capture/src/limiters/mod.rs | 2 + .../overflow.rs} | 29 ++++++----- capture/src/router.rs | 6 +-- capture/src/server.rs | 22 ++++---- capture/src/{sink.rs => sinks/kafka.rs} | 51 ++++--------------- capture/src/sinks/mod.rs | 13 +++++ capture/src/sinks/print.rs | 31 +++++++++++ capture/tests/django_compat.rs | 6 +-- 14 files changed, 104 insertions(+), 89 deletions(-) rename capture/src/{billing_limits.rs => limiters/billing.rs} (99%) create mode 100644 capture/src/limiters/mod.rs rename capture/src/{partition_limits.rs => limiters/overflow.rs} (80%) rename capture/src/{sink.rs => sinks/kafka.rs} (91%) create mode 100644 capture/src/sinks/mod.rs create mode 100644 capture/src/sinks/print.rs diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index ce31897583f16..214ecc80a62b5 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -28,8 +28,8 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { print_sink: false, address: SocketAddr::from_str("127.0.0.1:0").unwrap(), redis_url: "redis://localhost:6379/".to_string(), - burst_limit: NonZeroU32::new(5).unwrap(), - per_second_limit: NonZeroU32::new(10).unwrap(), + overflow_burst_limit: NonZeroU32::new(5).unwrap(), + overflow_per_second_limit: NonZeroU32::new(10).unwrap(), overflow_forced_keys: None, kafka: KafkaConfig { kafka_producer_linger_ms: 0, // Send messages as soon as possible diff --git a/capture-server/tests/events.rs b/capture-server/tests/events.rs index b38ac5a1a63fb..56fcdf79bf15d 100644 --- a/capture-server/tests/events.rs +++ b/capture-server/tests/events.rs @@ -77,7 +77,7 @@ async fn it_captures_a_batch() -> Result<()> { } #[tokio::test] -async fn it_is_limited_with_burst() -> Result<()> { +async fn it_overflows_events_on_burst() -> Result<()> { setup_tracing(); let token = random_string("token", 16); @@ -87,8 +87,8 @@ async fn it_is_limited_with_burst() -> Result<()> { let mut config = DEFAULT_CONFIG.clone(); config.kafka.kafka_topic = topic.topic_name().to_string(); - config.burst_limit = NonZeroU32::new(2).unwrap(); - config.per_second_limit = NonZeroU32::new(1).unwrap(); + config.overflow_burst_limit = NonZeroU32::new(2).unwrap(); + config.overflow_per_second_limit = NonZeroU32::new(1).unwrap(); let server = ServerHandle::for_config(config); @@ -125,7 +125,7 @@ async fn it_is_limited_with_burst() -> Result<()> { } #[tokio::test] -async fn it_does_not_partition_limit_different_ids() -> Result<()> { +async fn it_does_not_overflow_team_with_different_ids() -> Result<()> { setup_tracing(); let token = random_string("token", 16); @@ -136,8 +136,8 @@ async fn it_does_not_partition_limit_different_ids() -> Result<()> { let mut config = DEFAULT_CONFIG.clone(); config.kafka.kafka_topic = topic.topic_name().to_string(); - config.burst_limit = NonZeroU32::new(1).unwrap(); - config.per_second_limit = NonZeroU32::new(1).unwrap(); + config.overflow_burst_limit = NonZeroU32::new(1).unwrap(); + config.overflow_per_second_limit = NonZeroU32::new(1).unwrap(); let server = ServerHandle::for_config(config); diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 37e2872a9f9b4..6a903787a2c32 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -15,14 +15,14 @@ use metrics::counter; use time::OffsetDateTime; use tracing::instrument; -use crate::billing_limits::QuotaResource; use crate::event::{Compression, ProcessingContext}; +use crate::limiters::billing::QuotaResource; use crate::prometheus::report_dropped_events; use crate::token::validate_token; use crate::{ api::{CaptureError, CaptureResponse, CaptureResponseCode}, event::{EventFormData, EventQuery, ProcessedEvent, RawEvent}, - router, sink, + router, sinks, utils::uuid_v7, }; @@ -209,7 +209,7 @@ pub fn extract_and_verify_token(events: &[RawEvent]) -> Result( - sink: Arc, + sink: Arc, events: &'a [RawEvent], context: &'a ProcessingContext, ) -> Result<(), CaptureError> { diff --git a/capture/src/config.rs b/capture/src/config.rs index 69a085dd927b0..0c6ab1ce9eb62 100644 --- a/capture/src/config.rs +++ b/capture/src/config.rs @@ -14,10 +14,10 @@ pub struct Config { pub otel_url: Option, #[envconfig(default = "100")] - pub per_second_limit: NonZeroU32, + pub overflow_per_second_limit: NonZeroU32, #[envconfig(default = "1000")] - pub burst_limit: NonZeroU32, + pub overflow_burst_limit: NonZeroU32, pub overflow_forced_keys: Option, // Coma-delimited keys diff --git a/capture/src/lib.rs b/capture/src/lib.rs index eea915c307f71..058e994186edb 100644 --- a/capture/src/lib.rs +++ b/capture/src/lib.rs @@ -1,15 +1,14 @@ pub mod api; -pub mod billing_limits; pub mod capture; pub mod config; pub mod event; pub mod health; -pub mod partition_limits; +pub mod limiters; pub mod prometheus; pub mod redis; pub mod router; pub mod server; -pub mod sink; +pub mod sinks; pub mod time; pub mod token; pub mod utils; diff --git a/capture/src/billing_limits.rs b/capture/src/limiters/billing.rs similarity index 99% rename from capture/src/billing_limits.rs rename to capture/src/limiters/billing.rs index 9fa0fdd0e953e..b908519dda265 100644 --- a/capture/src/billing_limits.rs +++ b/capture/src/limiters/billing.rs @@ -166,7 +166,7 @@ mod tests { use time::Duration; use crate::{ - billing_limits::{BillingLimiter, QuotaResource}, + limiters::billing::{BillingLimiter, QuotaResource}, redis::MockRedisClient, }; diff --git a/capture/src/limiters/mod.rs b/capture/src/limiters/mod.rs new file mode 100644 index 0000000000000..58b2dcc1a5c8c --- /dev/null +++ b/capture/src/limiters/mod.rs @@ -0,0 +1,2 @@ +pub mod billing; +pub mod overflow; diff --git a/capture/src/partition_limits.rs b/capture/src/limiters/overflow.rs similarity index 80% rename from capture/src/partition_limits.rs rename to capture/src/limiters/overflow.rs index 7059b452ce51f..0e91a99d213b7 100644 --- a/capture/src/partition_limits.rs +++ b/capture/src/limiters/overflow.rs @@ -1,11 +1,12 @@ -/// When a customer is writing too often to the same key, we get hot partitions. This negatively -/// affects our write latency and cluster health. We try to provide ordering guarantees wherever -/// possible, but this does require that we map key -> partition. +/// The analytics ingestion pipeline provides ordering guarantees for events of the same +/// token and distinct_id. We currently achieve this through a locality constraint on the +/// Kafka partition (consistent partition hashing through a computed key). /// -/// If the write-rate reaches a certain amount, we need to be able to handle the hot partition -/// before it causes a negative impact. In this case, instead of passing the error to the customer -/// with a 429, we relax our ordering constraints and temporarily override the key, meaning the -/// customers data will be spread across all partitions. +/// Volume spikes to a given key can create lag on the destination partition and induce +/// ingestion lag. To protect the downstream systems, capture can relax this locality +/// constraint when bursts are detected. When that happens, the excess traffic will be +/// spread across all partitions and be processed by the overflow consumer, without +/// strict ordering guarantees. use std::collections::HashSet; use std::num::NonZeroU32; use std::sync::Arc; @@ -16,12 +17,12 @@ use rand::Rng; // See: https://docs.rs/governor/latest/governor/_guide/index.html#usage-in-multiple-threads #[derive(Clone)] -pub struct PartitionLimiter { +pub struct OverflowLimiter { limiter: Arc, clock::DefaultClock>>, forced_keys: HashSet, } -impl PartitionLimiter { +impl OverflowLimiter { pub fn new(per_second: NonZeroU32, burst: NonZeroU32, forced_keys: Option) -> Self { let quota = Quota::per_second(per_second).allow_burst(burst); let limiter = Arc::new(governor::RateLimiter::dashmap(quota)); @@ -31,7 +32,7 @@ impl PartitionLimiter { Some(values) => values.split(',').map(String::from).collect(), }; - PartitionLimiter { + OverflowLimiter { limiter, forced_keys, } @@ -71,12 +72,12 @@ impl PartitionLimiter { #[cfg(test)] mod tests { - use crate::partition_limits::PartitionLimiter; + use crate::limiters::overflow::OverflowLimiter; use std::num::NonZeroU32; #[tokio::test] async fn low_limits() { - let limiter = PartitionLimiter::new( + let limiter = OverflowLimiter::new( NonZeroU32::new(1).unwrap(), NonZeroU32::new(1).unwrap(), None, @@ -89,7 +90,7 @@ mod tests { #[tokio::test] async fn bursting() { - let limiter = PartitionLimiter::new( + let limiter = OverflowLimiter::new( NonZeroU32::new(1).unwrap(), NonZeroU32::new(3).unwrap(), None, @@ -109,7 +110,7 @@ mod tests { let key_three = String::from("three"); let forced_keys = Some(String::from("one,three")); - let limiter = PartitionLimiter::new( + let limiter = OverflowLimiter::new( NonZeroU32::new(1).unwrap(), NonZeroU32::new(1).unwrap(), forced_keys, diff --git a/capture/src/router.rs b/capture/src/router.rs index 6f2f044f88c67..d02e63faaad5d 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -10,13 +10,13 @@ use tower_http::cors::{AllowHeaders, AllowOrigin, CorsLayer}; use tower_http::trace::TraceLayer; use crate::health::HealthRegistry; -use crate::{billing_limits::BillingLimiter, capture, redis::Client, sink, time::TimeSource}; +use crate::{capture, limiters::billing::BillingLimiter, redis::Client, sinks, time::TimeSource}; use crate::prometheus::{setup_metrics_recorder, track_metrics}; #[derive(Clone)] pub struct State { - pub sink: Arc, + pub sink: Arc, pub timesource: Arc, pub redis: Arc, pub billing: BillingLimiter, @@ -28,7 +28,7 @@ async fn index() -> &'static str { pub fn router< TZ: TimeSource + Send + Sync + 'static, - S: sink::EventSink + Send + Sync + 'static, + S: sinks::Event + Send + Sync + 'static, R: Client + Send + Sync + 'static, >( timesource: TZ, diff --git a/capture/src/server.rs b/capture/src/server.rs index 9b8d60cbc23c4..22a1f3bc0bf04 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -4,12 +4,14 @@ use std::sync::Arc; use time::Duration; -use crate::billing_limits::BillingLimiter; use crate::config::Config; use crate::health::{ComponentStatus, HealthRegistry}; -use crate::partition_limits::PartitionLimiter; +use crate::limiters::billing::BillingLimiter; +use crate::limiters::overflow::OverflowLimiter; use crate::redis::RedisClient; -use crate::{router, sink}; +use crate::router; +use crate::sinks::kafka::KafkaSink; +use crate::sinks::print::PrintSink; pub async fn serve(config: Config, listener: TcpListener, shutdown: F) where @@ -34,7 +36,7 @@ where router::router( crate::time::SystemTime {}, liveness, - sink::PrintSink {}, + PrintSink {}, redis_client, billing, config.export_prometheus, @@ -44,9 +46,9 @@ where .register("rdkafka".to_string(), Duration::seconds(30)) .await; - let partition = PartitionLimiter::new( - config.per_second_limit, - config.burst_limit, + let partition = OverflowLimiter::new( + config.overflow_per_second_limit, + config.overflow_burst_limit, config.overflow_forced_keys, ); if config.export_prometheus { @@ -55,18 +57,14 @@ where partition.report_metrics().await; }); } - { // Ensure that the rate limiter state does not grow unbounded - let partition = partition.clone(); - tokio::spawn(async move { partition.clean_state().await; }); } - - let sink = sink::KafkaSink::new(config.kafka, sink_liveness, partition) + let sink = KafkaSink::new(config.kafka, sink_liveness, partition) .expect("failed to start Kafka sink"); router::router( diff --git a/capture/src/sink.rs b/capture/src/sinks/kafka.rs similarity index 91% rename from capture/src/sink.rs rename to capture/src/sinks/kafka.rs index af83e20c1a763..dc57c11e458be 100644 --- a/capture/src/sink.rs +++ b/capture/src/sinks/kafka.rs @@ -2,11 +2,10 @@ use std::time::Duration; use async_trait::async_trait; use metrics::{absolute_counter, counter, gauge, histogram}; -use rdkafka::config::ClientConfig; use rdkafka::error::{KafkaError, RDKafkaErrorCode}; -use rdkafka::producer::future_producer::{FutureProducer, FutureRecord}; -use rdkafka::producer::{DeliveryFuture, Producer}; +use rdkafka::producer::{DeliveryFuture, FutureProducer, FutureRecord, Producer}; use rdkafka::util::Timeout; +use rdkafka::ClientConfig; use tokio::task::JoinSet; use tracing::log::{debug, error, info}; use tracing::{info_span, instrument, Instrument}; @@ -15,38 +14,9 @@ use crate::api::CaptureError; use crate::config::KafkaConfig; use crate::event::ProcessedEvent; use crate::health::HealthHandle; -use crate::partition_limits::PartitionLimiter; +use crate::limiters::overflow::OverflowLimiter; use crate::prometheus::report_dropped_events; - -#[async_trait] -pub trait EventSink { - async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError>; - async fn send_batch(&self, events: Vec) -> Result<(), CaptureError>; -} - -pub struct PrintSink {} - -#[async_trait] -impl EventSink for PrintSink { - async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { - info!("single event: {:?}", event); - counter!("capture_events_ingested_total", 1); - - Ok(()) - } - async fn send_batch(&self, events: Vec) -> Result<(), CaptureError> { - let span = tracing::span!(tracing::Level::INFO, "batch of events"); - let _enter = span.enter(); - - histogram!("capture_event_batch_size", events.len() as f64); - counter!("capture_events_ingested_total", events.len() as u64); - for event in events { - info!("event: {:?}", event); - } - - Ok(()) - } -} +use crate::sinks::Event; struct KafkaContext { liveness: HealthHandle, @@ -113,14 +83,14 @@ impl rdkafka::ClientContext for KafkaContext { pub struct KafkaSink { producer: FutureProducer, topic: String, - partition: PartitionLimiter, + partition: OverflowLimiter, } impl KafkaSink { pub fn new( config: KafkaConfig, liveness: HealthHandle, - partition: PartitionLimiter, + partition: OverflowLimiter, ) -> anyhow::Result { info!("connecting to Kafka brokers at {}...", config.kafka_hosts); @@ -234,7 +204,7 @@ impl KafkaSink { } #[async_trait] -impl EventSink for KafkaSink { +impl Event for KafkaSink { #[instrument(skip_all)] async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { let limited = self.partition.is_limited(&event.key()); @@ -294,8 +264,9 @@ mod tests { use crate::config; use crate::event::ProcessedEvent; use crate::health::HealthRegistry; - use crate::partition_limits::PartitionLimiter; - use crate::sink::{EventSink, KafkaSink}; + use crate::limiters::overflow::OverflowLimiter; + use crate::sinks::kafka::KafkaSink; + use crate::sinks::Event; use crate::utils::uuid_v7; use rand::distributions::Alphanumeric; use rand::Rng; @@ -310,7 +281,7 @@ mod tests { let handle = registry .register("one".to_string(), Duration::seconds(30)) .await; - let limiter = PartitionLimiter::new( + let limiter = OverflowLimiter::new( NonZeroU32::new(10).unwrap(), NonZeroU32::new(10).unwrap(), None, diff --git a/capture/src/sinks/mod.rs b/capture/src/sinks/mod.rs new file mode 100644 index 0000000000000..0747f0e222a79 --- /dev/null +++ b/capture/src/sinks/mod.rs @@ -0,0 +1,13 @@ +use async_trait::async_trait; + +use crate::api::CaptureError; +use crate::event::ProcessedEvent; + +pub mod kafka; +pub mod print; + +#[async_trait] +pub trait Event { + async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError>; + async fn send_batch(&self, events: Vec) -> Result<(), CaptureError>; +} diff --git a/capture/src/sinks/print.rs b/capture/src/sinks/print.rs new file mode 100644 index 0000000000000..50bc1ade690ef --- /dev/null +++ b/capture/src/sinks/print.rs @@ -0,0 +1,31 @@ +use async_trait::async_trait; +use metrics::{counter, histogram}; +use tracing::log::info; + +use crate::api::CaptureError; +use crate::event::ProcessedEvent; +use crate::sinks::Event; + +pub struct PrintSink {} + +#[async_trait] +impl Event for PrintSink { + async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { + info!("single event: {:?}", event); + counter!("capture_events_ingested_total", 1); + + Ok(()) + } + async fn send_batch(&self, events: Vec) -> Result<(), CaptureError> { + let span = tracing::span!(tracing::Level::INFO, "batch of events"); + let _enter = span.enter(); + + histogram!("capture_event_batch_size", events.len() as f64); + counter!("capture_events_ingested_total", events.len() as u64); + for event in events { + info!("event: {:?}", event); + } + + Ok(()) + } +} diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index d1d075bdab97c..5d778997a89a9 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -5,12 +5,12 @@ use axum_test_helper::TestClient; use base64::engine::general_purpose; use base64::Engine; use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode}; -use capture::billing_limits::BillingLimiter; use capture::event::ProcessedEvent; use capture::health::HealthRegistry; +use capture::limiters::billing::BillingLimiter; use capture::redis::MockRedisClient; use capture::router::router; -use capture::sink::EventSink; +use capture::sinks::Event; use capture::time::TimeSource; use serde::Deserialize; use serde_json::{json, Value}; @@ -61,7 +61,7 @@ impl MemorySink { } #[async_trait] -impl EventSink for MemorySink { +impl Event for MemorySink { async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { self.events.lock().unwrap().push(event); Ok(()) From 56dfe44303ab694e7a649b1a4bccc6ae04962642 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 13 Dec 2023 15:51:35 +0100 Subject: [PATCH 114/247] return rdkafka to 0.36 (#73) --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- capture-server/tests/common.rs | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e3b8737e2daa..6bf02dfc43650 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1531,9 +1531,9 @@ dependencies = [ [[package]] name = "rdkafka" -version = "0.34.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053adfa02fab06e86c01d586cc68aa47ee0ff4489a59469081dc12cbcde578bf" +checksum = "d54f02a5a40220f8a2dfa47ddb38ba9064475a5807a69504b6f91711df2eea63" dependencies = [ "futures-channel", "futures-util", @@ -1549,9 +1549,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.6.0+2.2.0" +version = "4.7.0+2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad63c279fca41a27c231c450a2d2ad18288032e9cbb159ad16c9d96eba35aaaf" +checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615" dependencies = [ "cmake", "libc", diff --git a/Cargo.toml b/Cargo.toml index ecd9b6a0ad78b..faf9d2a0088be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ uuid = { version = "1.3.3", features = ["serde"] } async-trait = "0.1.68" serde_urlencoded = "0.7.1" rand = "0.8.5" -rdkafka = { version = "0.34.0", features = ["cmake-build", "ssl"] } +rdkafka = { version = "0.36.0", features = ["cmake-build", "ssl"] } metrics = "0.21.1" metrics-exporter-prometheus = "0.12.1" thiserror = "1.0.48" diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index 214ecc80a62b5..fa8688156e650 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -177,7 +177,6 @@ impl EphemeralTopic { impl Drop for EphemeralTopic { fn drop(&mut self) { debug!("dropping EphemeralTopic {}...", self.topic_name); - _ = self.consumer.unassign(); self.consumer.unsubscribe(); match futures::executor::block_on(timeout( Duration::from_secs(10), From 74c52079098463680207a05ed43e8650d0779cd5 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 12 Dec 2023 11:44:21 -0700 Subject: [PATCH 115/247] Add indexes, drop redundant column --- migrations/20231129172339_job_queue_table.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/migrations/20231129172339_job_queue_table.sql b/migrations/20231129172339_job_queue_table.sql index 4631f0b0a5d3f..8627556c5542f 100644 --- a/migrations/20231129172339_job_queue_table.sql +++ b/migrations/20231129172339_job_queue_table.sql @@ -10,7 +10,6 @@ CREATE TABLE job_queue( attempt INT NOT NULL DEFAULT 0, attempted_at TIMESTAMPTZ DEFAULT NULL, attempted_by TEXT[] DEFAULT ARRAY[]::TEXT[], - completed_at TIMESTAMPTZ DEFAULT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), errors jsonb[], max_attempts INT NOT NULL DEFAULT 1, @@ -21,3 +20,9 @@ CREATE TABLE job_queue( status job_status NOT NULL DEFAULT 'available'::job_status, target TEXT NOT NULL ); + +-- Needed for `dequeue` queries +CREATE INDEX idx_queue_scheduled_at ON job_queue(queue, status, scheduled_at); + +-- Needed for UPDATE-ing incomplete jobs with a specific target (i.e. slow destinations) +CREATE INDEX idx_queue_target ON job_queue(queue, status, target); \ No newline at end of file From c00bc04db16994971934a11686cc43521999b6e5 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 14 Dec 2023 17:24:43 -0700 Subject: [PATCH 116/247] Append final errors to error array and treat request errors as retryable --- hook-common/src/pgqueue.rs | 4 ++++ hook-consumer/src/consumer.rs | 9 +++++++-- hook-consumer/src/error.rs | 2 -- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index fb2211b33b02f..47938ecd83760 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -248,6 +248,7 @@ UPDATE SET finished_at = NOW(), status = 'failed'::job_status + errors = array_append("{0}".errors, $3) WHERE "{0}".id = $2 AND queue = $1 @@ -261,6 +262,7 @@ RETURNING sqlx::query(&base_query) .bind(&failed_job.queue) .bind(failed_job.id) + .bind(&failed_job.error) .execute(&mut *self.connection) .await .map_err(|error| PgJobError::QueryError { @@ -394,6 +396,7 @@ UPDATE SET finished_at = NOW(), status = 'failed'::job_status + errors = array_append("{0}".errors, $3) WHERE "{0}".id = $2 AND queue = $1 @@ -406,6 +409,7 @@ RETURNING sqlx::query(&base_query) .bind(&failed_job.queue) .bind(failed_job.id) + .bind(&failed_job.error) .execute(&mut *self.transaction) .await .map_err(|error| PgJobError::QueryError { diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 406dd732ae797..dd8ab4297900b 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -42,7 +42,8 @@ impl<'p> WebhookConsumer<'p> { let client = reqwest::Client::builder() .default_headers(headers) .timeout(request_timeout) - .build()?; + .build() + .expect("failed to construct reqwest client for webhook consumer"); Ok(Self { name: name.to_owned(), @@ -174,7 +175,11 @@ async fn send_webhook( .headers(headers) .body(body) .send() - .await?; + .await + .map_err(|e| WebhookConsumerError::RetryableWebhookError { + reason: e.to_string(), + retry_after: None, + })?; let status = response.status(); diff --git a/hook-consumer/src/error.rs b/hook-consumer/src/error.rs index 34f0619ada4f0..a19664359caf7 100644 --- a/hook-consumer/src/error.rs +++ b/hook-consumer/src/error.rs @@ -18,8 +18,6 @@ pub enum WebhookConsumerError { QueueError(#[from] pgqueue::PgQueueError), #[error("an error occurred in the underlying job")] PgJobError(String), - #[error("an error occurred when attempting to send a request")] - RequestError(#[from] reqwest::Error), #[error("a webhook could not be delivered but it could be retried later: {reason}")] RetryableWebhookError { reason: String, From 9084c6e5c519013c2a4ee65f8cd5dd2b2fc0d844 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 15 Dec 2023 05:35:26 -0700 Subject: [PATCH 117/247] Remove Result from WebhookConsumer::new --- hook-consumer/src/consumer.rs | 10 +++++----- hook-consumer/src/main.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index dd8ab4297900b..59b8e9f24d949 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -32,7 +32,7 @@ impl<'p> WebhookConsumer<'p> { poll_interval: time::Duration, request_timeout: time::Duration, max_concurrent_jobs: usize, - ) -> Result { + ) -> Self { let mut headers = header::HeaderMap::new(); headers.insert( header::CONTENT_TYPE, @@ -45,13 +45,13 @@ impl<'p> WebhookConsumer<'p> { .build() .expect("failed to construct reqwest client for webhook consumer"); - Ok(Self { + Self { name: name.to_owned(), queue, poll_interval, client, max_concurrent_jobs, - }) + } } /// Wait until a job becomes available in our queue. @@ -333,8 +333,8 @@ mod tests { time::Duration::from_millis(100), time::Duration::from_millis(5000), 10, - ) - .expect("consumer failed to initialize"); + ); + let consumed_job = consumer .wait_for_job() .await diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index f165b32409de1..bf76503041d3a 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -29,7 +29,7 @@ async fn main() -> Result<(), WebhookConsumerError> { config.poll_interval.0, config.request_timeout.0, config.max_concurrent_jobs, - )?; + ); let _ = consumer.run().await; From 71bf5531cefa8e71a64dad3e4e887072ea9cd83b Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 13 Dec 2023 12:03:43 -0700 Subject: [PATCH 118/247] Add hook-janitor skeleton --- Cargo.lock | 139 ++++++++++++++++++ Cargo.toml | 23 +-- hook-common/Cargo.toml | 3 + hook-common/src/lib.rs | 1 + {hook-producer => hook-common}/src/metrics.rs | 0 hook-janitor/Cargo.toml | 28 ++++ hook-janitor/src/cleanup.rs | 36 +++++ hook-janitor/src/config.rs | 66 +++++++++ hook-janitor/src/handlers/app.rs | 21 +++ hook-janitor/src/handlers/mod.rs | 3 + hook-janitor/src/kafka_producer.rs | 48 ++++++ hook-janitor/src/main.rs | 86 +++++++++++ hook-janitor/src/webhooks.rs | 54 +++++++ hook-producer/Cargo.toml | 2 +- hook-producer/src/handlers/app.rs | 7 +- hook-producer/src/main.rs | 4 +- 16 files changed, 506 insertions(+), 15 deletions(-) rename {hook-producer => hook-common}/src/metrics.rs (100%) create mode 100644 hook-janitor/Cargo.toml create mode 100644 hook-janitor/src/cleanup.rs create mode 100644 hook-janitor/src/config.rs create mode 100644 hook-janitor/src/handlers/app.rs create mode 100644 hook-janitor/src/handlers/mod.rs create mode 100644 hook-janitor/src/kafka_producer.rs create mode 100644 hook-janitor/src/main.rs create mode 100644 hook-janitor/src/webhooks.rs diff --git a/Cargo.lock b/Cargo.lock index 73fd7cb4b7f75..c86f175f1c487 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -404,6 +404,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "concurrent-queue" version = "2.4.0" @@ -985,8 +994,11 @@ dependencies = [ name = "hook-common" version = "0.1.0" dependencies = [ + "axum", "chrono", "http 0.2.11", + "metrics", + "metrics-exporter-prometheus", "serde", "serde_derive", "sqlx", @@ -1013,6 +1025,32 @@ dependencies = [ "url", ] +[[package]] +name = "hook-janitor" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum", + "envconfig", + "eyre", + "futures", + "hook-common", + "http-body-util", + "metrics", + "metrics-exporter-prometheus", + "rdkafka", + "serde", + "serde_derive", + "serde_json", + "sqlx", + "thiserror", + "tokio", + "tower", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "hook-producer" version = "0.1.0" @@ -1329,6 +1367,18 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-sys" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -1594,6 +1644,27 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "object" version = "0.32.1" @@ -1821,6 +1892,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.70" @@ -1894,6 +1975,38 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "rdkafka" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f16c17f411935214a5870e40aff9291f8b40a73e97bf8de29e5959c473d5ef33" +dependencies = [ + "futures-channel", + "futures-util", + "libc", + "log", + "rdkafka-sys", + "serde", + "serde_derive", + "serde_json", + "slab", + "tokio", +] + +[[package]] +name = "rdkafka-sys" +version = "4.7.0+2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615" +dependencies = [ + "cmake", + "libc", + "libz-sys", + "num_enum", + "openssl-sys", + "pkg-config", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -2619,6 +2732,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -3063,6 +3193,15 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +[[package]] +name = "winnow" +version = "0.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c830786f7720c2fd27a1a0e27a709dbd3c4d009b56d098fc742d4f4eab91fe2" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" diff --git a/Cargo.toml b/Cargo.toml index 60a7219d66630..535c2429412c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,14 +1,24 @@ [workspace] resolver = "2" -members = ["hook-common", "hook-producer", "hook-consumer"] +members = ["hook-common", "hook-producer", "hook-consumer", "hook-janitor"] [workspace.dependencies] +async-trait = "0.1.74" +axum = { version = "0.7.1", features = ["http2"] } chrono = { version = "0.4" } +envconfig = "0.10.0" +eyre = "0.6.9" +futures = { version = "0.3.29" } +http = { version = "0.2" } +http-body-util = "0.1.0" +metrics = "0.21.1" +metrics-exporter-prometheus = "0.12.1" +rdkafka = { version = "0.35.0", features = ["cmake-build", "ssl"] } +regex = "1.10.2" serde = { version = "1.0" } serde_derive = { version = "1.0" } serde_json = { version = "1.0" } -thiserror = { version = "1.0" } sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", @@ -17,14 +27,9 @@ sqlx = { version = "0.7", features = [ "json", "chrono", ] } +thiserror = { version = "1.0" } tokio = { version = "1.34.0", features = ["full"] } -eyre = "0.6.9" +tower = "0.4.13" tracing = "0.1.40" tracing-subscriber = "0.3.18" -envconfig = "0.10.0" -metrics = "0.21.1" -metrics-exporter-prometheus = "0.12.1" -http = { version = "0.2" } url = { version = "2.5.0 " } -tower = "0.4.13" -http-body-util = "0.1.0" diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index 24d1a0d663a5f..213d2e98f7634 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -6,8 +6,11 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +axum = { workspace = true, features = ["http2"] } chrono = { workspace = true } http = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } sqlx = { workspace = true } diff --git a/hook-common/src/lib.rs b/hook-common/src/lib.rs index 3138f087a1ebc..3b154c834224a 100644 --- a/hook-common/src/lib.rs +++ b/hook-common/src/lib.rs @@ -1,2 +1,3 @@ +pub mod metrics; pub mod pgqueue; pub mod webhook; diff --git a/hook-producer/src/metrics.rs b/hook-common/src/metrics.rs similarity index 100% rename from hook-producer/src/metrics.rs rename to hook-common/src/metrics.rs diff --git a/hook-janitor/Cargo.toml b/hook-janitor/Cargo.toml new file mode 100644 index 0000000000000..f23626bea05a6 --- /dev/null +++ b/hook-janitor/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "hook-janitor" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-trait = { workspace = true } +axum = { workspace = true } +envconfig = { workspace = true } +eyre = { workspace = true } +futures = { workspace = true } +hook-common = { path = "../hook-common" } +http-body-util = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } +rdkafka = { workspace = true } +serde = { workspace = true } +serde_derive = { workspace = true } +serde_json = { workspace = true } +sqlx = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tower = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = { workspace = true } diff --git a/hook-janitor/src/cleanup.rs b/hook-janitor/src/cleanup.rs new file mode 100644 index 0000000000000..e6e91e0be922f --- /dev/null +++ b/hook-janitor/src/cleanup.rs @@ -0,0 +1,36 @@ +use async_trait::async_trait; +use std::result::Result; +use std::str::FromStr; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum CleanerError { + #[error("pool creation failed with: {error}")] + PoolCreationError { error: sqlx::Error }, + #[error("invalid cleaner mode")] + InvalidCleanerMode, +} + +// Mode names, used by config/environment parsing to verify the mode is supported. +#[derive(Debug)] +pub enum CleanerModeName { + Webhooks, +} + +impl FromStr for CleanerModeName { + type Err = CleanerError; + + fn from_str(s: &str) -> Result { + match s { + "webhooks" => Ok(CleanerModeName::Webhooks), + _ => Err(CleanerError::InvalidCleanerMode), + } + } +} + +// Right now, all this trait does is allow us to call `cleanup` in a loop in `main.rs`. There may +// be other benefits as we build this out, or we could remove it if it doesn't end up being useful. +#[async_trait] +pub trait Cleaner { + async fn cleanup(&self); +} diff --git a/hook-janitor/src/config.rs b/hook-janitor/src/config.rs new file mode 100644 index 0000000000000..89621a2354617 --- /dev/null +++ b/hook-janitor/src/config.rs @@ -0,0 +1,66 @@ +use envconfig::Envconfig; + +#[derive(Envconfig)] +pub struct Config { + #[envconfig(from = "BIND_HOST", default = "0.0.0.0")] + pub host: String, + + #[envconfig(from = "BIND_PORT", default = "8000")] + pub port: u16, + + #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] + pub database_url: String, + + #[envconfig(default = "job_queue")] + pub table_name: String, + + #[envconfig(default = "default")] + pub queue_name: String, + + #[envconfig(default = "30")] + pub cleanup_interval_secs: u64, + + #[envconfig(default = "10000")] + pub cleanup_batch_size: u32, + + // The cleanup task needs to have special knowledge of the queue it's cleaning up. This is so it + // can do things like flush the proper app_metrics or plugin_log_entries, and so it knows what + // to expect in the job's payload JSONB column. + #[envconfig(default = "webhooks")] + pub mode: String, + + #[envconfig(nested = true)] + pub kafka: KafkaConfig, +} + +#[derive(Envconfig, Clone)] +pub struct KafkaConfig { + #[envconfig(default = "20")] + pub kafka_producer_linger_ms: u32, // Maximum time between producer batches during low traffic + + #[envconfig(default = "400")] + pub kafka_producer_queue_mib: u32, // Size of the in-memory producer queue in mebibytes + + #[envconfig(default = "20000")] + pub kafka_message_timeout_ms: u32, // Time before we stop retrying producing a message: 20 seconds + + #[envconfig(default = "none")] + pub kafka_compression_codec: String, // none, gzip, snappy, lz4, zstd + + #[envconfig(default = "false")] + pub kafka_tls: bool, + + #[envconfig(default = "app_metrics")] + pub app_metrics_topic: String, + + #[envconfig(default = "plugin_log_entries")] + pub plugin_log_entries_topic: String, + + pub kafka_hosts: String, +} + +impl Config { + pub fn bind(&self) -> String { + format!("{}:{}", self.host, self.port) + } +} diff --git a/hook-janitor/src/handlers/app.rs b/hook-janitor/src/handlers/app.rs new file mode 100644 index 0000000000000..279fa0e35923e --- /dev/null +++ b/hook-janitor/src/handlers/app.rs @@ -0,0 +1,21 @@ +use axum::{routing, Router}; +use metrics_exporter_prometheus::PrometheusHandle; + +use hook_common::metrics; + +pub fn app(metrics: Option) -> Router { + Router::new() + .route("/", routing::get(index)) + .route( + "/metrics", + routing::get(move || match metrics { + Some(ref recorder_handle) => std::future::ready(recorder_handle.render()), + None => std::future::ready("no metrics recorder installed".to_owned()), + }), + ) + .layer(axum::middleware::from_fn(metrics::track_metrics)) +} + +pub async fn index() -> &'static str { + "rusty-hook janitor" +} diff --git a/hook-janitor/src/handlers/mod.rs b/hook-janitor/src/handlers/mod.rs new file mode 100644 index 0000000000000..a884c04897bf9 --- /dev/null +++ b/hook-janitor/src/handlers/mod.rs @@ -0,0 +1,3 @@ +mod app; + +pub use app::app; diff --git a/hook-janitor/src/kafka_producer.rs b/hook-janitor/src/kafka_producer.rs new file mode 100644 index 0000000000000..4e905b3e9bae2 --- /dev/null +++ b/hook-janitor/src/kafka_producer.rs @@ -0,0 +1,48 @@ +use crate::config::KafkaConfig; + +use rdkafka::error::{KafkaError, RDKafkaErrorCode}; +use rdkafka::producer::{DeliveryFuture, FutureProducer, FutureRecord, Producer}; +use rdkafka::util::Timeout; +use rdkafka::ClientConfig; +use std::{str::FromStr, time::Duration}; +use tokio::sync::Semaphore; +use tracing::debug; + +// TODO: Take stats recording pieces that we want from `capture-rs`. +pub struct KafkaContext {} + +impl rdkafka::ClientContext for KafkaContext {} + +pub async fn create_kafka_producer( + config: &KafkaConfig, +) -> Result, KafkaError> { + let mut client_config = ClientConfig::new(); + client_config + .set("bootstrap.servers", &config.kafka_hosts) + .set("statistics.interval.ms", "10000") + .set("linger.ms", config.kafka_producer_linger_ms.to_string()) + .set( + "message.timeout.ms", + config.kafka_message_timeout_ms.to_string(), + ) + .set( + "compression.codec", + config.kafka_compression_codec.to_owned(), + ) + .set( + "queue.buffering.max.kbytes", + (config.kafka_producer_queue_mib * 1024).to_string(), + ); + + if config.kafka_tls { + client_config + .set("security.protocol", "ssl") + .set("enable.ssl.certificate.verification", "false"); + }; + + debug!("rdkafka configuration: {:?}", client_config); + let producer: FutureProducer = + client_config.create_with_context(KafkaContext {})?; + + Ok(producer) +} diff --git a/hook-janitor/src/main.rs b/hook-janitor/src/main.rs new file mode 100644 index 0000000000000..b487fda796690 --- /dev/null +++ b/hook-janitor/src/main.rs @@ -0,0 +1,86 @@ +use axum::Router; +use cleanup::{Cleaner, CleanerModeName}; +use config::Config; +use envconfig::Envconfig; +use eyre::Result; +use futures::future::{select, Either}; +use kafka_producer::create_kafka_producer; +use std::{str::FromStr, time::Duration}; +use tokio::sync::Semaphore; +use webhooks::WebhookCleaner; + +use hook_common::metrics; + +mod cleanup; +mod config; +mod handlers; +mod kafka_producer; +mod webhooks; + +async fn listen(app: Router, bind: String) -> Result<()> { + let listener = tokio::net::TcpListener::bind(bind).await?; + + axum::serve(listener, app).await?; + + Ok(()) +} + +async fn cleanup_loop(cleaner: Box, interval_secs: u64) -> Result<()> { + let semaphore = Semaphore::new(1); + let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + + loop { + let _permit = semaphore.acquire().await; + interval.tick().await; + cleaner.cleanup().await; + drop(_permit); + } +} + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + + let config = Config::init_from_env().expect("failed to load configuration from env"); + + let mode_name = CleanerModeName::from_str(&config.mode) + .unwrap_or_else(|_| panic!("invalid cleaner mode: {}", config.mode)); + + let cleaner = match mode_name { + CleanerModeName::Webhooks => { + let kafka_producer = create_kafka_producer(&config.kafka) + .await + .expect("failed to create kafka producer"); + + Box::new( + WebhookCleaner::new( + &config.queue_name, + &config.table_name, + &config.database_url, + config.cleanup_batch_size, + kafka_producer, + config.kafka.app_metrics_topic.to_owned(), + config.kafka.plugin_log_entries_topic.to_owned(), + ) + .expect("unable to create webhook cleaner"), + ) + } + }; + + let cleanup_loop = Box::pin(cleanup_loop(cleaner, config.cleanup_interval_secs)); + + let recorder_handle = metrics::setup_metrics_recorder(); + let app = handlers::app(Some(recorder_handle)); + let http_server = Box::pin(listen(app, config.bind())); + + match select(http_server, cleanup_loop).await { + Either::Left((listen_result, _)) => match listen_result { + Ok(_) => {} + Err(e) => tracing::error!("failed to start hook-janitor http server, {}", e), + }, + Either::Right((cleanup_result, _)) => match cleanup_result { + Ok(_) => {} + Err(e) => tracing::error!("hook-janitor cleanup task exited, {}", e), + }, + }; +} diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs new file mode 100644 index 0000000000000..a6cd9ff803ea5 --- /dev/null +++ b/hook-janitor/src/webhooks.rs @@ -0,0 +1,54 @@ +use async_trait::async_trait; + +use rdkafka::producer::FutureProducer; +use sqlx::postgres::{PgPool, PgPoolOptions}; + +use crate::cleanup::{Cleaner, CleanerError}; +use crate::kafka_producer::KafkaContext; + +pub struct WebhookCleaner { + queue_name: String, + table_name: String, + pg_pool: PgPool, + batch_size: u32, + kafka_producer: FutureProducer, + app_metrics_topic: String, + plugin_log_entries_topic: String, +} + +impl WebhookCleaner { + pub fn new( + queue_name: &str, + table_name: &str, + database_url: &str, + batch_size: u32, + kafka_producer: FutureProducer, + app_metrics_topic: String, + plugin_log_entries_topic: String, + ) -> Result { + let queue_name = queue_name.to_owned(); + let table_name = table_name.to_owned(); + let pg_pool = PgPoolOptions::new() + .connect_lazy(database_url) + .map_err(|error| CleanerError::PoolCreationError { error })?; + + Ok(Self { + queue_name, + table_name, + pg_pool, + batch_size, + kafka_producer, + app_metrics_topic, + plugin_log_entries_topic, + }) + } +} + +#[async_trait] +impl Cleaner for WebhookCleaner { + async fn cleanup(&self) { + // TODO: collect stats on completed/failed rows + // TODO: push metrics about those rows into `app_metrics` + // TODO: delete those completed/failed rows + } +} diff --git a/hook-producer/Cargo.toml b/hook-producer/Cargo.toml index ef1a24b01820a..f4b116563fd12 100644 --- a/hook-producer/Cargo.toml +++ b/hook-producer/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axum = { version = "0.7.1", features = ["http2"] } +axum = { workspace = true } envconfig = { workspace = true } eyre = { workspace = true } hook-common = { path = "../hook-common" } diff --git a/hook-producer/src/handlers/app.rs b/hook-producer/src/handlers/app.rs index 911a04d397416..1666676fee1bb 100644 --- a/hook-producer/src/handlers/app.rs +++ b/hook-producer/src/handlers/app.rs @@ -1,6 +1,7 @@ use axum::{routing, Router}; use metrics_exporter_prometheus::PrometheusHandle; +use hook_common::metrics; use hook_common::pgqueue::PgQueue; use super::webhook; @@ -16,11 +17,11 @@ pub fn app(pg_pool: PgQueue, metrics: Option) -> Router { }), ) .route("/webhook", routing::post(webhook::post).with_state(pg_pool)) - .layer(axum::middleware::from_fn(crate::metrics::track_metrics)) + .layer(axum::middleware::from_fn(metrics::track_metrics)) } pub async fn index() -> &'static str { - "rusty hook" + "rusty-hook producer" } #[cfg(test)] @@ -55,6 +56,6 @@ mod tests { assert_eq!(response.status(), StatusCode::OK); let body = response.into_body().collect().await.unwrap().to_bytes(); - assert_eq!(&body[..], b"rusty hook"); + assert_eq!(&body[..], b"rusty-hook producer"); } } diff --git a/hook-producer/src/main.rs b/hook-producer/src/main.rs index 29da8dd6efe0e..7c2b73c0f12bb 100644 --- a/hook-producer/src/main.rs +++ b/hook-producer/src/main.rs @@ -3,11 +3,11 @@ use config::Config; use envconfig::Envconfig; use eyre::Result; +use hook_common::metrics; use hook_common::pgqueue::{PgQueue, RetryPolicy}; mod config; mod handlers; -mod metrics; async fn listen(app: Router, bind: String) -> Result<()> { let listener = tokio::net::TcpListener::bind(bind).await?; @@ -35,7 +35,7 @@ async fn main() { .await .expect("failed to initialize queue"); - let recorder_handle = crate::metrics::setup_metrics_recorder(); + let recorder_handle = metrics::setup_metrics_recorder(); let app = handlers::app(pg_queue, Some(recorder_handle)); From d2d929d95f45c2e4f0649c70a0dcf95655bcc29c Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 14 Dec 2023 13:13:30 -0700 Subject: [PATCH 119/247] Add Kafka message types for app_metrics and plugin_log_entries --- Cargo.lock | 51 +++++++ Cargo.toml | 1 + hook-common/Cargo.toml | 3 + hook-common/src/kafka_messages/app_metrics.rs | 123 +++++++++++++++++ hook-common/src/kafka_messages/mod.rs | 20 +++ hook-common/src/kafka_messages/plugin_logs.rs | 130 ++++++++++++++++++ hook-common/src/lib.rs | 1 + 7 files changed, 329 insertions(+) create mode 100644 hook-common/src/kafka_messages/app_metrics.rs create mode 100644 hook-common/src/kafka_messages/mod.rs create mode 100644 hook-common/src/kafka_messages/plugin_logs.rs diff --git a/Cargo.lock b/Cargo.lock index c86f175f1c487..5b0aedcfa6038 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,6 +30,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.16" @@ -215,6 +224,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" + [[package]] name = "atomic-waker" version = "1.1.2" @@ -999,11 +1014,14 @@ dependencies = [ "http 0.2.11", "metrics", "metrics-exporter-prometheus", + "regex", "serde", "serde_derive", + "serde_json", "sqlx", "thiserror", "tokio", + "uuid", ] [[package]] @@ -2016,6 +2034,35 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "reqwest" version = "0.11.22" @@ -2902,6 +2949,10 @@ name = "uuid" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +dependencies = [ + "atomic", + "getrandom", +] [[package]] name = "valuable" diff --git a/Cargo.toml b/Cargo.toml index 535c2429412c7..faf8644a85dfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,3 +33,4 @@ tower = "0.4.13" tracing = "0.1.40" tracing-subscriber = "0.3.18" url = { version = "2.5.0 " } +uuid = { version = "1.6.1", features = ["v7"] } diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index 213d2e98f7634..2c32d9d35c1b1 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -11,10 +11,13 @@ chrono = { workspace = true } http = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } +regex = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } +serde_json = { workspace = true } sqlx = { workspace = true } thiserror = { workspace = true } +uuid = { workspace = true } [dev-dependencies] tokio = { workspace = true } # We need a runtime for async tests diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs new file mode 100644 index 0000000000000..a7530644280f6 --- /dev/null +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -0,0 +1,123 @@ +use chrono::{DateTime, Utc}; +use serde::{Serialize, Serializer}; +use uuid::Uuid; + +use super::{serialize_datetime, serialize_uuid}; + +#[derive(Serialize)] +pub enum AppMetricCategory { + ProcessEvent, + OnEvent, + ScheduledTask, + Webhook, + ComposeWebhook, +} + +#[derive(Serialize)] +pub enum ErrorType { + Timeout, + Connection, + HttpStatus(u16), +} + +#[derive(Serialize)] +pub struct ErrorDetails { + pub error: Error, + // TODO: The plugin-server sends the entire raw event with errors. In order to do this, we'll + // have to pass the entire event when we enqueue items, and store it in the Parameters JSONB + // column. We should see if it's possible to work around this before we commit to it. + // + // event: Value, +} + +#[derive(Serialize)] +pub struct Error { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub message: Option, + // TODO: Realistically, it doesn't seem likely that we'll generate Rust stack traces and put + // them here. I think this was more useful in plugin-server when the stack could come from + // plugin code. + #[serde(skip_serializing_if = "Option::is_none")] + pub stack: Option, +} + +#[derive(Serialize)] +pub struct AppMetric { + #[serde(serialize_with = "serialize_datetime")] + pub timestamp: DateTime, + pub team_id: u32, + pub plugin_config_id: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub job_id: Option, + #[serde(serialize_with = "serialize_category")] + pub category: AppMetricCategory, + pub successes: u32, + pub successes_on_retry: u32, + pub failures: u32, + #[serde(serialize_with = "serialize_uuid")] + pub error_uuid: Uuid, + #[serde(serialize_with = "serialize_error_type")] + pub error_type: ErrorType, + pub error_details: Error, +} + +fn serialize_category(category: &AppMetricCategory, serializer: S) -> Result +where + S: Serializer, +{ + let category_str = match category { + AppMetricCategory::ProcessEvent => "processEvent", + AppMetricCategory::OnEvent => "onEvent", + AppMetricCategory::ScheduledTask => "scheduledTask", + AppMetricCategory::Webhook => "webhook", + AppMetricCategory::ComposeWebhook => "composeWebhook", + }; + serializer.serialize_str(category_str) +} + +fn serialize_error_type(error_type: &ErrorType, serializer: S) -> Result +where + S: Serializer, +{ + let error_type = match error_type { + ErrorType::Connection => "Connection Error".to_owned(), + ErrorType::Timeout => "Timeout".to_owned(), + ErrorType::HttpStatus(s) => format!("HTTP Status: {}", s), + }; + serializer.serialize_str(&error_type) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_app_metric_serialization() { + use chrono::prelude::*; + + let app_metric = AppMetric { + timestamp: Utc.with_ymd_and_hms(2023, 12, 14, 12, 2, 0).unwrap(), + team_id: 123, + plugin_config_id: 456, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 10, + successes_on_retry: 0, + failures: 2, + error_uuid: Uuid::parse_str("550e8400-e29b-41d4-a716-446655447777").unwrap(), + error_type: ErrorType::Connection, + error_details: Error { + name: "FooError".to_owned(), + message: Some("Error Message".to_owned()), + stack: None, + }, + }; + + let serialized_json = serde_json::to_string(&app_metric).unwrap(); + + let expected_json = r#"{"timestamp":"2023-12-14 12:02:00","team_id":123,"plugin_config_id":456,"category":"webhook","successes":10,"successes_on_retry":0,"failures":2,"error_uuid":"550e8400-e29b-41d4-a716-446655447777","error_type":"Connection Error","error_details":{"name":"FooError","message":"Error Message"}}"#; + + assert_eq!(serialized_json, expected_json); + } +} diff --git a/hook-common/src/kafka_messages/mod.rs b/hook-common/src/kafka_messages/mod.rs new file mode 100644 index 0000000000000..1449f56faeaa7 --- /dev/null +++ b/hook-common/src/kafka_messages/mod.rs @@ -0,0 +1,20 @@ +pub mod app_metrics; +pub mod plugin_logs; + +use chrono::{DateTime, Utc}; +use serde::Serializer; +use uuid::Uuid; + +fn serialize_uuid(uuid: &Uuid, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_str(&uuid.to_string()) +} + +fn serialize_datetime(datetime: &DateTime, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_str(&datetime.format("%Y-%m-%d %H:%M:%S%.f").to_string()) +} diff --git a/hook-common/src/kafka_messages/plugin_logs.rs b/hook-common/src/kafka_messages/plugin_logs.rs new file mode 100644 index 0000000000000..8f8bb43efea96 --- /dev/null +++ b/hook-common/src/kafka_messages/plugin_logs.rs @@ -0,0 +1,130 @@ +use chrono::{DateTime, Utc}; +use serde::{Serialize, Serializer}; +use uuid::Uuid; + +use super::{serialize_datetime, serialize_uuid}; + +#[allow(dead_code)] +#[derive(Serialize)] +pub enum PluginLogEntrySource { + System, + Plugin, + Console, +} + +#[allow(dead_code)] +#[derive(Serialize)] +pub enum PluginLogEntryType { + Debug, + Log, + Info, + Warn, + Error, +} + +#[derive(Serialize)] +pub struct PluginLogEntry { + #[serde(serialize_with = "serialize_source")] + pub source: PluginLogEntrySource, + #[serde(rename = "type", serialize_with = "serialize_type")] + pub type_: PluginLogEntryType, + #[serde(serialize_with = "serialize_uuid")] + pub id: Uuid, + pub team_id: u32, + pub plugin_id: u32, + pub plugin_config_id: u32, + #[serde(serialize_with = "serialize_datetime")] + pub timestamp: DateTime, + #[serde(serialize_with = "serialize_message")] + pub message: String, + #[serde(serialize_with = "serialize_uuid")] + pub instance_id: Uuid, +} + +fn serialize_source(source: &PluginLogEntrySource, serializer: S) -> Result +where + S: Serializer, +{ + let source_str = match source { + PluginLogEntrySource::System => "SYSTEM", + PluginLogEntrySource::Plugin => "PLUGIN", + PluginLogEntrySource::Console => "CONSOLE", + }; + serializer.serialize_str(source_str) +} + +fn serialize_type(type_: &PluginLogEntryType, serializer: S) -> Result +where + S: Serializer, +{ + let type_str = match type_ { + PluginLogEntryType::Debug => "DEBUG", + PluginLogEntryType::Log => "LOG", + PluginLogEntryType::Info => "INFO", + PluginLogEntryType::Warn => "WARN", + PluginLogEntryType::Error => "ERROR", + }; + serializer.serialize_str(type_str) +} + +fn serialize_message(msg: &String, serializer: S) -> Result +where + S: Serializer, +{ + if msg.len() > 50_000 { + return Err(serde::ser::Error::custom( + "Message is too long for ClickHouse", + )); + } + + serializer.serialize_str(msg) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_plugin_log_entry_serialization() { + use chrono::prelude::*; + + let log_entry = PluginLogEntry { + source: PluginLogEntrySource::Plugin, + type_: PluginLogEntryType::Warn, + id: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(), + team_id: 4, + plugin_id: 5, + plugin_config_id: 6, + timestamp: Utc.with_ymd_and_hms(2023, 12, 14, 12, 2, 0).unwrap(), + message: "My message!".to_string(), + instance_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), + }; + + let serialized_json = serde_json::to_string(&log_entry).unwrap(); + + assert_eq!( + serialized_json, + r#"{"source":"PLUGIN","type":"WARN","id":"550e8400-e29b-41d4-a716-446655440000","team_id":4,"plugin_id":5,"plugin_config_id":6,"timestamp":"2023-12-14 12:02:00","message":"My message!","instance_id":"00000000-0000-0000-0000-000000000000"}"# + ); + } + + #[test] + fn test_plugin_log_entry_message_too_long() { + use chrono::prelude::*; + + let log_entry = PluginLogEntry { + source: PluginLogEntrySource::Plugin, + type_: PluginLogEntryType::Warn, + id: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(), + team_id: 4, + plugin_id: 5, + plugin_config_id: 6, + timestamp: Utc.with_ymd_and_hms(2023, 12, 14, 12, 2, 0).unwrap(), + message: "My message!".repeat(10_000).to_string(), + instance_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), + }; + + let err = serde_json::to_string(&log_entry).unwrap_err(); + assert_eq!(err.to_string(), "Message is too long for ClickHouse"); + } +} diff --git a/hook-common/src/lib.rs b/hook-common/src/lib.rs index 3b154c834224a..7d9ef37e84606 100644 --- a/hook-common/src/lib.rs +++ b/hook-common/src/lib.rs @@ -1,3 +1,4 @@ +pub mod kafka_messages; pub mod metrics; pub mod pgqueue; pub mod webhook; From a938a107faf8f48cfa176887177b52971f81de14 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Mon, 18 Dec 2023 09:40:15 -0700 Subject: [PATCH 120/247] Squelch clippy complaints --- hook-janitor/src/kafka_producer.rs | 7 ++----- hook-janitor/src/webhooks.rs | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/hook-janitor/src/kafka_producer.rs b/hook-janitor/src/kafka_producer.rs index 4e905b3e9bae2..4845e9410df56 100644 --- a/hook-janitor/src/kafka_producer.rs +++ b/hook-janitor/src/kafka_producer.rs @@ -1,11 +1,8 @@ use crate::config::KafkaConfig; -use rdkafka::error::{KafkaError, RDKafkaErrorCode}; -use rdkafka::producer::{DeliveryFuture, FutureProducer, FutureRecord, Producer}; -use rdkafka::util::Timeout; +use rdkafka::error::KafkaError; +use rdkafka::producer::FutureProducer; use rdkafka::ClientConfig; -use std::{str::FromStr, time::Duration}; -use tokio::sync::Semaphore; use tracing::debug; // TODO: Take stats recording pieces that we want from `capture-rs`. diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index a6cd9ff803ea5..e8895f1ba0781 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -6,6 +6,7 @@ use sqlx::postgres::{PgPool, PgPoolOptions}; use crate::cleanup::{Cleaner, CleanerError}; use crate::kafka_producer::KafkaContext; +#[allow(dead_code)] pub struct WebhookCleaner { queue_name: String, table_name: String, From 5a5c3bed8c3f6c78bbe869b8a83cbd288cd90eb2 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Mon, 18 Dec 2023 10:32:35 -0700 Subject: [PATCH 121/247] Remove pointless Result --- hook-janitor/src/main.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hook-janitor/src/main.rs b/hook-janitor/src/main.rs index b487fda796690..46223aa7aafb3 100644 --- a/hook-janitor/src/main.rs +++ b/hook-janitor/src/main.rs @@ -25,7 +25,7 @@ async fn listen(app: Router, bind: String) -> Result<()> { Ok(()) } -async fn cleanup_loop(cleaner: Box, interval_secs: u64) -> Result<()> { +async fn cleanup_loop(cleaner: Box, interval_secs: u64) { let semaphore = Semaphore::new(1); let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); @@ -78,9 +78,8 @@ async fn main() { Ok(_) => {} Err(e) => tracing::error!("failed to start hook-janitor http server, {}", e), }, - Either::Right((cleanup_result, _)) => match cleanup_result { - Ok(_) => {} - Err(e) => tracing::error!("hook-janitor cleanup task exited, {}", e), - }, + Either::Right((_, _)) => { + tracing::error!("hook-janitor cleanup task exited") + } }; } From 769c8f22fc83f348ef44a805111b2395e4322e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Mon, 18 Dec 2023 11:34:38 +0100 Subject: [PATCH 122/247] refactor: Add a metadata field to job queue --- hook-common/src/pgqueue.rs | 110 ++++++++++++------ hook-common/src/webhook.rs | 14 +-- hook-consumer/src/consumer.rs | 41 ++++--- hook-producer/src/handlers/webhook.rs | 106 +++++++++++------ migrations/20231129172339_job_queue_table.sql | 5 +- 5 files changed, 179 insertions(+), 97 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 47938ecd83760..5288ade106f47 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -74,9 +74,12 @@ impl FromStr for JobStatus { /// JobParameters are stored and read to and from a JSONB field, so we accept anything that fits `sqlx::types::Json`. pub type JobParameters = sqlx::types::Json; +/// JobMetadata are stored and read to and from a JSONB field, so we accept anything that fits `sqlx::types::Json`. +pub type JobMetadata = sqlx::types::Json; + /// A Job to be executed by a worker dequeueing a PgQueue. #[derive(sqlx::FromRow, Debug)] -pub struct Job { +pub struct Job { /// A unique id identifying a job. pub id: i64, /// A number corresponding to the current job attempt. @@ -89,6 +92,8 @@ pub struct Job { pub created_at: chrono::DateTime, /// The current job's number of max attempts. pub max_attempts: i32, + /// Arbitrary job metadata stored as JSON. + pub metadata: JobMetadata, /// Arbitrary job parameters stored as JSON. pub parameters: JobParameters, /// The queue this job belongs to. @@ -99,7 +104,7 @@ pub struct Job { pub target: String, } -impl Job { +impl Job { /// Return true if this job attempt is greater or equal to the maximum number of possible attempts. pub fn is_gte_max_attempts(&self) -> bool { self.attempt >= self.max_attempts @@ -146,19 +151,19 @@ impl Job { /// A Job that can be updated in PostgreSQL. #[derive(Debug)] -pub struct PgJob { - pub job: Job, +pub struct PgJob { + pub job: Job, pub table: String, pub connection: sqlx::pool::PoolConnection, pub retry_policy: RetryPolicy, } -impl PgJob { +impl PgJob { pub async fn retry( mut self, error: E, preferred_retry_interval: Option, - ) -> Result, PgJobError>> { + ) -> Result, PgJobError>> { if self.job.is_gte_max_attempts() { return Err(PgJobError::RetryInvalidError { job: self, @@ -203,7 +208,7 @@ RETURNING Ok(retryable_job) } - pub async fn complete(mut self) -> Result>> { + pub async fn complete(mut self) -> Result>> { let completed_job = self.job.complete(); let base_query = format!( @@ -238,7 +243,7 @@ RETURNING pub async fn fail( mut self, error: E, - ) -> Result, PgJobError>> { + ) -> Result, PgJobError>> { let failed_job = self.job.fail(error); let base_query = format!( @@ -277,19 +282,19 @@ RETURNING /// A Job within an open PostgreSQL transaction. /// This implementation allows 'hiding' the job from any other workers running SKIP LOCKED queries. #[derive(Debug)] -pub struct PgTransactionJob<'c, J> { - pub job: Job, +pub struct PgTransactionJob<'c, J, M> { + pub job: Job, pub table: String, pub transaction: sqlx::Transaction<'c, sqlx::postgres::Postgres>, pub retry_policy: RetryPolicy, } -impl<'c, J> PgTransactionJob<'c, J> { +impl<'c, J, M> PgTransactionJob<'c, J, M> { pub async fn retry( mut self, error: E, preferred_retry_interval: Option, - ) -> Result, PgJobError>> { + ) -> Result, PgJobError>> { if self.job.is_gte_max_attempts() { return Err(PgJobError::RetryInvalidError { job: self, @@ -343,7 +348,9 @@ RETURNING Ok(retryable_job) } - pub async fn complete(mut self) -> Result>> { + pub async fn complete( + mut self, + ) -> Result>> { let completed_job = self.job.complete(); let base_query = format!( @@ -386,7 +393,7 @@ RETURNING pub async fn fail( mut self, error: E, - ) -> Result, PgJobError>> { + ) -> Result, PgJobError>> { let failed_job = self.job.fail(error); let base_query = format!( @@ -461,19 +468,22 @@ pub struct FailedJob { } /// A NewJob to be enqueued into a PgQueue. -pub struct NewJob { +pub struct NewJob { /// The maximum amount of attempts this NewJob has to complete. pub max_attempts: i32, /// The JSON-deserializable parameters for this NewJob. + pub metadata: JobMetadata, + /// The JSON-deserializable parameters for this NewJob. pub parameters: JobParameters, /// The target of the NewJob. E.g. an endpoint or service we are trying to reach. pub target: String, } -impl NewJob { - pub fn new(max_attempts: i32, parameters: J, target: &str) -> Self { +impl NewJob { + pub fn new(max_attempts: i32, metadata: M, parameters: J, target: &str) -> Self { Self { max_attempts, + metadata: sqlx::types::Json(metadata), parameters: sqlx::types::Json(parameters), target: target.to_owned(), } @@ -583,10 +593,11 @@ impl PgQueue { /// Dequeue a Job from this PgQueue to work on it. pub async fn dequeue< J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, + M: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, >( &self, attempted_by: &str, - ) -> PgQueueResult>> { + ) -> PgQueueResult>> { let mut connection = self .pool .acquire() @@ -628,7 +639,7 @@ RETURNING &self.table ); - let query_result: Result, sqlx::Error> = sqlx::query_as(&base_query) + let query_result: Result, sqlx::Error> = sqlx::query_as(&base_query) .bind(&self.name) .bind(attempted_by) .fetch_one(&mut *connection) @@ -662,10 +673,11 @@ RETURNING pub async fn dequeue_tx< 'a, J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, + M: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, >( &self, attempted_by: &str, - ) -> PgQueueResult>> { + ) -> PgQueueResult>> { let mut tx = self .pool .begin() @@ -707,7 +719,7 @@ RETURNING &self.table ); - let query_result: Result, sqlx::Error> = sqlx::query_as(&base_query) + let query_result: Result, sqlx::Error> = sqlx::query_as(&base_query) .bind(&self.name) .bind(attempted_by) .fetch_one(&mut *tx) @@ -732,23 +744,27 @@ RETURNING /// Enqueue a Job into this PgQueue. /// We take ownership of NewJob to enforce a specific NewJob is only enqueued once. - pub async fn enqueue( + pub async fn enqueue< + J: serde::Serialize + std::marker::Sync, + M: serde::Serialize + std::marker::Sync, + >( &self, - job: NewJob, + job: NewJob, ) -> PgQueueResult<()> { // TODO: Escaping. I think sqlx doesn't support identifiers. let base_query = format!( r#" INSERT INTO {} - (attempt, created_at, scheduled_at, max_attempts, parameters, queue, status, target) + (attempt, created_at, scheduled_at, max_attempts, metadata, parameters, queue, status, target) VALUES - (0, NOW(), NOW(), $1, $2, $3, 'available'::job_status, $4) + (0, NOW(), NOW(), $1, $2, $3, $4, 'available'::job_status, $5) "#, &self.table ); sqlx::query(&base_query) .bind(job.max_attempts) + .bind(&job.metadata) .bind(&job.parameters) .bind(&self.name) .bind(&job.target) @@ -767,6 +783,23 @@ VALUES mod tests { use super::*; + #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] + struct JobMetadata { + team_id: u32, + plugin_config_id: u32, + plugin_id: u32, + } + + impl Default for JobMetadata { + fn default() -> Self { + Self { + team_id: 0, + plugin_config_id: 1, + plugin_id: 2, + } + } + } + #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] struct JobParameters { method: String, @@ -798,8 +831,9 @@ mod tests { async fn test_can_dequeue_job() { let job_target = job_target(); let job_parameters = JobParameters::default(); + let job_metadata = JobMetadata::default(); let worker_id = worker_id(); - let new_job = NewJob::new(1, job_parameters, &job_target); + let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); let queue = PgQueue::new( "test_can_dequeue_job", @@ -812,7 +846,7 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); - let pg_job: PgJob = queue + let pg_job: PgJob = queue .dequeue(&worker_id) .await .expect("failed to dequeue job") @@ -839,7 +873,7 @@ mod tests { .await .expect("failed to connect to local test postgresql database"); - let pg_job: Option> = queue + let pg_job: Option> = queue .dequeue(&worker_id) .await .expect("failed to dequeue job"); @@ -850,9 +884,10 @@ mod tests { #[tokio::test] async fn test_can_dequeue_tx_job() { let job_target = job_target(); + let job_metadata = JobMetadata::default(); let job_parameters = JobParameters::default(); let worker_id = worker_id(); - let new_job = NewJob::new(1, job_parameters, &job_target); + let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); let queue = PgQueue::new( "test_can_dequeue_tx_job", @@ -865,7 +900,7 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); - let tx_job: PgTransactionJob<'_, JobParameters> = queue + let tx_job: PgTransactionJob<'_, JobParameters, JobMetadata> = queue .dequeue_tx(&worker_id) .await .expect("failed to dequeue job") @@ -875,6 +910,7 @@ mod tests { assert!(tx_job.job.attempted_by.contains(&worker_id)); assert_eq!(tx_job.job.attempted_by.len(), 1); assert_eq!(tx_job.job.max_attempts, 1); + assert_eq!(*tx_job.job.metadata.as_ref(), JobMetadata::default()); assert_eq!(*tx_job.job.parameters.as_ref(), JobParameters::default()); assert_eq!(tx_job.job.status, JobStatus::Running); assert_eq!(tx_job.job.target, job_target); @@ -892,7 +928,7 @@ mod tests { .await .expect("failed to connect to local test postgresql database"); - let tx_job: Option> = queue + let tx_job: Option> = queue .dequeue_tx(&worker_id) .await .expect("failed to dequeue job"); @@ -904,8 +940,9 @@ mod tests { async fn test_can_retry_job_with_remaining_attempts() { let job_target = job_target(); let job_parameters = JobParameters::default(); + let job_metadata = JobMetadata::default(); let worker_id = worker_id(); - let new_job = NewJob::new(2, job_parameters, &job_target); + let new_job = NewJob::new(2, job_metadata, job_parameters, &job_target); let retry_policy = RetryPolicy { backoff_coefficient: 0, initial_interval: time::Duration::from_secs(0), @@ -922,7 +959,7 @@ mod tests { .expect("failed to connect to local test postgresql database"); queue.enqueue(new_job).await.expect("failed to enqueue job"); - let job: PgJob = queue + let job: PgJob = queue .dequeue(&worker_id) .await .expect("failed to dequeue job") @@ -931,7 +968,7 @@ mod tests { .retry("a very reasonable failure reason", None) .await .expect("failed to retry job"); - let retried_job: PgJob = queue + let retried_job: PgJob = queue .dequeue(&worker_id) .await .expect("failed to dequeue job") @@ -954,8 +991,9 @@ mod tests { async fn test_cannot_retry_job_without_remaining_attempts() { let job_target = job_target(); let job_parameters = JobParameters::default(); + let job_metadata = JobMetadata::default(); let worker_id = worker_id(); - let new_job = NewJob::new(1, job_parameters, &job_target); + let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); let retry_policy = RetryPolicy { backoff_coefficient: 0, initial_interval: time::Duration::from_secs(0), @@ -973,7 +1011,7 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); - let job: PgJob = queue + let job: PgJob = queue .dequeue(&worker_id) .await .expect("failed to dequeue job") diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index b17959ce0f6da..64968fc51deb3 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -124,16 +124,14 @@ pub struct WebhookJobParameters { pub headers: collections::HashMap, pub method: HttpMethod, pub url: String, +} - // These should be set if the Webhook is associated with a plugin `composeWebhook` invocation. +/// `JobParameters` required for the `WebhookConsumer` to execute a webhook. +/// These parameters should match the exported Webhook interface that PostHog plugins. +/// implement. See: https://github.com/PostHog/plugin-scaffold/blob/main/src/types.ts#L15. +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct WebhookJobMetadata { pub team_id: Option, pub plugin_id: Option, pub plugin_config_id: Option, - - #[serde(default = "default_max_attempts")] - pub max_attempts: i32, -} - -fn default_max_attempts() -> i32 { - 3 } diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 59b8e9f24d949..9e4b3b798405e 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -4,7 +4,7 @@ use std::time; use async_std::task; use hook_common::pgqueue::{PgJobError, PgQueue, PgQueueError, PgTransactionJob}; -use hook_common::webhook::{HttpMethod, WebhookJobParameters}; +use hook_common::webhook::{HttpMethod, WebhookJobMetadata, WebhookJobParameters}; use http::StatusCode; use reqwest::header; use tokio::sync; @@ -57,7 +57,8 @@ impl<'p> WebhookConsumer<'p> { /// Wait until a job becomes available in our queue. async fn wait_for_job<'a>( &self, - ) -> Result, WebhookConsumerError> { + ) -> Result, WebhookConsumerError> + { loop { if let Some(job) = self.queue.dequeue_tx(&self.name).await? { return Ok(job); @@ -102,7 +103,7 @@ impl<'p> WebhookConsumer<'p> { /// * `request_timeout`: A timeout for the HTTP request. async fn process_webhook_job( client: reqwest::Client, - webhook_job: PgTransactionJob<'_, WebhookJobParameters>, + webhook_job: PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata>, ) -> Result<(), WebhookConsumerError> { match send_webhook( client, @@ -261,9 +262,10 @@ mod tests { queue: &PgQueue, max_attempts: i32, job_parameters: WebhookJobParameters, + job_metadata: WebhookJobMetadata, ) -> Result<(), PgQueueError> { let job_target = job_parameters.url.to_owned(); - let new_job = NewJob::new(max_attempts, job_parameters, &job_target); + let new_job = NewJob::new(max_attempts, job_metadata, job_parameters, &job_target); queue.enqueue(new_job).await?; Ok(()) } @@ -308,25 +310,29 @@ mod tests { .await .expect("failed to connect to PG"); - let webhook_job = WebhookJobParameters { + let webhook_job_parameters = WebhookJobParameters { body: "a webhook job body. much wow.".to_owned(), headers: collections::HashMap::new(), method: HttpMethod::POST, url: "localhost".to_owned(), - - team_id: Some(1), - plugin_id: Some(2), - plugin_config_id: Some(3), - - max_attempts: 1, + }; + let webhook_job_metadata = WebhookJobMetadata { + team_id: None, + plugin_id: None, + plugin_config_id: None, }; // enqueue takes ownership of the job enqueued to avoid bugs that can cause duplicate jobs. // Normally, a separate application would be enqueueing jobs for us to consume, so no ownership // conflicts would arise. However, in this test we need to do the enqueueing ourselves. // So, we clone the job to keep it around and assert the values returned by wait_for_job. - enqueue_job(&queue, 1, webhook_job.clone()) - .await - .expect("failed to enqueue job"); + enqueue_job( + &queue, + 1, + webhook_job_parameters.clone(), + webhook_job_metadata, + ) + .await + .expect("failed to enqueue job"); let consumer = WebhookConsumer::new( &worker_id, &queue, @@ -344,9 +350,12 @@ mod tests { assert!(consumed_job.job.attempted_by.contains(&worker_id)); assert_eq!(consumed_job.job.attempted_by.len(), 1); assert_eq!(consumed_job.job.max_attempts, 1); - assert_eq!(*consumed_job.job.parameters.as_ref(), webhook_job); + assert_eq!( + *consumed_job.job.parameters.as_ref(), + webhook_job_parameters + ); assert_eq!(consumed_job.job.status, JobStatus::Running); - assert_eq!(consumed_job.job.target, webhook_job.url); + assert_eq!(consumed_job.job.target, webhook_job_parameters.url); consumed_job .complete() diff --git a/hook-producer/src/handlers/webhook.rs b/hook-producer/src/handlers/webhook.rs index 7de1126fe4db8..394732094a3c6 100644 --- a/hook-producer/src/handlers/webhook.rs +++ b/hook-producer/src/handlers/webhook.rs @@ -1,5 +1,5 @@ use axum::{extract::State, http::StatusCode, Json}; -use hook_common::webhook::WebhookJobParameters; +use hook_common::webhook::{WebhookJobMetadata, WebhookJobParameters}; use serde_derive::Deserialize; use url::Url; @@ -15,13 +15,27 @@ pub struct WebhookPostResponse { error: Option, } +/// The body of a request made to create a webhook Job. +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct WebhookPostRequestBody { + parameters: WebhookJobParameters, + metadata: WebhookJobMetadata, + + #[serde(default = "default_max_attempts")] + max_attempts: u32, +} + +fn default_max_attempts() -> u32 { + 3 +} + pub async fn post( State(pg_queue): State, - Json(payload): Json, + Json(payload): Json, ) -> Result, (StatusCode, Json)> { debug!("received payload: {:?}", payload); - if payload.body.len() > MAX_BODY_SIZE { + if payload.parameters.body.len() > MAX_BODY_SIZE { return Err(( StatusCode::BAD_REQUEST, Json(WebhookPostResponse { @@ -30,8 +44,22 @@ pub async fn post( )); } - let url_hostname = get_hostname(&payload.url)?; - let job = NewJob::new(payload.max_attempts, payload, url_hostname.as_str()); + let url_hostname = get_hostname(&payload.parameters.url)?; + // We could cast to i32, but this ensures we are not wrapping. + let max_attempts = i32::try_from(payload.max_attempts).map_err(|_| { + ( + StatusCode::BAD_REQUEST, + Json(WebhookPostResponse { + error: Some("invalid number of max attempts".to_owned()), + }), + ) + })?; + let job = NewJob::new( + max_attempts, + payload.metadata, + payload.parameters, + url_hostname.as_str(), + ); pg_queue.enqueue(job).await.map_err(internal_error)?; @@ -74,6 +102,8 @@ fn get_hostname(url_str: &str) -> Result Date: Mon, 18 Dec 2023 11:43:11 +0100 Subject: [PATCH 123/247] fix: Update webhook.rs docs --- hook-common/src/webhook.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index 64968fc51deb3..ce7d3ae2ad4d0 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -126,9 +126,8 @@ pub struct WebhookJobParameters { pub url: String, } -/// `JobParameters` required for the `WebhookConsumer` to execute a webhook. -/// These parameters should match the exported Webhook interface that PostHog plugins. -/// implement. See: https://github.com/PostHog/plugin-scaffold/blob/main/src/types.ts#L15. +/// `JobMetadata` required for the `WebhookConsumer` to execute a webhook. +/// These should be set if the Webhook is associated with a plugin `composeWebhook` invocation. #[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] pub struct WebhookJobMetadata { pub team_id: Option, From 491c2697d29aa995e972478964b4d8cfc85bff20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Mon, 18 Dec 2023 12:08:08 +0100 Subject: [PATCH 124/247] feat: Use new WebhookJobError struct for error reporting --- Cargo.lock | 156 +++++++++--------- Cargo.toml | 1 + hook-common/Cargo.toml | 1 + hook-common/src/kafka_messages/app_metrics.rs | 10 +- hook-common/src/kafka_messages/mod.rs | 4 +- hook-common/src/webhook.rs | 116 +++++++++++++ hook-consumer/Cargo.toml | 2 +- hook-consumer/src/consumer.rs | 117 +++++++------ hook-consumer/src/error.rs | 27 +-- hook-consumer/src/main.rs | 4 +- 10 files changed, 294 insertions(+), 144 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b0aedcfa6038..ba9f4bb10b67d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -100,13 +100,13 @@ dependencies = [ [[package]] name = "async-global-executor" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4353121d5644cdf2beb5726ab752e79a8db1ebb52031770ec47db31d245526" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ "async-channel 2.1.1", "async-executor", - "async-io 2.2.1", + "async-io 2.2.2", "async-lock 3.2.0", "blocking", "futures-lite 2.1.0", @@ -135,9 +135,9 @@ dependencies = [ [[package]] name = "async-io" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6d3b15875ba253d1110c740755e246537483f152fa334f91abd7fe84c88b3ff" +checksum = "6afaa937395a620e33dc6a742c593c01aced20aa376ffb0f628121198578ccc7" dependencies = [ "async-lock 3.2.0", "cfg-if", @@ -146,7 +146,7 @@ dependencies = [ "futures-lite 2.1.0", "parking", "polling 3.3.1", - "rustix 0.38.27", + "rustix 0.38.28", "slab", "tracing", "windows-sys 0.52.0", @@ -200,9 +200,9 @@ dependencies = [ [[package]] name = "async-task" -version = "4.5.0" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" +checksum = "e1d90cd0b264dfdd8eb5bad0a2c217c1f88fa96a8573f40e7b12de23fb468f46" [[package]] name = "async-trait" @@ -212,7 +212,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -265,7 +265,7 @@ dependencies = [ "http 1.0.0", "http-body 1.0.0", "http-body-util", - "hyper 1.0.1", + "hyper 1.1.0", "hyper-util", "itoa", "matchit", @@ -439,9 +439,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "core-foundation" @@ -485,22 +485,21 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" dependencies = [ "cfg-if", "crossbeam-utils", @@ -508,9 +507,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" dependencies = [ "cfg-if", ] @@ -648,9 +647,9 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbb8258be8305fb0237d7b295f47bb24ff1b136a535f473baf40e70468515aa" +checksum = "b6267a1fa6f59179ea4afc8e50fd8612a3cc60bc858f786ff877a4a8cb042799" dependencies = [ "indenter", "once_cell", @@ -813,7 +812,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -980,9 +979,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hkdf" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ "hmac", ] @@ -998,11 +997,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1015,6 +1014,7 @@ dependencies = [ "metrics", "metrics-exporter-prometheus", "regex", + "reqwest", "serde", "serde_derive", "serde_json", @@ -1115,9 +1115,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http 0.2.11", @@ -1161,9 +1161,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -1171,12 +1171,12 @@ dependencies = [ "futures-util", "h2 0.3.22", "http 0.2.11", - "http-body 0.4.5", + "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -1185,9 +1185,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56" +checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" dependencies = [ "bytes", "futures-channel", @@ -1209,7 +1209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.27", + "hyper 0.14.28", "native-tls", "tokio", "tokio-native-tls", @@ -1226,7 +1226,7 @@ dependencies = [ "futures-util", "http 1.0.0", "http-body 1.0.0", - "hyper 1.0.1", + "hyper 1.1.0", "pin-project-lite", "socket2 0.5.5", "tokio", @@ -1331,9 +1331,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" @@ -1364,9 +1364,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.150" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libm" @@ -1481,12 +1481,12 @@ dependencies = [ [[package]] name = "metrics-exporter-prometheus" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" +checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" dependencies = [ "base64", - "hyper 0.14.27", + "hyper 0.14.28", "indexmap 1.9.3", "ipnet", "metrics", @@ -1505,7 +1505,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -1721,7 +1721,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -1815,7 +1815,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -1893,7 +1893,7 @@ dependencies = [ "cfg-if", "concurrent-queue", "pin-project-lite", - "rustix 0.38.27", + "rustix 0.38.28", "tracing", "windows-sys 0.52.0", ] @@ -2065,9 +2065,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "base64", "bytes", @@ -2076,8 +2076,8 @@ dependencies = [ "futures-util", "h2 0.3.22", "http 0.2.11", - "http-body 0.4.5", - "hyper 0.14.27", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-tls", "ipnet", "js-sys", @@ -2143,9 +2143,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.27" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfeae074e687625746172d639330f1de242a178bf3189b51e35a7a21573513ac" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ "bitflags 2.4.1", "errno", @@ -2162,9 +2162,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "schannel" @@ -2221,7 +2221,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2620,9 +2620,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" dependencies = [ "proc-macro2", "quote", @@ -2665,28 +2665,28 @@ dependencies = [ "cfg-if", "fastrand 2.0.1", "redox_syscall", - "rustix 0.38.27", + "rustix 0.38.28", "windows-sys 0.48.0", ] [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2716,9 +2716,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.34.0" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" dependencies = [ "backtrace", "bytes", @@ -2741,7 +2741,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2844,7 +2844,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -3020,7 +3020,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", "wasm-bindgen-shared", ] @@ -3054,7 +3054,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3246,9 +3246,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.28" +version = "0.5.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c830786f7720c2fd27a1a0e27a709dbd3c4d009b56d098fc742d4f4eab91fe2" +checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5" dependencies = [ "memchr", ] @@ -3265,22 +3265,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.29" +version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d075cf85bbb114e933343e087b92f2146bac0d55b534cbb8188becf0039948e" +checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.29" +version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86cd5ca076997b97ef09d3ad65efe811fa68c9e874cb636ccb211223a813b0c2" +checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index faf8644a85dfb..0a0a2f578560b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ http-body-util = "0.1.0" metrics = "0.21.1" metrics-exporter-prometheus = "0.12.1" rdkafka = { version = "0.35.0", features = ["cmake-build", "ssl"] } +reqwest = { version = "0.11" } regex = "1.10.2" serde = { version = "1.0" } serde_derive = { version = "1.0" } diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index 2c32d9d35c1b1..6350ba4b8f7c1 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -11,6 +11,7 @@ chrono = { workspace = true } http = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } +reqwest = { workspace = true } regex = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs index a7530644280f6..327f4b89fb6a6 100644 --- a/hook-common/src/kafka_messages/app_metrics.rs +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -13,14 +13,16 @@ pub enum AppMetricCategory { ComposeWebhook, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub enum ErrorType { Timeout, Connection, HttpStatus(u16), + Parse, + MaxAttempts, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub struct ErrorDetails { pub error: Error, // TODO: The plugin-server sends the entire raw event with errors. In order to do this, we'll @@ -30,7 +32,7 @@ pub struct ErrorDetails { // event: Value, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub struct Error { pub name: String, #[serde(skip_serializing_if = "Option::is_none")] @@ -84,6 +86,8 @@ where ErrorType::Connection => "Connection Error".to_owned(), ErrorType::Timeout => "Timeout".to_owned(), ErrorType::HttpStatus(s) => format!("HTTP Status: {}", s), + ErrorType::Parse => "Parse Error".to_owned(), + ErrorType::MaxAttempts => "Maximum attempts exceeded".to_owned(), }; serializer.serialize_str(&error_type) } diff --git a/hook-common/src/kafka_messages/mod.rs b/hook-common/src/kafka_messages/mod.rs index 1449f56faeaa7..a29e2193d327b 100644 --- a/hook-common/src/kafka_messages/mod.rs +++ b/hook-common/src/kafka_messages/mod.rs @@ -5,14 +5,14 @@ use chrono::{DateTime, Utc}; use serde::Serializer; use uuid::Uuid; -fn serialize_uuid(uuid: &Uuid, serializer: S) -> Result +pub fn serialize_uuid(uuid: &Uuid, serializer: S) -> Result where S: Serializer, { serializer.serialize_str(&uuid.to_string()) } -fn serialize_datetime(datetime: &DateTime, serializer: S) -> Result +pub fn serialize_datetime(datetime: &DateTime, serializer: S) -> Result where S: Serializer, { diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index ce7d3ae2ad4d0..ea7dfb4724159 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -1,9 +1,11 @@ use std::collections; +use std::convert::From; use std::fmt; use std::str::FromStr; use serde::{de::Visitor, Deserialize, Serialize}; +use crate::kafka_messages::{app_metrics, serialize_uuid}; use crate::pgqueue::PgQueueError; /// Supported HTTP methods for webhooks. @@ -134,3 +136,117 @@ pub struct WebhookJobMetadata { pub plugin_id: Option, pub plugin_config_id: Option, } + +/// An error originating during a Webhook Job invocation. +#[derive(Serialize, Debug)] +pub struct WebhookJobError { + pub r#type: app_metrics::ErrorType, + pub details: app_metrics::ErrorDetails, + #[serde(serialize_with = "serialize_uuid")] + pub uuid: uuid::Uuid, +} + +impl From for WebhookJobError { + fn from(error: reqwest::Error) -> Self { + if error.is_body() || error.is_decode() { + WebhookJobError::new_parse(&error.to_string()) + } else if error.is_timeout() { + WebhookJobError::new_timeout(&error.to_string()) + } else if error.is_status() { + WebhookJobError::new_http_status( + error.status().expect("status code is defined").into(), + &error.to_string(), + ) + } else if error.is_connect() + || error.is_builder() + || error.is_request() + || error.is_redirect() + { + // Builder errors seem to be related to unable to setup TLS, so I'm bundling them in connection. + WebhookJobError::new_connection(&error.to_string()) + } else { + // We can't match on Kind as some types do not have an associated variant in Kind (e.g. Timeout). + unreachable!("We have covered all reqwest::Error types.") + } + } +} + +impl WebhookJobError { + pub fn new_timeout(message: &str) -> Self { + let error_details = app_metrics::Error { + name: "timeout".to_owned(), + message: Some(message.to_owned()), + stack: None, + }; + Self { + r#type: app_metrics::ErrorType::Timeout, + details: app_metrics::ErrorDetails { + error: error_details, + }, + uuid: uuid::Uuid::now_v7(), + } + } + + pub fn new_connection(message: &str) -> Self { + let error_details = app_metrics::Error { + name: "connection error".to_owned(), + message: Some(message.to_owned()), + stack: None, + }; + Self { + r#type: app_metrics::ErrorType::Connection, + details: app_metrics::ErrorDetails { + error: error_details, + }, + uuid: uuid::Uuid::now_v7(), + } + } + + pub fn new_http_status(status_code: u16, message: &str) -> Self { + let error_details = app_metrics::Error { + name: "http status".to_owned(), + message: Some(message.to_owned()), + stack: None, + }; + Self { + r#type: app_metrics::ErrorType::HttpStatus(status_code), + details: app_metrics::ErrorDetails { + error: error_details, + }, + uuid: uuid::Uuid::now_v7(), + } + } + + pub fn new_parse(message: &str) -> Self { + let error_details = app_metrics::Error { + name: "parse error".to_owned(), + message: Some(message.to_owned()), + stack: None, + }; + Self { + r#type: app_metrics::ErrorType::Parse, + details: app_metrics::ErrorDetails { + error: error_details, + }, + uuid: uuid::Uuid::now_v7(), + } + } + + pub fn new_max_attempts(max_attempts: i32) -> Self { + let error_details = app_metrics::Error { + name: "maximum attempts exceeded".to_owned(), + message: Some(format!( + "Exceeded maximum number of attempts ({}) for webhook", + max_attempts + )), + stack: None, + }; + Self { + r#type: app_metrics::ErrorType::MaxAttempts, + details: app_metrics::ErrorDetails { + error: error_details, + }, + uuid: uuid::Uuid::now_v7(), + } + } +} diff --git a/hook-consumer/Cargo.toml b/hook-consumer/Cargo.toml index 5ff1eb08eb660..35c64b5e7c7a4 100644 --- a/hook-consumer/Cargo.toml +++ b/hook-consumer/Cargo.toml @@ -10,7 +10,7 @@ envconfig = { workspace = true } futures = "0.3" hook-common = { path = "../hook-common" } http = { version = "0.2" } -reqwest = { version = "0.11" } +reqwest = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } sqlx = { workspace = true } diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 9e4b3b798405e..bf7573fec4058 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -4,12 +4,12 @@ use std::time; use async_std::task; use hook_common::pgqueue::{PgJobError, PgQueue, PgQueueError, PgTransactionJob}; -use hook_common::webhook::{HttpMethod, WebhookJobMetadata, WebhookJobParameters}; +use hook_common::webhook::{HttpMethod, WebhookJobError, WebhookJobMetadata, WebhookJobParameters}; use http::StatusCode; use reqwest::header; use tokio::sync; -use crate::error::WebhookConsumerError; +use crate::error::{ConsumerError, WebhookError}; /// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. pub struct WebhookConsumer<'p> { @@ -57,8 +57,7 @@ impl<'p> WebhookConsumer<'p> { /// Wait until a job becomes available in our queue. async fn wait_for_job<'a>( &self, - ) -> Result, WebhookConsumerError> - { + ) -> Result, ConsumerError> { loop { if let Some(job) = self.queue.dequeue_tx(&self.name).await? { return Ok(job); @@ -69,7 +68,7 @@ impl<'p> WebhookConsumer<'p> { } /// Run this consumer to continuously process any jobs that become available. - pub async fn run(&self) -> Result<(), WebhookConsumerError> { + pub async fn run(&self) -> Result<(), ConsumerError> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); loop { @@ -104,7 +103,7 @@ impl<'p> WebhookConsumer<'p> { async fn process_webhook_job( client: reqwest::Client, webhook_job: PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata>, -) -> Result<(), WebhookConsumerError> { +) -> Result<(), ConsumerError> { match send_webhook( client, &webhook_job.job.parameters.method, @@ -118,31 +117,54 @@ async fn process_webhook_job( webhook_job .complete() .await - .map_err(|error| WebhookConsumerError::PgJobError(error.to_string()))?; + .map_err(|error| ConsumerError::PgJobError(error.to_string()))?; + Ok(()) + } + Err(WebhookError::ParseHeadersError(e)) => { + webhook_job + .fail(WebhookJobError::new_parse(&e.to_string())) + .await + .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; Ok(()) } - Err(WebhookConsumerError::RetryableWebhookError { - reason, - retry_after, - }) => match webhook_job.retry(reason.to_string(), retry_after).await { - Ok(_) => Ok(()), - Err(PgJobError::RetryInvalidError { - job: webhook_job, - error: fail_error, - }) => { - webhook_job - .fail(fail_error.to_string()) - .await - .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; - Ok(()) + Err(WebhookError::ParseHttpMethodError(e)) => { + webhook_job + .fail(WebhookJobError::new_parse(&e)) + .await + .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + Ok(()) + } + Err(WebhookError::ParseUrlError(e)) => { + webhook_job + .fail(WebhookJobError::new_parse(&e.to_string())) + .await + .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + Ok(()) + } + Err(WebhookError::RetryableRequestError { error, retry_after }) => { + match webhook_job + .retry(WebhookJobError::from(error), retry_after) + .await + { + Ok(_) => Ok(()), + Err(PgJobError::RetryInvalidError { + job: webhook_job, .. + }) => { + let max_attempts = webhook_job.job.max_attempts; + webhook_job + .fail(WebhookJobError::new_max_attempts(max_attempts)) + .await + .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + Ok(()) + } + Err(job_error) => Err(ConsumerError::PgJobError(job_error.to_string())), } - Err(job_error) => Err(WebhookConsumerError::PgJobError(job_error.to_string())), - }, - Err(error) => { + } + Err(WebhookError::NonRetryableRetryableRequestError(error)) => { webhook_job - .fail(error.to_string()) + .fail(WebhookJobError::from(error)) .await - .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; Ok(()) } } @@ -163,12 +185,12 @@ async fn send_webhook( url: &str, headers: &collections::HashMap, body: String, -) -> Result { +) -> Result<(), WebhookError> { let method: http::Method = method.into(); - let url: reqwest::Url = (url).parse().map_err(WebhookConsumerError::ParseUrlError)?; + let url: reqwest::Url = (url).parse().map_err(WebhookError::ParseUrlError)?; let headers: reqwest::header::HeaderMap = (headers) .try_into() - .map_err(WebhookConsumerError::ParseHeadersError)?; + .map_err(WebhookError::ParseHeadersError)?; let body = reqwest::Body::from(body); let response = client @@ -177,27 +199,28 @@ async fn send_webhook( .body(body) .send() .await - .map_err(|e| WebhookConsumerError::RetryableWebhookError { - reason: e.to_string(), + .map_err(|e| WebhookError::RetryableRequestError { + error: e, retry_after: None, })?; - let status = response.status(); - - if status.is_success() { - Ok(response) - } else if is_retryable_status(status) { - let retry_after = parse_retry_after_header(response.headers()); - - Err(WebhookConsumerError::RetryableWebhookError { - reason: format!("retryable status code {}", status), - retry_after, - }) - } else { - Err(WebhookConsumerError::NonRetryableWebhookError(format!( - "non-retryable status code {}", - status - ))) + match response.error_for_status_ref() { + Ok(_) => Ok(()), + Err(err) => { + if is_retryable_status( + err.status() + .expect("status code is set as error is generated from a response"), + ) { + let retry_after = parse_retry_after_header(response.headers()); + + Err(WebhookError::RetryableRequestError { + error: err, + retry_after, + }) + } else { + Err(WebhookError::NonRetryableRetryableRequestError(err)) + } + } } } diff --git a/hook-consumer/src/error.rs b/hook-consumer/src/error.rs index a19664359caf7..b05d476e22849 100644 --- a/hook-consumer/src/error.rs +++ b/hook-consumer/src/error.rs @@ -3,26 +3,31 @@ use std::time; use hook_common::pgqueue; use thiserror::Error; -/// Enumeration of errors for operations with WebhookConsumer. +/// Enumeration of errors related to webhook job processing in the WebhookConsumer. #[derive(Error, Debug)] -pub enum WebhookConsumerError { - #[error("timed out while waiting for jobs to be available")] - TimeoutError, +pub enum WebhookError { #[error("{0} is not a valid HttpMethod")] ParseHttpMethodError(String), #[error("error parsing webhook headers")] ParseHeadersError(http::Error), #[error("error parsing webhook url")] ParseUrlError(url::ParseError), + #[error("a webhook could not be delivered but it could be retried later: {error}")] + RetryableRequestError { + error: reqwest::Error, + retry_after: Option, + }, + #[error("a webhook could not be delivered and it cannot be retried further: {0}")] + NonRetryableRetryableRequestError(reqwest::Error), +} + +/// Enumeration of errors related to initialization and consumption of webhook jobs. +#[derive(Error, Debug)] +pub enum ConsumerError { + #[error("timed out while waiting for jobs to be available")] + TimeoutError, #[error("an error occurred in the underlying queue")] QueueError(#[from] pgqueue::PgQueueError), #[error("an error occurred in the underlying job")] PgJobError(String), - #[error("a webhook could not be delivered but it could be retried later: {reason}")] - RetryableWebhookError { - reason: String, - retry_after: Option, - }, - #[error("a webhook could not be delivered and it cannot be retried further: {0}")] - NonRetryableWebhookError(String), } diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index bf76503041d3a..bb02526c23b76 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -3,10 +3,10 @@ use envconfig::Envconfig; use hook_common::pgqueue::{PgQueue, RetryPolicy}; use hook_consumer::config::Config; use hook_consumer::consumer::WebhookConsumer; -use hook_consumer::error::WebhookConsumerError; +use hook_consumer::error::ConsumerError; #[tokio::main] -async fn main() -> Result<(), WebhookConsumerError> { +async fn main() -> Result<(), ConsumerError> { let config = Config::init_from_env().expect("Invalid configuration:"); let retry_policy = RetryPolicy::new( From c64c3d054c7f9701dfc170e79e6f77f9f36a995d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Tue, 19 Dec 2023 12:15:08 +0100 Subject: [PATCH 125/247] fix: Return response from send_webhook --- hook-consumer/src/consumer.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index bf7573fec4058..9430126f1f45a 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -185,7 +185,7 @@ async fn send_webhook( url: &str, headers: &collections::HashMap, body: String, -) -> Result<(), WebhookError> { +) -> Result { let method: http::Method = method.into(); let url: reqwest::Url = (url).parse().map_err(WebhookError::ParseUrlError)?; let headers: reqwest::header::HeaderMap = (headers) @@ -204,15 +204,15 @@ async fn send_webhook( retry_after: None, })?; - match response.error_for_status_ref() { - Ok(_) => Ok(()), + let retry_after = parse_retry_after_header(response.headers()); + + match response.error_for_status() { + Ok(response) => Ok(response), Err(err) => { if is_retryable_status( err.status() .expect("status code is set as error is generated from a response"), ) { - let retry_after = parse_retry_after_header(response.headers()); - Err(WebhookError::RetryableRequestError { error: err, retry_after, From d80644b2fa8325dcd121f71656d13a6860f0cb2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Tue, 19 Dec 2023 15:02:18 +0100 Subject: [PATCH 126/247] refactor: Re-use underlying error when failing after retry --- hook-common/src/kafka_messages/app_metrics.rs | 2 -- hook-common/src/webhook.rs | 22 ++----------------- hook-consumer/src/consumer.rs | 7 +++--- 3 files changed, 5 insertions(+), 26 deletions(-) diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs index 327f4b89fb6a6..8964144bf65b6 100644 --- a/hook-common/src/kafka_messages/app_metrics.rs +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -19,7 +19,6 @@ pub enum ErrorType { Connection, HttpStatus(u16), Parse, - MaxAttempts, } #[derive(Serialize, Debug)] @@ -87,7 +86,6 @@ where ErrorType::Timeout => "Timeout".to_owned(), ErrorType::HttpStatus(s) => format!("HTTP Status: {}", s), ErrorType::Parse => "Parse Error".to_owned(), - ErrorType::MaxAttempts => "Maximum attempts exceeded".to_owned(), }; serializer.serialize_str(&error_type) } diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index ea7dfb4724159..d8c174d3f6cd6 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -146,8 +146,8 @@ pub struct WebhookJobError { pub uuid: uuid::Uuid, } -impl From for WebhookJobError { - fn from(error: reqwest::Error) -> Self { +impl From<&reqwest::Error> for WebhookJobError { + fn from(error: &reqwest::Error) -> Self { if error.is_body() || error.is_decode() { WebhookJobError::new_parse(&error.to_string()) } else if error.is_timeout() { @@ -231,22 +231,4 @@ impl WebhookJobError { uuid: uuid::Uuid::now_v7(), } } - - pub fn new_max_attempts(max_attempts: i32) -> Self { - let error_details = app_metrics::Error { - name: "maximum attempts exceeded".to_owned(), - message: Some(format!( - "Exceeded maximum number of attempts ({}) for webhook", - max_attempts - )), - stack: None, - }; - Self { - r#type: app_metrics::ErrorType::MaxAttempts, - details: app_metrics::ErrorDetails { - error: error_details, - }, - uuid: uuid::Uuid::now_v7(), - } - } } diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 9430126f1f45a..8a2ecdc026f39 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -143,16 +143,15 @@ async fn process_webhook_job( } Err(WebhookError::RetryableRequestError { error, retry_after }) => { match webhook_job - .retry(WebhookJobError::from(error), retry_after) + .retry(WebhookJobError::from(&error), retry_after) .await { Ok(_) => Ok(()), Err(PgJobError::RetryInvalidError { job: webhook_job, .. }) => { - let max_attempts = webhook_job.job.max_attempts; webhook_job - .fail(WebhookJobError::new_max_attempts(max_attempts)) + .fail(WebhookJobError::from(&error)) .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; Ok(()) @@ -162,7 +161,7 @@ async fn process_webhook_job( } Err(WebhookError::NonRetryableRetryableRequestError(error)) => { webhook_job - .fail(WebhookJobError::from(error)) + .fail(WebhookJobError::from(&error)) .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; Ok(()) From e4ee369fb24c2f6ee410a8349da0b78953d90e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Tue, 19 Dec 2023 16:45:58 +0100 Subject: [PATCH 127/247] chore: Update docs --- hook-common/src/webhook.rs | 4 ++++ hook-consumer/src/consumer.rs | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index d8c174d3f6cd6..488d52786b125 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -138,6 +138,7 @@ pub struct WebhookJobMetadata { } /// An error originating during a Webhook Job invocation. +/// This is to be serialized to be stored as an error whenever retrying or failing a webhook job. #[derive(Serialize, Debug)] pub struct WebhookJobError { pub r#type: app_metrics::ErrorType, @@ -146,6 +147,9 @@ pub struct WebhookJobError { pub uuid: uuid::Uuid, } +/// Webhook jobs boil down to an HTTP request, so it's useful to have a way to convert from &reqwest::Error. +/// For the convertion we check all possible error types with the associated is_* methods provided by reqwest. +/// Some precision may be lost as our app_metrics::ErrorType does not support the same number of variants. impl From<&reqwest::Error> for WebhookJobError { fn from(error: &reqwest::Error) -> Self { if error.is_body() || error.is_decode() { diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 8a2ecdc026f39..12d4b38b9a56f 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -98,8 +98,8 @@ impl<'p> WebhookConsumer<'p> { /// /// # Arguments /// +/// * `client`: An HTTP client to execute the webhook job request. /// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. -/// * `request_timeout`: A timeout for the HTTP request. async fn process_webhook_job( client: reqwest::Client, webhook_job: PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata>, @@ -173,11 +173,11 @@ async fn process_webhook_job( /// /// # Arguments /// +/// * `client`: An HTTP client to execute the HTTP request. /// * `method`: The HTTP method to use in the HTTP request. /// * `url`: The URL we are targetting with our request. Parsing this URL fail. /// * `headers`: Key, value pairs of HTTP headers in a `std::collections::HashMap`. Can fail if headers are not valid. /// * `body`: The body of the request. Ownership is required. -/// * `timeout`: A timeout for the HTTP request. async fn send_webhook( client: reqwest::Client, method: &HttpMethod, From f4803070a5aca3803bb9d56ec5574341c85f8d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Tue, 19 Dec 2023 17:10:37 +0100 Subject: [PATCH 128/247] refactor: Have Connection Error catch all reqwest errors --- hook-common/src/webhook.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index 488d52786b125..2bf8db34f9fd5 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -152,25 +152,18 @@ pub struct WebhookJobError { /// Some precision may be lost as our app_metrics::ErrorType does not support the same number of variants. impl From<&reqwest::Error> for WebhookJobError { fn from(error: &reqwest::Error) -> Self { - if error.is_body() || error.is_decode() { - WebhookJobError::new_parse(&error.to_string()) - } else if error.is_timeout() { + if error.is_timeout() { WebhookJobError::new_timeout(&error.to_string()) } else if error.is_status() { WebhookJobError::new_http_status( error.status().expect("status code is defined").into(), &error.to_string(), ) - } else if error.is_connect() - || error.is_builder() - || error.is_request() - || error.is_redirect() - { - // Builder errors seem to be related to unable to setup TLS, so I'm bundling them in connection. - WebhookJobError::new_connection(&error.to_string()) } else { - // We can't match on Kind as some types do not have an associated variant in Kind (e.g. Timeout). - unreachable!("We have covered all reqwest::Error types.") + // Catch all other errors as `app_metrics::ErrorType::Connection` errors. + // Not all of `reqwest::Error` may strictly be connection errors, so our supported error types may need an extension + // depending on how strict error reporting has to be. + WebhookJobError::new_connection(&error.to_string()) } } } From f9b32ae4c04add0281f7f0d5390a61dc8bcdfc27 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 19 Dec 2023 09:53:20 -0700 Subject: [PATCH 129/247] Implement Webhook cleanup --- hook-common/src/kafka_messages/app_metrics.rs | 66 ++-- hook-common/src/kafka_messages/mod.rs | 10 + hook-common/src/webhook.rs | 10 +- hook-janitor/src/cleanup.rs | 2 - hook-janitor/src/config.rs | 3 - hook-janitor/src/main.rs | 2 - hook-janitor/src/webhooks.rs | 328 +++++++++++++++++- 7 files changed, 361 insertions(+), 60 deletions(-) diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs index 8964144bf65b6..439664342fa35 100644 --- a/hook-common/src/kafka_messages/app_metrics.rs +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -1,10 +1,10 @@ use chrono::{DateTime, Utc}; -use serde::{Serialize, Serializer}; +use serde::{Deserialize, Serialize, Serializer}; use uuid::Uuid; -use super::{serialize_datetime, serialize_uuid}; +use super::{serialize_datetime, serialize_optional_uuid}; -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub enum AppMetricCategory { ProcessEvent, OnEvent, @@ -13,7 +13,7 @@ pub enum AppMetricCategory { ComposeWebhook, } -#[derive(Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug)] pub enum ErrorType { Timeout, Connection, @@ -21,29 +21,23 @@ pub enum ErrorType { Parse, } -#[derive(Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug)] pub struct ErrorDetails { pub error: Error, - // TODO: The plugin-server sends the entire raw event with errors. In order to do this, we'll - // have to pass the entire event when we enqueue items, and store it in the Parameters JSONB - // column. We should see if it's possible to work around this before we commit to it. - // - // event: Value, } -#[derive(Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug)] pub struct Error { pub name: String, #[serde(skip_serializing_if = "Option::is_none")] pub message: Option, - // TODO: Realistically, it doesn't seem likely that we'll generate Rust stack traces and put - // them here. I think this was more useful in plugin-server when the stack could come from - // plugin code. + // This field will only be useful if we start running plugins in Rust (via a WASM runtime or + // something) and want to provide the user with stack traces like we do for TypeScript plugins. #[serde(skip_serializing_if = "Option::is_none")] pub stack: Option, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub struct AppMetric { #[serde(serialize_with = "serialize_datetime")] pub timestamp: DateTime, @@ -56,11 +50,18 @@ pub struct AppMetric { pub successes: u32, pub successes_on_retry: u32, pub failures: u32, - #[serde(serialize_with = "serialize_uuid")] - pub error_uuid: Uuid, - #[serde(serialize_with = "serialize_error_type")] - pub error_type: ErrorType, - pub error_details: Error, + #[serde( + serialize_with = "serialize_optional_uuid", + skip_serializing_if = "Option::is_none" + )] + pub error_uuid: Option, + #[serde( + serialize_with = "serialize_error_type", + skip_serializing_if = "Option::is_none" + )] + pub error_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_details: Option, } fn serialize_category(category: &AppMetricCategory, serializer: S) -> Result @@ -77,10 +78,15 @@ where serializer.serialize_str(category_str) } -fn serialize_error_type(error_type: &ErrorType, serializer: S) -> Result +fn serialize_error_type(error_type: &Option, serializer: S) -> Result where S: Serializer, { + let error_type = match error_type { + Some(error_type) => error_type, + None => return serializer.serialize_none(), + }; + let error_type = match error_type { ErrorType::Connection => "Connection Error".to_owned(), ErrorType::Timeout => "Timeout".to_owned(), @@ -107,18 +113,20 @@ mod tests { successes: 10, successes_on_retry: 0, failures: 2, - error_uuid: Uuid::parse_str("550e8400-e29b-41d4-a716-446655447777").unwrap(), - error_type: ErrorType::Connection, - error_details: Error { - name: "FooError".to_owned(), - message: Some("Error Message".to_owned()), - stack: None, - }, + error_uuid: Some(Uuid::parse_str("550e8400-e29b-41d4-a716-446655447777").unwrap()), + error_type: Some(ErrorType::Connection), + error_details: Some(ErrorDetails { + error: Error { + name: "FooError".to_owned(), + message: Some("Error Message".to_owned()), + stack: None, + }, + }), }; let serialized_json = serde_json::to_string(&app_metric).unwrap(); - let expected_json = r#"{"timestamp":"2023-12-14 12:02:00","team_id":123,"plugin_config_id":456,"category":"webhook","successes":10,"successes_on_retry":0,"failures":2,"error_uuid":"550e8400-e29b-41d4-a716-446655447777","error_type":"Connection Error","error_details":{"name":"FooError","message":"Error Message"}}"#; + let expected_json = r#"{"timestamp":"2023-12-14 12:02:00","team_id":123,"plugin_config_id":456,"category":"webhook","successes":10,"successes_on_retry":0,"failures":2,"error_uuid":"550e8400-e29b-41d4-a716-446655447777","error_type":"Connection Error","error_details":{"error":{"name":"FooError","message":"Error Message"}}}"#; assert_eq!(serialized_json, expected_json); } diff --git a/hook-common/src/kafka_messages/mod.rs b/hook-common/src/kafka_messages/mod.rs index a29e2193d327b..72b49e1e45059 100644 --- a/hook-common/src/kafka_messages/mod.rs +++ b/hook-common/src/kafka_messages/mod.rs @@ -12,6 +12,16 @@ where serializer.serialize_str(&uuid.to_string()) } +pub fn serialize_optional_uuid(uuid: &Option, serializer: S) -> Result +where + S: Serializer, +{ + match uuid { + Some(uuid) => serializer.serialize_str(&uuid.to_string()), + None => serializer.serialize_none(), + } +} + pub fn serialize_datetime(datetime: &DateTime, serializer: S) -> Result where S: Serializer, diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index 2bf8db34f9fd5..d320ce0c4aac7 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -5,7 +5,7 @@ use std::str::FromStr; use serde::{de::Visitor, Deserialize, Serialize}; -use crate::kafka_messages::{app_metrics, serialize_uuid}; +use crate::kafka_messages::app_metrics; use crate::pgqueue::PgQueueError; /// Supported HTTP methods for webhooks. @@ -139,12 +139,10 @@ pub struct WebhookJobMetadata { /// An error originating during a Webhook Job invocation. /// This is to be serialized to be stored as an error whenever retrying or failing a webhook job. -#[derive(Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug)] pub struct WebhookJobError { pub r#type: app_metrics::ErrorType, pub details: app_metrics::ErrorDetails, - #[serde(serialize_with = "serialize_uuid")] - pub uuid: uuid::Uuid, } /// Webhook jobs boil down to an HTTP request, so it's useful to have a way to convert from &reqwest::Error. @@ -180,7 +178,6 @@ impl WebhookJobError { details: app_metrics::ErrorDetails { error: error_details, }, - uuid: uuid::Uuid::now_v7(), } } @@ -195,7 +192,6 @@ impl WebhookJobError { details: app_metrics::ErrorDetails { error: error_details, }, - uuid: uuid::Uuid::now_v7(), } } @@ -210,7 +206,6 @@ impl WebhookJobError { details: app_metrics::ErrorDetails { error: error_details, }, - uuid: uuid::Uuid::now_v7(), } } @@ -225,7 +220,6 @@ impl WebhookJobError { details: app_metrics::ErrorDetails { error: error_details, }, - uuid: uuid::Uuid::now_v7(), } } } diff --git a/hook-janitor/src/cleanup.rs b/hook-janitor/src/cleanup.rs index e6e91e0be922f..82b91303a721b 100644 --- a/hook-janitor/src/cleanup.rs +++ b/hook-janitor/src/cleanup.rs @@ -5,8 +5,6 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum CleanerError { - #[error("pool creation failed with: {error}")] - PoolCreationError { error: sqlx::Error }, #[error("invalid cleaner mode")] InvalidCleanerMode, } diff --git a/hook-janitor/src/config.rs b/hook-janitor/src/config.rs index 89621a2354617..c1efb85d38ee5 100644 --- a/hook-janitor/src/config.rs +++ b/hook-janitor/src/config.rs @@ -20,9 +20,6 @@ pub struct Config { #[envconfig(default = "30")] pub cleanup_interval_secs: u64, - #[envconfig(default = "10000")] - pub cleanup_batch_size: u32, - // The cleanup task needs to have special knowledge of the queue it's cleaning up. This is so it // can do things like flush the proper app_metrics or plugin_log_entries, and so it knows what // to expect in the job's payload JSONB column. diff --git a/hook-janitor/src/main.rs b/hook-janitor/src/main.rs index 46223aa7aafb3..5de3ec4d93978 100644 --- a/hook-janitor/src/main.rs +++ b/hook-janitor/src/main.rs @@ -57,10 +57,8 @@ async fn main() { &config.queue_name, &config.table_name, &config.database_url, - config.cleanup_batch_size, kafka_producer, config.kafka.app_metrics_topic.to_owned(), - config.kafka.plugin_log_entries_topic.to_owned(), ) .expect("unable to create webhook cleaner"), ) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index e8895f1ba0781..6b10ce0d322dd 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -1,20 +1,87 @@ -use async_trait::async_trait; +use std::time::Duration; -use rdkafka::producer::FutureProducer; -use sqlx::postgres::{PgPool, PgPoolOptions}; +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use futures::future::join_all; +use hook_common::webhook::WebhookJobError; +use rdkafka::error::KafkaError; +use rdkafka::producer::{FutureProducer, FutureRecord}; +use serde_json::error::Error as SerdeError; +use sqlx::postgres::{PgPool, PgPoolOptions, Postgres}; +use sqlx::types::{chrono, Uuid}; +use sqlx::Transaction; +use thiserror::Error; +use tracing::{debug, error}; -use crate::cleanup::{Cleaner, CleanerError}; +use crate::cleanup::Cleaner; use crate::kafka_producer::KafkaContext; -#[allow(dead_code)] +use hook_common::kafka_messages::app_metrics::{AppMetric, AppMetricCategory}; + +#[derive(Error, Debug)] +pub enum WebhookCleanerError { + #[error("failed to create postgres pool: {error}")] + PoolCreationError { error: sqlx::Error }, + #[error("failed to acquire conn and start txn: {error}")] + StartTxnError { error: sqlx::Error }, + #[error("failed to get completed rows: {error}")] + GetCompletedRowsError { error: sqlx::Error }, + #[error("failed to get failed rows: {error}")] + GetFailedRowsError { error: sqlx::Error }, + #[error("failed to serialize rows: {error}")] + SerializeRowsError { error: SerdeError }, + #[error("failed to produce to kafka: {error}")] + KafkaProduceError { error: KafkaError }, + #[error("failed to produce to kafka (timeout)")] + KafkaProduceCanceled, + #[error("failed to delete rows: {error}")] + DeleteRowsError { error: sqlx::Error }, + #[error("failed to commit txn: {error}")] + CommitTxnError { error: sqlx::Error }, +} + +type Result = std::result::Result; + pub struct WebhookCleaner { queue_name: String, table_name: String, pg_pool: PgPool, - batch_size: u32, kafka_producer: FutureProducer, app_metrics_topic: String, - plugin_log_entries_topic: String, +} + +#[derive(sqlx::FromRow, Debug)] +struct CompletedRow { + // App Metrics truncates/aggregates rows on the hour, so we take advantage of that to GROUP BY + // and aggregate to select fewer rows. + hour: DateTime, + // A note about the `try_from`s: Postgres returns all of those types as `bigint` (i64), but + // we know their true sizes, and so we can convert them to the correct types here. If this + // ever fails then something has gone wrong. + #[sqlx(try_from = "i64")] + team_id: u32, + #[sqlx(try_from = "i64")] + plugin_config_id: u32, + #[sqlx(try_from = "i64")] + successes: u32, +} + +#[derive(sqlx::FromRow, Debug)] +struct FailedRow { + // App Metrics truncates/aggregates rows on the hour, so we take advantage of that to GROUP BY + // and aggregate to select fewer rows. + hour: DateTime, + // A note about the `try_from`s: Postgres returns all of those types as `bigint` (i64), but + // we know their true sizes, and so we can convert them to the correct types here. If this + // ever fails then something has gone wrong. + #[sqlx(try_from = "i64")] + team_id: u32, + #[sqlx(try_from = "i64")] + plugin_config_id: u32, + #[sqlx(json)] + last_error: WebhookJobError, + #[sqlx(try_from = "i64")] + failures: u32, } impl WebhookCleaner { @@ -22,34 +89,263 @@ impl WebhookCleaner { queue_name: &str, table_name: &str, database_url: &str, - batch_size: u32, kafka_producer: FutureProducer, app_metrics_topic: String, - plugin_log_entries_topic: String, - ) -> Result { + ) -> Result { let queue_name = queue_name.to_owned(); let table_name = table_name.to_owned(); let pg_pool = PgPoolOptions::new() + .acquire_timeout(Duration::from_secs(10)) .connect_lazy(database_url) - .map_err(|error| CleanerError::PoolCreationError { error })?; + .map_err(|error| WebhookCleanerError::PoolCreationError { error })?; Ok(Self { queue_name, table_name, pg_pool, - batch_size, kafka_producer, app_metrics_topic, - plugin_log_entries_topic, }) } + + async fn start_serializable_txn(&self) -> Result> { + let mut tx = self + .pg_pool + .begin() + .await + .map_err(|e| WebhookCleanerError::StartTxnError { error: e })?; + + // We use serializable isolation so that we observe a snapshot of the DB at the time we + // start the cleanup process. This prevents us from accidentally deleting rows that are + // added (or become 'completed' or 'failed') after we start the cleanup process. + sqlx::query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE") + .execute(&mut *tx) + .await + .map_err(|e| WebhookCleanerError::StartTxnError { error: e })?; + + Ok(tx) + } + + async fn get_completed_rows( + &self, + tx: &mut Transaction<'_, Postgres>, + ) -> Result> { + let base_query = format!( + r#" + SELECT DATE_TRUNC('hour', finished_at) AS hour, + metadata->>'team_id' AS team_id, + metadata->>'plugin_config_id' AS plugin_config_id, + count(*) as successes + FROM {0} + WHERE status = 'completed' + AND queue = $1 + GROUP BY hour, team_id, plugin_config_id + ORDER BY hour, team_id, plugin_config_id; + "#, + self.table_name + ); + + let rows = sqlx::query_as::<_, CompletedRow>(&base_query) + .bind(&self.queue_name) + .fetch_all(&mut **tx) + .await + .map_err(|e| WebhookCleanerError::GetCompletedRowsError { error: e })?; + + Ok(rows) + } + + async fn serialize_completed_rows( + &self, + completed_rows: Vec, + ) -> Result> { + let mut payloads = Vec::new(); + + for row in completed_rows { + let app_metric = AppMetric { + timestamp: row.hour, + team_id: row.team_id, + plugin_config_id: row.plugin_config_id, + job_id: None, + category: AppMetricCategory::Webhook, + successes: row.successes, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + }; + + let payload = serde_json::to_string(&app_metric) + .map_err(|e| WebhookCleanerError::SerializeRowsError { error: e })?; + + payloads.push(payload) + } + + Ok(payloads) + } + + async fn get_failed_rows(&self, tx: &mut Transaction<'_, Postgres>) -> Result> { + let base_query = format!( + r#" + SELECT DATE_TRUNC('hour', finished_at) AS hour, + metadata->>'team_id' AS team_id, + metadata->>'plugin_config_id' AS plugin_config_id, + errors[-1] AS last_error, + count(*) as failures + FROM {0} + WHERE status = 'failed' + AND queue = $1 + GROUP BY hour, team_id, plugin_config_id, last_error + ORDER BY hour, team_id, plugin_config_id, last_error; + "#, + self.table_name + ); + + let rows = sqlx::query_as::<_, FailedRow>(&base_query) + .bind(&self.queue_name) + .fetch_all(&mut **tx) + .await + .map_err(|e| WebhookCleanerError::GetFailedRowsError { error: e })?; + + Ok(rows) + } + + async fn serialize_failed_rows(&self, failed_rows: Vec) -> Result> { + let mut payloads = Vec::new(); + + for row in failed_rows { + let app_metric = AppMetric { + timestamp: row.hour, + team_id: row.team_id, + plugin_config_id: row.plugin_config_id, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: row.failures, + error_uuid: Some(Uuid::now_v7()), + error_type: Some(row.last_error.r#type), + error_details: Some(row.last_error.details), + }; + + let payload = serde_json::to_string(&app_metric) + .map_err(|e| WebhookCleanerError::SerializeRowsError { error: e })?; + + payloads.push(payload) + } + + Ok(payloads) + } + + async fn send_messages_to_kafka(&self, payloads: Vec) -> Result<()> { + let mut delivery_futures = Vec::new(); + + for payload in payloads { + match self.kafka_producer.send_result(FutureRecord { + topic: self.app_metrics_topic.as_str(), + payload: Some(&payload), + partition: None, + key: None::<&str>, + timestamp: None, + headers: None, + }) { + Ok(future) => delivery_futures.push(future), + Err((error, _)) => return Err(WebhookCleanerError::KafkaProduceError { error }), + } + } + + for result in join_all(delivery_futures).await { + match result { + Ok(Ok(_)) => {} + Ok(Err((error, _))) => { + return Err(WebhookCleanerError::KafkaProduceError { error }) + } + Err(_) => { + // Cancelled due to timeout while retrying + return Err(WebhookCleanerError::KafkaProduceCanceled); + } + } + } + + Ok(()) + } + + async fn delete_observed_rows(&self, tx: &mut Transaction<'_, Postgres>) -> Result { + // This DELETE is only safe because we are in serializable isolation mode, see the note + // in `start_serializable_txn`. + let base_query = format!( + r#" + DELETE FROM {0} + WHERE status IN ('failed', 'completed') + AND queue = $1; + "#, + self.table_name + ); + + let result = sqlx::query(&base_query) + .bind(&self.queue_name) + .execute(&mut **tx) + .await + .map_err(|e| WebhookCleanerError::DeleteRowsError { error: e })?; + + Ok(result.rows_affected()) + } + + async fn commit_txn(&self, tx: Transaction<'_, Postgres>) -> Result<()> { + tx.commit() + .await + .map_err(|e| WebhookCleanerError::CommitTxnError { error: e })?; + + Ok(()) + } + + async fn cleanup_impl(&self) -> Result<()> { + debug!("WebhookCleaner starting cleanup"); + + // Note that we select all completed and failed rows without any pagination at the moment. + // We aggregrate as much as possible with GROUP BY, truncating the timestamp down to the + // hour just like App Metrics does. A completed row is 24 bytes (and aggregates an entire + // hour per `plugin_config_id`), and a failed row is 104 + the message length (and + // aggregates an entire hour per `plugin_config_id` per `error`), so we can fit a lot of + // rows in memory. It seems unlikely we'll need to paginate, but that can be added in the + // future if necessary. + + let mut tx = self.start_serializable_txn().await?; + let completed_rows = self.get_completed_rows(&mut tx).await?; + let mut payloads = self.serialize_completed_rows(completed_rows).await?; + let failed_rows = self.get_failed_rows(&mut tx).await?; + let mut failed_payloads = self.serialize_failed_rows(failed_rows).await?; + payloads.append(&mut failed_payloads); + let mut rows_deleted: u64 = 0; + if !payloads.is_empty() { + self.send_messages_to_kafka(payloads).await?; + rows_deleted = self.delete_observed_rows(&mut tx).await?; + self.commit_txn(tx).await?; + } + + debug!( + "WebhookCleaner finished cleanup, deleted {} rows", + rows_deleted + ); + + Ok(()) + } } #[async_trait] impl Cleaner for WebhookCleaner { async fn cleanup(&self) { - // TODO: collect stats on completed/failed rows - // TODO: push metrics about those rows into `app_metrics` - // TODO: delete those completed/failed rows + match self.cleanup_impl().await { + Ok(_) => {} + Err(error) => { + error!(error = ?error, "WebhookCleaner::cleanup failed"); + } + } } } + +#[cfg(test)] +mod tests { + #[tokio::test] + async fn test() {} +} From 72aa509cccef0a6df87e5e69a63ce4e9db020d95 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 19 Dec 2023 09:59:14 -0700 Subject: [PATCH 130/247] Drop unnecessary asyncs --- hook-janitor/src/webhooks.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 6b10ce0d322dd..7e6f540837844 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -154,10 +154,7 @@ impl WebhookCleaner { Ok(rows) } - async fn serialize_completed_rows( - &self, - completed_rows: Vec, - ) -> Result> { + fn serialize_completed_rows(&self, completed_rows: Vec) -> Result> { let mut payloads = Vec::new(); for row in completed_rows { @@ -210,7 +207,7 @@ impl WebhookCleaner { Ok(rows) } - async fn serialize_failed_rows(&self, failed_rows: Vec) -> Result> { + fn serialize_failed_rows(&self, failed_rows: Vec) -> Result> { let mut payloads = Vec::new(); for row in failed_rows { @@ -312,9 +309,9 @@ impl WebhookCleaner { let mut tx = self.start_serializable_txn().await?; let completed_rows = self.get_completed_rows(&mut tx).await?; - let mut payloads = self.serialize_completed_rows(completed_rows).await?; + let mut payloads = self.serialize_completed_rows(completed_rows)?; let failed_rows = self.get_failed_rows(&mut tx).await?; - let mut failed_payloads = self.serialize_failed_rows(failed_rows).await?; + let mut failed_payloads = self.serialize_failed_rows(failed_rows)?; payloads.append(&mut failed_payloads); let mut rows_deleted: u64 = 0; if !payloads.is_empty() { From 4fe10877dc9a0345f7904bda99a32bd70bac773d Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 19 Dec 2023 12:39:25 -0700 Subject: [PATCH 131/247] Make WebhookJobMetadata fields required --- hook-common/src/webhook.rs | 6 +++--- hook-consumer/src/consumer.rs | 6 +++--- hook-producer/src/handlers/webhook.rs | 18 +++++++++--------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index d320ce0c4aac7..9a21b83cd713b 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -132,9 +132,9 @@ pub struct WebhookJobParameters { /// These should be set if the Webhook is associated with a plugin `composeWebhook` invocation. #[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] pub struct WebhookJobMetadata { - pub team_id: Option, - pub plugin_id: Option, - pub plugin_config_id: Option, + pub team_id: u32, + pub plugin_id: u32, + pub plugin_config_id: u32, } /// An error originating during a Webhook Job invocation. diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 12d4b38b9a56f..633381a140def 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -339,9 +339,9 @@ mod tests { url: "localhost".to_owned(), }; let webhook_job_metadata = WebhookJobMetadata { - team_id: None, - plugin_id: None, - plugin_config_id: None, + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, }; // enqueue takes ownership of the job enqueued to avoid bugs that can cause duplicate jobs. // Normally, a separate application would be enqueueing jobs for us to consume, so no ownership diff --git a/hook-producer/src/handlers/webhook.rs b/hook-producer/src/handlers/webhook.rs index 394732094a3c6..18aebf39ce991 100644 --- a/hook-producer/src/handlers/webhook.rs +++ b/hook-producer/src/handlers/webhook.rs @@ -146,9 +146,9 @@ mod tests { body: r#"{"a": "b"}"#.to_owned(), }, metadata: WebhookJobMetadata { - team_id: Some(1), - plugin_id: Some(2), - plugin_config_id: Some(3), + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, }, max_attempts: 1, }) @@ -193,9 +193,9 @@ mod tests { body: r#"{"a": "b"}"#.to_owned(), }, metadata: WebhookJobMetadata { - team_id: Some(1), - plugin_id: Some(2), - plugin_config_id: Some(3), + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, }, max_attempts: 1, }) @@ -296,9 +296,9 @@ mod tests { body: long_string.to_string(), }, metadata: WebhookJobMetadata { - team_id: Some(1), - plugin_id: Some(2), - plugin_config_id: Some(3), + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, }, max_attempts: 1, }) From 9ad59d24504e58d13ba35503d1cd2ad1d46d26e3 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 19 Dec 2023 13:28:50 -0700 Subject: [PATCH 132/247] Switch to sqlx::test for per-test DBs and fixtures --- .env | 1 + Cargo.toml | 7 ++- hook-common/src/pgqueue.rs | 65 +++++++++++++-------- hook-consumer/src/consumer.rs | 13 +++-- hook-producer/src/handlers/app.rs | 17 +++--- hook-producer/src/handlers/webhook.rs | 83 +++++++++++---------------- 6 files changed, 92 insertions(+), 94 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000000000..43eda2a13040b --- /dev/null +++ b/.env @@ -0,0 +1 @@ +DATABASE_URL=postgres://posthog:posthog@localhost:15432/test_database diff --git a/Cargo.toml b/Cargo.toml index 0a0a2f578560b..2481c1dfbbefa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,12 +21,13 @@ serde = { version = "1.0" } serde_derive = { version = "1.0" } serde_json = { version = "1.0" } sqlx = { version = "0.7", features = [ + "chrono", + "json", + "migrate", + "postgres", "runtime-tokio", "tls-native-tls", - "postgres", "uuid", - "json", - "chrono", ] } thiserror = { version = "1.0" } tokio = { version = "1.34.0", features = ["full"] } diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 5288ade106f47..8b36b6d5fb167 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -590,6 +590,23 @@ impl PgQueue { }) } + pub async fn new_from_pool( + queue_name: &str, + table_name: &str, + pool: PgPool, + retry_policy: RetryPolicy, + ) -> PgQueueResult { + let name = queue_name.to_owned(); + let table = table_name.to_owned(); + + Ok(Self { + name, + pool, + retry_policy, + table, + }) + } + /// Dequeue a Job from this PgQueue to work on it. pub async fn dequeue< J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, @@ -827,18 +844,18 @@ mod tests { "https://myhost/endpoint".to_owned() } - #[tokio::test] - async fn test_can_dequeue_job() { + #[sqlx::test(migrations = "../migrations")] + async fn test_can_dequeue_job(db: PgPool) { let job_target = job_target(); let job_parameters = JobParameters::default(); let job_metadata = JobMetadata::default(); let worker_id = worker_id(); let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); - let queue = PgQueue::new( + let queue = PgQueue::new_from_pool( "test_can_dequeue_job", "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", + db, RetryPolicy::default(), ) .await @@ -861,13 +878,13 @@ mod tests { assert_eq!(pg_job.job.target, job_target); } - #[tokio::test] - async fn test_dequeue_returns_none_on_no_jobs() { + #[sqlx::test(migrations = "../migrations")] + async fn test_dequeue_returns_none_on_no_jobs(db: PgPool) { let worker_id = worker_id(); - let queue = PgQueue::new( + let queue = PgQueue::new_from_pool( "test_dequeue_returns_none_on_no_jobs", "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", + db, RetryPolicy::default(), ) .await @@ -881,18 +898,18 @@ mod tests { assert!(pg_job.is_none()); } - #[tokio::test] - async fn test_can_dequeue_tx_job() { + #[sqlx::test(migrations = "../migrations")] + async fn test_can_dequeue_tx_job(db: PgPool) { let job_target = job_target(); let job_metadata = JobMetadata::default(); let job_parameters = JobParameters::default(); let worker_id = worker_id(); let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); - let queue = PgQueue::new( + let queue = PgQueue::new_from_pool( "test_can_dequeue_tx_job", "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", + db, RetryPolicy::default(), ) .await @@ -916,13 +933,13 @@ mod tests { assert_eq!(tx_job.job.target, job_target); } - #[tokio::test] - async fn test_dequeue_tx_returns_none_on_no_jobs() { + #[sqlx::test(migrations = "../migrations")] + async fn test_dequeue_tx_returns_none_on_no_jobs(db: PgPool) { let worker_id = worker_id(); - let queue = PgQueue::new( + let queue = PgQueue::new_from_pool( "test_dequeue_tx_returns_none_on_no_jobs", "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", + db, RetryPolicy::default(), ) .await @@ -936,8 +953,8 @@ mod tests { assert!(tx_job.is_none()); } - #[tokio::test] - async fn test_can_retry_job_with_remaining_attempts() { + #[sqlx::test(migrations = "../migrations")] + async fn test_can_retry_job_with_remaining_attempts(db: PgPool) { let job_target = job_target(); let job_parameters = JobParameters::default(); let job_metadata = JobMetadata::default(); @@ -949,10 +966,10 @@ mod tests { maximum_interval: None, }; - let queue = PgQueue::new( + let queue = PgQueue::new_from_pool( "test_can_retry_job_with_remaining_attempts", "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", + db, retry_policy, ) .await @@ -986,9 +1003,9 @@ mod tests { assert_eq!(retried_job.job.target, job_target); } - #[tokio::test] + #[sqlx::test(migrations = "../migrations")] #[should_panic(expected = "failed to retry job")] - async fn test_cannot_retry_job_without_remaining_attempts() { + async fn test_cannot_retry_job_without_remaining_attempts(db: PgPool) { let job_target = job_target(); let job_parameters = JobParameters::default(); let job_metadata = JobMetadata::default(); @@ -1000,10 +1017,10 @@ mod tests { maximum_interval: None, }; - let queue = PgQueue::new( + let queue = PgQueue::new_from_pool( "test_cannot_retry_job_without_remaining_attempts", "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", + db, retry_policy, ) .await diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 633381a140def..d17578b5afe55 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -272,6 +272,8 @@ mod tests { // See: https://github.com/rust-lang/rust/issues/46379. #[allow(unused_imports)] use hook_common::pgqueue::{JobStatus, NewJob, RetryPolicy}; + #[allow(unused_imports)] + use sqlx::PgPool; /// Use process id as a worker id for tests. #[allow(dead_code)] @@ -322,13 +324,12 @@ mod tests { assert_eq!(duration, None); } - #[tokio::test] - async fn test_wait_for_job() { + #[sqlx::test(migrations = "../migrations")] + async fn test_wait_for_job(db: PgPool) { let worker_id = worker_id(); let queue_name = "test_wait_for_job".to_string(); let table_name = "job_queue".to_string(); - let db_url = "postgres://posthog:posthog@localhost:15432/test_database".to_string(); - let queue = PgQueue::new(&queue_name, &table_name, &db_url, RetryPolicy::default()) + let queue = PgQueue::new_from_pool(&queue_name, &table_name, db, RetryPolicy::default()) .await .expect("failed to connect to PG"); @@ -385,8 +386,8 @@ mod tests { .expect("job not successfully completed"); } - #[tokio::test] - async fn test_send_webhook() { + #[sqlx::test(migrations = "../migrations")] + async fn test_send_webhook(_: PgPool) { let method = HttpMethod::POST; let url = "http://localhost:18081/echo"; let headers = collections::HashMap::new(); diff --git a/hook-producer/src/handlers/app.rs b/hook-producer/src/handlers/app.rs index 1666676fee1bb..c3309dea9efb3 100644 --- a/hook-producer/src/handlers/app.rs +++ b/hook-producer/src/handlers/app.rs @@ -33,18 +33,15 @@ mod tests { }; use hook_common::pgqueue::{PgQueue, RetryPolicy}; use http_body_util::BodyExt; // for `collect` + use sqlx::PgPool; use tower::ServiceExt; // for `call`, `oneshot`, and `ready` - #[tokio::test] - async fn index() { - let pg_queue = PgQueue::new( - "test_index", - "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), - ) - .await - .expect("failed to construct pg_queue"); + #[sqlx::test(migrations = "../migrations")] + async fn index(db: PgPool) { + let pg_queue = + PgQueue::new_from_pool("test_index", "job_queue", db, RetryPolicy::default()) + .await + .expect("failed to construct pg_queue"); let app = app(pg_queue, None); diff --git a/hook-producer/src/handlers/webhook.rs b/hook-producer/src/handlers/webhook.rs index 18aebf39ce991..e2864e8aa0d43 100644 --- a/hook-producer/src/handlers/webhook.rs +++ b/hook-producer/src/handlers/webhook.rs @@ -110,22 +110,19 @@ mod tests { }; use hook_common::pgqueue::{PgQueue, RetryPolicy}; use hook_common::webhook::{HttpMethod, WebhookJobParameters}; - use http_body_util::BodyExt; // for `collect` + use http_body_util::BodyExt; + use sqlx::PgPool; // for `collect` use std::collections; use tower::ServiceExt; // for `call`, `oneshot`, and `ready` use crate::handlers::app; - #[tokio::test] - async fn webhook_success() { - let pg_queue = PgQueue::new( - "test_index", - "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), - ) - .await - .expect("failed to construct pg_queue"); + #[sqlx::test(migrations = "../migrations")] + async fn webhook_success(db: PgPool) { + let pg_queue = + PgQueue::new_from_pool("test_index", "job_queue", db, RetryPolicy::default()) + .await + .expect("failed to construct pg_queue"); let app = app(pg_queue, None); @@ -165,16 +162,12 @@ mod tests { assert_eq!(&body[..], b"{}"); } - #[tokio::test] - async fn webhook_bad_url() { - let pg_queue = PgQueue::new( - "test_index", - "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), - ) - .await - .expect("failed to construct pg_queue"); + #[sqlx::test(migrations = "../migrations")] + async fn webhook_bad_url(db: PgPool) { + let pg_queue = + PgQueue::new_from_pool("test_index", "job_queue", db, RetryPolicy::default()) + .await + .expect("failed to construct pg_queue"); let app = app(pg_queue, None); @@ -209,16 +202,12 @@ mod tests { assert_eq!(response.status(), StatusCode::BAD_REQUEST); } - #[tokio::test] - async fn webhook_payload_missing_fields() { - let pg_queue = PgQueue::new( - "test_index", - "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), - ) - .await - .expect("failed to construct pg_queue"); + #[sqlx::test(migrations = "../migrations")] + async fn webhook_payload_missing_fields(db: PgPool) { + let pg_queue = + PgQueue::new_from_pool("test_index", "job_queue", db, RetryPolicy::default()) + .await + .expect("failed to construct pg_queue"); let app = app(pg_queue, None); @@ -237,16 +226,12 @@ mod tests { assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); } - #[tokio::test] - async fn webhook_payload_not_json() { - let pg_queue = PgQueue::new( - "test_index", - "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), - ) - .await - .expect("failed to construct pg_queue"); + #[sqlx::test(migrations = "../migrations")] + async fn webhook_payload_not_json(db: PgPool) { + let pg_queue = + PgQueue::new_from_pool("test_index", "job_queue", db, RetryPolicy::default()) + .await + .expect("failed to construct pg_queue"); let app = app(pg_queue, None); @@ -265,16 +250,12 @@ mod tests { assert_eq!(response.status(), StatusCode::BAD_REQUEST); } - #[tokio::test] - async fn webhook_payload_body_too_large() { - let pg_queue = PgQueue::new( - "test_index", - "job_queue", - "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), - ) - .await - .expect("failed to construct pg_queue"); + #[sqlx::test(migrations = "../migrations")] + async fn webhook_payload_body_too_large(db: PgPool) { + let pg_queue = + PgQueue::new_from_pool("test_index", "job_queue", db, RetryPolicy::default()) + .await + .expect("failed to construct pg_queue"); let app = app(pg_queue, None); From 9359e7c846152aec9a43cf0c2734fabb7a8a9f0f Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 19 Dec 2023 15:55:21 -0700 Subject: [PATCH 133/247] Add tests, fix some bugs found by said tests --- Cargo.lock | 1 + Cargo.toml | 2 +- hook-janitor/src/fixtures/webhook_cleanup.sql | 81 ++++++++++ hook-janitor/src/webhooks.rs | 140 +++++++++++++++--- 4 files changed, 206 insertions(+), 18 deletions(-) create mode 100644 hook-janitor/src/fixtures/webhook_cleanup.sql diff --git a/Cargo.lock b/Cargo.lock index ba9f4bb10b67d..fbfcc50d98ee8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2009,6 +2009,7 @@ dependencies = [ "serde_json", "slab", "tokio", + "tracing", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2481c1dfbbefa..a29806a2fa730 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ http = { version = "0.2" } http-body-util = "0.1.0" metrics = "0.21.1" metrics-exporter-prometheus = "0.12.1" -rdkafka = { version = "0.35.0", features = ["cmake-build", "ssl"] } +rdkafka = { version = "0.35.0", features = ["cmake-build", "ssl", "tracing"] } reqwest = { version = "0.11" } regex = "1.10.2" serde = { version = "1.0" } diff --git a/hook-janitor/src/fixtures/webhook_cleanup.sql b/hook-janitor/src/fixtures/webhook_cleanup.sql new file mode 100644 index 0000000000000..6f29d302b8462 --- /dev/null +++ b/hook-janitor/src/fixtures/webhook_cleanup.sql @@ -0,0 +1,81 @@ +INSERT INTO + job_queue ( + errors, + metadata, + finished_at, + parameters, + queue, + status, + target + ) +VALUES + -- team:1, plugin_config:2, completed in hour 20 + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- another team:1, plugin_config:2, completed in hour 20 + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, completed in hour 21 (different hour) + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 21:01:18.799371+00', + '{}', + 'webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:3, completed in hour 20 (different plugin_config) + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3}', + '2023-12-19 20:01:18.80335+00', + '{}', + 'webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, completed but in a different queue + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'not-webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:2, plugin_config:4, completed in hour 20 (different team) + ( + NULL, + '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'completed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, failed in hour 20 + ( + ARRAY ['{"type":"Timeout","details":{"error":{"name":"timeout"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ); \ No newline at end of file diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 7e6f540837844..e30f71ac861b3 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -108,6 +108,26 @@ impl WebhookCleaner { }) } + #[allow(dead_code)] // This is used in tests. + pub fn new_from_pool( + queue_name: &str, + table_name: &str, + pg_pool: PgPool, + kafka_producer: FutureProducer, + app_metrics_topic: String, + ) -> Result { + let queue_name = queue_name.to_owned(); + let table_name = table_name.to_owned(); + + Ok(Self { + queue_name, + table_name, + pg_pool, + kafka_producer, + app_metrics_topic, + }) + } + async fn start_serializable_txn(&self) -> Result> { let mut tx = self .pg_pool @@ -118,6 +138,9 @@ impl WebhookCleaner { // We use serializable isolation so that we observe a snapshot of the DB at the time we // start the cleanup process. This prevents us from accidentally deleting rows that are // added (or become 'completed' or 'failed') after we start the cleanup process. + // + // If we find that this has a significant performance impact, we could instead move + // rows to a temporary table for processing and then deletion. sqlx::query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE") .execute(&mut *tx) .await @@ -133,8 +156,8 @@ impl WebhookCleaner { let base_query = format!( r#" SELECT DATE_TRUNC('hour', finished_at) AS hour, - metadata->>'team_id' AS team_id, - metadata->>'plugin_config_id' AS plugin_config_id, + (metadata->>'team_id')::bigint AS team_id, + (metadata->>'plugin_config_id')::bigint AS plugin_config_id, count(*) as successes FROM {0} WHERE status = 'completed' @@ -185,9 +208,13 @@ impl WebhookCleaner { let base_query = format!( r#" SELECT DATE_TRUNC('hour', finished_at) AS hour, - metadata->>'team_id' AS team_id, - metadata->>'plugin_config_id' AS plugin_config_id, - errors[-1] AS last_error, + (metadata->>'team_id')::bigint AS team_id, + (metadata->>'plugin_config_id')::bigint AS plugin_config_id, + CASE + WHEN array_length(errors, 1) > 1 + THEN errors[array_length(errors, 1)] + ELSE errors[1] + END AS last_error, count(*) as failures FROM {0} WHERE status = 'failed' @@ -302,27 +329,34 @@ impl WebhookCleaner { // Note that we select all completed and failed rows without any pagination at the moment. // We aggregrate as much as possible with GROUP BY, truncating the timestamp down to the // hour just like App Metrics does. A completed row is 24 bytes (and aggregates an entire - // hour per `plugin_config_id`), and a failed row is 104 + the message length (and - // aggregates an entire hour per `plugin_config_id` per `error`), so we can fit a lot of - // rows in memory. It seems unlikely we'll need to paginate, but that can be added in the + // hour per `plugin_config_id`), and a failed row is 104 bytes + the error message length + // (and aggregates an entire hour per `plugin_config_id` per `error`), so we can fit a lot + // of rows in memory. It seems unlikely we'll need to paginate, but that can be added in the // future if necessary. let mut tx = self.start_serializable_txn().await?; + let completed_rows = self.get_completed_rows(&mut tx).await?; - let mut payloads = self.serialize_completed_rows(completed_rows)?; + let completed_agg_row_count = completed_rows.len(); + let completed_kafka_payloads = self.serialize_completed_rows(completed_rows)?; + let failed_rows = self.get_failed_rows(&mut tx).await?; - let mut failed_payloads = self.serialize_failed_rows(failed_rows)?; - payloads.append(&mut failed_payloads); + let failed_agg_row_count = failed_rows.len(); + let mut failed_kafka_payloads = self.serialize_failed_rows(failed_rows)?; + + let mut all_kafka_payloads = completed_kafka_payloads; + all_kafka_payloads.append(&mut failed_kafka_payloads); + let mut rows_deleted: u64 = 0; - if !payloads.is_empty() { - self.send_messages_to_kafka(payloads).await?; + if !all_kafka_payloads.is_empty() { + self.send_messages_to_kafka(all_kafka_payloads).await?; rows_deleted = self.delete_observed_rows(&mut tx).await?; self.commit_txn(tx).await?; } debug!( - "WebhookCleaner finished cleanup, deleted {} rows", - rows_deleted + "WebhookCleaner finished cleanup, deleted {} rows ({} completed+aggregated, {} failed+aggregated)", + rows_deleted, completed_agg_row_count, failed_agg_row_count ); Ok(()) @@ -343,6 +377,78 @@ impl Cleaner for WebhookCleaner { #[cfg(test)] mod tests { - #[tokio::test] - async fn test() {} + use super::*; + use crate::config; + use crate::kafka_producer::{create_kafka_producer, KafkaContext}; + use rdkafka::mocking::MockCluster; + use rdkafka::producer::{DefaultProducerContext, FutureProducer}; + use sqlx::PgPool; + + const APP_METRICS_TOPIC: &str = "app_metrics"; + + async fn create_mock_kafka() -> ( + MockCluster<'static, DefaultProducerContext>, + FutureProducer, + ) { + let cluster = MockCluster::new(1).expect("failed to create mock brokers"); + + let config = config::KafkaConfig { + kafka_producer_linger_ms: 0, + kafka_producer_queue_mib: 50, + kafka_message_timeout_ms: 5000, + kafka_compression_codec: "none".to_string(), + kafka_hosts: cluster.bootstrap_servers(), + app_metrics_topic: APP_METRICS_TOPIC.to_string(), + plugin_log_entries_topic: "plugin_log_entries".to_string(), + kafka_tls: false, + }; + + ( + cluster, + create_kafka_producer(&config) + .await + .expect("failed to create mocked kafka producer"), + ) + } + + #[sqlx::test(migrations = "../migrations", fixtures("webhook_cleanup"))] + async fn test_cleanup_impl(db: PgPool) { + let (mock_cluster, mock_producer) = create_mock_kafka().await; + mock_cluster + .create_topic(APP_METRICS_TOPIC, 1, 1) + .expect("failed to create mock app_metrics topic"); + + let table_name = "job_queue"; + let queue_name = "webhooks"; + + let webhook_cleaner = WebhookCleaner::new_from_pool( + &queue_name, + &table_name, + db, + mock_producer, + APP_METRICS_TOPIC.to_owned(), + ) + .expect("unable to create webhook cleaner"); + + let _ = webhook_cleaner + .cleanup_impl() + .await + .expect("webbook cleanup_impl failed"); + + // TODO: I spent a lot of time trying to get the mock Kafka consumer to work, but I think + // I've identified an issue with the rust-rdkafka library: + // https://github.com/fede1024/rust-rdkafka/issues/629#issuecomment-1863555417 + // + // I wanted to test the messages put on the AppMetrics topic, but I think we need to figure + // out that issue about in order to do so. (Capture uses the MockProducer but not a + // Consumer, fwiw.) + // + // For now, I'll probably have to make `cleanup_impl` return the row information so at + // least we can inspect that for correctness. + } + + // #[sqlx::test] + // async fn test_serializable_isolation() { + // TODO: I'm going to add a test that verifies new rows aren't visible during the txn. + // } } From cc206a54216cf30d72900cedb5904ec1d2685cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 16:50:39 +0100 Subject: [PATCH 134/247] feat: Give non-transactional consumer a chance --- hook-consumer/src/config.rs | 3 +++ hook-consumer/src/consumer.rs | 20 +++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/hook-consumer/src/config.rs b/hook-consumer/src/config.rs index 8e4bde9d956be..36c120ad421cc 100644 --- a/hook-consumer/src/config.rs +++ b/hook-consumer/src/config.rs @@ -26,6 +26,9 @@ pub struct Config { #[envconfig(nested = true)] pub retry_policy: RetryPolicyConfig, + #[envconfig(default = "true")] + pub transactional: bool, + #[envconfig(default = "job_queue")] pub table_name: String, } diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 12d4b38b9a56f..04600f138e590 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -23,6 +23,8 @@ pub struct WebhookConsumer<'p> { client: reqwest::Client, /// Maximum number of concurrent jobs being processed. max_concurrent_jobs: usize, + /// Indicates whether we are holding an open transaction while processing or not. + transactional: bool, } impl<'p> WebhookConsumer<'p> { @@ -54,6 +56,19 @@ impl<'p> WebhookConsumer<'p> { } } + /// Wait until a job becomes available in our queue. + async fn wait_for_job_tx<'a>( + &self, + ) -> Result, WebhookConsumerError> { + loop { + if let Some(job) = self.queue.dequeue_tx(&self.name).await? { + return Ok(job); + } else { + task::sleep(self.poll_interval).await; + } + } + } + /// Wait until a job becomes available in our queue. async fn wait_for_job<'a>( &self, @@ -72,7 +87,10 @@ impl<'p> WebhookConsumer<'p> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); loop { - let webhook_job = self.wait_for_job().await?; + let webhook_job = match self.transactional { + true => self.wait_for_job_tx().await, + false => self.wait_for_job().await, + }?; // reqwest::Client internally wraps with Arc, so this allocation is cheap. let client = self.client.clone(); From bd4dc4bdc5b01d52047cc439d4e1723d9c05e0f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 14 Dec 2023 12:30:31 +0100 Subject: [PATCH 135/247] refactor: Two clients one for each mode --- hook-consumer/src/consumer.rs | 137 +++++++++++++++++++++++++++++++--- 1 file changed, 126 insertions(+), 11 deletions(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 04600f138e590..83ea319d2c555 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -23,8 +23,6 @@ pub struct WebhookConsumer<'p> { client: reqwest::Client, /// Maximum number of concurrent jobs being processed. max_concurrent_jobs: usize, - /// Indicates whether we are holding an open transaction while processing or not. - transactional: bool, } impl<'p> WebhookConsumer<'p> { @@ -57,11 +55,9 @@ impl<'p> WebhookConsumer<'p> { } /// Wait until a job becomes available in our queue. - async fn wait_for_job_tx<'a>( - &self, - ) -> Result, WebhookConsumerError> { + async fn wait_for_job<'a>(&self) -> Result, WebhookConsumerError> { loop { - if let Some(job) = self.queue.dequeue_tx(&self.name).await? { + if let Some(job) = self.queue.dequeue(&self.name).await? { return Ok(job); } else { task::sleep(self.poll_interval).await; @@ -69,6 +65,68 @@ impl<'p> WebhookConsumer<'p> { } } + /// Run this consumer to continuously process any jobs that become available. + pub async fn run(&self) -> Result<(), WebhookConsumerError> { + let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); + + loop { + let webhook_job = self.wait_for_job().await?; + + // reqwest::Client internally wraps with Arc, so this allocation is cheap. + let client = self.client.clone(); + let permit = semaphore.clone().acquire_owned().await.unwrap(); + + tokio::spawn(async move { + let result = process_webhook_job(client, webhook_job).await; + drop(permit); + result + }); + } + } +} + +/// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. +pub struct WebhookTransactionConsumer<'p> { + /// An identifier for this consumer. Used to mark jobs we have consumed. + name: String, + /// The queue we will be dequeuing jobs from. + queue: &'p PgQueue, + /// The interval for polling the queue. + poll_interval: time::Duration, + /// The client used for HTTP requests. + client: reqwest::Client, + /// Maximum number of concurrent jobs being processed. + max_concurrent_jobs: usize, +} + +impl<'p> WebhookTransactionConsumer<'p> { + pub fn new( + name: &str, + queue: &'p PgQueue, + poll_interval: time::Duration, + request_timeout: time::Duration, + max_concurrent_jobs: usize, + ) -> Result { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ); + + let client = reqwest::Client::builder() + .default_headers(headers) + .timeout(request_timeout) + .build()?; + + Ok(Self { + name: name.to_owned(), + queue, + poll_interval, + client, + max_concurrent_jobs, + }) + } + /// Wait until a job becomes available in our queue. async fn wait_for_job<'a>( &self, @@ -87,17 +145,14 @@ impl<'p> WebhookConsumer<'p> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); loop { - let webhook_job = match self.transactional { - true => self.wait_for_job_tx().await, - false => self.wait_for_job().await, - }?; + let webhook_job = self.wait_for_job().await?; // reqwest::Client internally wraps with Arc, so this allocation is cheap. let client = self.client.clone(); let permit = semaphore.clone().acquire_owned().await.unwrap(); tokio::spawn(async move { - let result = process_webhook_job(client, webhook_job).await; + let result = process_webhook_job_tx(client, webhook_job).await; drop(permit); result }); @@ -187,6 +242,66 @@ async fn process_webhook_job( } } +/// Process a webhook job by transitioning it to its appropriate state after its request is sent. +/// After we finish, the webhook job will be set as completed (if the request was successful), retryable (if the request +/// was unsuccessful but we can still attempt a retry), or failed (if the request was unsuccessful and no more retries +/// may be attempted). +/// +/// A webhook job is considered retryable after a failing request if: +/// 1. The job has attempts remaining (i.e. hasn't reached `max_attempts`), and... +/// 2. The status code indicates retrying at a later point could resolve the issue. This means: 429 and any 5XX. +/// +/// # Arguments +/// +/// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. +/// * `request_timeout`: A timeout for the HTTP request. +async fn process_webhook_job( + client: reqwest::Client, + webhook_job: PgJob, +) -> Result<(), WebhookConsumerError> { + match send_webhook( + client, + &webhook_job.job.parameters.method, + &webhook_job.job.parameters.url, + &webhook_job.job.parameters.headers, + webhook_job.job.parameters.body.clone(), + ) + .await + { + Ok(_) => { + webhook_job + .complete() + .await + .map_err(|error| WebhookConsumerError::PgJobError(error.to_string()))?; + Ok(()) + } + Err(WebhookConsumerError::RetryableWebhookError { + reason, + retry_after, + }) => match webhook_job.retry(reason.to_string(), retry_after).await { + Ok(_) => Ok(()), + Err(PgJobError::RetryInvalidError { + job: webhook_job, + error: fail_error, + }) => { + webhook_job + .fail(fail_error.to_string()) + .await + .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; + Ok(()) + } + Err(job_error) => Err(WebhookConsumerError::PgJobError(job_error.to_string())), + }, + Err(error) => { + webhook_job + .fail(error.to_string()) + .await + .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; + Ok(()) + } + } +} + /// Make an HTTP request to a webhook endpoint. /// /// # Arguments From 3747c3450bc1a87240ed16a8d6d4ac20a3f216f1 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 20 Dec 2023 07:04:59 -0700 Subject: [PATCH 136/247] Handle some feedback --- README.md | 3 +++ hook-janitor/src/webhooks.rs | 10 +++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a3a674c28ff38..f579cd34fed49 100644 --- a/README.md +++ b/README.md @@ -15,5 +15,8 @@ docker compose -f docker-compose.yml up -d --wait 2. Test: ```bash +# Note that tests require a DATABASE_URL environment variable to be set, e.g.: +# export DATABASE_URL=postgres://posthog:posthog@localhost:15432/test_database +# But there is an .env file in the project root that should be used automatically. cargo test ``` diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index e30f71ac861b3..a9b05e560e737 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -210,11 +210,7 @@ impl WebhookCleaner { SELECT DATE_TRUNC('hour', finished_at) AS hour, (metadata->>'team_id')::bigint AS team_id, (metadata->>'plugin_config_id')::bigint AS plugin_config_id, - CASE - WHEN array_length(errors, 1) > 1 - THEN errors[array_length(errors, 1)] - ELSE errors[1] - END AS last_error, + errors[array_upper(errors, 1)] AS last_error, count(*) as failures FROM {0} WHERE status = 'failed' @@ -342,10 +338,10 @@ impl WebhookCleaner { let failed_rows = self.get_failed_rows(&mut tx).await?; let failed_agg_row_count = failed_rows.len(); - let mut failed_kafka_payloads = self.serialize_failed_rows(failed_rows)?; + let failed_kafka_payloads = self.serialize_failed_rows(failed_rows)?; let mut all_kafka_payloads = completed_kafka_payloads; - all_kafka_payloads.append(&mut failed_kafka_payloads); + all_kafka_payloads.extend(failed_kafka_payloads.into_iter()); let mut rows_deleted: u64 = 0; if !all_kafka_payloads.is_empty() { From d28dfef201c40d2cfe4f708f5230bd83b418b123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 20 Dec 2023 15:25:56 +0100 Subject: [PATCH 137/247] refactor: Support both modes in single client with WebhookJob trait --- Cargo.lock | 1 + hook-common/Cargo.toml | 1 + hook-common/src/pgqueue.rs | 179 +++++++++++++++++--------------- hook-consumer/src/consumer.rs | 187 +++++++++++----------------------- hook-consumer/src/main.rs | 6 +- 5 files changed, 164 insertions(+), 210 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba9f4bb10b67d..130f765cff81d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1008,6 +1008,7 @@ dependencies = [ name = "hook-common" version = "0.1.0" dependencies = [ + "async-trait", "axum", "chrono", "http 0.2.11", diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index 6350ba4b8f7c1..9b20396a388c9 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-trait = { workspace = true } axum = { workspace = true, features = ["http2"] } chrono = { workspace = true } http = { workspace = true } diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 5288ade106f47..78fe2fc21c3f7 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -6,6 +6,7 @@ use std::default::Default; use std::str::FromStr; use std::time; +use async_trait::async_trait; use chrono; use serde; use sqlx::postgres::{PgPool, PgPoolOptions}; @@ -149,6 +150,22 @@ impl Job { } } +#[async_trait] +pub trait PgQueueJob { + async fn complete(mut self) -> Result>>; + + async fn fail( + mut self, + error: E, + ) -> Result, PgJobError>>; + + async fn retry( + mut self, + error: E, + preferred_retry_interval: Option, + ) -> Result, PgJobError>>; +} + /// A Job that can be updated in PostgreSQL. #[derive(Debug)] pub struct PgJob { @@ -158,22 +175,10 @@ pub struct PgJob { pub retry_policy: RetryPolicy, } -impl PgJob { - pub async fn retry( - mut self, - error: E, - preferred_retry_interval: Option, - ) -> Result, PgJobError>> { - if self.job.is_gte_max_attempts() { - return Err(PgJobError::RetryInvalidError { - job: self, - error: "Maximum attempts reached".to_owned(), - }); - } - let retryable_job = self.job.retry(error); - let retry_interval = self - .retry_policy - .time_until_next_retry(&retryable_job, preferred_retry_interval); +#[async_trait] +impl PgQueueJob for PgJob { + async fn complete(mut self) -> Result>>> { + let completed_job = self.job.complete(); let base_query = format!( r#" @@ -181,9 +186,7 @@ UPDATE "{0}" SET finished_at = NOW(), - status = 'available'::job_status, - scheduled_at = NOW() + $3, - errors = array_append("{0}".errors, $4) + status = 'completed'::job_status WHERE "{0}".id = $2 AND queue = $1 @@ -194,10 +197,8 @@ RETURNING ); sqlx::query(&base_query) - .bind(&retryable_job.queue) - .bind(retryable_job.id) - .bind(retry_interval) - .bind(&retryable_job.error) + .bind(&completed_job.queue) + .bind(completed_job.id) .execute(&mut *self.connection) .await .map_err(|error| PgJobError::QueryError { @@ -205,11 +206,14 @@ RETURNING error, })?; - Ok(retryable_job) + Ok(completed_job) } - pub async fn complete(mut self) -> Result>> { - let completed_job = self.job.complete(); + async fn fail( + mut self, + error: E, + ) -> Result, PgJobError>>> { + let failed_job = self.job.fail(error); let base_query = format!( r#" @@ -217,19 +221,22 @@ UPDATE "{0}" SET finished_at = NOW(), - status = 'completed'::job_status + status = 'failed'::job_status + errors = array_append("{0}".errors, $3) WHERE "{0}".id = $2 AND queue = $1 RETURNING "{0}".* + "#, &self.table ); sqlx::query(&base_query) - .bind(&completed_job.queue) - .bind(completed_job.id) + .bind(&failed_job.queue) + .bind(failed_job.id) + .bind(&failed_job.error) .execute(&mut *self.connection) .await .map_err(|error| PgJobError::QueryError { @@ -237,14 +244,24 @@ RETURNING error, })?; - Ok(completed_job) + Ok(failed_job) } - pub async fn fail( + async fn retry( mut self, error: E, - ) -> Result, PgJobError>> { - let failed_job = self.job.fail(error); + preferred_retry_interval: Option, + ) -> Result, PgJobError>>> { + if self.job.is_gte_max_attempts() { + return Err(PgJobError::RetryInvalidError { + job: Box::new(self), + error: "Maximum attempts reached".to_owned(), + }); + } + let retryable_job = self.job.retry(error); + let retry_interval = self + .retry_policy + .time_until_next_retry(&retryable_job, preferred_retry_interval); let base_query = format!( r#" @@ -252,22 +269,23 @@ UPDATE "{0}" SET finished_at = NOW(), - status = 'failed'::job_status - errors = array_append("{0}".errors, $3) + status = 'available'::job_status, + scheduled_at = NOW() + $3, + errors = array_append("{0}".errors, $4) WHERE "{0}".id = $2 AND queue = $1 RETURNING "{0}".* - "#, &self.table ); sqlx::query(&base_query) - .bind(&failed_job.queue) - .bind(failed_job.id) - .bind(&failed_job.error) + .bind(&retryable_job.queue) + .bind(retryable_job.id) + .bind(retry_interval) + .bind(&retryable_job.error) .execute(&mut *self.connection) .await .map_err(|error| PgJobError::QueryError { @@ -275,7 +293,7 @@ RETURNING error, })?; - Ok(failed_job) + Ok(retryable_job) } } @@ -289,22 +307,12 @@ pub struct PgTransactionJob<'c, J, M> { pub retry_policy: RetryPolicy, } -impl<'c, J, M> PgTransactionJob<'c, J, M> { - pub async fn retry( +#[async_trait] +impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactionJob<'c, J, M> { + async fn complete( mut self, - error: E, - preferred_retry_interval: Option, - ) -> Result, PgJobError>> { - if self.job.is_gte_max_attempts() { - return Err(PgJobError::RetryInvalidError { - job: self, - error: "Maximum attempts reached".to_owned(), - }); - } - let retryable_job = self.job.retry(error); - let retry_interval = self - .retry_policy - .time_until_next_retry(&retryable_job, preferred_retry_interval); + ) -> Result>>> { + let completed_job = self.job.complete(); let base_query = format!( r#" @@ -312,24 +320,19 @@ UPDATE "{0}" SET finished_at = NOW(), - status = 'available'::job_status, - scheduled_at = NOW() + $3, - errors = array_append("{0}".errors, $4) + status = 'completed'::job_status WHERE "{0}".id = $2 AND queue = $1 RETURNING "{0}".* - "#, &self.table ); sqlx::query(&base_query) - .bind(&retryable_job.queue) - .bind(retryable_job.id) - .bind(retry_interval) - .bind(&retryable_job.error) + .bind(&completed_job.queue) + .bind(completed_job.id) .execute(&mut *self.transaction) .await .map_err(|error| PgJobError::QueryError { @@ -345,13 +348,14 @@ RETURNING error, })?; - Ok(retryable_job) + Ok(completed_job) } - pub async fn complete( + async fn fail( mut self, - ) -> Result>> { - let completed_job = self.job.complete(); + error: E, + ) -> Result, PgJobError>>> { + let failed_job = self.job.fail(error); let base_query = format!( r#" @@ -359,7 +363,8 @@ UPDATE "{0}" SET finished_at = NOW(), - status = 'completed'::job_status + status = 'failed'::job_status + errors = array_append("{0}".errors, $3) WHERE "{0}".id = $2 AND queue = $1 @@ -370,8 +375,9 @@ RETURNING ); sqlx::query(&base_query) - .bind(&completed_job.queue) - .bind(completed_job.id) + .bind(&failed_job.queue) + .bind(failed_job.id) + .bind(&failed_job.error) .execute(&mut *self.transaction) .await .map_err(|error| PgJobError::QueryError { @@ -387,14 +393,24 @@ RETURNING error, })?; - Ok(completed_job) + Ok(failed_job) } - pub async fn fail( + async fn retry( mut self, error: E, - ) -> Result, PgJobError>> { - let failed_job = self.job.fail(error); + preferred_retry_interval: Option, + ) -> Result, PgJobError>>> { + if self.job.is_gte_max_attempts() { + return Err(PgJobError::RetryInvalidError { + job: Box::new(self), + error: "Maximum attempts reached".to_owned(), + }); + } + let retryable_job = self.job.retry(error); + let retry_interval = self + .retry_policy + .time_until_next_retry(&retryable_job, preferred_retry_interval); let base_query = format!( r#" @@ -402,21 +418,24 @@ UPDATE "{0}" SET finished_at = NOW(), - status = 'failed'::job_status - errors = array_append("{0}".errors, $3) + status = 'available'::job_status, + scheduled_at = NOW() + $3, + errors = array_append("{0}".errors, $4) WHERE "{0}".id = $2 AND queue = $1 RETURNING "{0}".* + "#, &self.table ); sqlx::query(&base_query) - .bind(&failed_job.queue) - .bind(failed_job.id) - .bind(&failed_job.error) + .bind(&retryable_job.queue) + .bind(retryable_job.id) + .bind(retry_interval) + .bind(&retryable_job.error) .execute(&mut *self.transaction) .await .map_err(|error| PgJobError::QueryError { @@ -432,7 +451,7 @@ RETURNING error, })?; - Ok(failed_job) + Ok(retryable_job) } } diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 83ea319d2c555..4fdc8e476d745 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -3,7 +3,9 @@ use std::sync::Arc; use std::time; use async_std::task; -use hook_common::pgqueue::{PgJobError, PgQueue, PgQueueError, PgTransactionJob}; +use hook_common::pgqueue::{ + PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob, +}; use hook_common::webhook::{HttpMethod, WebhookJobError, WebhookJobMetadata, WebhookJobParameters}; use http::StatusCode; use reqwest::header; @@ -11,6 +13,32 @@ use tokio::sync; use crate::error::{ConsumerError, WebhookError}; +/// A WebhookJob is any PgQueueJob that returns webhook required parameters and metadata. +trait WebhookJob: PgQueueJob + std::marker::Send { + fn parameters<'a>(&'a self) -> &'a WebhookJobParameters; + fn metadata<'a>(&'a self) -> &'a WebhookJobMetadata; +} + +impl WebhookJob for PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata> { + fn parameters<'a>(&'a self) -> &'a WebhookJobParameters { + &self.job.parameters + } + + fn metadata<'a>(&'a self) -> &'a WebhookJobMetadata { + &self.job.metadata + } +} + +impl WebhookJob for PgJob { + fn parameters<'a>(&'a self) -> &'a WebhookJobParameters { + &self.job.parameters + } + + fn metadata<'a>(&'a self) -> &'a WebhookJobMetadata { + &self.job.metadata + } +} + /// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. pub struct WebhookConsumer<'p> { /// An identifier for this consumer. Used to mark jobs we have consumed. @@ -55,7 +83,9 @@ impl<'p> WebhookConsumer<'p> { } /// Wait until a job becomes available in our queue. - async fn wait_for_job<'a>(&self) -> Result, WebhookConsumerError> { + async fn wait_for_job<'a>( + &self, + ) -> Result, ConsumerError> { loop { if let Some(job) = self.queue.dequeue(&self.name).await? { return Ok(job); @@ -65,8 +95,21 @@ impl<'p> WebhookConsumer<'p> { } } + /// Wait until a job becomes available in our queue in transactional mode. + async fn wait_for_job_tx<'a>( + &self, + ) -> Result, ConsumerError> { + loop { + if let Some(job) = self.queue.dequeue_tx(&self.name).await? { + return Ok(job); + } else { + task::sleep(self.poll_interval).await; + } + } + } + /// Run this consumer to continuously process any jobs that become available. - pub async fn run(&self) -> Result<(), WebhookConsumerError> { + pub async fn run(&self) -> Result<(), ConsumerError> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); loop { @@ -83,76 +126,20 @@ impl<'p> WebhookConsumer<'p> { }); } } -} - -/// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. -pub struct WebhookTransactionConsumer<'p> { - /// An identifier for this consumer. Used to mark jobs we have consumed. - name: String, - /// The queue we will be dequeuing jobs from. - queue: &'p PgQueue, - /// The interval for polling the queue. - poll_interval: time::Duration, - /// The client used for HTTP requests. - client: reqwest::Client, - /// Maximum number of concurrent jobs being processed. - max_concurrent_jobs: usize, -} - -impl<'p> WebhookTransactionConsumer<'p> { - pub fn new( - name: &str, - queue: &'p PgQueue, - poll_interval: time::Duration, - request_timeout: time::Duration, - max_concurrent_jobs: usize, - ) -> Result { - let mut headers = header::HeaderMap::new(); - headers.insert( - header::CONTENT_TYPE, - header::HeaderValue::from_static("application/json"), - ); - - let client = reqwest::Client::builder() - .default_headers(headers) - .timeout(request_timeout) - .build()?; - - Ok(Self { - name: name.to_owned(), - queue, - poll_interval, - client, - max_concurrent_jobs, - }) - } - - /// Wait until a job becomes available in our queue. - async fn wait_for_job<'a>( - &self, - ) -> Result, ConsumerError> { - loop { - if let Some(job) = self.queue.dequeue_tx(&self.name).await? { - return Ok(job); - } else { - task::sleep(self.poll_interval).await; - } - } - } - /// Run this consumer to continuously process any jobs that become available. - pub async fn run(&self) -> Result<(), ConsumerError> { + /// Run this consumer to continuously process any jobs that become available in transactional mode. + pub async fn run_tx(&self) -> Result<(), ConsumerError> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); loop { - let webhook_job = self.wait_for_job().await?; + let webhook_job = self.wait_for_job_tx().await?; // reqwest::Client internally wraps with Arc, so this allocation is cheap. let client = self.client.clone(); let permit = semaphore.clone().acquire_owned().await.unwrap(); tokio::spawn(async move { - let result = process_webhook_job_tx(client, webhook_job).await; + let result = process_webhook_job(client, webhook_job).await; drop(permit); result }); @@ -173,16 +160,18 @@ impl<'p> WebhookTransactionConsumer<'p> { /// /// * `client`: An HTTP client to execute the webhook job request. /// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. -async fn process_webhook_job( +async fn process_webhook_job( client: reqwest::Client, - webhook_job: PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata>, + webhook_job: W, ) -> Result<(), ConsumerError> { + let parameters = webhook_job.parameters(); + match send_webhook( client, - &webhook_job.job.parameters.method, - &webhook_job.job.parameters.url, - &webhook_job.job.parameters.headers, - webhook_job.job.parameters.body.clone(), + ¶meters.method, + ¶meters.url, + ¶meters.headers, + parameters.body.clone(), ) .await { @@ -242,66 +231,6 @@ async fn process_webhook_job( } } -/// Process a webhook job by transitioning it to its appropriate state after its request is sent. -/// After we finish, the webhook job will be set as completed (if the request was successful), retryable (if the request -/// was unsuccessful but we can still attempt a retry), or failed (if the request was unsuccessful and no more retries -/// may be attempted). -/// -/// A webhook job is considered retryable after a failing request if: -/// 1. The job has attempts remaining (i.e. hasn't reached `max_attempts`), and... -/// 2. The status code indicates retrying at a later point could resolve the issue. This means: 429 and any 5XX. -/// -/// # Arguments -/// -/// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. -/// * `request_timeout`: A timeout for the HTTP request. -async fn process_webhook_job( - client: reqwest::Client, - webhook_job: PgJob, -) -> Result<(), WebhookConsumerError> { - match send_webhook( - client, - &webhook_job.job.parameters.method, - &webhook_job.job.parameters.url, - &webhook_job.job.parameters.headers, - webhook_job.job.parameters.body.clone(), - ) - .await - { - Ok(_) => { - webhook_job - .complete() - .await - .map_err(|error| WebhookConsumerError::PgJobError(error.to_string()))?; - Ok(()) - } - Err(WebhookConsumerError::RetryableWebhookError { - reason, - retry_after, - }) => match webhook_job.retry(reason.to_string(), retry_after).await { - Ok(_) => Ok(()), - Err(PgJobError::RetryInvalidError { - job: webhook_job, - error: fail_error, - }) => { - webhook_job - .fail(fail_error.to_string()) - .await - .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; - Ok(()) - } - Err(job_error) => Err(WebhookConsumerError::PgJobError(job_error.to_string())), - }, - Err(error) => { - webhook_job - .fail(error.to_string()) - .await - .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; - Ok(()) - } - } -} - /// Make an HTTP request to a webhook endpoint. /// /// # Arguments diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index bb02526c23b76..49c2e76a8610f 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -31,7 +31,11 @@ async fn main() -> Result<(), ConsumerError> { config.max_concurrent_jobs, ); - let _ = consumer.run().await; + if config.transactional { + consumer.run_tx().await?; + } else { + consumer.run().await?; + } Ok(()) } From b747d4c242b5abd0b959190a99797c717b3c7239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 20 Dec 2023 15:31:21 +0100 Subject: [PATCH 138/247] fix: Elide lifetimes --- hook-consumer/src/consumer.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 4fdc8e476d745..73bac1cffba87 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -13,28 +13,28 @@ use tokio::sync; use crate::error::{ConsumerError, WebhookError}; -/// A WebhookJob is any PgQueueJob that returns webhook required parameters and metadata. +/// A WebhookJob is any PgQueueJob that returns a reference to webhook parameters and metadata. trait WebhookJob: PgQueueJob + std::marker::Send { - fn parameters<'a>(&'a self) -> &'a WebhookJobParameters; - fn metadata<'a>(&'a self) -> &'a WebhookJobMetadata; + fn parameters(&self) -> &WebhookJobParameters; + fn metadata(&self) -> &WebhookJobMetadata; } impl WebhookJob for PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata> { - fn parameters<'a>(&'a self) -> &'a WebhookJobParameters { + fn parameters(&self) -> &WebhookJobParameters { &self.job.parameters } - fn metadata<'a>(&'a self) -> &'a WebhookJobMetadata { + fn metadata(&self) -> &WebhookJobMetadata { &self.job.metadata } } impl WebhookJob for PgJob { - fn parameters<'a>(&'a self) -> &'a WebhookJobParameters { + fn parameters(&self) -> &WebhookJobParameters { &self.job.parameters } - fn metadata<'a>(&'a self) -> &'a WebhookJobMetadata { + fn metadata(&self) -> &WebhookJobMetadata { &self.job.metadata } } From 059fc99cb9bc4e381534a5cdcd7361b269c4b87a Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 20 Dec 2023 07:54:54 -0700 Subject: [PATCH 139/247] Add SerializableTxn for a little safety --- hook-janitor/src/webhooks.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index a9b05e560e737..71fe567dd30e3 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -84,6 +84,10 @@ struct FailedRow { failures: u32, } +// A simple wrapper type that ensures we don't use any old Transaction object when we need one +// that has set the isolation level to serializable. +struct SerializableTxn<'a>(Transaction<'a, Postgres>); + impl WebhookCleaner { pub fn new( queue_name: &str, @@ -128,7 +132,7 @@ impl WebhookCleaner { }) } - async fn start_serializable_txn(&self) -> Result> { + async fn start_serializable_txn(&self) -> Result { let mut tx = self .pg_pool .begin() @@ -146,13 +150,10 @@ impl WebhookCleaner { .await .map_err(|e| WebhookCleanerError::StartTxnError { error: e })?; - Ok(tx) + Ok(SerializableTxn(tx)) } - async fn get_completed_rows( - &self, - tx: &mut Transaction<'_, Postgres>, - ) -> Result> { + async fn get_completed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { let base_query = format!( r#" SELECT DATE_TRUNC('hour', finished_at) AS hour, @@ -170,7 +171,7 @@ impl WebhookCleaner { let rows = sqlx::query_as::<_, CompletedRow>(&base_query) .bind(&self.queue_name) - .fetch_all(&mut **tx) + .fetch_all(&mut *tx.0) .await .map_err(|e| WebhookCleanerError::GetCompletedRowsError { error: e })?; @@ -204,7 +205,7 @@ impl WebhookCleaner { Ok(payloads) } - async fn get_failed_rows(&self, tx: &mut Transaction<'_, Postgres>) -> Result> { + async fn get_failed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { let base_query = format!( r#" SELECT DATE_TRUNC('hour', finished_at) AS hour, @@ -223,7 +224,7 @@ impl WebhookCleaner { let rows = sqlx::query_as::<_, FailedRow>(&base_query) .bind(&self.queue_name) - .fetch_all(&mut **tx) + .fetch_all(&mut *tx.0) .await .map_err(|e| WebhookCleanerError::GetFailedRowsError { error: e })?; @@ -290,7 +291,7 @@ impl WebhookCleaner { Ok(()) } - async fn delete_observed_rows(&self, tx: &mut Transaction<'_, Postgres>) -> Result { + async fn delete_observed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result { // This DELETE is only safe because we are in serializable isolation mode, see the note // in `start_serializable_txn`. let base_query = format!( @@ -304,15 +305,15 @@ impl WebhookCleaner { let result = sqlx::query(&base_query) .bind(&self.queue_name) - .execute(&mut **tx) + .execute(&mut *tx.0) .await .map_err(|e| WebhookCleanerError::DeleteRowsError { error: e })?; Ok(result.rows_affected()) } - async fn commit_txn(&self, tx: Transaction<'_, Postgres>) -> Result<()> { - tx.commit() + async fn commit_txn(&self, tx: SerializableTxn<'_>) -> Result<()> { + tx.0.commit() .await .map_err(|e| WebhookCleanerError::CommitTxnError { error: e })?; From 8aeaad7e474e5592f6be47a53a1044c5314005f3 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 20 Dec 2023 08:18:04 -0700 Subject: [PATCH 140/247] Cleanup AppMetric creation and serialization --- hook-janitor/src/webhooks.rs | 139 +++++++++++++++++------------------ 1 file changed, 66 insertions(+), 73 deletions(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 71fe567dd30e3..d4b62550d1e61 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -66,6 +66,24 @@ struct CompletedRow { successes: u32, } +impl From for AppMetric { + fn from(row: CompletedRow) -> Self { + AppMetric { + timestamp: row.hour, + team_id: row.team_id, + plugin_config_id: row.plugin_config_id, + job_id: None, + category: AppMetricCategory::Webhook, + successes: row.successes, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + } + } +} + #[derive(sqlx::FromRow, Debug)] struct FailedRow { // App Metrics truncates/aggregates rows on the hour, so we take advantage of that to GROUP BY @@ -84,6 +102,24 @@ struct FailedRow { failures: u32, } +impl From for AppMetric { + fn from(row: FailedRow) -> Self { + AppMetric { + timestamp: row.hour, + team_id: row.team_id, + plugin_config_id: row.plugin_config_id, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: row.failures, + error_uuid: Some(Uuid::now_v7()), + error_type: Some(row.last_error.r#type), + error_details: Some(row.last_error.details), + } + } +} + // A simple wrapper type that ensures we don't use any old Transaction object when we need one // that has set the isolation level to serializable. struct SerializableTxn<'a>(Transaction<'a, Postgres>); @@ -178,33 +214,6 @@ impl WebhookCleaner { Ok(rows) } - fn serialize_completed_rows(&self, completed_rows: Vec) -> Result> { - let mut payloads = Vec::new(); - - for row in completed_rows { - let app_metric = AppMetric { - timestamp: row.hour, - team_id: row.team_id, - plugin_config_id: row.plugin_config_id, - job_id: None, - category: AppMetricCategory::Webhook, - successes: row.successes, - successes_on_retry: 0, - failures: 0, - error_uuid: None, - error_type: None, - error_details: None, - }; - - let payload = serde_json::to_string(&app_metric) - .map_err(|e| WebhookCleanerError::SerializeRowsError { error: e })?; - - payloads.push(payload) - } - - Ok(payloads) - } - async fn get_failed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { let base_query = format!( r#" @@ -231,34 +240,13 @@ impl WebhookCleaner { Ok(rows) } - fn serialize_failed_rows(&self, failed_rows: Vec) -> Result> { - let mut payloads = Vec::new(); - - for row in failed_rows { - let app_metric = AppMetric { - timestamp: row.hour, - team_id: row.team_id, - plugin_config_id: row.plugin_config_id, - job_id: None, - category: AppMetricCategory::Webhook, - successes: 0, - successes_on_retry: 0, - failures: row.failures, - error_uuid: Some(Uuid::now_v7()), - error_type: Some(row.last_error.r#type), - error_details: Some(row.last_error.details), - }; - - let payload = serde_json::to_string(&app_metric) - .map_err(|e| WebhookCleanerError::SerializeRowsError { error: e })?; - - payloads.push(payload) - } + async fn send_metrics_to_kafka(&self, metrics: Vec) -> Result<()> { + let payloads: Vec = metrics + .into_iter() + .map(|metric| serde_json::to_string(&metric)) + .collect::, SerdeError>>() + .map_err(|e| WebhookCleanerError::SerializeRowsError { error: e })?; - Ok(payloads) - } - - async fn send_messages_to_kafka(&self, payloads: Vec) -> Result<()> { let mut delivery_futures = Vec::new(); for payload in payloads { @@ -332,30 +320,35 @@ impl WebhookCleaner { // future if necessary. let mut tx = self.start_serializable_txn().await?; + let mut rows_processed = 0; + + { + let completed_rows = self.get_completed_rows(&mut tx).await?; + rows_processed += completed_rows.len(); + let completed_app_metrics: Vec = + completed_rows.into_iter().map(Into::into).collect(); + self.send_metrics_to_kafka(completed_app_metrics).await?; + } - let completed_rows = self.get_completed_rows(&mut tx).await?; - let completed_agg_row_count = completed_rows.len(); - let completed_kafka_payloads = self.serialize_completed_rows(completed_rows)?; - - let failed_rows = self.get_failed_rows(&mut tx).await?; - let failed_agg_row_count = failed_rows.len(); - let failed_kafka_payloads = self.serialize_failed_rows(failed_rows)?; - - let mut all_kafka_payloads = completed_kafka_payloads; - all_kafka_payloads.extend(failed_kafka_payloads.into_iter()); + { + let failed_rows = self.get_failed_rows(&mut tx).await?; + rows_processed += failed_rows.len(); + let failed_app_metrics: Vec = + failed_rows.into_iter().map(Into::into).collect(); + self.send_metrics_to_kafka(failed_app_metrics).await?; + } - let mut rows_deleted: u64 = 0; - if !all_kafka_payloads.is_empty() { - self.send_messages_to_kafka(all_kafka_payloads).await?; - rows_deleted = self.delete_observed_rows(&mut tx).await?; + if rows_processed != 0 { + let rows_deleted = self.delete_observed_rows(&mut tx).await?; self.commit_txn(tx).await?; + debug!( + "WebhookCleaner finished cleanup, processed and deleted {} rows", + rows_deleted + ); + } else { + debug!("WebhookCleaner finished cleanup, no-op"); } - debug!( - "WebhookCleaner finished cleanup, deleted {} rows ({} completed+aggregated, {} failed+aggregated)", - rows_deleted, completed_agg_row_count, failed_agg_row_count - ); - Ok(()) } } From a94ee2f6dec7fdac47a524b9caf56c01ed362584 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 20 Dec 2023 08:31:13 -0700 Subject: [PATCH 141/247] Clean up row counts --- hook-janitor/src/webhooks.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index d4b62550d1e61..5ccf6dca701c3 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -241,6 +241,10 @@ impl WebhookCleaner { } async fn send_metrics_to_kafka(&self, metrics: Vec) -> Result<()> { + if metrics.is_empty() { + return Ok(()); + } + let payloads: Vec = metrics .into_iter() .map(|metric| serde_json::to_string(&metric)) @@ -320,33 +324,34 @@ impl WebhookCleaner { // future if necessary. let mut tx = self.start_serializable_txn().await?; - let mut rows_processed = 0; - { + let completed_agg_row_count = { let completed_rows = self.get_completed_rows(&mut tx).await?; - rows_processed += completed_rows.len(); + let row_count = completed_rows.len(); let completed_app_metrics: Vec = completed_rows.into_iter().map(Into::into).collect(); self.send_metrics_to_kafka(completed_app_metrics).await?; - } + row_count + }; - { + let failed_agg_row_count = { let failed_rows = self.get_failed_rows(&mut tx).await?; - rows_processed += failed_rows.len(); + let row_count = failed_rows.len(); let failed_app_metrics: Vec = failed_rows.into_iter().map(Into::into).collect(); self.send_metrics_to_kafka(failed_app_metrics).await?; - } + row_count + }; - if rows_processed != 0 { + if completed_agg_row_count + failed_agg_row_count != 0 { let rows_deleted = self.delete_observed_rows(&mut tx).await?; self.commit_txn(tx).await?; debug!( - "WebhookCleaner finished cleanup, processed and deleted {} rows", - rows_deleted + "WebhookCleaner finished cleanup, processed and deleted {} rows ({}/{} aggregated completed/failed rows)", + rows_deleted, completed_agg_row_count, failed_agg_row_count ); } else { - debug!("WebhookCleaner finished cleanup, no-op"); + debug!("WebhookCleaner finished cleanup, there were no rows to process"); } Ok(()) From 66ea83428f7bd5502475a21e8d0992d868a7645f Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 20 Dec 2023 08:53:47 -0700 Subject: [PATCH 142/247] Tweak ErrorType, add some notes about stability of serialized types --- hook-common/src/kafka_messages/app_metrics.rs | 24 ++++++++++++------- hook-common/src/webhook.rs | 16 ++++++------- hook-janitor/src/fixtures/webhook_cleanup.sql | 14 +++++++++-- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs index 439664342fa35..cecf6049a8637 100644 --- a/hook-common/src/kafka_messages/app_metrics.rs +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -13,14 +13,20 @@ pub enum AppMetricCategory { ComposeWebhook, } +// NOTE: These are stored in Postgres and deserialized by the cleanup/janitor process, so these +// names need to remain stable, or new variants need to be deployed to the cleanup/janitor +// process before they are used. #[derive(Deserialize, Serialize, Debug)] pub enum ErrorType { - Timeout, - Connection, - HttpStatus(u16), - Parse, + TimeoutError, + ConnectionError, + BadHttpStatus(u16), + ParseError, } +// NOTE: This is stored in Postgres and deserialized by the cleanup/janitor process, so this +// shouldn't change. It is intended to replicate the shape of `error_details` used in the +// plugin-server and by the frontend. #[derive(Deserialize, Serialize, Debug)] pub struct ErrorDetails { pub error: Error, @@ -88,10 +94,10 @@ where }; let error_type = match error_type { - ErrorType::Connection => "Connection Error".to_owned(), - ErrorType::Timeout => "Timeout".to_owned(), - ErrorType::HttpStatus(s) => format!("HTTP Status: {}", s), - ErrorType::Parse => "Parse Error".to_owned(), + ErrorType::ConnectionError => "Connection Error".to_owned(), + ErrorType::TimeoutError => "Timeout".to_owned(), + ErrorType::BadHttpStatus(s) => format!("HTTP Status: {}", s), + ErrorType::ParseError => "Parse Error".to_owned(), }; serializer.serialize_str(&error_type) } @@ -114,7 +120,7 @@ mod tests { successes_on_retry: 0, failures: 2, error_uuid: Some(Uuid::parse_str("550e8400-e29b-41d4-a716-446655447777").unwrap()), - error_type: Some(ErrorType::Connection), + error_type: Some(ErrorType::ConnectionError), error_details: Some(ErrorDetails { error: Error { name: "FooError".to_owned(), diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index 9a21b83cd713b..bb1b5be04390a 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -169,12 +169,12 @@ impl From<&reqwest::Error> for WebhookJobError { impl WebhookJobError { pub fn new_timeout(message: &str) -> Self { let error_details = app_metrics::Error { - name: "timeout".to_owned(), + name: "Timeout Error".to_owned(), message: Some(message.to_owned()), stack: None, }; Self { - r#type: app_metrics::ErrorType::Timeout, + r#type: app_metrics::ErrorType::TimeoutError, details: app_metrics::ErrorDetails { error: error_details, }, @@ -183,12 +183,12 @@ impl WebhookJobError { pub fn new_connection(message: &str) -> Self { let error_details = app_metrics::Error { - name: "connection error".to_owned(), + name: "Connection Error".to_owned(), message: Some(message.to_owned()), stack: None, }; Self { - r#type: app_metrics::ErrorType::Connection, + r#type: app_metrics::ErrorType::ConnectionError, details: app_metrics::ErrorDetails { error: error_details, }, @@ -197,12 +197,12 @@ impl WebhookJobError { pub fn new_http_status(status_code: u16, message: &str) -> Self { let error_details = app_metrics::Error { - name: "http status".to_owned(), + name: "Bad Http Status".to_owned(), message: Some(message.to_owned()), stack: None, }; Self { - r#type: app_metrics::ErrorType::HttpStatus(status_code), + r#type: app_metrics::ErrorType::BadHttpStatus(status_code), details: app_metrics::ErrorDetails { error: error_details, }, @@ -211,12 +211,12 @@ impl WebhookJobError { pub fn new_parse(message: &str) -> Self { let error_details = app_metrics::Error { - name: "parse error".to_owned(), + name: "Parse Error".to_owned(), message: Some(message.to_owned()), stack: None, }; Self { - r#type: app_metrics::ErrorType::Parse, + r#type: app_metrics::ErrorType::ParseError, details: app_metrics::ErrorDetails { error: error_details, }, diff --git a/hook-janitor/src/fixtures/webhook_cleanup.sql b/hook-janitor/src/fixtures/webhook_cleanup.sql index 6f29d302b8462..e4ea082ebf188 100644 --- a/hook-janitor/src/fixtures/webhook_cleanup.sql +++ b/hook-janitor/src/fixtures/webhook_cleanup.sql @@ -19,7 +19,7 @@ VALUES 'completed', 'https://myhost/endpoint' ), - -- another team:1, plugin_config:2, completed in hour 20 + -- team:1, plugin_config:2, completed in hour 20 (purposeful duplicate) ( NULL, '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', @@ -71,7 +71,17 @@ VALUES ), -- team:1, plugin_config:2, failed in hour 20 ( - ARRAY ['{"type":"Timeout","details":{"error":{"name":"timeout"}}}'::jsonb], + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"timeout"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, failed in hour 20 (purposeful duplicate) + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"timeout"}}}'::jsonb], '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '{}', From dd8faee798efe8796d2fb17df5f054c1f7f75c9d Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 20 Dec 2023 15:41:25 -0700 Subject: [PATCH 143/247] Finish up tests --- Cargo.lock | 1 + Cargo.toml | 2 +- hook-common/src/kafka_messages/app_metrics.rs | 94 ++++- hook-common/src/kafka_messages/mod.rs | 29 +- hook-common/src/kafka_messages/plugin_logs.rs | 4 +- hook-janitor/src/fixtures/webhook_cleanup.sql | 74 +++- hook-janitor/src/webhooks.rs | 372 ++++++++++++++++-- 7 files changed, 510 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fbfcc50d98ee8..c36b1da60cae4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2953,6 +2953,7 @@ checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" dependencies = [ "atomic", "getrandom", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a29806a2fa730..1f6a38b6e3ed1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,4 +35,4 @@ tower = "0.4.13" tracing = "0.1.40" tracing-subscriber = "0.3.18" url = { version = "2.5.0 " } -uuid = { version = "1.6.1", features = ["v7"] } +uuid = { version = "1.6.1", features = ["v7", "serde"] } diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs index cecf6049a8637..5aff62cd9fedd 100644 --- a/hook-common/src/kafka_messages/app_metrics.rs +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -1,10 +1,10 @@ use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize, Serializer}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use uuid::Uuid; -use super::{serialize_datetime, serialize_optional_uuid}; +use super::{deserialize_datetime, serialize_datetime}; -#[derive(Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] pub enum AppMetricCategory { ProcessEvent, OnEvent, @@ -16,7 +16,7 @@ pub enum AppMetricCategory { // NOTE: These are stored in Postgres and deserialized by the cleanup/janitor process, so these // names need to remain stable, or new variants need to be deployed to the cleanup/janitor // process before they are used. -#[derive(Deserialize, Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] pub enum ErrorType { TimeoutError, ConnectionError, @@ -27,12 +27,12 @@ pub enum ErrorType { // NOTE: This is stored in Postgres and deserialized by the cleanup/janitor process, so this // shouldn't change. It is intended to replicate the shape of `error_details` used in the // plugin-server and by the frontend. -#[derive(Deserialize, Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] pub struct ErrorDetails { pub error: Error, } -#[derive(Deserialize, Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] pub struct Error { pub name: String, #[serde(skip_serializing_if = "Option::is_none")] @@ -43,26 +43,30 @@ pub struct Error { pub stack: Option, } -#[derive(Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] pub struct AppMetric { - #[serde(serialize_with = "serialize_datetime")] + #[serde( + serialize_with = "serialize_datetime", + deserialize_with = "deserialize_datetime" + )] pub timestamp: DateTime, pub team_id: u32, pub plugin_config_id: u32, #[serde(skip_serializing_if = "Option::is_none")] pub job_id: Option, - #[serde(serialize_with = "serialize_category")] + #[serde( + serialize_with = "serialize_category", + deserialize_with = "deserialize_category" + )] pub category: AppMetricCategory, pub successes: u32, pub successes_on_retry: u32, pub failures: u32, - #[serde( - serialize_with = "serialize_optional_uuid", - skip_serializing_if = "Option::is_none" - )] pub error_uuid: Option, #[serde( serialize_with = "serialize_error_type", + deserialize_with = "deserialize_error_type", + default, skip_serializing_if = "Option::is_none" )] pub error_type: Option, @@ -84,6 +88,35 @@ where serializer.serialize_str(category_str) } +fn deserialize_category<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s: String = Deserialize::deserialize(deserializer)?; + + let category = match &s[..] { + "processEvent" => AppMetricCategory::ProcessEvent, + "onEvent" => AppMetricCategory::OnEvent, + "scheduledTask" => AppMetricCategory::ScheduledTask, + "webhook" => AppMetricCategory::Webhook, + "composeWebhook" => AppMetricCategory::ComposeWebhook, + _ => { + return Err(serde::de::Error::unknown_variant( + &s, + &[ + "processEvent", + "onEvent", + "scheduledTask", + "webhook", + "composeWebhook", + ], + )) + } + }; + + Ok(category) +} + fn serialize_error_type(error_type: &Option, serializer: S) -> Result where S: Serializer, @@ -102,6 +135,41 @@ where serializer.serialize_str(&error_type) } +fn deserialize_error_type<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let opt = Option::::deserialize(deserializer)?; + let error_type = match opt { + Some(s) => { + let error_type = match &s[..] { + "Connection Error" => ErrorType::ConnectionError, + "Timeout" => ErrorType::TimeoutError, + _ if s.starts_with("HTTP Status:") => { + let status = &s["HTTP Status:".len()..]; + ErrorType::BadHttpStatus(status.parse().map_err(serde::de::Error::custom)?) + } + "Parse Error" => ErrorType::ParseError, + _ => { + return Err(serde::de::Error::unknown_variant( + &s, + &[ + "Connection Error", + "Timeout", + "HTTP Status: ", + "Parse Error", + ], + )) + } + }; + Some(error_type) + } + None => None, + }; + + Ok(error_type) +} + #[cfg(test)] mod tests { use super::*; diff --git a/hook-common/src/kafka_messages/mod.rs b/hook-common/src/kafka_messages/mod.rs index 72b49e1e45059..f548563af5ba1 100644 --- a/hook-common/src/kafka_messages/mod.rs +++ b/hook-common/src/kafka_messages/mod.rs @@ -1,30 +1,25 @@ pub mod app_metrics; pub mod plugin_logs; -use chrono::{DateTime, Utc}; -use serde::Serializer; -use uuid::Uuid; +use chrono::{DateTime, NaiveDateTime, Utc}; +use serde::{Deserialize, Deserializer, Serializer}; -pub fn serialize_uuid(uuid: &Uuid, serializer: S) -> Result +pub fn serialize_datetime(datetime: &DateTime, serializer: S) -> Result where S: Serializer, { - serializer.serialize_str(&uuid.to_string()) + serializer.serialize_str(&datetime.format("%Y-%m-%d %H:%M:%S").to_string()) } -pub fn serialize_optional_uuid(uuid: &Option, serializer: S) -> Result +pub fn deserialize_datetime<'de, D>(deserializer: D) -> Result, D::Error> where - S: Serializer, + D: Deserializer<'de>, { - match uuid { - Some(uuid) => serializer.serialize_str(&uuid.to_string()), - None => serializer.serialize_none(), - } -} + let formatted: String = Deserialize::deserialize(deserializer)?; + let datetime = match NaiveDateTime::parse_from_str(&formatted, "%Y-%m-%d %H:%M:%S") { + Ok(d) => d.and_utc(), + Err(_) => return Err(serde::de::Error::custom("Invalid datetime format")), + }; -pub fn serialize_datetime(datetime: &DateTime, serializer: S) -> Result -where - S: Serializer, -{ - serializer.serialize_str(&datetime.format("%Y-%m-%d %H:%M:%S%.f").to_string()) + Ok(datetime) } diff --git a/hook-common/src/kafka_messages/plugin_logs.rs b/hook-common/src/kafka_messages/plugin_logs.rs index 8f8bb43efea96..e761fa40799ea 100644 --- a/hook-common/src/kafka_messages/plugin_logs.rs +++ b/hook-common/src/kafka_messages/plugin_logs.rs @@ -2,7 +2,7 @@ use chrono::{DateTime, Utc}; use serde::{Serialize, Serializer}; use uuid::Uuid; -use super::{serialize_datetime, serialize_uuid}; +use super::serialize_datetime; #[allow(dead_code)] #[derive(Serialize)] @@ -28,7 +28,6 @@ pub struct PluginLogEntry { pub source: PluginLogEntrySource, #[serde(rename = "type", serialize_with = "serialize_type")] pub type_: PluginLogEntryType, - #[serde(serialize_with = "serialize_uuid")] pub id: Uuid, pub team_id: u32, pub plugin_id: u32, @@ -37,7 +36,6 @@ pub struct PluginLogEntry { pub timestamp: DateTime, #[serde(serialize_with = "serialize_message")] pub message: String, - #[serde(serialize_with = "serialize_uuid")] pub instance_id: Uuid, } diff --git a/hook-janitor/src/fixtures/webhook_cleanup.sql b/hook-janitor/src/fixtures/webhook_cleanup.sql index e4ea082ebf188..4aeb231febbd3 100644 --- a/hook-janitor/src/fixtures/webhook_cleanup.sql +++ b/hook-janitor/src/fixtures/webhook_cleanup.sql @@ -71,7 +71,7 @@ VALUES ), -- team:1, plugin_config:2, failed in hour 20 ( - ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"timeout"}}}'::jsonb], + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '{}', @@ -81,11 +81,81 @@ VALUES ), -- team:1, plugin_config:2, failed in hour 20 (purposeful duplicate) ( - ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"timeout"}}}'::jsonb], + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '{}', 'webhooks', 'failed', 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, failed in hour 20 (different error) + ( + ARRAY ['{"type":"ConnectionError","details":{"error":{"name":"Connection Error"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, failed in hour 21 (different hour) + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 21:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:3, failed in hour 20 (different plugin_config) + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, failed but in a different queue + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'not-webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:2, plugin_config:4, failed in hour 20 (purposeful duplicate) + ( + ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], + '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'failed', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, available + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '{"body": "hello world", "headers": {}, "method": "POST", "url": "https://myhost/endpoint"}', + 'webhooks', + 'available', + 'https://myhost/endpoint' + ), + -- team:1, plugin_config:2, running + ( + NULL, + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '2023-12-19 20:01:18.799371+00', + '{}', + 'webhooks', + 'running', + 'https://myhost/endpoint' ); \ No newline at end of file diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 5ccf6dca701c3..fb48a57ea2421 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -124,6 +124,12 @@ impl From for AppMetric { // that has set the isolation level to serializable. struct SerializableTxn<'a>(Transaction<'a, Postgres>); +struct CleanupStats { + rows_processed: u64, + completed_agg_row_count: usize, + failed_agg_row_count: usize, +} + impl WebhookCleaner { pub fn new( queue_name: &str, @@ -312,7 +318,7 @@ impl WebhookCleaner { Ok(()) } - async fn cleanup_impl(&self) -> Result<()> { + async fn cleanup_impl(&self) -> Result { debug!("WebhookCleaner starting cleanup"); // Note that we select all completed and failed rows without any pagination at the moment. @@ -343,18 +349,17 @@ impl WebhookCleaner { row_count }; + let mut rows_processed = 0; if completed_agg_row_count + failed_agg_row_count != 0 { - let rows_deleted = self.delete_observed_rows(&mut tx).await?; + rows_processed = self.delete_observed_rows(&mut tx).await?; self.commit_txn(tx).await?; - debug!( - "WebhookCleaner finished cleanup, processed and deleted {} rows ({}/{} aggregated completed/failed rows)", - rows_deleted, completed_agg_row_count, failed_agg_row_count - ); - } else { - debug!("WebhookCleaner finished cleanup, there were no rows to process"); } - Ok(()) + Ok(CleanupStats { + rows_processed, + completed_agg_row_count, + failed_agg_row_count, + }) } } @@ -362,7 +367,18 @@ impl WebhookCleaner { impl Cleaner for WebhookCleaner { async fn cleanup(&self) { match self.cleanup_impl().await { - Ok(_) => {} + Ok(stats) => { + if stats.rows_processed > 0 { + debug!( + rows_processed = stats.rows_processed, + completed_agg_row_count = stats.completed_agg_row_count, + failed_agg_row_count = stats.failed_agg_row_count, + "WebhookCleaner::cleanup finished" + ); + } else { + debug!("WebhookCleaner finished cleanup, there were no rows to process"); + } + } Err(error) => { error!(error = ?error, "WebhookCleaner::cleanup failed"); } @@ -375,9 +391,18 @@ mod tests { use super::*; use crate::config; use crate::kafka_producer::{create_kafka_producer, KafkaContext}; + use hook_common::kafka_messages::app_metrics::{ + Error as WebhookError, ErrorDetails, ErrorType, + }; + use hook_common::pgqueue::{NewJob, PgJob, PgQueue, RetryPolicy}; + use hook_common::webhook::{HttpMethod, WebhookJobMetadata, WebhookJobParameters}; + use rdkafka::consumer::{Consumer, StreamConsumer}; use rdkafka::mocking::MockCluster; use rdkafka::producer::{DefaultProducerContext, FutureProducer}; - use sqlx::PgPool; + use rdkafka::{ClientConfig, Message}; + use sqlx::{PgPool, Row}; + use std::collections::HashMap; + use std::str::FromStr; const APP_METRICS_TOPIC: &str = "app_metrics"; @@ -406,6 +431,18 @@ mod tests { ) } + fn check_app_metric_vector_equality(v1: &[AppMetric], v2: &[AppMetric]) { + // Ignores `error_uuid`s. + assert_eq!(v1.len(), v2.len()); + for (item1, item2) in v1.iter().zip(v2) { + let mut item1 = item1.clone(); + item1.error_uuid = None; + let mut item2 = item2.clone(); + item2.error_uuid = None; + assert_eq!(item1, item2); + } + } + #[sqlx::test(migrations = "../migrations", fixtures("webhook_cleanup"))] async fn test_cleanup_impl(db: PgPool) { let (mock_cluster, mock_producer) = create_mock_kafka().await; @@ -413,37 +450,312 @@ mod tests { .create_topic(APP_METRICS_TOPIC, 1, 1) .expect("failed to create mock app_metrics topic"); - let table_name = "job_queue"; - let queue_name = "webhooks"; + let consumer: StreamConsumer = ClientConfig::new() + .set("bootstrap.servers", mock_cluster.bootstrap_servers()) + .set("group.id", "mock") + .set("auto.offset.reset", "earliest") + .create() + .expect("failed to create mock consumer"); + consumer.subscribe(&[APP_METRICS_TOPIC]).unwrap(); let webhook_cleaner = WebhookCleaner::new_from_pool( - &queue_name, - &table_name, + &"webhooks", + &"job_queue", db, mock_producer, APP_METRICS_TOPIC.to_owned(), ) .expect("unable to create webhook cleaner"); - let _ = webhook_cleaner + let cleanup_stats = webhook_cleaner .cleanup_impl() .await .expect("webbook cleanup_impl failed"); - // TODO: I spent a lot of time trying to get the mock Kafka consumer to work, but I think - // I've identified an issue with the rust-rdkafka library: - // https://github.com/fede1024/rust-rdkafka/issues/629#issuecomment-1863555417 - // - // I wanted to test the messages put on the AppMetrics topic, but I think we need to figure - // out that issue about in order to do so. (Capture uses the MockProducer but not a - // Consumer, fwiw.) - // - // For now, I'll probably have to make `cleanup_impl` return the row information so at - // least we can inspect that for correctness. + // Rows from other queues and rows that are not 'completed' or 'failed' should not be + // processed. + assert_eq!(cleanup_stats.rows_processed, 11); + + let mut received_app_metrics = Vec::new(); + for _ in 0..(cleanup_stats.completed_agg_row_count + cleanup_stats.failed_agg_row_count) { + let kafka_msg = consumer.recv().await.unwrap(); + let payload_str = String::from_utf8(kafka_msg.payload().unwrap().to_vec()).unwrap(); + let app_metric: AppMetric = serde_json::from_str(&payload_str).unwrap(); + received_app_metrics.push(app_metric); + } + + let expected_app_metrics = vec![ + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 2, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 2, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 3, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 1, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 2, + plugin_config_id: 4, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 1, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T21:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 2, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 1, + successes_on_retry: 0, + failures: 0, + error_uuid: None, + error_type: None, + error_details: None, + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 2, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: 1, + error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), + error_type: Some(ErrorType::ConnectionError), + error_details: Some(ErrorDetails { + error: WebhookError { + name: "Connection Error".to_owned(), + message: None, + stack: None, + }, + }), + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 2, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: 2, + error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), + error_type: Some(ErrorType::TimeoutError), + error_details: Some(ErrorDetails { + error: WebhookError { + name: "Timeout".to_owned(), + message: None, + stack: None, + }, + }), + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 3, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: 1, + error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), + error_type: Some(ErrorType::TimeoutError), + error_details: Some(ErrorDetails { + error: WebhookError { + name: "Timeout".to_owned(), + message: None, + stack: None, + }, + }), + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T20:00:00Z").unwrap(), + team_id: 2, + plugin_config_id: 4, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: 1, + error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), + error_type: Some(ErrorType::TimeoutError), + error_details: Some(ErrorDetails { + error: WebhookError { + name: "Timeout".to_owned(), + message: None, + stack: None, + }, + }), + }, + AppMetric { + timestamp: DateTime::::from_str("2023-12-19T21:00:00Z").unwrap(), + team_id: 1, + plugin_config_id: 2, + job_id: None, + category: AppMetricCategory::Webhook, + successes: 0, + successes_on_retry: 0, + failures: 1, + error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), + error_type: Some(ErrorType::TimeoutError), + error_details: Some(ErrorDetails { + error: WebhookError { + name: "Timeout".to_owned(), + message: None, + stack: None, + }, + }), + }, + ]; + + check_app_metric_vector_equality(&expected_app_metrics, &received_app_metrics); } - // #[sqlx::test] - // async fn test_serializable_isolation() { - // TODO: I'm going to add a test that verifies new rows aren't visible during the txn. - // } + #[sqlx::test(migrations = "../migrations", fixtures("webhook_cleanup"))] + async fn test_serializable_isolation(db: PgPool) { + let (_, mock_producer) = create_mock_kafka().await; + let webhook_cleaner = WebhookCleaner::new_from_pool( + &"webhooks", + &"job_queue", + db.clone(), + mock_producer, + APP_METRICS_TOPIC.to_owned(), + ) + .expect("unable to create webhook cleaner"); + + let queue = + PgQueue::new_from_pool("webhooks", "job_queue", db.clone(), RetryPolicy::default()) + .await + .expect("failed to connect to local test postgresql database"); + + async fn get_count_from_new_conn(db: &PgPool, status: &str) -> i64 { + let mut conn = db.acquire().await.unwrap(); + let count: i64 = sqlx::query( + "SELECT count(*) FROM job_queue WHERE queue = 'webhooks' AND status = $1::job_status", + ) + .bind(&status) + .fetch_one(&mut *conn) + .await + .unwrap() + .get(0); + count + } + + // Important! Serializable txn is started here. + let mut tx = webhook_cleaner.start_serializable_txn().await.unwrap(); + webhook_cleaner.get_completed_rows(&mut tx).await.unwrap(); + webhook_cleaner.get_failed_rows(&mut tx).await.unwrap(); + + // All 13 rows in the queue are visible from outside the txn. + // The 11 the cleaner will process, plus 1 available and 1 running. + assert_eq!(get_count_from_new_conn(&db, "completed").await, 5); + assert_eq!(get_count_from_new_conn(&db, "failed").await, 6); + assert_eq!(get_count_from_new_conn(&db, "available").await, 1); + assert_eq!(get_count_from_new_conn(&db, "running").await, 1); + + { + // The fixtures include an available job, so let's complete it while the txn is open. + let webhook_job: PgJob = queue + .dequeue(&"worker_id") + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + webhook_job + .complete() + .await + .expect("failed to complete job"); + } + + { + // Enqueue and complete another job while the txn is open. + let job_parameters = WebhookJobParameters { + body: "foo".to_owned(), + headers: HashMap::new(), + method: HttpMethod::POST, + url: "http://example.com".to_owned(), + }; + let job_metadata = WebhookJobMetadata { + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, + }; + let new_job = NewJob::new(1, job_metadata, job_parameters, &"target"); + queue.enqueue(new_job).await.expect("failed to enqueue job"); + let webhook_job: PgJob = queue + .dequeue(&"worker_id") + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + webhook_job + .complete() + .await + .expect("failed to complete job"); + } + + { + // Enqueue another available job while the txn is open. + let job_parameters = WebhookJobParameters { + body: "foo".to_owned(), + headers: HashMap::new(), + method: HttpMethod::POST, + url: "http://example.com".to_owned(), + }; + let job_metadata = WebhookJobMetadata { + team_id: 1, + plugin_id: 2, + plugin_config_id: 3, + }; + let new_job = NewJob::new(1, job_metadata, job_parameters, &"target"); + queue.enqueue(new_job).await.expect("failed to enqueue job"); + } + + // There are now 2 more completed rows (jobs added above) than before, visible from outside the txn. + assert_eq!(get_count_from_new_conn(&db, "completed").await, 7); + assert_eq!(get_count_from_new_conn(&db, "available").await, 1); + + let rows_processed = webhook_cleaner.delete_observed_rows(&mut tx).await.unwrap(); + // The 11 rows that were in the queue when the txn started should be deleted. + assert_eq!(rows_processed, 11); + + // We haven't committed, so the rows are still visible from outside the txn. + assert_eq!(get_count_from_new_conn(&db, "completed").await, 7); + assert_eq!(get_count_from_new_conn(&db, "available").await, 1); + + webhook_cleaner.commit_txn(tx).await.unwrap(); + + // We have committed, what remains are: + // * The 1 available job we completed while the txn was open. + // * The 2 brand new jobs we added while the txn was open. + // * The 1 running job that didn't change. + assert_eq!(get_count_from_new_conn(&db, "completed").await, 2); + assert_eq!(get_count_from_new_conn(&db, "failed").await, 0); + assert_eq!(get_count_from_new_conn(&db, "available").await, 1); + assert_eq!(get_count_from_new_conn(&db, "running").await, 1); + } } From d6035105e36ac604dad7bfa8805ed5608c92be22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 21 Dec 2023 12:10:26 +0100 Subject: [PATCH 144/247] refactor: Only one run method --- hook-consumer/src/consumer.rs | 73 +++++++++++++++++++++-------------- hook-consumer/src/main.rs | 6 +-- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 73bac1cffba87..921a0651bc2ba 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -109,42 +109,55 @@ impl<'p> WebhookConsumer<'p> { } /// Run this consumer to continuously process any jobs that become available. - pub async fn run(&self) -> Result<(), ConsumerError> { + pub async fn run(&self, transactional: bool) -> Result<(), ConsumerError> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); - loop { - let webhook_job = self.wait_for_job().await?; - - // reqwest::Client internally wraps with Arc, so this allocation is cheap. - let client = self.client.clone(); - let permit = semaphore.clone().acquire_owned().await.unwrap(); - - tokio::spawn(async move { - let result = process_webhook_job(client, webhook_job).await; - drop(permit); - result - }); + if transactional { + loop { + let webhook_job = self.wait_for_job_tx().await?; + spawn_webhook_job_processing_task( + self.client.clone(), + semaphore.clone(), + webhook_job, + ) + .await; + } + } else { + loop { + let webhook_job = self.wait_for_job().await?; + spawn_webhook_job_processing_task( + self.client.clone(), + semaphore.clone(), + webhook_job, + ) + .await; + } } } +} - /// Run this consumer to continuously process any jobs that become available in transactional mode. - pub async fn run_tx(&self) -> Result<(), ConsumerError> { - let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); - - loop { - let webhook_job = self.wait_for_job_tx().await?; - - // reqwest::Client internally wraps with Arc, so this allocation is cheap. - let client = self.client.clone(); - let permit = semaphore.clone().acquire_owned().await.unwrap(); +/// Spawn a Tokio task to process a Webhook Job once we successfully acquire a permit. +/// +/// # Arguments +/// +/// * `client`: An HTTP client to execute the webhook job request. +/// * `semaphore`: A semaphore used for rate limiting purposes. This function will panic if this semaphore is closed. +/// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. +async fn spawn_webhook_job_processing_task( + client: reqwest::Client, + semaphore: Arc, + webhook_job: W, +) -> tokio::task::JoinHandle> { + let permit = semaphore + .acquire_owned() + .await + .expect("semaphore has been closed"); - tokio::spawn(async move { - let result = process_webhook_job(client, webhook_job).await; - drop(permit); - result - }); - } - } + tokio::spawn(async move { + let result = process_webhook_job(client, webhook_job).await; + drop(permit); + result + }) } /// Process a webhook job by transitioning it to its appropriate state after its request is sent. diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index 49c2e76a8610f..86b8094a36218 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -31,11 +31,7 @@ async fn main() -> Result<(), ConsumerError> { config.max_concurrent_jobs, ); - if config.transactional { - consumer.run_tx().await?; - } else { - consumer.run().await?; - } + consumer.run(config.transactional).await?; Ok(()) } From fe972c7a65208bea3412f9d0b59d2e467897a3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 8 Dec 2023 16:50:39 +0100 Subject: [PATCH 145/247] feat: Give non-transactional consumer a chance --- hook-consumer/src/consumer.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 921a0651bc2ba..19fa0a831300e 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -51,6 +51,8 @@ pub struct WebhookConsumer<'p> { client: reqwest::Client, /// Maximum number of concurrent jobs being processed. max_concurrent_jobs: usize, + /// Indicates whether we are holding an open transaction while processing or not. + transactional: bool, } impl<'p> WebhookConsumer<'p> { @@ -82,6 +84,19 @@ impl<'p> WebhookConsumer<'p> { } } + /// Wait until a job becomes available in our queue. + async fn wait_for_job_tx<'a>( + &self, + ) -> Result, WebhookConsumerError> { + loop { + if let Some(job) = self.queue.dequeue_tx(&self.name).await? { + return Ok(job); + } else { + task::sleep(self.poll_interval).await; + } + } + } + /// Wait until a job becomes available in our queue. async fn wait_for_job<'a>( &self, From dc97d8c168924ac372d487489d843fbdddaf2e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 14 Dec 2023 12:30:31 +0100 Subject: [PATCH 146/247] refactor: Two clients one for each mode --- hook-consumer/src/consumer.rs | 130 ++++++++++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 6 deletions(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 19fa0a831300e..0127d25a8284c 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -51,8 +51,6 @@ pub struct WebhookConsumer<'p> { client: reqwest::Client, /// Maximum number of concurrent jobs being processed. max_concurrent_jobs: usize, - /// Indicates whether we are holding an open transaction while processing or not. - transactional: bool, } impl<'p> WebhookConsumer<'p> { @@ -85,11 +83,9 @@ impl<'p> WebhookConsumer<'p> { } /// Wait until a job becomes available in our queue. - async fn wait_for_job_tx<'a>( - &self, - ) -> Result, WebhookConsumerError> { + async fn wait_for_job<'a>(&self) -> Result, WebhookConsumerError> { loop { - if let Some(job) = self.queue.dequeue_tx(&self.name).await? { + if let Some(job) = self.queue.dequeue(&self.name).await? { return Ok(job); } else { task::sleep(self.poll_interval).await; @@ -97,6 +93,68 @@ impl<'p> WebhookConsumer<'p> { } } + /// Run this consumer to continuously process any jobs that become available. + pub async fn run(&self) -> Result<(), WebhookConsumerError> { + let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); + + loop { + let webhook_job = self.wait_for_job().await?; + + // reqwest::Client internally wraps with Arc, so this allocation is cheap. + let client = self.client.clone(); + let permit = semaphore.clone().acquire_owned().await.unwrap(); + + tokio::spawn(async move { + let result = process_webhook_job(client, webhook_job).await; + drop(permit); + result + }); + } + } +} + +/// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. +pub struct WebhookTransactionConsumer<'p> { + /// An identifier for this consumer. Used to mark jobs we have consumed. + name: String, + /// The queue we will be dequeuing jobs from. + queue: &'p PgQueue, + /// The interval for polling the queue. + poll_interval: time::Duration, + /// The client used for HTTP requests. + client: reqwest::Client, + /// Maximum number of concurrent jobs being processed. + max_concurrent_jobs: usize, +} + +impl<'p> WebhookTransactionConsumer<'p> { + pub fn new( + name: &str, + queue: &'p PgQueue, + poll_interval: time::Duration, + request_timeout: time::Duration, + max_concurrent_jobs: usize, + ) -> Result { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ); + + let client = reqwest::Client::builder() + .default_headers(headers) + .timeout(request_timeout) + .build()?; + + Ok(Self { + name: name.to_owned(), + queue, + poll_interval, + client, + max_concurrent_jobs, + }) + } + /// Wait until a job becomes available in our queue. async fn wait_for_job<'a>( &self, @@ -259,6 +317,66 @@ async fn process_webhook_job( } } +/// Process a webhook job by transitioning it to its appropriate state after its request is sent. +/// After we finish, the webhook job will be set as completed (if the request was successful), retryable (if the request +/// was unsuccessful but we can still attempt a retry), or failed (if the request was unsuccessful and no more retries +/// may be attempted). +/// +/// A webhook job is considered retryable after a failing request if: +/// 1. The job has attempts remaining (i.e. hasn't reached `max_attempts`), and... +/// 2. The status code indicates retrying at a later point could resolve the issue. This means: 429 and any 5XX. +/// +/// # Arguments +/// +/// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. +/// * `request_timeout`: A timeout for the HTTP request. +async fn process_webhook_job( + client: reqwest::Client, + webhook_job: PgJob, +) -> Result<(), WebhookConsumerError> { + match send_webhook( + client, + &webhook_job.job.parameters.method, + &webhook_job.job.parameters.url, + &webhook_job.job.parameters.headers, + webhook_job.job.parameters.body.clone(), + ) + .await + { + Ok(_) => { + webhook_job + .complete() + .await + .map_err(|error| WebhookConsumerError::PgJobError(error.to_string()))?; + Ok(()) + } + Err(WebhookConsumerError::RetryableWebhookError { + reason, + retry_after, + }) => match webhook_job.retry(reason.to_string(), retry_after).await { + Ok(_) => Ok(()), + Err(PgJobError::RetryInvalidError { + job: webhook_job, + error: fail_error, + }) => { + webhook_job + .fail(fail_error.to_string()) + .await + .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; + Ok(()) + } + Err(job_error) => Err(WebhookConsumerError::PgJobError(job_error.to_string())), + }, + Err(error) => { + webhook_job + .fail(error.to_string()) + .await + .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; + Ok(()) + } + } +} + /// Make an HTTP request to a webhook endpoint. /// /// # Arguments From acd0d1379768fa035baa9fb73fadecca149a6c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 14 Dec 2023 14:49:50 +0100 Subject: [PATCH 147/247] refactor: PgQueue no longer takes a RetryPolicy --- hook-common/src/pgqueue.rs | 97 +++-------------------------------- hook-common/src/retry.rs | 55 ++++++++++++++++++++ hook-consumer/src/consumer.rs | 48 +++++++++++------ hook-consumer/src/main.rs | 13 ++--- 4 files changed, 98 insertions(+), 115 deletions(-) create mode 100644 hook-common/src/retry.rs diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 78fe2fc21c3f7..509d84e35efb2 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -1,8 +1,6 @@ //! # PgQueue //! //! A job queue implementation backed by a PostgreSQL table. - -use std::default::Default; use std::str::FromStr; use std::time; @@ -172,7 +170,6 @@ pub struct PgJob { pub job: Job, pub table: String, pub connection: sqlx::pool::PoolConnection, - pub retry_policy: RetryPolicy, } #[async_trait] @@ -259,9 +256,6 @@ RETURNING }); } let retryable_job = self.job.retry(error); - let retry_interval = self - .retry_policy - .time_until_next_retry(&retryable_job, preferred_retry_interval); let base_query = format!( r#" @@ -304,7 +298,6 @@ pub struct PgTransactionJob<'c, J, M> { pub job: Job, pub table: String, pub transaction: sqlx::Transaction<'c, sqlx::postgres::Postgres>, - pub retry_policy: RetryPolicy, } #[async_trait] @@ -408,9 +401,6 @@ RETURNING }); } let retryable_job = self.job.retry(error); - let retry_interval = self - .retry_policy - .time_until_next_retry(&retryable_job, preferred_retry_interval); let base_query = format!( r#" @@ -509,61 +499,6 @@ impl NewJob { } } -#[derive(Copy, Clone, Debug)] -/// The retry policy that PgQueue will use to determine how to set scheduled_at when enqueuing a retry. -pub struct RetryPolicy { - /// Coefficient to multiply initial_interval with for every past attempt. - backoff_coefficient: u32, - /// The backoff interval for the first retry. - initial_interval: time::Duration, - /// The maximum possible backoff between retries. - maximum_interval: Option, -} - -impl RetryPolicy { - pub fn new( - backoff_coefficient: u32, - initial_interval: time::Duration, - maximum_interval: Option, - ) -> Self { - Self { - backoff_coefficient, - initial_interval, - maximum_interval, - } - } - - /// Calculate the time until the next retry for a given RetryableJob. - pub fn time_until_next_retry( - &self, - job: &RetryableJob, - preferred_retry_interval: Option, - ) -> time::Duration { - let candidate_interval = - self.initial_interval * self.backoff_coefficient.pow(job.attempt as u32); - - match (preferred_retry_interval, self.maximum_interval) { - (Some(duration), Some(max_interval)) => std::cmp::min( - std::cmp::max(std::cmp::min(candidate_interval, max_interval), duration), - max_interval, - ), - (Some(duration), None) => std::cmp::max(candidate_interval, duration), - (None, Some(max_interval)) => std::cmp::min(candidate_interval, max_interval), - (None, None) => candidate_interval, - } - } -} - -impl Default for RetryPolicy { - fn default() -> Self { - Self { - backoff_coefficient: 2, - initial_interval: time::Duration::from_secs(1), - maximum_interval: None, - } - } -} - /// A queue implemented on top of a PostgreSQL table. #[derive(Clone)] pub struct PgQueue { @@ -571,8 +506,6 @@ pub struct PgQueue { name: String, /// A connection pool used to connect to the PostgreSQL database. pool: PgPool, - /// The retry policy to be assigned to Jobs in this PgQueue. - retry_policy: RetryPolicy, /// The identifier of the PostgreSQL table this queue runs on. table: String, } @@ -588,25 +521,14 @@ impl PgQueue { /// * `table_name`: The name for the table the queue will use in PostgreSQL. /// * `url`: A URL pointing to where the PostgreSQL database is hosted. /// * `worker_name`: The name of the worker that is operating with this queue. - /// * `retry_policy`: A retry policy to pass to jobs from this queue. - pub async fn new( - queue_name: &str, - table_name: &str, - url: &str, - retry_policy: RetryPolicy, - ) -> PgQueueResult { + pub async fn new(queue_name: &str, table_name: &str, url: &str) -> PgQueueResult { let name = queue_name.to_owned(); let table = table_name.to_owned(); let pool = PgPoolOptions::new() .connect_lazy(url) .map_err(|error| PgQueueError::PoolCreationError { error })?; - Ok(Self { - name, - pool, - retry_policy, - table, - }) + Ok(Self { name, pool, table }) } /// Dequeue a Job from this PgQueue to work on it. @@ -669,7 +591,6 @@ RETURNING job, table: self.table.to_owned(), connection, - retry_policy: self.retry_policy, })), // Although connection would be closed once it goes out of scope, sqlx recommends explicitly calling close(). @@ -749,7 +670,6 @@ RETURNING job, table: self.table.to_owned(), transaction: tx, - retry_policy: self.retry_policy, })), // Transaction is rolledback on drop. @@ -801,6 +721,7 @@ VALUES #[cfg(test)] mod tests { use super::*; + use crate::retry::RetryPolicy; #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] struct JobMetadata { @@ -858,7 +779,6 @@ mod tests { "test_can_dequeue_job", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), ) .await .expect("failed to connect to local test postgresql database"); @@ -887,7 +807,6 @@ mod tests { "test_dequeue_returns_none_on_no_jobs", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), ) .await .expect("failed to connect to local test postgresql database"); @@ -912,7 +831,6 @@ mod tests { "test_can_dequeue_tx_job", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), ) .await .expect("failed to connect to local test postgresql database"); @@ -942,7 +860,6 @@ mod tests { "test_dequeue_tx_returns_none_on_no_jobs", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), ) .await .expect("failed to connect to local test postgresql database"); @@ -972,7 +889,6 @@ mod tests { "test_can_retry_job_with_remaining_attempts", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - retry_policy, ) .await .expect("failed to connect to local test postgresql database"); @@ -983,8 +899,9 @@ mod tests { .await .expect("failed to dequeue job") .expect("didn't find a job to dequeue"); + let retry_interval = retry_policy.time_until_next_retry(job.job.attempt as u32, None); let _ = job - .retry("a very reasonable failure reason", None) + .retry("a very reasonable failure reason", retry_interval) .await .expect("failed to retry job"); let retried_job: PgJob = queue @@ -1023,7 +940,6 @@ mod tests { "test_cannot_retry_job_without_remaining_attempts", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - retry_policy, ) .await .expect("failed to connect to local test postgresql database"); @@ -1035,7 +951,8 @@ mod tests { .await .expect("failed to dequeue job") .expect("didn't find a job to dequeue"); - job.retry("a very reasonable failure reason", None) + let retry_interval = retry_policy.time_until_next_retry(job.job.attempt as u32, None); + job.retry("a very reasonable failure reason", retry_interval) .await .expect("failed to retry job"); } diff --git a/hook-common/src/retry.rs b/hook-common/src/retry.rs new file mode 100644 index 0000000000000..140da192a8b32 --- /dev/null +++ b/hook-common/src/retry.rs @@ -0,0 +1,55 @@ +use std::time; + +#[derive(Copy, Clone, Debug)] +/// The retry policy that PgQueue will use to determine how to set scheduled_at when enqueuing a retry. +pub struct RetryPolicy { + /// Coefficient to multiply initial_interval with for every past attempt. + backoff_coefficient: u32, + /// The backoff interval for the first retry. + initial_interval: time::Duration, + /// The maximum possible backoff between retries. + maximum_interval: Option, +} + +impl RetryPolicy { + pub fn new( + backoff_coefficient: u32, + initial_interval: time::Duration, + maximum_interval: Option, + ) -> Self { + Self { + backoff_coefficient, + initial_interval, + maximum_interval, + } + } + + /// Calculate the time until the next retry for a given RetryableJob. + pub fn time_until_next_retry( + &self, + attempt: u32, + preferred_retry_interval: Option, + ) -> time::Duration { + let candidate_interval = self.initial_interval * self.backoff_coefficient.pow(attempt); + + match (preferred_retry_interval, self.maximum_interval) { + (Some(duration), Some(max_interval)) => std::cmp::min( + std::cmp::max(std::cmp::min(candidate_interval, max_interval), duration), + max_interval, + ), + (Some(duration), None) => std::cmp::max(candidate_interval, duration), + (None, Some(max_interval)) => std::cmp::min(candidate_interval, max_interval), + (None, None) => candidate_interval, + } + } +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + backoff_coefficient: 2, + initial_interval: time::Duration::from_secs(1), + maximum_interval: None, + } + } +} diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 0127d25a8284c..d6ebac7517e06 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -51,6 +51,8 @@ pub struct WebhookConsumer<'p> { client: reqwest::Client, /// Maximum number of concurrent jobs being processed. max_concurrent_jobs: usize, + /// The retry policy used to calculate retry intervals when a job fails with a retryable error. + retry_policy: RetryPolicy, } impl<'p> WebhookConsumer<'p> { @@ -96,6 +98,7 @@ impl<'p> WebhookConsumer<'p> { /// Run this consumer to continuously process any jobs that become available. pub async fn run(&self) -> Result<(), WebhookConsumerError> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); + let retry_policy = self.retry_policy.clone(); loop { let webhook_job = self.wait_for_job().await?; @@ -105,7 +108,7 @@ impl<'p> WebhookConsumer<'p> { let permit = semaphore.clone().acquire_owned().await.unwrap(); tokio::spawn(async move { - let result = process_webhook_job(client, webhook_job).await; + let result = process_webhook_job(client, webhook_job, &retry_policy).await; drop(permit); result }); @@ -125,6 +128,8 @@ pub struct WebhookTransactionConsumer<'p> { client: reqwest::Client, /// Maximum number of concurrent jobs being processed. max_concurrent_jobs: usize, + /// The retry policy used to calculate retry intervals when a job fails with a retryable error. + retry_policy: RetryPolicy, } impl<'p> WebhookTransactionConsumer<'p> { @@ -134,6 +139,7 @@ impl<'p> WebhookTransactionConsumer<'p> { poll_interval: time::Duration, request_timeout: time::Duration, max_concurrent_jobs: usize, + retry_policy: RetryPolicy, ) -> Result { let mut headers = header::HeaderMap::new(); headers.insert( @@ -152,6 +158,7 @@ impl<'p> WebhookTransactionConsumer<'p> { poll_interval, client, max_concurrent_jobs, + retry_policy, }) } @@ -184,6 +191,7 @@ impl<'p> WebhookTransactionConsumer<'p> { /// Run this consumer to continuously process any jobs that become available. pub async fn run(&self, transactional: bool) -> Result<(), ConsumerError> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); + let retry_policy = self.retry_policy.clone(); if transactional { loop { @@ -333,6 +341,7 @@ async fn process_webhook_job( async fn process_webhook_job( client: reqwest::Client, webhook_job: PgJob, + retry_policy: &RetryPolicy, ) -> Result<(), WebhookConsumerError> { match send_webhook( client, @@ -353,20 +362,27 @@ async fn process_webhook_job( Err(WebhookConsumerError::RetryableWebhookError { reason, retry_after, - }) => match webhook_job.retry(reason.to_string(), retry_after).await { - Ok(_) => Ok(()), - Err(PgJobError::RetryInvalidError { - job: webhook_job, - error: fail_error, - }) => { - webhook_job - .fail(fail_error.to_string()) - .await - .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; - Ok(()) + }) => { + let retry_interval = + retry_policy.time_until_next_retry(webhook_job.job.attempt as u32, retry_after); + + match webhook_job.retry(reason.to_string(), retry_interval).await { + Ok(_) => Ok(()), + Err(PgJobError::RetryInvalidError { + job: webhook_job, + error: fail_error, + }) => { + webhook_job + .fail(fail_error.to_string()) + .await + .map_err(|job_error| { + WebhookConsumerError::PgJobError(job_error.to_string()) + })?; + Ok(()) + } + Err(job_error) => Err(WebhookConsumerError::PgJobError(job_error.to_string())), } - Err(job_error) => Err(WebhookConsumerError::PgJobError(job_error.to_string())), - }, + } Err(error) => { webhook_job .fail(error.to_string()) @@ -479,7 +495,7 @@ mod tests { // This is due to a long-standing cargo bug that reports imports and helper functions as unused. // See: https://github.com/rust-lang/rust/issues/46379. #[allow(unused_imports)] - use hook_common::pgqueue::{JobStatus, NewJob, RetryPolicy}; + use hook_common::pgqueue::{JobStatus, NewJob}; /// Use process id as a worker id for tests. #[allow(dead_code)] @@ -536,7 +552,7 @@ mod tests { let queue_name = "test_wait_for_job".to_string(); let table_name = "job_queue".to_string(); let db_url = "postgres://posthog:posthog@localhost:15432/test_database".to_string(); - let queue = PgQueue::new(&queue_name, &table_name, &db_url, RetryPolicy::default()) + let queue = PgQueue::new(&queue_name, &table_name, &db_url) .await .expect("failed to connect to PG"); diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index 86b8094a36218..dd9d3e7979e14 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -1,6 +1,6 @@ use envconfig::Envconfig; -use hook_common::pgqueue::{PgQueue, RetryPolicy}; +use hook_common::{pgqueue::PgQueue, retry::RetryPolicy}; use hook_consumer::config::Config; use hook_consumer::consumer::WebhookConsumer; use hook_consumer::error::ConsumerError; @@ -14,14 +14,9 @@ async fn main() -> Result<(), ConsumerError> { config.retry_policy.initial_interval.0, Some(config.retry_policy.maximum_interval.0), ); - let queue = PgQueue::new( - &config.queue_name, - &config.table_name, - &config.database_url, - retry_policy, - ) - .await - .expect("failed to initialize queue"); + let queue = PgQueue::new(&config.queue_name, &config.table_name, &config.database_url) + .await + .expect("failed to initialize queue"); let consumer = WebhookConsumer::new( &config.consumer_name, From 0f61af3ccda26f905dcf44c9b30cb35f4965e764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 21 Dec 2023 12:36:40 +0100 Subject: [PATCH 148/247] fix: Rebase on changes and fix all conflicts --- hook-common/src/lib.rs | 1 + hook-common/src/pgqueue.rs | 6 +- hook-common/src/retry.rs | 6 +- hook-consumer/src/consumer.rs | 177 ++++---------------------- hook-consumer/src/main.rs | 1 + hook-producer/src/handlers/app.rs | 3 +- hook-producer/src/handlers/webhook.rs | 7 +- hook-producer/src/main.rs | 4 +- 8 files changed, 36 insertions(+), 169 deletions(-) diff --git a/hook-common/src/lib.rs b/hook-common/src/lib.rs index 7d9ef37e84606..8e63ded5a7bf2 100644 --- a/hook-common/src/lib.rs +++ b/hook-common/src/lib.rs @@ -1,4 +1,5 @@ pub mod kafka_messages; pub mod metrics; pub mod pgqueue; +pub mod retry; pub mod webhook; diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 509d84e35efb2..00ce57f0c35b4 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -160,7 +160,7 @@ pub trait PgQueueJob { async fn retry( mut self, error: E, - preferred_retry_interval: Option, + retry_interval: time::Duration, ) -> Result, PgJobError>>; } @@ -247,7 +247,7 @@ RETURNING async fn retry( mut self, error: E, - preferred_retry_interval: Option, + retry_interval: time::Duration, ) -> Result, PgJobError>>> { if self.job.is_gte_max_attempts() { return Err(PgJobError::RetryInvalidError { @@ -392,7 +392,7 @@ RETURNING async fn retry( mut self, error: E, - preferred_retry_interval: Option, + retry_interval: time::Duration, ) -> Result, PgJobError>>> { if self.job.is_gte_max_attempts() { return Err(PgJobError::RetryInvalidError { diff --git a/hook-common/src/retry.rs b/hook-common/src/retry.rs index 140da192a8b32..f72b0d166fdc8 100644 --- a/hook-common/src/retry.rs +++ b/hook-common/src/retry.rs @@ -4,11 +4,11 @@ use std::time; /// The retry policy that PgQueue will use to determine how to set scheduled_at when enqueuing a retry. pub struct RetryPolicy { /// Coefficient to multiply initial_interval with for every past attempt. - backoff_coefficient: u32, + pub backoff_coefficient: u32, /// The backoff interval for the first retry. - initial_interval: time::Duration, + pub initial_interval: time::Duration, /// The maximum possible backoff between retries. - maximum_interval: Option, + pub maximum_interval: Option, } impl RetryPolicy { diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index d6ebac7517e06..b1e507177a924 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -3,10 +3,11 @@ use std::sync::Arc; use std::time; use async_std::task; -use hook_common::pgqueue::{ - PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob, +use hook_common::{ + pgqueue::{PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, + retry::RetryPolicy, + webhook::{HttpMethod, WebhookJobError, WebhookJobMetadata, WebhookJobParameters}, }; -use hook_common::webhook::{HttpMethod, WebhookJobError, WebhookJobMetadata, WebhookJobParameters}; use http::StatusCode; use reqwest::header; use tokio::sync; @@ -17,6 +18,7 @@ use crate::error::{ConsumerError, WebhookError}; trait WebhookJob: PgQueueJob + std::marker::Send { fn parameters(&self) -> &WebhookJobParameters; fn metadata(&self) -> &WebhookJobMetadata; + fn attempt(&self) -> i32; } impl WebhookJob for PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata> { @@ -27,6 +29,10 @@ impl WebhookJob for PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadat fn metadata(&self) -> &WebhookJobMetadata { &self.job.metadata } + + fn attempt(&self) -> i32 { + self.job.attempt + } } impl WebhookJob for PgJob { @@ -37,6 +43,10 @@ impl WebhookJob for PgJob { fn metadata(&self) -> &WebhookJobMetadata { &self.job.metadata } + + fn attempt(&self) -> i32 { + self.job.attempt + } } /// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. @@ -62,6 +72,7 @@ impl<'p> WebhookConsumer<'p> { poll_interval: time::Duration, request_timeout: time::Duration, max_concurrent_jobs: usize, + retry_policy: RetryPolicy, ) -> Self { let mut headers = header::HeaderMap::new(); headers.insert( @@ -81,85 +92,8 @@ impl<'p> WebhookConsumer<'p> { poll_interval, client, max_concurrent_jobs, - } - } - - /// Wait until a job becomes available in our queue. - async fn wait_for_job<'a>(&self) -> Result, WebhookConsumerError> { - loop { - if let Some(job) = self.queue.dequeue(&self.name).await? { - return Ok(job); - } else { - task::sleep(self.poll_interval).await; - } - } - } - - /// Run this consumer to continuously process any jobs that become available. - pub async fn run(&self) -> Result<(), WebhookConsumerError> { - let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); - let retry_policy = self.retry_policy.clone(); - - loop { - let webhook_job = self.wait_for_job().await?; - - // reqwest::Client internally wraps with Arc, so this allocation is cheap. - let client = self.client.clone(); - let permit = semaphore.clone().acquire_owned().await.unwrap(); - - tokio::spawn(async move { - let result = process_webhook_job(client, webhook_job, &retry_policy).await; - drop(permit); - result - }); - } - } -} - -/// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. -pub struct WebhookTransactionConsumer<'p> { - /// An identifier for this consumer. Used to mark jobs we have consumed. - name: String, - /// The queue we will be dequeuing jobs from. - queue: &'p PgQueue, - /// The interval for polling the queue. - poll_interval: time::Duration, - /// The client used for HTTP requests. - client: reqwest::Client, - /// Maximum number of concurrent jobs being processed. - max_concurrent_jobs: usize, - /// The retry policy used to calculate retry intervals when a job fails with a retryable error. - retry_policy: RetryPolicy, -} - -impl<'p> WebhookTransactionConsumer<'p> { - pub fn new( - name: &str, - queue: &'p PgQueue, - poll_interval: time::Duration, - request_timeout: time::Duration, - max_concurrent_jobs: usize, - retry_policy: RetryPolicy, - ) -> Result { - let mut headers = header::HeaderMap::new(); - headers.insert( - header::CONTENT_TYPE, - header::HeaderValue::from_static("application/json"), - ); - - let client = reqwest::Client::builder() - .default_headers(headers) - .timeout(request_timeout) - .build()?; - - Ok(Self { - name: name.to_owned(), - queue, - poll_interval, - client, - max_concurrent_jobs, retry_policy, - }) + } } /// Wait until a job becomes available in our queue. @@ -191,7 +125,6 @@ impl<'p> WebhookTransactionConsumer<'p> { /// Run this consumer to continuously process any jobs that become available. pub async fn run(&self, transactional: bool) -> Result<(), ConsumerError> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); - let retry_policy = self.retry_policy.clone(); if transactional { loop { @@ -199,6 +132,7 @@ impl<'p> WebhookTransactionConsumer<'p> { spawn_webhook_job_processing_task( self.client.clone(), semaphore.clone(), + self.retry_policy, webhook_job, ) .await; @@ -209,6 +143,7 @@ impl<'p> WebhookTransactionConsumer<'p> { spawn_webhook_job_processing_task( self.client.clone(), semaphore.clone(), + self.retry_policy, webhook_job, ) .await; @@ -227,6 +162,7 @@ impl<'p> WebhookTransactionConsumer<'p> { async fn spawn_webhook_job_processing_task( client: reqwest::Client, semaphore: Arc, + retry_policy: RetryPolicy, webhook_job: W, ) -> tokio::task::JoinHandle> { let permit = semaphore @@ -235,7 +171,7 @@ async fn spawn_webhook_job_processing_task( .expect("semaphore has been closed"); tokio::spawn(async move { - let result = process_webhook_job(client, webhook_job).await; + let result = process_webhook_job(client, webhook_job, &retry_policy).await; drop(permit); result }) @@ -257,6 +193,7 @@ async fn spawn_webhook_job_processing_task( async fn process_webhook_job( client: reqwest::Client, webhook_job: W, + retry_policy: &RetryPolicy, ) -> Result<(), ConsumerError> { let parameters = webhook_job.parameters(); @@ -298,8 +235,11 @@ async fn process_webhook_job( Ok(()) } Err(WebhookError::RetryableRequestError { error, retry_after }) => { + let retry_interval = + retry_policy.time_until_next_retry(webhook_job.attempt() as u32, retry_after); + match webhook_job - .retry(WebhookJobError::from(&error), retry_after) + .retry(WebhookJobError::from(&error), retry_interval) .await { Ok(_) => Ok(()), @@ -325,74 +265,6 @@ async fn process_webhook_job( } } -/// Process a webhook job by transitioning it to its appropriate state after its request is sent. -/// After we finish, the webhook job will be set as completed (if the request was successful), retryable (if the request -/// was unsuccessful but we can still attempt a retry), or failed (if the request was unsuccessful and no more retries -/// may be attempted). -/// -/// A webhook job is considered retryable after a failing request if: -/// 1. The job has attempts remaining (i.e. hasn't reached `max_attempts`), and... -/// 2. The status code indicates retrying at a later point could resolve the issue. This means: 429 and any 5XX. -/// -/// # Arguments -/// -/// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. -/// * `request_timeout`: A timeout for the HTTP request. -async fn process_webhook_job( - client: reqwest::Client, - webhook_job: PgJob, - retry_policy: &RetryPolicy, -) -> Result<(), WebhookConsumerError> { - match send_webhook( - client, - &webhook_job.job.parameters.method, - &webhook_job.job.parameters.url, - &webhook_job.job.parameters.headers, - webhook_job.job.parameters.body.clone(), - ) - .await - { - Ok(_) => { - webhook_job - .complete() - .await - .map_err(|error| WebhookConsumerError::PgJobError(error.to_string()))?; - Ok(()) - } - Err(WebhookConsumerError::RetryableWebhookError { - reason, - retry_after, - }) => { - let retry_interval = - retry_policy.time_until_next_retry(webhook_job.job.attempt as u32, retry_after); - - match webhook_job.retry(reason.to_string(), retry_interval).await { - Ok(_) => Ok(()), - Err(PgJobError::RetryInvalidError { - job: webhook_job, - error: fail_error, - }) => { - webhook_job - .fail(fail_error.to_string()) - .await - .map_err(|job_error| { - WebhookConsumerError::PgJobError(job_error.to_string()) - })?; - Ok(()) - } - Err(job_error) => Err(WebhookConsumerError::PgJobError(job_error.to_string())), - } - } - Err(error) => { - webhook_job - .fail(error.to_string()) - .await - .map_err(|job_error| WebhookConsumerError::PgJobError(job_error.to_string()))?; - Ok(()) - } - } -} - /// Make an HTTP request to a webhook endpoint. /// /// # Arguments @@ -585,6 +457,7 @@ mod tests { time::Duration::from_millis(100), time::Duration::from_millis(5000), 10, + RetryPolicy::default(), ); let consumed_job = consumer diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index dd9d3e7979e14..3cefc1d04f42d 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -24,6 +24,7 @@ async fn main() -> Result<(), ConsumerError> { config.poll_interval.0, config.request_timeout.0, config.max_concurrent_jobs, + retry_policy, ); consumer.run(config.transactional).await?; diff --git a/hook-producer/src/handlers/app.rs b/hook-producer/src/handlers/app.rs index 1666676fee1bb..78d4dc5663fc7 100644 --- a/hook-producer/src/handlers/app.rs +++ b/hook-producer/src/handlers/app.rs @@ -31,7 +31,7 @@ mod tests { body::Body, http::{Request, StatusCode}, }; - use hook_common::pgqueue::{PgQueue, RetryPolicy}; + use hook_common::pgqueue::PgQueue; use http_body_util::BodyExt; // for `collect` use tower::ServiceExt; // for `call`, `oneshot`, and `ready` @@ -41,7 +41,6 @@ mod tests { "test_index", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), ) .await .expect("failed to construct pg_queue"); diff --git a/hook-producer/src/handlers/webhook.rs b/hook-producer/src/handlers/webhook.rs index 394732094a3c6..ca0cc3792bef5 100644 --- a/hook-producer/src/handlers/webhook.rs +++ b/hook-producer/src/handlers/webhook.rs @@ -108,7 +108,7 @@ mod tests { body::Body, http::{self, Request, StatusCode}, }; - use hook_common::pgqueue::{PgQueue, RetryPolicy}; + use hook_common::pgqueue::PgQueue; use hook_common::webhook::{HttpMethod, WebhookJobParameters}; use http_body_util::BodyExt; // for `collect` use std::collections; @@ -122,7 +122,6 @@ mod tests { "test_index", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), ) .await .expect("failed to construct pg_queue"); @@ -171,7 +170,6 @@ mod tests { "test_index", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), ) .await .expect("failed to construct pg_queue"); @@ -215,7 +213,6 @@ mod tests { "test_index", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), ) .await .expect("failed to construct pg_queue"); @@ -243,7 +240,6 @@ mod tests { "test_index", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), ) .await .expect("failed to construct pg_queue"); @@ -271,7 +267,6 @@ mod tests { "test_index", "job_queue", "postgres://posthog:posthog@localhost:15432/test_database", - RetryPolicy::default(), ) .await .expect("failed to construct pg_queue"); diff --git a/hook-producer/src/main.rs b/hook-producer/src/main.rs index 7c2b73c0f12bb..39f45004271bc 100644 --- a/hook-producer/src/main.rs +++ b/hook-producer/src/main.rs @@ -4,7 +4,7 @@ use envconfig::Envconfig; use eyre::Result; use hook_common::metrics; -use hook_common::pgqueue::{PgQueue, RetryPolicy}; +use hook_common::pgqueue::PgQueue; mod config; mod handlers; @@ -29,8 +29,6 @@ async fn main() { &config.queue_name, &config.table_name, &config.database_url, - // TODO: It seems unnecessary that the producer side needs to know about the retry policy. - RetryPolicy::default(), ) .await .expect("failed to initialize queue"); From 9b7ab99d19327b274adf6c9b2bfb877c8eb0a59d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 21 Dec 2023 15:15:59 +0100 Subject: [PATCH 149/247] feat: Provide a function to start serving metrics --- Cargo.lock | 1 + hook-common/Cargo.toml | 1 + hook-common/src/metrics.rs | 16 ++++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 130f765cff81d..ac2c8fc525e44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1035,6 +1035,7 @@ dependencies = [ "futures", "hook-common", "http 0.2.11", + "metrics", "reqwest", "serde", "serde_derive", diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index 9b20396a388c9..00c7bd2296a49 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -18,6 +18,7 @@ serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } +tokio = { workspace = true } thiserror = { workspace = true } uuid = { workspace = true } diff --git a/hook-common/src/metrics.rs b/hook-common/src/metrics.rs index dbdc7b1fa1107..7d881ea06bc40 100644 --- a/hook-common/src/metrics.rs +++ b/hook-common/src/metrics.rs @@ -2,9 +2,25 @@ use std::time::Instant; use axum::{ body::Body, extract::MatchedPath, http::Request, middleware::Next, response::IntoResponse, + routing::get, Router, }; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; +/// Bind a TcpListener on the provided bind address to serve metrics on it. +pub async fn serve_metrics(bind: &str) -> Result<(), std::io::Error> { + let recorder_handle = setup_metrics_recorder(); + + let router = Router::new() + .route("/metrics", get(recorder_handle.render())) + .layer(axum::middleware::from_fn(track_metrics)); + + let listener = tokio::net::TcpListener::bind(bind).await?; + + axum::serve(listener, router).await?; + + Ok(()) +} + pub fn setup_metrics_recorder() -> PrometheusHandle { const EXPONENTIAL_SECONDS: &[f64] = &[ 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, From ec3728606fa18ff0c6b8db5d8be1660c740d2779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 21 Dec 2023 15:16:12 +0100 Subject: [PATCH 150/247] feat: Serve basic count metrics on consumer --- hook-consumer/Cargo.toml | 1 + hook-consumer/src/config.rs | 13 +++++++++ hook-consumer/src/consumer.rs | 54 ++++++++++++++++++++++++++++++++++- hook-consumer/src/main.rs | 9 +++++- 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/hook-consumer/Cargo.toml b/hook-consumer/Cargo.toml index 35c64b5e7c7a4..2733a59b91109 100644 --- a/hook-consumer/Cargo.toml +++ b/hook-consumer/Cargo.toml @@ -10,6 +10,7 @@ envconfig = { workspace = true } futures = "0.3" hook-common = { path = "../hook-common" } http = { version = "0.2" } +metrics = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } diff --git a/hook-consumer/src/config.rs b/hook-consumer/src/config.rs index 36c120ad421cc..91695128f2192 100644 --- a/hook-consumer/src/config.rs +++ b/hook-consumer/src/config.rs @@ -5,6 +5,12 @@ use envconfig::Envconfig; #[derive(Envconfig, Clone)] pub struct Config { + #[envconfig(from = "BIND_HOST", default = "0.0.0.0")] + pub host: String, + + #[envconfig(from = "BIND_PORT", default = "8001")] + pub port: u16, + #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] pub database_url: String, @@ -33,6 +39,13 @@ pub struct Config { pub table_name: String, } +impl Config { + /// Produce a host:port address for binding a TcpListener. + pub fn bind(&self) -> String { + format!("{}:{}", self.host, self.port) + } +} + #[derive(Debug, Clone, Copy)] pub struct EnvMsDuration(pub time::Duration); diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index b1e507177a924..065555e15f61e 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -19,6 +19,8 @@ trait WebhookJob: PgQueueJob + std::marker::Send { fn parameters(&self) -> &WebhookJobParameters; fn metadata(&self) -> &WebhookJobMetadata; fn attempt(&self) -> i32; + fn queue(&self) -> String; + fn target(&self) -> String; } impl WebhookJob for PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata> { @@ -33,6 +35,14 @@ impl WebhookJob for PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadat fn attempt(&self) -> i32 { self.job.attempt } + + fn queue(&self) -> String { + self.job.queue.to_owned() + } + + fn target(&self) -> String { + self.job.target.to_owned() + } } impl WebhookJob for PgJob { @@ -47,6 +57,14 @@ impl WebhookJob for PgJob { fn attempt(&self) -> i32 { self.job.attempt } + + fn queue(&self) -> String { + self.job.queue.to_owned() + } + + fn target(&self) -> String { + self.job.target.to_owned() + } } /// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. @@ -170,6 +188,13 @@ async fn spawn_webhook_job_processing_task( .await .expect("semaphore has been closed"); + let labels = [ + ("queue", webhook_job.queue()), + ("target", webhook_job.target()), + ]; + + metrics::increment_counter!("webhook_jobs_total", &labels); + tokio::spawn(async move { let result = process_webhook_job(client, webhook_job, &retry_policy).await; drop(permit); @@ -197,6 +222,11 @@ async fn process_webhook_job( ) -> Result<(), ConsumerError> { let parameters = webhook_job.parameters(); + let labels = [ + ("queue", webhook_job.queue()), + ("target", webhook_job.target()), + ]; + match send_webhook( client, ¶meters.method, @@ -211,6 +241,9 @@ async fn process_webhook_job( .complete() .await .map_err(|error| ConsumerError::PgJobError(error.to_string()))?; + + metrics::increment_counter!("webhook_jobs_completed", &labels); + Ok(()) } Err(WebhookError::ParseHeadersError(e)) => { @@ -218,6 +251,9 @@ async fn process_webhook_job( .fail(WebhookJobError::new_parse(&e.to_string())) .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + + metrics::increment_counter!("webhook_jobs_failed", &labels); + Ok(()) } Err(WebhookError::ParseHttpMethodError(e)) => { @@ -225,6 +261,9 @@ async fn process_webhook_job( .fail(WebhookJobError::new_parse(&e)) .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + + metrics::increment_counter!("webhook_jobs_failed", &labels); + Ok(()) } Err(WebhookError::ParseUrlError(e)) => { @@ -232,6 +271,9 @@ async fn process_webhook_job( .fail(WebhookJobError::new_parse(&e.to_string())) .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + + metrics::increment_counter!("webhook_jobs_failed", &labels); + Ok(()) } Err(WebhookError::RetryableRequestError { error, retry_after }) => { @@ -242,7 +284,11 @@ async fn process_webhook_job( .retry(WebhookJobError::from(&error), retry_interval) .await { - Ok(_) => Ok(()), + Ok(_) => { + metrics::increment_counter!("webhook_jobs_retried", &labels); + + Ok(()) + } Err(PgJobError::RetryInvalidError { job: webhook_job, .. }) => { @@ -250,6 +296,9 @@ async fn process_webhook_job( .fail(WebhookJobError::from(&error)) .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + + metrics::increment_counter!("webhook_jobs_failed", &labels); + Ok(()) } Err(job_error) => Err(ConsumerError::PgJobError(job_error.to_string())), @@ -260,6 +309,9 @@ async fn process_webhook_job( .fail(WebhookJobError::from(&error)) .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + + metrics::increment_counter!("webhook_jobs_failed", &labels); + Ok(()) } } diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index 3cefc1d04f42d..b3ed22ce36ff3 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -1,6 +1,6 @@ use envconfig::Envconfig; -use hook_common::{pgqueue::PgQueue, retry::RetryPolicy}; +use hook_common::{metrics::serve_metrics, pgqueue::PgQueue, retry::RetryPolicy}; use hook_consumer::config::Config; use hook_consumer::consumer::WebhookConsumer; use hook_consumer::error::ConsumerError; @@ -27,6 +27,13 @@ async fn main() -> Result<(), ConsumerError> { retry_policy, ); + let bind = config.bind(); + tokio::task::spawn(async move { + serve_metrics(&bind) + .await + .expect("failed to start serving metrics"); + }); + consumer.run(config.transactional).await?; Ok(()) From 0a014cb2da24f98b649c11d053bd79b840070791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 21 Dec 2023 15:20:30 +0100 Subject: [PATCH 151/247] refactor: Use tokio::time::interval instead of async_std::task::sleep --- Cargo.lock | 381 +--------------------------------- hook-consumer/Cargo.toml | 1 - hook-consumer/src/consumer.rs | 13 +- 3 files changed, 15 insertions(+), 380 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac2c8fc525e44..7cc66455c70db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,150 +60,6 @@ dependencies = [ "libc", ] -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - -[[package]] -name = "async-channel" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" -dependencies = [ - "concurrent-queue", - "event-listener 4.0.0", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-executor" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" -dependencies = [ - "async-lock 3.2.0", - "async-task", - "concurrent-queue", - "fastrand 2.0.1", - "futures-lite 2.1.0", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" -dependencies = [ - "async-channel 2.1.1", - "async-executor", - "async-io 2.2.2", - "async-lock 3.2.0", - "blocking", - "futures-lite 2.1.0", - "once_cell", -] - -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock 2.8.0", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite 1.13.0", - "log", - "parking", - "polling 2.8.0", - "rustix 0.37.27", - "slab", - "socket2 0.4.10", - "waker-fn", -] - -[[package]] -name = "async-io" -version = "2.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afaa937395a620e33dc6a742c593c01aced20aa376ffb0f628121198578ccc7" -dependencies = [ - "async-lock 3.2.0", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite 2.1.0", - "parking", - "polling 3.3.1", - "rustix 0.38.28", - "slab", - "tracing", - "windows-sys 0.52.0", -] - -[[package]] -name = "async-lock" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" -dependencies = [ - "event-listener 2.5.3", -] - -[[package]] -name = "async-lock" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" -dependencies = [ - "event-listener 4.0.0", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-std" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" -dependencies = [ - "async-channel 1.9.0", - "async-global-executor", - "async-io 1.13.0", - "async-lock 2.8.0", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite 1.13.0", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - -[[package]] -name = "async-task" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d90cd0b264dfdd8eb5bad0a2c217c1f88fa96a8573f40e7b12de23fb468f46" - [[package]] name = "async-trait" version = "0.1.74" @@ -230,12 +86,6 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "atomic-write-file" version = "0.1.2" @@ -356,22 +206,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "blocking" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" -dependencies = [ - "async-channel 2.1.1", - "async-lock 3.2.0", - "async-task", - "fastrand 2.0.1", - "futures-io", - "futures-lite 2.1.0", - "piper", - "tracing", -] - [[package]] name = "bumpalo" version = "3.14.0" @@ -428,15 +262,6 @@ dependencies = [ "cc", ] -[[package]] -name = "concurrent-queue" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "const-oid" version = "0.9.6" @@ -624,27 +449,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "event-listener" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener 4.0.0", - "pin-project-lite", -] - [[package]] name = "eyre" version = "0.6.11" @@ -655,15 +459,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.0.1" @@ -776,34 +571,6 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - -[[package]] -name = "futures-lite" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143" -dependencies = [ - "fastrand 2.0.1", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - [[package]] name = "futures-macro" version = "0.3.29" @@ -872,18 +639,6 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" -[[package]] -name = "gloo-timers" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "h2" version = "0.3.22" @@ -1029,7 +784,6 @@ dependencies = [ name = "hook-consumer" version = "0.1.0" dependencies = [ - "async-std", "chrono", "envconfig", "futures", @@ -1178,7 +932,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.5", + "socket2", "tokio", "tower-service", "tracing", @@ -1230,7 +984,7 @@ dependencies = [ "http-body 1.0.0", "hyper 1.1.0", "pin-project-lite", - "socket2 0.5.5", + "socket2", "tokio", "tower", "tower-service", @@ -1296,26 +1050,6 @@ dependencies = [ "hashbrown 0.14.3", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "ipnet" version = "2.9.0" @@ -1346,15 +1080,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -1399,12 +1124,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - [[package]] name = "linux-raw-sys" version = "0.4.12" @@ -1426,9 +1145,6 @@ name = "log" version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -dependencies = [ - "value-bag", -] [[package]] name = "mach2" @@ -1750,12 +1466,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parking" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" - [[package]] name = "parking_lot" version = "0.12.1" @@ -1832,17 +1542,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "piper" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" -dependencies = [ - "atomic-waker", - "fastrand 2.0.1", - "futures-io", -] - [[package]] name = "pkcs1" version = "0.7.5" @@ -1870,36 +1569,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags 1.3.2", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", -] - -[[package]] -name = "polling" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf63fa624ab313c11656b4cda960bfc46c410187ad493c41f6ba2d8c1e991c9e" -dependencies = [ - "cfg-if", - "concurrent-queue", - "pin-project-lite", - "rustix 0.38.28", - "tracing", - "windows-sys 0.52.0", -] - [[package]] name = "portable-atomic" version = "1.6.0" @@ -2129,20 +1798,6 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" -[[package]] -name = "rustix" -version = "0.37.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", -] - [[package]] name = "rustix" version = "0.38.28" @@ -2152,7 +1807,7 @@ dependencies = [ "bitflags 2.4.1", "errno", "libc", - "linux-raw-sys 0.4.12", + "linux-raw-sys", "windows-sys 0.52.0", ] @@ -2330,16 +1985,6 @@ version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "socket2" version = "0.5.5" @@ -2414,7 +2059,7 @@ dependencies = [ "crossbeam-queue", "dotenvy", "either", - "event-listener 2.5.3", + "event-listener", "futures-channel", "futures-core", "futures-intrusive", @@ -2665,9 +2310,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", - "fastrand 2.0.1", + "fastrand", "redox_syscall", - "rustix 0.38.28", + "rustix", "windows-sys 0.48.0", ] @@ -2730,7 +2375,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.5", + "socket2", "tokio-macros", "windows-sys 0.48.0", ] @@ -2962,12 +2607,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "value-bag" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" - [[package]] name = "vcpkg" version = "0.2.15" @@ -2980,12 +2619,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "waker-fn" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" - [[package]] name = "want" version = "0.3.1" diff --git a/hook-consumer/Cargo.toml b/hook-consumer/Cargo.toml index 2733a59b91109..fc8ee4a797fac 100644 --- a/hook-consumer/Cargo.toml +++ b/hook-consumer/Cargo.toml @@ -4,7 +4,6 @@ version = "0.1.0" edition = "2021" [dependencies] -async-std = { version = "1.12" } chrono = { workspace = true } envconfig = { workspace = true } futures = "0.3" diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 065555e15f61e..de857c55f777f 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -2,7 +2,6 @@ use std::collections; use std::sync::Arc; use std::time; -use async_std::task; use hook_common::{ pgqueue::{PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, retry::RetryPolicy, @@ -118,11 +117,13 @@ impl<'p> WebhookConsumer<'p> { async fn wait_for_job<'a>( &self, ) -> Result, ConsumerError> { + let mut interval = tokio::time::interval(self.poll_interval); + loop { + interval.tick().await; + if let Some(job) = self.queue.dequeue(&self.name).await? { return Ok(job); - } else { - task::sleep(self.poll_interval).await; } } } @@ -131,11 +132,13 @@ impl<'p> WebhookConsumer<'p> { async fn wait_for_job_tx<'a>( &self, ) -> Result, ConsumerError> { + let mut interval = tokio::time::interval(self.poll_interval); + loop { + interval.tick().await; + if let Some(job) = self.queue.dequeue_tx(&self.name).await? { return Ok(job); - } else { - task::sleep(self.poll_interval).await; } } } From f3e1252aed2a91f23a642bf38e466f4f3e123ceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 21 Dec 2023 15:43:42 +0100 Subject: [PATCH 152/247] feat: Track webhook job processing --- hook-common/src/metrics.rs | 5 +--- hook-consumer/src/consumer.rs | 49 +++++++++++++++++------------------ 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/hook-common/src/metrics.rs b/hook-common/src/metrics.rs index 7d881ea06bc40..f83edb5b062c9 100644 --- a/hook-common/src/metrics.rs +++ b/hook-common/src/metrics.rs @@ -27,10 +27,7 @@ pub fn setup_metrics_recorder() -> PrometheusHandle { ]; PrometheusBuilder::new() - .set_buckets_for_metric( - Matcher::Full("http_requests_duration_seconds".to_string()), - EXPONENTIAL_SECONDS, - ) + .set_buckets(EXPONENTIAL_SECONDS) .unwrap() .install_recorder() .unwrap() diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index de857c55f777f..8e67949466c9f 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::time; use hook_common::{ - pgqueue::{PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, + pgqueue::{Job, PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, retry::RetryPolicy, webhook::{HttpMethod, WebhookJobError, WebhookJobMetadata, WebhookJobParameters}, }; @@ -13,38 +13,26 @@ use tokio::sync; use crate::error::{ConsumerError, WebhookError}; -/// A WebhookJob is any PgQueueJob that returns a reference to webhook parameters and metadata. +/// A WebhookJob is any PgQueueJob with WebhookJobParameters and WebhookJobMetadata. trait WebhookJob: PgQueueJob + std::marker::Send { fn parameters(&self) -> &WebhookJobParameters; fn metadata(&self) -> &WebhookJobMetadata; - fn attempt(&self) -> i32; - fn queue(&self) -> String; - fn target(&self) -> String; -} - -impl WebhookJob for PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata> { - fn parameters(&self) -> &WebhookJobParameters { - &self.job.parameters - } - - fn metadata(&self) -> &WebhookJobMetadata { - &self.job.metadata - } + fn job(&self) -> &Job; fn attempt(&self) -> i32 { - self.job.attempt + self.job().attempt } fn queue(&self) -> String { - self.job.queue.to_owned() + self.job().queue.to_owned() } fn target(&self) -> String { - self.job.target.to_owned() + self.job().target.to_owned() } } -impl WebhookJob for PgJob { +impl WebhookJob for PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadata> { fn parameters(&self) -> &WebhookJobParameters { &self.job.parameters } @@ -53,16 +41,22 @@ impl WebhookJob for PgJob { &self.job.metadata } - fn attempt(&self) -> i32 { - self.job.attempt + fn job(&self) -> &Job { + &self.job } +} - fn queue(&self) -> String { - self.job.queue.to_owned() +impl WebhookJob for PgJob { + fn parameters(&self) -> &WebhookJobParameters { + &self.job.parameters } - fn target(&self) -> String { - self.job.target.to_owned() + fn metadata(&self) -> &WebhookJobMetadata { + &self.job.metadata + } + + fn job(&self) -> &Job { + &self.job } } @@ -199,8 +193,13 @@ async fn spawn_webhook_job_processing_task( metrics::increment_counter!("webhook_jobs_total", &labels); tokio::spawn(async move { + let now = tokio::time::Instant::now(); let result = process_webhook_job(client, webhook_job, &retry_policy).await; drop(permit); + + let elapsed = now.elapsed().as_secs_f64(); + metrics::histogram!("webhook_jobs_processing_duration_seconds", elapsed, &labels); + result }) } From 0e54ceaa639c6ae5103a818d89caa953e9b15003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 21 Dec 2023 15:53:58 +0100 Subject: [PATCH 153/247] fix: Remove unused imports --- hook-common/src/metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook-common/src/metrics.rs b/hook-common/src/metrics.rs index f83edb5b062c9..4411fead9e5a7 100644 --- a/hook-common/src/metrics.rs +++ b/hook-common/src/metrics.rs @@ -4,7 +4,7 @@ use axum::{ body::Body, extract::MatchedPath, http::Request, middleware::Next, response::IntoResponse, routing::get, Router, }; -use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; +use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; /// Bind a TcpListener on the provided bind address to serve metrics on it. pub async fn serve_metrics(bind: &str) -> Result<(), std::io::Error> { From 1f8088bd7c7d101f3e99775f0351bf9baf866dfb Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 21 Dec 2023 09:58:27 -0700 Subject: [PATCH 154/247] Update hook-common/src/kafka_messages/app_metrics.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tomás Farías Santana --- hook-common/src/kafka_messages/app_metrics.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs index 5aff62cd9fedd..145d4afee020a 100644 --- a/hook-common/src/kafka_messages/app_metrics.rs +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -128,8 +128,8 @@ where let error_type = match error_type { ErrorType::ConnectionError => "Connection Error".to_owned(), - ErrorType::TimeoutError => "Timeout".to_owned(), - ErrorType::BadHttpStatus(s) => format!("HTTP Status: {}", s), + ErrorType::TimeoutError => "Timeout Error".to_owned(), + ErrorType::BadHttpStatus(s) => format!("Bad HTTP Status: {}", s), ErrorType::ParseError => "Parse Error".to_owned(), }; serializer.serialize_str(&error_type) From 2220b6bf2a331765b852f7ae4ff78d031a7f33bb Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 21 Dec 2023 09:59:21 -0700 Subject: [PATCH 155/247] Update hook-common/src/kafka_messages/app_metrics.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tomás Farías Santana --- hook-common/src/kafka_messages/app_metrics.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs index 145d4afee020a..94ec24ee5bc43 100644 --- a/hook-common/src/kafka_messages/app_metrics.rs +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -144,9 +144,9 @@ where Some(s) => { let error_type = match &s[..] { "Connection Error" => ErrorType::ConnectionError, - "Timeout" => ErrorType::TimeoutError, - _ if s.starts_with("HTTP Status:") => { - let status = &s["HTTP Status:".len()..]; + "Timeout Error" => ErrorType::TimeoutError, + _ if s.starts_with("Bad HTTP Status:") => { + let status = &s["Bad HTTP Status:".len()..]; ErrorType::BadHttpStatus(status.parse().map_err(serde::de::Error::custom)?) } "Parse Error" => ErrorType::ParseError, From 50884295fa660f78510c42493ce88bd61b669ba4 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 21 Dec 2023 09:59:30 -0700 Subject: [PATCH 156/247] Update hook-common/src/kafka_messages/app_metrics.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tomás Farías Santana --- hook-common/src/kafka_messages/app_metrics.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs index 94ec24ee5bc43..9acc4112e1113 100644 --- a/hook-common/src/kafka_messages/app_metrics.rs +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -155,8 +155,8 @@ where &s, &[ "Connection Error", - "Timeout", - "HTTP Status: ", + "Timeout Error", + "Bad HTTP Status: ", "Parse Error", ], )) From db076bf7a61fa0f0d5548ea1d37581fbaa072e5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 22 Dec 2023 11:43:21 +0100 Subject: [PATCH 157/247] refactor: Split-up router creation from metrics serving to allow callers to add more routes --- hook-common/src/metrics.rs | 20 ++++++++++++-------- hook-consumer/src/main.rs | 7 +++++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/hook-common/src/metrics.rs b/hook-common/src/metrics.rs index 4411fead9e5a7..3d9e4c02655a5 100644 --- a/hook-common/src/metrics.rs +++ b/hook-common/src/metrics.rs @@ -6,14 +6,9 @@ use axum::{ }; use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; -/// Bind a TcpListener on the provided bind address to serve metrics on it. -pub async fn serve_metrics(bind: &str) -> Result<(), std::io::Error> { - let recorder_handle = setup_metrics_recorder(); - - let router = Router::new() - .route("/metrics", get(recorder_handle.render())) - .layer(axum::middleware::from_fn(track_metrics)); - +/// Bind a `TcpListener` on the provided bind address to serve a `Router` on it. +/// This function is intended to take a Router as returned by `setup_metrics_router`, potentially with more routes added by the caller. +pub async fn serve(router: Router, bind: &str) -> Result<(), std::io::Error> { let listener = tokio::net::TcpListener::bind(bind).await?; axum::serve(listener, router).await?; @@ -21,6 +16,15 @@ pub async fn serve_metrics(bind: &str) -> Result<(), std::io::Error> { Ok(()) } +/// Build a Router for a metrics endpoint. +pub fn setup_metrics_router() -> Router { + let recorder_handle = setup_metrics_recorder(); + + Router::new() + .route("/metrics", get(recorder_handle.render())) + .layer(axum::middleware::from_fn(track_metrics)) +} + pub fn setup_metrics_recorder() -> PrometheusHandle { const EXPONENTIAL_SECONDS: &[f64] = &[ 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index b3ed22ce36ff3..38c2ee23042de 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -1,6 +1,8 @@ use envconfig::Envconfig; -use hook_common::{metrics::serve_metrics, pgqueue::PgQueue, retry::RetryPolicy}; +use hook_common::{ + metrics::serve, metrics::setup_metrics_router, pgqueue::PgQueue, retry::RetryPolicy, +}; use hook_consumer::config::Config; use hook_consumer::consumer::WebhookConsumer; use hook_consumer::error::ConsumerError; @@ -29,7 +31,8 @@ async fn main() -> Result<(), ConsumerError> { let bind = config.bind(); tokio::task::spawn(async move { - serve_metrics(&bind) + let router = setup_metrics_router(); + serve(router, &bind) .await .expect("failed to start serving metrics"); }); From 232c54084a7db5f6baea351045b04f72e848edf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 22 Dec 2023 14:24:23 +0100 Subject: [PATCH 158/247] refactor: Only track duration seconds on success --- hook-consumer/src/consumer.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 8e67949466c9f..ecfa4b0ba42bd 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -193,13 +193,8 @@ async fn spawn_webhook_job_processing_task( metrics::increment_counter!("webhook_jobs_total", &labels); tokio::spawn(async move { - let now = tokio::time::Instant::now(); let result = process_webhook_job(client, webhook_job, &retry_policy).await; drop(permit); - - let elapsed = now.elapsed().as_secs_f64(); - metrics::histogram!("webhook_jobs_processing_duration_seconds", elapsed, &labels); - result }) } @@ -229,15 +224,20 @@ async fn process_webhook_job( ("target", webhook_job.target()), ]; - match send_webhook( + let now = tokio::time::Instant::now(); + + let send_result = send_webhook( client, ¶meters.method, ¶meters.url, ¶meters.headers, parameters.body.clone(), ) - .await - { + .await; + + let elapsed = now.elapsed().as_secs_f64(); + + match send_result { Ok(_) => { webhook_job .complete() @@ -245,6 +245,7 @@ async fn process_webhook_job( .map_err(|error| ConsumerError::PgJobError(error.to_string()))?; metrics::increment_counter!("webhook_jobs_completed", &labels); + metrics::histogram!("webhook_jobs_processing_duration_seconds", elapsed, &labels); Ok(()) } From 83426b8613c82ef26098339f04135f35b13c1bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 3 Jan 2024 11:06:30 +0100 Subject: [PATCH 159/247] feat: Implement support for retrying jobs to different queue (#19) Co-authored-by: Brett Hoerner --- .dockerignore | 5 + .github/workflows/docker-hook-consumer.yml | 63 +++ .github/workflows/docker-hook-janitor.yml | 63 +++ .github/workflows/docker-hook-producer.yml | 63 +++ Dockerfile | 31 ++ hook-common/src/pgqueue.rs | 522 +++++++++++++-------- hook-common/src/retry.rs | 208 +++++++- hook-consumer/src/config.rs | 3 + hook-consumer/src/consumer.rs | 15 +- hook-consumer/src/main.rs | 9 +- 10 files changed, 747 insertions(+), 235 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/docker-hook-consumer.yml create mode 100644 .github/workflows/docker-hook-janitor.yml create mode 100644 .github/workflows/docker-hook-producer.yml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000..9879089557b41 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +target +docker +.env +.git +.github diff --git a/.github/workflows/docker-hook-consumer.yml b/.github/workflows/docker-hook-consumer.yml new file mode 100644 index 0000000000000..5975120e3ff1a --- /dev/null +++ b/.github/workflows/docker-hook-consumer.yml @@ -0,0 +1,63 @@ +name: Build hook-consumer docker image + +on: + workflow_dispatch: + push: + branches: + - 'main' + +permissions: + packages: write + +jobs: + build: + name: build and publish hook-consumer image + runs-on: buildjet-4vcpu-ubuntu-2204-arm + steps: + + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/hook-consumer + tags: | + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push consumer + id: docker_build_hook_consumer + uses: docker/build-push-action@v4 + with: + context: ./ + file: ./Dockerfile + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: BIN=hook-consumer + + - name: Hook-consumer image digest + run: echo ${{ steps.docker_build_hook_consumer.outputs.digest }} diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml new file mode 100644 index 0000000000000..2649821f697b3 --- /dev/null +++ b/.github/workflows/docker-hook-janitor.yml @@ -0,0 +1,63 @@ +name: Build hook-janitor docker image + +on: + workflow_dispatch: + push: + branches: + - 'main' + +permissions: + packages: write + +jobs: + build: + name: build and publish hook-janitor image + runs-on: buildjet-4vcpu-ubuntu-2204-arm + steps: + + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/hook-janitor + tags: | + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push janitor + id: docker_build_hook_janitor + uses: docker/build-push-action@v4 + with: + context: ./ + file: ./Dockerfile + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: BIN=hook-janitor + + - name: Hook-janitor image digest + run: echo ${{ steps.docker_build_hook_janitor.outputs.digest }} diff --git a/.github/workflows/docker-hook-producer.yml b/.github/workflows/docker-hook-producer.yml new file mode 100644 index 0000000000000..d5e131feb014b --- /dev/null +++ b/.github/workflows/docker-hook-producer.yml @@ -0,0 +1,63 @@ +name: Build hook-producer docker image + +on: + workflow_dispatch: + push: + branches: + - 'main' + +permissions: + packages: write + +jobs: + build: + name: build and publish hook-producer image + runs-on: buildjet-4vcpu-ubuntu-2204-arm + steps: + + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/hook-producer + tags: | + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push producer + id: docker_build_hook_producer + uses: docker/build-push-action@v4 + with: + context: ./ + file: ./Dockerfile + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: BIN=hook-producer + + - name: Hook-producer image digest + run: echo ${{ steps.docker_build_hook_producer.outputs.digest }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000..959fd17180fb5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM docker.io/lukemathwalker/cargo-chef:latest-rust-1.74.0-buster AS chef +ARG BIN +WORKDIR app + +FROM chef AS planner +ARG BIN + +COPY . . +RUN cargo chef prepare --recipe-path recipe.json --bin $BIN + +FROM chef AS builder +ARG BIN + +# Ensure working C compile setup (not installed by default in arm64 images) +RUN apt update && apt install build-essential cmake -y + +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json + +COPY . . +RUN cargo build --release --bin $BIN + +FROM debian:bullseye-20230320-slim AS runtime +ARG BIN +ENV ENTRYPOINT=/usr/local/bin/$BIN +WORKDIR app + +USER nobody + +COPY --from=builder /app/target/release/$BIN /usr/local/bin +ENTRYPOINT [ $ENTRYPOINT ] diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index a47864d9e104d..39a09ce7fda27 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -109,42 +109,103 @@ impl Job { self.attempt >= self.max_attempts } - /// Consume Job to retry it. - /// This returns a RetryableJob that can be enqueued by PgQueue. - /// - /// # Arguments - /// - /// * `error`: Any JSON-serializable value to be stored as an error. - fn retry(self, error: E) -> RetryableJob { + /// Consume `Job` to transition it to a `RetryableJob`, i.e. a `Job` that may be retried. + fn retryable(self) -> RetryableJob { RetryableJob { id: self.id, attempt: self.attempt, - error: sqlx::types::Json(error), queue: self.queue, + retry_queue: None, } } - /// Consume Job to complete it. - /// This returns a CompletedJob that can be marked as completed by PgQueue. - fn complete(self) -> CompletedJob { - CompletedJob { + /// Consume `Job` to complete it. + /// A `CompletedJob` is finalized and cannot be used further; it is returned for reporting or inspection. + /// + /// # Arguments + /// + /// * `table`: The table where this job will be marked as completed. + /// * `executor`: Any sqlx::Executor that can execute the UPDATE query required to mark this `Job` as completed. + async fn complete<'c, E>(self, table: &str, executor: E) -> Result + where + E: sqlx::Executor<'c, Database = sqlx::Postgres>, + { + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + status = 'completed'::job_status +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + "#, + table + ); + + sqlx::query(&base_query) + .bind(&self.queue) + .bind(self.id) + .execute(executor) + .await?; + + Ok(CompletedJob { id: self.id, queue: self.queue, - } + }) } - /// Consume Job to fail it. - /// This returns a FailedJob that can be marked as failed by PgQueue. + /// Consume `Job` to fail it. + /// A `FailedJob` is finalized and cannot be used further; it is returned for reporting or inspection. /// /// # Arguments /// /// * `error`: Any JSON-serializable value to be stored as an error. - fn fail(self, error: E) -> FailedJob { - FailedJob { + /// * `table`: The table where this job will be marked as failed. + /// * `executor`: Any sqlx::Executor that can execute the UPDATE query required to mark this `Job` as failed. + async fn fail<'c, E, S>( + self, + error: S, + table: &str, + executor: E, + ) -> Result, sqlx::Error> + where + S: serde::Serialize + std::marker::Sync + std::marker::Send, + E: sqlx::Executor<'c, Database = sqlx::Postgres>, + { + let json_error = sqlx::types::Json(error); + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + status = 'failed'::job_status + errors = array_append("{0}".errors, $3) +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + "#, + &table + ); + + sqlx::query(&base_query) + .bind(&self.queue) + .bind(self.id) + .bind(&json_error) + .execute(executor) + .await?; + + Ok(FailedJob { id: self.id, - error: sqlx::types::Json(error), + error: json_error, queue: self.queue, - } + }) } } @@ -161,7 +222,8 @@ pub trait PgQueueJob { mut self, error: E, retry_interval: time::Duration, - ) -> Result, PgJobError>>; + queue: &str, + ) -> Result>>; } /// A Job that can be updated in PostgreSQL. @@ -175,28 +237,9 @@ pub struct PgJob { #[async_trait] impl PgQueueJob for PgJob { async fn complete(mut self) -> Result>>> { - let completed_job = self.job.complete(); - - let base_query = format!( - r#" -UPDATE - "{0}" -SET - finished_at = NOW(), - status = 'completed'::job_status -WHERE - "{0}".id = $2 - AND queue = $1 -RETURNING - "{0}".* - "#, - &self.table - ); - - sqlx::query(&base_query) - .bind(&completed_job.queue) - .bind(completed_job.id) - .execute(&mut *self.connection) + let completed_job = self + .job + .complete(&self.table, &mut *self.connection) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -210,31 +253,9 @@ RETURNING mut self, error: E, ) -> Result, PgJobError>>> { - let failed_job = self.job.fail(error); - - let base_query = format!( - r#" -UPDATE - "{0}" -SET - finished_at = NOW(), - status = 'failed'::job_status - errors = array_append("{0}".errors, $3) -WHERE - "{0}".id = $2 - AND queue = $1 -RETURNING - "{0}".* - - "#, - &self.table - ); - - sqlx::query(&base_query) - .bind(&failed_job.queue) - .bind(failed_job.id) - .bind(&failed_job.error) - .execute(&mut *self.connection) + let failed_job = self + .job + .fail(error, &self.table, &mut *self.connection) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -248,46 +269,27 @@ RETURNING mut self, error: E, retry_interval: time::Duration, - ) -> Result, PgJobError>>> { + queue: &str, + ) -> Result>>> { if self.job.is_gte_max_attempts() { return Err(PgJobError::RetryInvalidError { job: Box::new(self), error: "Maximum attempts reached".to_owned(), }); } - let retryable_job = self.job.retry(error); - - let base_query = format!( - r#" -UPDATE - "{0}" -SET - finished_at = NOW(), - status = 'available'::job_status, - scheduled_at = NOW() + $3, - errors = array_append("{0}".errors, $4) -WHERE - "{0}".id = $2 - AND queue = $1 -RETURNING - "{0}".* - "#, - &self.table - ); - sqlx::query(&base_query) - .bind(&retryable_job.queue) - .bind(retryable_job.id) - .bind(retry_interval) - .bind(&retryable_job.error) - .execute(&mut *self.connection) + let retried_job = self + .job + .retryable() + .queue(queue) + .retry(error, retry_interval, &self.table, &mut *self.connection) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), error, })?; - Ok(retryable_job) + Ok(retried_job) } } @@ -305,28 +307,9 @@ impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactio async fn complete( mut self, ) -> Result>>> { - let completed_job = self.job.complete(); - - let base_query = format!( - r#" -UPDATE - "{0}" -SET - finished_at = NOW(), - status = 'completed'::job_status -WHERE - "{0}".id = $2 - AND queue = $1 -RETURNING - "{0}".* - "#, - &self.table - ); - - sqlx::query(&base_query) - .bind(&completed_job.queue) - .bind(completed_job.id) - .execute(&mut *self.transaction) + let completed_job = self + .job + .complete(&self.table, &mut *self.transaction) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -344,34 +327,13 @@ RETURNING Ok(completed_job) } - async fn fail( + async fn fail( mut self, - error: E, - ) -> Result, PgJobError>>> { - let failed_job = self.job.fail(error); - - let base_query = format!( - r#" -UPDATE - "{0}" -SET - finished_at = NOW(), - status = 'failed'::job_status - errors = array_append("{0}".errors, $3) -WHERE - "{0}".id = $2 - AND queue = $1 -RETURNING - "{0}".* - "#, - &self.table - ); - - sqlx::query(&base_query) - .bind(&failed_job.queue) - .bind(failed_job.id) - .bind(&failed_job.error) - .execute(&mut *self.transaction) + error: S, + ) -> Result, PgJobError>>> { + let failed_job = self + .job + .fail(error, &self.table, &mut *self.transaction) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -393,40 +355,22 @@ RETURNING mut self, error: E, retry_interval: time::Duration, - ) -> Result, PgJobError>>> { + queue: &str, + ) -> Result>>> { + // Ideally, the transition to RetryableJob should be fallible. + // But taking ownership of self when we return this error makes things difficult. if self.job.is_gte_max_attempts() { return Err(PgJobError::RetryInvalidError { job: Box::new(self), error: "Maximum attempts reached".to_owned(), }); } - let retryable_job = self.job.retry(error); - - let base_query = format!( - r#" -UPDATE - "{0}" -SET - finished_at = NOW(), - status = 'available'::job_status, - scheduled_at = NOW() + $3, - errors = array_append("{0}".errors, $4) -WHERE - "{0}".id = $2 - AND queue = $1 -RETURNING - "{0}".* - "#, - &self.table - ); - - sqlx::query(&base_query) - .bind(&retryable_job.queue) - .bind(retryable_job.id) - .bind(retry_interval) - .bind(&retryable_job.error) - .execute(&mut *self.transaction) + let retried_job = self + .job + .retryable() + .queue(queue) + .retry(error, retry_interval, &self.table, &mut *self.transaction) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -441,42 +385,127 @@ RETURNING error, })?; - Ok(retryable_job) + Ok(retried_job) } } /// A Job that has failed but can still be enqueued into a PgQueue to be retried at a later point. /// The time until retry will depend on the PgQueue's RetryPolicy. -pub struct RetryableJob { +pub struct RetryableJob { /// A unique id identifying a job. - id: i64, + pub id: i64, /// A number corresponding to the current job attempt. pub attempt: i32, - /// Any JSON-serializable value to be stored as an error. - pub error: sqlx::types::Json, /// A unique id identifying a job queue. - pub queue: String, + queue: String, + /// An optional separate queue where to enqueue this job when retrying. + retry_queue: Option, +} + +impl RetryableJob { + /// Set the queue for a `RetryableJob`. + /// If not set, `Job` will be retried to its original queue on calling `retry`. + fn queue(mut self, queue: &str) -> Self { + self.retry_queue = Some(queue.to_owned()); + self + } + + /// Return the queue that a `Job` is to be retried into. + fn retry_queue(&self) -> &str { + self.retry_queue.as_ref().unwrap_or(&self.queue) + } + + /// Consume `Job` to retry it. + /// A `RetriedJob` cannot be used further; it is returned for reporting or inspection. + /// + /// # Arguments + /// + /// * `error`: Any JSON-serializable value to be stored as an error. + /// * `retry_interval`: The duration until the `Job` is to be retried again. Used to set `scheduled_at`. + /// * `table`: The table where this job will be marked as completed. + /// * `executor`: Any sqlx::Executor that can execute the UPDATE query required to mark this `Job` as completed. + async fn retry<'c, S, E>( + self, + error: S, + retry_interval: time::Duration, + table: &str, + executor: E, + ) -> Result + where + S: serde::Serialize + std::marker::Sync + std::marker::Send, + E: sqlx::Executor<'c, Database = sqlx::Postgres>, + { + let json_error = sqlx::types::Json(error); + let base_query = format!( + r#" +UPDATE + "{0}" +SET + finished_at = NOW(), + errors = array_append("{0}".errors, $4), + queue = $5, + status = 'available'::job_status, + scheduled_at = NOW() + $3 +WHERE + "{0}".id = $2 + AND queue = $1 +RETURNING + "{0}".* + "#, + &table + ); + + sqlx::query(&base_query) + .bind(&self.queue) + .bind(self.id) + .bind(retry_interval) + .bind(&json_error) + .bind(self.retry_queue()) + .execute(executor) + .await?; + + Ok(RetriedJob { + id: self.id, + table: table.to_owned(), + queue: self.queue, + retry_queue: self.retry_queue.to_owned(), + }) + } } -/// A Job that has completed to be enqueued into a PgQueue and marked as completed. +/// State a `Job` is transitioned to after successfully completing. +#[derive(Debug)] pub struct CompletedJob { /// A unique id identifying a job. - id: i64, + pub id: i64, /// A unique id identifying a job queue. pub queue: String, } -/// A Job that has failed to be enqueued into a PgQueue and marked as failed. +/// State a `Job` is transitioned to after it has been enqueued for retrying. +#[derive(Debug)] +pub struct RetriedJob { + /// A unique id identifying a job. + pub id: i64, + /// A unique id identifying a job queue. + pub queue: String, + pub retry_queue: Option, + pub table: String, +} + +/// State a `Job` is transitioned to after exhausting all of their attempts. +#[derive(Debug)] pub struct FailedJob { /// A unique id identifying a job. - id: i64, + pub id: i64, /// Any JSON-serializable value to be stored as an error. pub error: sqlx::types::Json, /// A unique id identifying a job queue. pub queue: String, } -/// A NewJob to be enqueued into a PgQueue. +/// This struct represents a new job being created to be enqueued into a `PgQueue`. +#[derive(Debug)] pub struct NewJob { /// The maximum amount of attempts this NewJob has to complete. pub max_attempts: i32, @@ -513,14 +542,13 @@ pub struct PgQueue { pub type PgQueueResult = std::result::Result; impl PgQueue { - /// Initialize a new PgQueue backed by table in PostgreSQL. + /// Initialize a new PgQueue backed by table in PostgreSQL by intializing a connection pool to the database in `url`. /// /// # Arguments /// /// * `queue_name`: A name for the queue we are going to initialize. /// * `table_name`: The name for the table the queue will use in PostgreSQL. /// * `url`: A URL pointing to where the PostgreSQL database is hosted. - /// * `worker_name`: The name of the worker that is operating with this queue. pub async fn new(queue_name: &str, table_name: &str, url: &str) -> PgQueueResult { let name = queue_name.to_owned(); let table = table_name.to_owned(); @@ -531,6 +559,13 @@ impl PgQueue { Ok(Self { name, pool, table }) } + /// Initialize a new PgQueue backed by table in PostgreSQL from a provided connection pool. + /// + /// # Arguments + /// + /// * `queue_name`: A name for the queue we are going to initialize. + /// * `table_name`: The name for the table the queue will use in PostgreSQL. + /// * `pool`: A database connection pool to be used by this queue. pub async fn new_from_pool( queue_name: &str, table_name: &str, @@ -542,7 +577,8 @@ impl PgQueue { Ok(Self { name, pool, table }) } - /// Dequeue a Job from this PgQueue to work on it. + /// Dequeue a `Job` from this `PgQueue`. + /// The `Job` will be updated to `'running'` status, so any other `dequeue` calls will skip it. pub async fn dequeue< J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, M: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, @@ -620,7 +656,9 @@ RETURNING } } - /// Dequeue a Job from this PgQueue to work on it. + /// Dequeue a `Job` from this `PgQueue` and hold the transaction. + /// Any other `dequeue_tx` calls will skip rows locked, so by holding a transaction we ensure only one worker can dequeue a job. + /// Holding a transaction open can have performance implications, but it means no `'running'` state is required. pub async fn dequeue_tx< 'a, J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, @@ -692,8 +730,8 @@ RETURNING } } - /// Enqueue a Job into this PgQueue. - /// We take ownership of NewJob to enforce a specific NewJob is only enqueued once. + /// Enqueue a `NewJob` into this PgQueue. + /// We take ownership of `NewJob` to enforce a specific `NewJob` is only enqueued once. pub async fn enqueue< J: serde::Serialize + std::marker::Sync, M: serde::Serialize + std::marker::Sync, @@ -875,19 +913,16 @@ mod tests { let job_metadata = JobMetadata::default(); let worker_id = worker_id(); let new_job = NewJob::new(2, job_metadata, job_parameters, &job_target); - let retry_policy = RetryPolicy { - backoff_coefficient: 0, - initial_interval: time::Duration::from_secs(0), - maximum_interval: None, - }; + let table_name = "job_queue".to_owned(); + let queue_name = "test_can_retry_job_with_remaining_attempts".to_owned(); - let queue = PgQueue::new_from_pool( - "test_can_retry_job_with_remaining_attempts", - "job_queue", - db, - ) - .await - .expect("failed to connect to local test postgresql database"); + let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)) + .queue(&queue_name) + .provide(); + + let queue = PgQueue::new_from_pool(&queue_name, &table_name, db) + .await + .expect("failed to connect to local test postgresql database"); queue.enqueue(new_job).await.expect("failed to enqueue job"); let job: PgJob = queue @@ -895,11 +930,18 @@ mod tests { .await .expect("failed to dequeue job") .expect("didn't find a job to dequeue"); - let retry_interval = retry_policy.time_until_next_retry(job.job.attempt as u32, None); + + let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); + let retry_queue = retry_policy.retry_queue(&job.job.queue).to_owned(); let _ = job - .retry("a very reasonable failure reason", retry_interval) + .retry( + "a very reasonable failure reason", + retry_interval, + &retry_queue, + ) .await .expect("failed to retry job"); + let retried_job: PgJob = queue .dequeue(&worker_id) .await @@ -918,6 +960,72 @@ mod tests { assert_eq!(retried_job.job.target, job_target); } + #[sqlx::test(migrations = "../migrations")] + async fn test_can_retry_job_to_different_queue(db: PgPool) { + let job_target = job_target(); + let job_parameters = JobParameters::default(); + let job_metadata = JobMetadata::default(); + let worker_id = worker_id(); + let new_job = NewJob::new(2, job_metadata, job_parameters, &job_target); + let table_name = "job_queue".to_owned(); + let queue_name = "test_can_retry_job_to_different_queue".to_owned(); + let retry_queue_name = "test_can_retry_job_to_different_queue_retry".to_owned(); + + let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)) + .queue(&retry_queue_name) + .provide(); + + let queue = PgQueue::new_from_pool(&queue_name, &table_name, db.clone()) + .await + .expect("failed to connect to queue in local test postgresql database"); + + queue.enqueue(new_job).await.expect("failed to enqueue job"); + let job: PgJob = queue + .dequeue(&worker_id) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + + let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); + let retry_queue = retry_policy.retry_queue(&job.job.queue).to_owned(); + let _ = job + .retry( + "a very reasonable failure reason", + retry_interval, + &retry_queue, + ) + .await + .expect("failed to retry job"); + + let retried_job_not_found: Option> = queue + .dequeue(&worker_id) + .await + .expect("failed to dequeue job"); + + assert!(retried_job_not_found.is_none()); + + let queue = PgQueue::new_from_pool(&retry_queue_name, &table_name, db) + .await + .expect("failed to connect to retry queue in local test postgresql database"); + + let retried_job: PgJob = queue + .dequeue(&worker_id) + .await + .expect("failed to dequeue job") + .expect("job not found in retry queue"); + + assert_eq!(retried_job.job.attempt, 2); + assert!(retried_job.job.attempted_by.contains(&worker_id)); + assert_eq!(retried_job.job.attempted_by.len(), 2); + assert_eq!(retried_job.job.max_attempts, 2); + assert_eq!( + *retried_job.job.parameters.as_ref(), + JobParameters::default() + ); + assert_eq!(retried_job.job.status, JobStatus::Running); + assert_eq!(retried_job.job.target, job_target); + } + #[sqlx::test(migrations = "../migrations")] #[should_panic(expected = "failed to retry job")] async fn test_cannot_retry_job_without_remaining_attempts(db: PgPool) { @@ -926,11 +1034,7 @@ mod tests { let job_metadata = JobMetadata::default(); let worker_id = worker_id(); let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); - let retry_policy = RetryPolicy { - backoff_coefficient: 0, - initial_interval: time::Duration::from_secs(0), - maximum_interval: None, - }; + let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)).provide(); let queue = PgQueue::new_from_pool( "test_cannot_retry_job_without_remaining_attempts", @@ -947,8 +1051,10 @@ mod tests { .await .expect("failed to dequeue job") .expect("didn't find a job to dequeue"); - let retry_interval = retry_policy.time_until_next_retry(job.job.attempt as u32, None); - job.retry("a very reasonable failure reason", retry_interval) + + let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); + + job.retry("a very reasonable failure reason", retry_interval, "any") .await .expect("failed to retry job"); } diff --git a/hook-common/src/retry.rs b/hook-common/src/retry.rs index f72b0d166fdc8..b00f967a7b6e5 100644 --- a/hook-common/src/retry.rs +++ b/hook-common/src/retry.rs @@ -1,7 +1,10 @@ +//! # Retry +//! +//! Module providing a `RetryPolicy` struct to configure job retrying. use std::time; -#[derive(Copy, Clone, Debug)] -/// The retry policy that PgQueue will use to determine how to set scheduled_at when enqueuing a retry. +#[derive(Clone, Debug)] +/// A retry policy to determine retry parameters for a job. pub struct RetryPolicy { /// Coefficient to multiply initial_interval with for every past attempt. pub backoff_coefficient: u32, @@ -9,47 +12,214 @@ pub struct RetryPolicy { pub initial_interval: time::Duration, /// The maximum possible backoff between retries. pub maximum_interval: Option, + /// An optional queue to send WebhookJob retries to. + pub queue: Option, } impl RetryPolicy { - pub fn new( - backoff_coefficient: u32, - initial_interval: time::Duration, - maximum_interval: Option, - ) -> Self { - Self { - backoff_coefficient, - initial_interval, - maximum_interval, - } + /// Initialize a `RetryPolicyBuilder`. + pub fn build(backoff_coefficient: u32, initial_interval: time::Duration) -> RetryPolicyBuilder { + RetryPolicyBuilder::new(backoff_coefficient, initial_interval) } - /// Calculate the time until the next retry for a given RetryableJob. - pub fn time_until_next_retry( + /// Determine interval for retrying at a given attempt number. + /// If not `None`, this method will respect `preferred_retry_interval` as long as it falls within `candidate_interval <= preferred_retry_interval <= maximum_interval`. + pub fn retry_interval( &self, attempt: u32, preferred_retry_interval: Option, ) -> time::Duration { - let candidate_interval = self.initial_interval * self.backoff_coefficient.pow(attempt); + let candidate_interval = + self.initial_interval * self.backoff_coefficient.pow(attempt.saturating_sub(1)); match (preferred_retry_interval, self.maximum_interval) { - (Some(duration), Some(max_interval)) => std::cmp::min( - std::cmp::max(std::cmp::min(candidate_interval, max_interval), duration), - max_interval, - ), + (Some(duration), Some(max_interval)) => { + let min_interval_allowed = std::cmp::min(candidate_interval, max_interval); + + if min_interval_allowed <= duration && duration <= max_interval { + duration + } else { + min_interval_allowed + } + } (Some(duration), None) => std::cmp::max(candidate_interval, duration), (None, Some(max_interval)) => std::cmp::min(candidate_interval, max_interval), (None, None) => candidate_interval, } } + + /// Determine the queue to be used for retrying. + /// Only whether a queue is configured in this RetryPolicy is used to determine which queue to use for retrying. + /// This may be extended in the future to support more decision parameters. + pub fn retry_queue<'s>(&'s self, current_queue: &'s str) -> &'s str { + if let Some(new_queue) = &self.queue { + new_queue + } else { + current_queue + } + } } impl Default for RetryPolicy { + fn default() -> Self { + RetryPolicyBuilder::default().provide() + } +} + +/// Builder pattern struct to provide a `RetryPolicy`. +pub struct RetryPolicyBuilder { + /// Coefficient to multiply initial_interval with for every past attempt. + pub backoff_coefficient: u32, + /// The backoff interval for the first retry. + pub initial_interval: time::Duration, + /// The maximum possible backoff between retries. + pub maximum_interval: Option, + /// An optional queue to send WebhookJob retries to. + pub queue: Option, +} + +impl Default for RetryPolicyBuilder { fn default() -> Self { Self { backoff_coefficient: 2, initial_interval: time::Duration::from_secs(1), maximum_interval: None, + queue: None, + } + } +} + +impl RetryPolicyBuilder { + pub fn new(backoff_coefficient: u32, initial_interval: time::Duration) -> Self { + Self { + backoff_coefficient, + initial_interval, + ..RetryPolicyBuilder::default() + } + } + + pub fn maximum_interval(mut self, interval: time::Duration) -> RetryPolicyBuilder { + self.maximum_interval = Some(interval); + self + } + + pub fn queue(mut self, queue: &str) -> RetryPolicyBuilder { + self.queue = Some(queue.to_owned()); + self + } + + /// Provide a `RetryPolicy` according to build parameters provided thus far. + pub fn provide(&self) -> RetryPolicy { + RetryPolicy { + backoff_coefficient: self.backoff_coefficient, + initial_interval: self.initial_interval, + maximum_interval: self.maximum_interval, + queue: self.queue.clone(), } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_constant_retry_interval() { + let retry_policy = RetryPolicy::build(1, time::Duration::from_secs(2)).provide(); + let first_interval = retry_policy.retry_interval(1, None); + let second_interval = retry_policy.retry_interval(2, None); + let third_interval = retry_policy.retry_interval(3, None); + + assert_eq!(first_interval, time::Duration::from_secs(2)); + assert_eq!(second_interval, time::Duration::from_secs(2)); + assert_eq!(third_interval, time::Duration::from_secs(2)); + } + + #[test] + fn test_retry_interval_never_exceeds_maximum() { + let retry_policy = RetryPolicy::build(2, time::Duration::from_secs(2)) + .maximum_interval(time::Duration::from_secs(4)) + .provide(); + let first_interval = retry_policy.retry_interval(1, None); + let second_interval = retry_policy.retry_interval(2, None); + let third_interval = retry_policy.retry_interval(3, None); + let fourth_interval = retry_policy.retry_interval(4, None); + + assert_eq!(first_interval, time::Duration::from_secs(2)); + assert_eq!(second_interval, time::Duration::from_secs(4)); + assert_eq!(third_interval, time::Duration::from_secs(4)); + assert_eq!(fourth_interval, time::Duration::from_secs(4)); + } + + #[test] + fn test_retry_interval_increases_with_coefficient() { + let retry_policy = RetryPolicy::build(2, time::Duration::from_secs(2)).provide(); + let first_interval = retry_policy.retry_interval(1, None); + let second_interval = retry_policy.retry_interval(2, None); + let third_interval = retry_policy.retry_interval(3, None); + + assert_eq!(first_interval, time::Duration::from_secs(2)); + assert_eq!(second_interval, time::Duration::from_secs(4)); + assert_eq!(third_interval, time::Duration::from_secs(8)); + } + + #[test] + fn test_retry_interval_respects_preferred() { + let retry_policy = RetryPolicy::build(1, time::Duration::from_secs(2)).provide(); + let preferred = time::Duration::from_secs(999); + let first_interval = retry_policy.retry_interval(1, Some(preferred)); + let second_interval = retry_policy.retry_interval(2, Some(preferred)); + let third_interval = retry_policy.retry_interval(3, Some(preferred)); + + assert_eq!(first_interval, preferred); + assert_eq!(second_interval, preferred); + assert_eq!(third_interval, preferred); + } + + #[test] + fn test_retry_interval_ignores_small_preferred() { + let retry_policy = RetryPolicy::build(1, time::Duration::from_secs(5)).provide(); + let preferred = time::Duration::from_secs(2); + let first_interval = retry_policy.retry_interval(1, Some(preferred)); + let second_interval = retry_policy.retry_interval(2, Some(preferred)); + let third_interval = retry_policy.retry_interval(3, Some(preferred)); + + assert_eq!(first_interval, time::Duration::from_secs(5)); + assert_eq!(second_interval, time::Duration::from_secs(5)); + assert_eq!(third_interval, time::Duration::from_secs(5)); + } + + #[test] + fn test_retry_interval_ignores_large_preferred() { + let retry_policy = RetryPolicy::build(2, time::Duration::from_secs(2)) + .maximum_interval(time::Duration::from_secs(4)) + .provide(); + let preferred = time::Duration::from_secs(10); + let first_interval = retry_policy.retry_interval(1, Some(preferred)); + let second_interval = retry_policy.retry_interval(2, Some(preferred)); + let third_interval = retry_policy.retry_interval(3, Some(preferred)); + + assert_eq!(first_interval, time::Duration::from_secs(2)); + assert_eq!(second_interval, time::Duration::from_secs(4)); + assert_eq!(third_interval, time::Duration::from_secs(4)); + } + + #[test] + fn test_returns_retry_queue_if_set() { + let retry_queue_name = "retry_queue".to_owned(); + let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)) + .queue(&retry_queue_name) + .provide(); + let current_queue = "queue".to_owned(); + + assert_eq!(retry_policy.retry_queue(¤t_queue), retry_queue_name); + } + + #[test] + fn test_returns_queue_if_retry_queue_not_set() { + let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)).provide(); + let current_queue = "queue".to_owned(); + + assert_eq!(retry_policy.retry_queue(¤t_queue), current_queue); + } +} diff --git a/hook-consumer/src/config.rs b/hook-consumer/src/config.rs index 91695128f2192..6525b25253943 100644 --- a/hook-consumer/src/config.rs +++ b/hook-consumer/src/config.rs @@ -72,4 +72,7 @@ pub struct RetryPolicyConfig { #[envconfig(default = "100000")] pub maximum_interval: EnvMsDuration, + + #[envconfig(default = "default")] + pub retry_queue_name: String, } diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 99e97f08d9724..42cb1ff77537e 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -13,7 +13,7 @@ use tokio::sync; use crate::error::{ConsumerError, WebhookError}; -/// A WebhookJob is any PgQueueJob with WebhookJobParameters and WebhookJobMetadata. +/// A WebhookJob is any `PgQueueJob` with `WebhookJobParameters` and `WebhookJobMetadata`. trait WebhookJob: PgQueueJob + std::marker::Send { fn parameters(&self) -> &WebhookJobParameters; fn metadata(&self) -> &WebhookJobMetadata; @@ -147,7 +147,7 @@ impl<'p> WebhookConsumer<'p> { spawn_webhook_job_processing_task( self.client.clone(), semaphore.clone(), - self.retry_policy, + self.retry_policy.clone(), webhook_job, ) .await; @@ -155,10 +155,11 @@ impl<'p> WebhookConsumer<'p> { } else { loop { let webhook_job = self.wait_for_job().await?; + spawn_webhook_job_processing_task( self.client.clone(), semaphore.clone(), - self.retry_policy, + self.retry_policy.clone(), webhook_job, ) .await; @@ -173,6 +174,7 @@ impl<'p> WebhookConsumer<'p> { /// /// * `client`: An HTTP client to execute the webhook job request. /// * `semaphore`: A semaphore used for rate limiting purposes. This function will panic if this semaphore is closed. +/// * `retry_policy`: The retry policy used to set retry parameters if a job fails and has remaining attempts. /// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. async fn spawn_webhook_job_processing_task( client: reqwest::Client, @@ -212,6 +214,7 @@ async fn spawn_webhook_job_processing_task( /// /// * `client`: An HTTP client to execute the webhook job request. /// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. +/// * `retry_policy`: The retry policy used to set retry parameters if a job fails and has remaining attempts. async fn process_webhook_job( client: reqwest::Client, webhook_job: W, @@ -281,10 +284,12 @@ async fn process_webhook_job( } Err(WebhookError::RetryableRequestError { error, retry_after }) => { let retry_interval = - retry_policy.time_until_next_retry(webhook_job.attempt() as u32, retry_after); + retry_policy.retry_interval(webhook_job.attempt() as u32, retry_after); + let current_queue = webhook_job.queue(); + let retry_queue = retry_policy.retry_queue(¤t_queue); match webhook_job - .retry(WebhookJobError::from(&error), retry_interval) + .retry(WebhookJobError::from(&error), retry_interval, retry_queue) .await { Ok(_) => { diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index 38c2ee23042de..c71b8eb91a7d0 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -1,3 +1,4 @@ +//! Consume `PgQueue` jobs to run webhook calls. use envconfig::Envconfig; use hook_common::{ @@ -11,11 +12,13 @@ use hook_consumer::error::ConsumerError; async fn main() -> Result<(), ConsumerError> { let config = Config::init_from_env().expect("Invalid configuration:"); - let retry_policy = RetryPolicy::new( + let retry_policy = RetryPolicy::build( config.retry_policy.backoff_coefficient, config.retry_policy.initial_interval.0, - Some(config.retry_policy.maximum_interval.0), - ); + ) + .maximum_interval(config.retry_policy.maximum_interval.0) + .queue(&config.retry_policy.retry_queue_name) + .provide(); let queue = PgQueue::new(&config.queue_name, &config.table_name, &config.database_url) .await .expect("failed to initialize queue"); From 76476efd75bcd8db3decae7454821ec3fdf17eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 3 Jan 2024 12:11:03 +0100 Subject: [PATCH 160/247] fix: Add org name to image path (#22) --- .github/workflows/docker-hook-consumer.yml | 2 +- .github/workflows/docker-hook-janitor.yml | 2 +- .github/workflows/docker-hook-producer.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-hook-consumer.yml b/.github/workflows/docker-hook-consumer.yml index 5975120e3ff1a..db920744f145c 100644 --- a/.github/workflows/docker-hook-consumer.yml +++ b/.github/workflows/docker-hook-consumer.yml @@ -25,7 +25,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/hook-consumer + images: ghcr.io/posthog/hook-consumer tags: | type=ref,event=pr type=ref,event=branch diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index 2649821f697b3..b426d51a5f526 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -25,7 +25,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/hook-janitor + images: ghcr.io/posthog/hook-janitor tags: | type=ref,event=pr type=ref,event=branch diff --git a/.github/workflows/docker-hook-producer.yml b/.github/workflows/docker-hook-producer.yml index d5e131feb014b..ec2594f1f3d46 100644 --- a/.github/workflows/docker-hook-producer.yml +++ b/.github/workflows/docker-hook-producer.yml @@ -25,7 +25,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/hook-producer + images: ghcr.io/posthog/hook-producer tags: | type=ref,event=pr type=ref,event=branch From a277c452190ab7397e70508b3441e1c94364f66c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 5 Jan 2024 11:28:19 +0100 Subject: [PATCH 161/247] refactor: Deploy the migration container too (#24) --- .github/workflows/docker-migrator.yml | 62 +++++++++++++++++++++++++++ Dockerfile.migrate | 16 +++++++ Dockerfile.sqlx | 5 --- bin/migrate | 4 ++ docker-compose.yml | 6 +-- 5 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/docker-migrator.yml create mode 100644 Dockerfile.migrate delete mode 100644 Dockerfile.sqlx create mode 100755 bin/migrate diff --git a/.github/workflows/docker-migrator.yml b/.github/workflows/docker-migrator.yml new file mode 100644 index 0000000000000..73d7afa20cb62 --- /dev/null +++ b/.github/workflows/docker-migrator.yml @@ -0,0 +1,62 @@ +name: Build hook-migrator docker image + +on: + workflow_dispatch: + push: + branches: + - 'main' + +permissions: + packages: write + +jobs: + build: + name: build and publish hook-migrator image + runs-on: buildjet-4vcpu-ubuntu-2204-arm + steps: + + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/posthog/hook-migrator + tags: | + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push migrator + id: docker_build_hook_migrator + uses: docker/build-push-action@v4 + with: + context: ./ + file: ./Dockerfile.migrate + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Hook-migrator image digest + run: echo ${{ steps.docker_build_hook_migrator.outputs.digest }} diff --git a/Dockerfile.migrate b/Dockerfile.migrate new file mode 100644 index 0000000000000..47790778f4396 --- /dev/null +++ b/Dockerfile.migrate @@ -0,0 +1,16 @@ +FROM docker.io/library/rust:1.74.0-buster as builder + +RUN apt update && apt install build-essential cmake -y +RUN cargo install sqlx-cli@0.7.3 --no-default-features --features native-tls,postgres --root /app/target/release/ + +FROM debian:bullseye-20230320-slim AS runtime +WORKDIR /sqlx + +ADD bin /sqlx/bin/ +ADD migrations /sqlx/migrations/ + +COPY --from=builder /app/target/release/bin/sqlx /usr/local/bin + +RUN chmod +x ./bin/migrate + +CMD ["./bin/migrate"] diff --git a/Dockerfile.sqlx b/Dockerfile.sqlx deleted file mode 100644 index c55dfaa8a960a..0000000000000 --- a/Dockerfile.sqlx +++ /dev/null @@ -1,5 +0,0 @@ -FROM docker.io/library/rust:1.74.0 - -RUN cargo install sqlx-cli --no-default-features --features native-tls,postgres - -WORKDIR /sqlx diff --git a/bin/migrate b/bin/migrate new file mode 100755 index 0000000000000..6e36fc40f8f9a --- /dev/null +++ b/bin/migrate @@ -0,0 +1,4 @@ +#!/bin/sh + +sqlx database create +sqlx migrate run diff --git a/docker-compose.yml b/docker-compose.yml index afaf48ef86d06..d95186492d63e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,18 +19,14 @@ services: container_name: setup-test-db build: context: . - dockerfile: Dockerfile.sqlx + dockerfile: Dockerfile.migrate restart: on-failure - command: > - sh -c "sqlx database create && sqlx migrate run" depends_on: db: condition: service_healthy restart: true environment: DATABASE_URL: postgres://posthog:posthog@db:5432/test_database - volumes: - - ./migrations:/sqlx/migrations/ echo_server: image: docker.io/library/caddy:2 From 3b5229dca92740e0fe0a1af833144c433f109278 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 5 Jan 2024 08:29:24 -0700 Subject: [PATCH 162/247] Change dequeue ORDER BY and rename finished_at (#23) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tomás Farías Santana --- hook-common/src/pgqueue.rs | 12 +++++++----- hook-janitor/src/fixtures/webhook_cleanup.sql | 2 +- hook-janitor/src/webhooks.rs | 6 +++--- migrations/20231129172339_job_queue_table.sql | 12 ++++++------ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 39a09ce7fda27..ec5b6843331dd 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -135,7 +135,7 @@ impl Job { UPDATE "{0}" SET - finished_at = NOW(), + last_attempt_finished_at = NOW(), status = 'completed'::job_status WHERE "{0}".id = $2 @@ -182,7 +182,7 @@ RETURNING UPDATE "{0}" SET - finished_at = NOW(), + last_attempt_finished_at = NOW(), status = 'failed'::job_status errors = array_append("{0}".errors, $3) WHERE @@ -441,7 +441,7 @@ impl RetryableJob { UPDATE "{0}" SET - finished_at = NOW(), + last_attempt_finished_at = NOW(), errors = array_append("{0}".errors, $4), queue = $5, status = 'available'::job_status, @@ -606,7 +606,8 @@ WITH available_in_queue AS ( AND scheduled_at <= NOW() AND queue = $1 ORDER BY - id + attempt, + scheduled_at LIMIT 1 FOR UPDATE SKIP LOCKED ) @@ -687,7 +688,8 @@ WITH available_in_queue AS ( AND scheduled_at <= NOW() AND queue = $1 ORDER BY - id + attempt, + scheduled_at LIMIT 1 FOR UPDATE SKIP LOCKED ) diff --git a/hook-janitor/src/fixtures/webhook_cleanup.sql b/hook-janitor/src/fixtures/webhook_cleanup.sql index 4aeb231febbd3..bddaf269763d5 100644 --- a/hook-janitor/src/fixtures/webhook_cleanup.sql +++ b/hook-janitor/src/fixtures/webhook_cleanup.sql @@ -2,7 +2,7 @@ INSERT INTO job_queue ( errors, metadata, - finished_at, + last_attempt_finished_at, parameters, queue, status, diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 57b984d7bae2f..b14dba1c34b34 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -198,13 +198,13 @@ impl WebhookCleaner { async fn get_completed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { let base_query = format!( r#" - SELECT DATE_TRUNC('hour', finished_at) AS hour, + SELECT DATE_TRUNC('hour', last_attempt_finished_at) AS hour, (metadata->>'team_id')::bigint AS team_id, (metadata->>'plugin_config_id')::bigint AS plugin_config_id, count(*) as successes FROM {0} WHERE status = 'completed' - AND queue = $1 + AND queue = $1 GROUP BY hour, team_id, plugin_config_id ORDER BY hour, team_id, plugin_config_id; "#, @@ -223,7 +223,7 @@ impl WebhookCleaner { async fn get_failed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { let base_query = format!( r#" - SELECT DATE_TRUNC('hour', finished_at) AS hour, + SELECT DATE_TRUNC('hour', last_attempt_finished_at) AS hour, (metadata->>'team_id')::bigint AS team_id, (metadata->>'plugin_config_id')::bigint AS plugin_config_id, errors[array_upper(errors, 1)] AS last_error, diff --git a/migrations/20231129172339_job_queue_table.sql b/migrations/20231129172339_job_queue_table.sql index 7efa154c85816..bf8c3df0a3706 100644 --- a/migrations/20231129172339_job_queue_table.sql +++ b/migrations/20231129172339_job_queue_table.sql @@ -9,21 +9,21 @@ CREATE TABLE job_queue( id BIGSERIAL PRIMARY KEY, attempt INT NOT NULL DEFAULT 0, attempted_at TIMESTAMPTZ DEFAULT NULL, - attempted_by TEXT[] DEFAULT ARRAY[]::TEXT[], + attempted_by TEXT [] DEFAULT ARRAY [] :: TEXT [], created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - errors JSONB[], + errors JSONB [], max_attempts INT NOT NULL DEFAULT 1, metadata JSONB, - finished_at TIMESTAMPTZ DEFAULT NULL, + last_attempt_finished_at TIMESTAMPTZ DEFAULT NULL, parameters JSONB, - queue TEXT NOT NULL DEFAULT 'default'::text, + queue TEXT NOT NULL DEFAULT 'default' :: text, scheduled_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - status job_status NOT NULL DEFAULT 'available'::job_status, + status job_status NOT NULL DEFAULT 'available' :: job_status, target TEXT NOT NULL ); -- Needed for `dequeue` queries -CREATE INDEX idx_queue_scheduled_at ON job_queue(queue, status, scheduled_at); +CREATE INDEX idx_queue_scheduled_at ON job_queue(queue, status, scheduled_at, attempt); -- Needed for UPDATE-ing incomplete jobs with a specific target (i.e. slow destinations) CREATE INDEX idx_queue_target ON job_queue(queue, status, target); From 7d519ab19981759aacc0a01ae63859996e0e131e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Mon, 8 Jan 2024 19:22:59 +0100 Subject: [PATCH 163/247] refactor: Remove table name config and hardcode table name (#25) --- hook-common/src/pgqueue.rs | 187 +++++++++----------------- hook-consumer/src/config.rs | 3 - hook-consumer/src/consumer.rs | 3 +- hook-consumer/src/main.rs | 2 +- hook-janitor/src/config.rs | 3 - hook-janitor/src/main.rs | 1 - hook-janitor/src/webhooks.rs | 44 ++---- hook-producer/src/config.rs | 3 - hook-producer/src/handlers/app.rs | 2 +- hook-producer/src/handlers/webhook.rs | 10 +- hook-producer/src/main.rs | 1 - 11 files changed, 88 insertions(+), 171 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index ec5b6843331dd..04573d53f9a41 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -124,29 +124,25 @@ impl Job { /// /// # Arguments /// - /// * `table`: The table where this job will be marked as completed. /// * `executor`: Any sqlx::Executor that can execute the UPDATE query required to mark this `Job` as completed. - async fn complete<'c, E>(self, table: &str, executor: E) -> Result + async fn complete<'c, E>(self, executor: E) -> Result where E: sqlx::Executor<'c, Database = sqlx::Postgres>, { - let base_query = format!( - r#" + let base_query = r#" UPDATE - "{0}" + "job_queue" SET last_attempt_finished_at = NOW(), status = 'completed'::job_status WHERE - "{0}".id = $2 + "job_queue".id = $2 AND queue = $1 RETURNING - "{0}".* - "#, - table - ); + "job_queue".* + "#; - sqlx::query(&base_query) + sqlx::query(base_query) .bind(&self.queue) .bind(self.id) .execute(executor) @@ -164,37 +160,28 @@ RETURNING /// # Arguments /// /// * `error`: Any JSON-serializable value to be stored as an error. - /// * `table`: The table where this job will be marked as failed. /// * `executor`: Any sqlx::Executor that can execute the UPDATE query required to mark this `Job` as failed. - async fn fail<'c, E, S>( - self, - error: S, - table: &str, - executor: E, - ) -> Result, sqlx::Error> + async fn fail<'c, E, S>(self, error: S, executor: E) -> Result, sqlx::Error> where S: serde::Serialize + std::marker::Sync + std::marker::Send, E: sqlx::Executor<'c, Database = sqlx::Postgres>, { let json_error = sqlx::types::Json(error); - let base_query = format!( - r#" + let base_query = r#" UPDATE - "{0}" + "job_queue" SET last_attempt_finished_at = NOW(), status = 'failed'::job_status - errors = array_append("{0}".errors, $3) + errors = array_append("job_queue".errors, $3) WHERE - "{0}".id = $2 + "job_queue".id = $2 AND queue = $1 RETURNING - "{0}".* - "#, - &table - ); + "job_queue".* + "#; - sqlx::query(&base_query) + sqlx::query(base_query) .bind(&self.queue) .bind(self.id) .bind(&json_error) @@ -230,7 +217,6 @@ pub trait PgQueueJob { #[derive(Debug)] pub struct PgJob { pub job: Job, - pub table: String, pub connection: sqlx::pool::PoolConnection, } @@ -239,7 +225,7 @@ impl PgQueueJob for PgJob { async fn complete(mut self) -> Result>>> { let completed_job = self .job - .complete(&self.table, &mut *self.connection) + .complete(&mut *self.connection) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -255,7 +241,7 @@ impl PgQueueJob for PgJob { ) -> Result, PgJobError>>> { let failed_job = self .job - .fail(error, &self.table, &mut *self.connection) + .fail(error, &mut *self.connection) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -282,7 +268,7 @@ impl PgQueueJob for PgJob { .job .retryable() .queue(queue) - .retry(error, retry_interval, &self.table, &mut *self.connection) + .retry(error, retry_interval, &mut *self.connection) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -298,7 +284,6 @@ impl PgQueueJob for PgJob { #[derive(Debug)] pub struct PgTransactionJob<'c, J, M> { pub job: Job, - pub table: String, pub transaction: sqlx::Transaction<'c, sqlx::postgres::Postgres>, } @@ -309,7 +294,7 @@ impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactio ) -> Result>>> { let completed_job = self .job - .complete(&self.table, &mut *self.transaction) + .complete(&mut *self.transaction) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -333,7 +318,7 @@ impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactio ) -> Result, PgJobError>>> { let failed_job = self .job - .fail(error, &self.table, &mut *self.transaction) + .fail(error, &mut *self.transaction) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -370,7 +355,7 @@ impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactio .job .retryable() .queue(queue) - .retry(error, retry_interval, &self.table, &mut *self.transaction) + .retry(error, retry_interval, &mut *self.transaction) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -422,13 +407,11 @@ impl RetryableJob { /// /// * `error`: Any JSON-serializable value to be stored as an error. /// * `retry_interval`: The duration until the `Job` is to be retried again. Used to set `scheduled_at`. - /// * `table`: The table where this job will be marked as completed. /// * `executor`: Any sqlx::Executor that can execute the UPDATE query required to mark this `Job` as completed. async fn retry<'c, S, E>( self, error: S, retry_interval: time::Duration, - table: &str, executor: E, ) -> Result where @@ -436,26 +419,23 @@ impl RetryableJob { E: sqlx::Executor<'c, Database = sqlx::Postgres>, { let json_error = sqlx::types::Json(error); - let base_query = format!( - r#" + let base_query = r#" UPDATE - "{0}" + "job_queue" SET last_attempt_finished_at = NOW(), - errors = array_append("{0}".errors, $4), + errors = array_append("job_queue".errors, $4), queue = $5, status = 'available'::job_status, scheduled_at = NOW() + $3 WHERE - "{0}".id = $2 + "job_queue".id = $2 AND queue = $1 RETURNING - "{0}".* - "#, - &table - ); + "job_queue".* + "#; - sqlx::query(&base_query) + sqlx::query(base_query) .bind(&self.queue) .bind(self.id) .bind(retry_interval) @@ -466,7 +446,6 @@ RETURNING Ok(RetriedJob { id: self.id, - table: table.to_owned(), queue: self.queue, retry_queue: self.retry_queue.to_owned(), }) @@ -490,7 +469,6 @@ pub struct RetriedJob { /// A unique id identifying a job queue. pub queue: String, pub retry_queue: Option, - pub table: String, } /// State a `Job` is transitioned to after exhausting all of their attempts. @@ -535,8 +513,6 @@ pub struct PgQueue { name: String, /// A connection pool used to connect to the PostgreSQL database. pool: PgPool, - /// The identifier of the PostgreSQL table this queue runs on. - table: String, } pub type PgQueueResult = std::result::Result; @@ -547,16 +523,14 @@ impl PgQueue { /// # Arguments /// /// * `queue_name`: A name for the queue we are going to initialize. - /// * `table_name`: The name for the table the queue will use in PostgreSQL. /// * `url`: A URL pointing to where the PostgreSQL database is hosted. - pub async fn new(queue_name: &str, table_name: &str, url: &str) -> PgQueueResult { + pub async fn new(queue_name: &str, url: &str) -> PgQueueResult { let name = queue_name.to_owned(); - let table = table_name.to_owned(); let pool = PgPoolOptions::new() .connect_lazy(url) .map_err(|error| PgQueueError::PoolCreationError { error })?; - Ok(Self { name, pool, table }) + Ok(Self { name, pool }) } /// Initialize a new PgQueue backed by table in PostgreSQL from a provided connection pool. @@ -564,17 +538,11 @@ impl PgQueue { /// # Arguments /// /// * `queue_name`: A name for the queue we are going to initialize. - /// * `table_name`: The name for the table the queue will use in PostgreSQL. /// * `pool`: A database connection pool to be used by this queue. - pub async fn new_from_pool( - queue_name: &str, - table_name: &str, - pool: PgPool, - ) -> PgQueueResult { + pub async fn new_from_pool(queue_name: &str, pool: PgPool) -> PgQueueResult { let name = queue_name.to_owned(); - let table = table_name.to_owned(); - Ok(Self { name, pool, table }) + Ok(Self { name, pool }) } /// Dequeue a `Job` from this `PgQueue`. @@ -594,13 +562,12 @@ impl PgQueue { // The query that follows uses a FOR UPDATE SKIP LOCKED clause. // For more details on this see: 2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5. - let base_query = format!( - r#" + let base_query = r#" WITH available_in_queue AS ( SELECT id FROM - "{0}" + "job_queue" WHERE status = 'available' AND scheduled_at <= NOW() @@ -612,34 +579,28 @@ WITH available_in_queue AS ( FOR UPDATE SKIP LOCKED ) UPDATE - "{0}" + "job_queue" SET attempted_at = NOW(), status = 'running'::job_status, - attempt = "{0}".attempt + 1, - attempted_by = array_append("{0}".attempted_by, $2::text) + attempt = "job_queue".attempt + 1, + attempted_by = array_append("job_queue".attempted_by, $2::text) FROM available_in_queue WHERE - "{0}".id = available_in_queue.id + "job_queue".id = available_in_queue.id RETURNING - "{0}".* - "#, - &self.table - ); + "job_queue".* + "#; - let query_result: Result, sqlx::Error> = sqlx::query_as(&base_query) + let query_result: Result, sqlx::Error> = sqlx::query_as(base_query) .bind(&self.name) .bind(attempted_by) .fetch_one(&mut *connection) .await; match query_result { - Ok(job) => Ok(Some(PgJob { - job, - table: self.table.to_owned(), - connection, - })), + Ok(job) => Ok(Some(PgJob { job, connection })), // Although connection would be closed once it goes out of scope, sqlx recommends explicitly calling close(). // See: https://docs.rs/sqlx/latest/sqlx/postgres/any/trait.AnyConnectionBackend.html#tymethod.close. @@ -676,13 +637,12 @@ RETURNING // The query that follows uses a FOR UPDATE SKIP LOCKED clause. // For more details on this see: 2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5. - let base_query = format!( - r#" + let base_query = r#" WITH available_in_queue AS ( SELECT id FROM - "{0}" + "job_queue" WHERE status = 'available' AND scheduled_at <= NOW() @@ -694,23 +654,21 @@ WITH available_in_queue AS ( FOR UPDATE SKIP LOCKED ) UPDATE - "{0}" + "job_queue" SET attempted_at = NOW(), status = 'running'::job_status, - attempt = "{0}".attempt + 1, - attempted_by = array_append("{0}".attempted_by, $2::text) + attempt = "job_queue".attempt + 1, + attempted_by = array_append("job_queue".attempted_by, $2::text) FROM available_in_queue WHERE - "{0}".id = available_in_queue.id + "job_queue".id = available_in_queue.id RETURNING - "{0}".* - "#, - &self.table - ); + "job_queue".* + "#; - let query_result: Result, sqlx::Error> = sqlx::query_as(&base_query) + let query_result: Result, sqlx::Error> = sqlx::query_as(base_query) .bind(&self.name) .bind(attempted_by) .fetch_one(&mut *tx) @@ -719,7 +677,6 @@ RETURNING match query_result { Ok(job) => Ok(Some(PgTransactionJob { job, - table: self.table.to_owned(), transaction: tx, })), @@ -742,17 +699,14 @@ RETURNING job: NewJob, ) -> PgQueueResult<()> { // TODO: Escaping. I think sqlx doesn't support identifiers. - let base_query = format!( - r#" -INSERT INTO {} + let base_query = r#" +INSERT INTO job_queue (attempt, created_at, scheduled_at, max_attempts, metadata, parameters, queue, status, target) VALUES (0, NOW(), NOW(), $1, $2, $3, $4, 'available'::job_status, $5) - "#, - &self.table - ); + "#; - sqlx::query(&base_query) + sqlx::query(base_query) .bind(job.max_attempts) .bind(&job.metadata) .bind(&job.parameters) @@ -826,7 +780,7 @@ mod tests { let worker_id = worker_id(); let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); - let queue = PgQueue::new_from_pool("test_can_dequeue_job", "job_queue", db) + let queue = PgQueue::new_from_pool("test_can_dequeue_job", db) .await .expect("failed to connect to local test postgresql database"); @@ -850,7 +804,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn test_dequeue_returns_none_on_no_jobs(db: PgPool) { let worker_id = worker_id(); - let queue = PgQueue::new_from_pool("test_dequeue_returns_none_on_no_jobs", "job_queue", db) + let queue = PgQueue::new_from_pool("test_dequeue_returns_none_on_no_jobs", db) .await .expect("failed to connect to local test postgresql database"); @@ -870,7 +824,7 @@ mod tests { let worker_id = worker_id(); let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); - let queue = PgQueue::new_from_pool("test_can_dequeue_tx_job", "job_queue", db) + let queue = PgQueue::new_from_pool("test_can_dequeue_tx_job", db) .await .expect("failed to connect to local test postgresql database"); @@ -895,10 +849,9 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn test_dequeue_tx_returns_none_on_no_jobs(db: PgPool) { let worker_id = worker_id(); - let queue = - PgQueue::new_from_pool("test_dequeue_tx_returns_none_on_no_jobs", "job_queue", db) - .await - .expect("failed to connect to local test postgresql database"); + let queue = PgQueue::new_from_pool("test_dequeue_tx_returns_none_on_no_jobs", db) + .await + .expect("failed to connect to local test postgresql database"); let tx_job: Option> = queue .dequeue_tx(&worker_id) @@ -915,14 +868,13 @@ mod tests { let job_metadata = JobMetadata::default(); let worker_id = worker_id(); let new_job = NewJob::new(2, job_metadata, job_parameters, &job_target); - let table_name = "job_queue".to_owned(); let queue_name = "test_can_retry_job_with_remaining_attempts".to_owned(); let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)) .queue(&queue_name) .provide(); - let queue = PgQueue::new_from_pool(&queue_name, &table_name, db) + let queue = PgQueue::new_from_pool(&queue_name, db) .await .expect("failed to connect to local test postgresql database"); @@ -969,7 +921,6 @@ mod tests { let job_metadata = JobMetadata::default(); let worker_id = worker_id(); let new_job = NewJob::new(2, job_metadata, job_parameters, &job_target); - let table_name = "job_queue".to_owned(); let queue_name = "test_can_retry_job_to_different_queue".to_owned(); let retry_queue_name = "test_can_retry_job_to_different_queue_retry".to_owned(); @@ -977,7 +928,7 @@ mod tests { .queue(&retry_queue_name) .provide(); - let queue = PgQueue::new_from_pool(&queue_name, &table_name, db.clone()) + let queue = PgQueue::new_from_pool(&queue_name, db.clone()) .await .expect("failed to connect to queue in local test postgresql database"); @@ -1006,7 +957,7 @@ mod tests { assert!(retried_job_not_found.is_none()); - let queue = PgQueue::new_from_pool(&retry_queue_name, &table_name, db) + let queue = PgQueue::new_from_pool(&retry_queue_name, db) .await .expect("failed to connect to retry queue in local test postgresql database"); @@ -1038,13 +989,9 @@ mod tests { let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)).provide(); - let queue = PgQueue::new_from_pool( - "test_cannot_retry_job_without_remaining_attempts", - "job_queue", - db, - ) - .await - .expect("failed to connect to local test postgresql database"); + let queue = PgQueue::new_from_pool("test_cannot_retry_job_without_remaining_attempts", db) + .await + .expect("failed to connect to local test postgresql database"); queue.enqueue(new_job).await.expect("failed to enqueue job"); diff --git a/hook-consumer/src/config.rs b/hook-consumer/src/config.rs index 6525b25253943..01f94e76bd428 100644 --- a/hook-consumer/src/config.rs +++ b/hook-consumer/src/config.rs @@ -34,9 +34,6 @@ pub struct Config { #[envconfig(default = "true")] pub transactional: bool, - - #[envconfig(default = "job_queue")] - pub table_name: String, } impl Config { diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 42cb1ff77537e..671c7b94325d3 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -484,8 +484,7 @@ mod tests { async fn test_wait_for_job(db: PgPool) { let worker_id = worker_id(); let queue_name = "test_wait_for_job".to_string(); - let table_name = "job_queue".to_string(); - let queue = PgQueue::new_from_pool(&queue_name, &table_name, db) + let queue = PgQueue::new_from_pool(&queue_name, db) .await .expect("failed to connect to PG"); diff --git a/hook-consumer/src/main.rs b/hook-consumer/src/main.rs index c71b8eb91a7d0..4182348121bd6 100644 --- a/hook-consumer/src/main.rs +++ b/hook-consumer/src/main.rs @@ -19,7 +19,7 @@ async fn main() -> Result<(), ConsumerError> { .maximum_interval(config.retry_policy.maximum_interval.0) .queue(&config.retry_policy.retry_queue_name) .provide(); - let queue = PgQueue::new(&config.queue_name, &config.table_name, &config.database_url) + let queue = PgQueue::new(&config.queue_name, &config.database_url) .await .expect("failed to initialize queue"); diff --git a/hook-janitor/src/config.rs b/hook-janitor/src/config.rs index c1efb85d38ee5..64db0e613f355 100644 --- a/hook-janitor/src/config.rs +++ b/hook-janitor/src/config.rs @@ -11,9 +11,6 @@ pub struct Config { #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] pub database_url: String, - #[envconfig(default = "job_queue")] - pub table_name: String, - #[envconfig(default = "default")] pub queue_name: String, diff --git a/hook-janitor/src/main.rs b/hook-janitor/src/main.rs index 5de3ec4d93978..7d7e2230e548e 100644 --- a/hook-janitor/src/main.rs +++ b/hook-janitor/src/main.rs @@ -55,7 +55,6 @@ async fn main() { Box::new( WebhookCleaner::new( &config.queue_name, - &config.table_name, &config.database_url, kafka_producer, config.kafka.app_metrics_topic.to_owned(), diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index b14dba1c34b34..18a21c9e5d403 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -44,7 +44,6 @@ type Result = std::result::Result; pub struct WebhookCleaner { queue_name: String, - table_name: String, pg_pool: PgPool, kafka_producer: FutureProducer, app_metrics_topic: String, @@ -133,13 +132,11 @@ struct CleanupStats { impl WebhookCleaner { pub fn new( queue_name: &str, - table_name: &str, database_url: &str, kafka_producer: FutureProducer, app_metrics_topic: String, ) -> Result { let queue_name = queue_name.to_owned(); - let table_name = table_name.to_owned(); let pg_pool = PgPoolOptions::new() .acquire_timeout(Duration::from_secs(10)) .connect_lazy(database_url) @@ -147,7 +144,6 @@ impl WebhookCleaner { Ok(Self { queue_name, - table_name, pg_pool, kafka_producer, app_metrics_topic, @@ -157,17 +153,14 @@ impl WebhookCleaner { #[allow(dead_code)] // This is used in tests. pub fn new_from_pool( queue_name: &str, - table_name: &str, pg_pool: PgPool, kafka_producer: FutureProducer, app_metrics_topic: String, ) -> Result { let queue_name = queue_name.to_owned(); - let table_name = table_name.to_owned(); Ok(Self { queue_name, - table_name, pg_pool, kafka_producer, app_metrics_topic, @@ -196,22 +189,19 @@ impl WebhookCleaner { } async fn get_completed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { - let base_query = format!( - r#" + let base_query = r#" SELECT DATE_TRUNC('hour', last_attempt_finished_at) AS hour, (metadata->>'team_id')::bigint AS team_id, (metadata->>'plugin_config_id')::bigint AS plugin_config_id, count(*) as successes - FROM {0} + FROM job_queue WHERE status = 'completed' AND queue = $1 GROUP BY hour, team_id, plugin_config_id ORDER BY hour, team_id, plugin_config_id; - "#, - self.table_name - ); + "#; - let rows = sqlx::query_as::<_, CompletedRow>(&base_query) + let rows = sqlx::query_as::<_, CompletedRow>(base_query) .bind(&self.queue_name) .fetch_all(&mut *tx.0) .await @@ -221,23 +211,20 @@ impl WebhookCleaner { } async fn get_failed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { - let base_query = format!( - r#" + let base_query = r#" SELECT DATE_TRUNC('hour', last_attempt_finished_at) AS hour, (metadata->>'team_id')::bigint AS team_id, (metadata->>'plugin_config_id')::bigint AS plugin_config_id, errors[array_upper(errors, 1)] AS last_error, count(*) as failures - FROM {0} + FROM job_queue WHERE status = 'failed' AND queue = $1 GROUP BY hour, team_id, plugin_config_id, last_error ORDER BY hour, team_id, plugin_config_id, last_error; - "#, - self.table_name - ); + "#; - let rows = sqlx::query_as::<_, FailedRow>(&base_query) + let rows = sqlx::query_as::<_, FailedRow>(base_query) .bind(&self.queue_name) .fetch_all(&mut *tx.0) .await @@ -292,16 +279,13 @@ impl WebhookCleaner { async fn delete_observed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result { // This DELETE is only safe because we are in serializable isolation mode, see the note // in `start_serializable_txn`. - let base_query = format!( - r#" - DELETE FROM {0} + let base_query = r#" + DELETE FROM job_queue WHERE status IN ('failed', 'completed') AND queue = $1; - "#, - self.table_name - ); + "#; - let result = sqlx::query(&base_query) + let result = sqlx::query(base_query) .bind(&self.queue_name) .execute(&mut *tx.0) .await @@ -460,7 +444,6 @@ mod tests { let webhook_cleaner = WebhookCleaner::new_from_pool( &"webhooks", - &"job_queue", db, mock_producer, APP_METRICS_TOPIC.to_owned(), @@ -642,14 +625,13 @@ mod tests { let (_, mock_producer) = create_mock_kafka().await; let webhook_cleaner = WebhookCleaner::new_from_pool( &"webhooks", - &"job_queue", db.clone(), mock_producer, APP_METRICS_TOPIC.to_owned(), ) .expect("unable to create webhook cleaner"); - let queue = PgQueue::new_from_pool("webhooks", "job_queue", db.clone()) + let queue = PgQueue::new_from_pool("webhooks", db.clone()) .await .expect("failed to connect to local test postgresql database"); diff --git a/hook-producer/src/config.rs b/hook-producer/src/config.rs index 87fad5d07dbb1..8daf04e4ca8b8 100644 --- a/hook-producer/src/config.rs +++ b/hook-producer/src/config.rs @@ -11,9 +11,6 @@ pub struct Config { #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] pub database_url: String, - #[envconfig(default = "job_queue")] - pub table_name: String, - #[envconfig(default = "default")] pub queue_name: String, } diff --git a/hook-producer/src/handlers/app.rs b/hook-producer/src/handlers/app.rs index e588dbdb06007..b4d099c96f241 100644 --- a/hook-producer/src/handlers/app.rs +++ b/hook-producer/src/handlers/app.rs @@ -38,7 +38,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn index(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", "job_queue", db) + let pg_queue = PgQueue::new_from_pool("test_index", db) .await .expect("failed to construct pg_queue"); diff --git a/hook-producer/src/handlers/webhook.rs b/hook-producer/src/handlers/webhook.rs index 72923b2441063..ab1552cf71033 100644 --- a/hook-producer/src/handlers/webhook.rs +++ b/hook-producer/src/handlers/webhook.rs @@ -119,7 +119,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn webhook_success(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", "job_queue", db) + let pg_queue = PgQueue::new_from_pool("test_index", db) .await .expect("failed to construct pg_queue"); @@ -163,7 +163,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn webhook_bad_url(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", "job_queue", db) + let pg_queue = PgQueue::new_from_pool("test_index", db) .await .expect("failed to construct pg_queue"); @@ -202,7 +202,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn webhook_payload_missing_fields(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", "job_queue", db) + let pg_queue = PgQueue::new_from_pool("test_index", db) .await .expect("failed to construct pg_queue"); @@ -225,7 +225,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn webhook_payload_not_json(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", "job_queue", db) + let pg_queue = PgQueue::new_from_pool("test_index", db) .await .expect("failed to construct pg_queue"); @@ -248,7 +248,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn webhook_payload_body_too_large(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", "job_queue", db) + let pg_queue = PgQueue::new_from_pool("test_index", db) .await .expect("failed to construct pg_queue"); diff --git a/hook-producer/src/main.rs b/hook-producer/src/main.rs index 39f45004271bc..d0190c2a37a54 100644 --- a/hook-producer/src/main.rs +++ b/hook-producer/src/main.rs @@ -27,7 +27,6 @@ async fn main() { // TODO: Coupling the queue name to the PgQueue object doesn't seem ideal from the producer // side, but we don't need more than one queue for now. &config.queue_name, - &config.table_name, &config.database_url, ) .await From ead1bf28ef8d092848cceedf8e2ae8867b02276d Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 9 Jan 2024 02:13:30 -0700 Subject: [PATCH 164/247] Minor query syntax/ordering cleanup (#26) --- hook-common/src/pgqueue.rs | 56 +++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 04573d53f9a41..35f5b4a9e47fb 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -131,15 +131,15 @@ impl Job { { let base_query = r#" UPDATE - "job_queue" + job_queue SET last_attempt_finished_at = NOW(), status = 'completed'::job_status WHERE - "job_queue".id = $2 - AND queue = $1 + queue = $1 + AND id = $2 RETURNING - "job_queue".* + job_queue.* "#; sqlx::query(base_query) @@ -169,16 +169,16 @@ RETURNING let json_error = sqlx::types::Json(error); let base_query = r#" UPDATE - "job_queue" + job_queue SET last_attempt_finished_at = NOW(), status = 'failed'::job_status - errors = array_append("job_queue".errors, $3) + errors = array_append(errors, $3) WHERE - "job_queue".id = $2 - AND queue = $1 + queue = $1 + AND id = $2 RETURNING - "job_queue".* + job_queue.* "#; sqlx::query(base_query) @@ -421,18 +421,18 @@ impl RetryableJob { let json_error = sqlx::types::Json(error); let base_query = r#" UPDATE - "job_queue" + job_queue SET last_attempt_finished_at = NOW(), - errors = array_append("job_queue".errors, $4), - queue = $5, status = 'available'::job_status, - scheduled_at = NOW() + $3 + scheduled_at = NOW() + $3, + errors = array_append(errors, $4), + queue = $5 WHERE - "job_queue".id = $2 - AND queue = $1 + queue = $1 + AND id = $2 RETURNING - "job_queue".* + job_queue.* "#; sqlx::query(base_query) @@ -567,7 +567,7 @@ WITH available_in_queue AS ( SELECT id FROM - "job_queue" + job_queue WHERE status = 'available' AND scheduled_at <= NOW() @@ -579,18 +579,18 @@ WITH available_in_queue AS ( FOR UPDATE SKIP LOCKED ) UPDATE - "job_queue" + job_queue SET attempted_at = NOW(), status = 'running'::job_status, - attempt = "job_queue".attempt + 1, - attempted_by = array_append("job_queue".attempted_by, $2::text) + attempt = attempt + 1, + attempted_by = array_append(attempted_by, $2::text) FROM available_in_queue WHERE - "job_queue".id = available_in_queue.id + job_queue.id = available_in_queue.id RETURNING - "job_queue".* + job_queue.* "#; let query_result: Result, sqlx::Error> = sqlx::query_as(base_query) @@ -642,7 +642,7 @@ WITH available_in_queue AS ( SELECT id FROM - "job_queue" + job_queue WHERE status = 'available' AND scheduled_at <= NOW() @@ -654,18 +654,18 @@ WITH available_in_queue AS ( FOR UPDATE SKIP LOCKED ) UPDATE - "job_queue" + job_queue SET attempted_at = NOW(), status = 'running'::job_status, - attempt = "job_queue".attempt + 1, - attempted_by = array_append("job_queue".attempted_by, $2::text) + attempt = attempt + 1, + attempted_by = array_append(attempted_by, $2::text) FROM available_in_queue WHERE - "job_queue".id = available_in_queue.id + job_queue.id = available_in_queue.id RETURNING - "job_queue".* + job_queue.* "#; let query_result: Result, sqlx::Error> = sqlx::query_as(base_query) From 892d30b5c53fc4f4a724997f1fc370da93c88db7 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 9 Jan 2024 09:23:09 -0700 Subject: [PATCH 165/247] Add metrics, fix metrics endpoint, bump deps (#27) --- Cargo.lock | 271 +++++++++++--------------- Cargo.toml | 4 +- hook-common/src/metrics.rs | 9 +- hook-consumer/src/consumer.rs | 19 +- hook-janitor/src/webhooks.rs | 17 +- hook-producer/src/handlers/app.rs | 16 +- hook-producer/src/handlers/mod.rs | 2 +- hook-producer/src/handlers/webhook.rs | 20 +- hook-producer/src/main.rs | 7 +- 9 files changed, 172 insertions(+), 193 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdef24de2af58..e4da8161adcc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", "getrandom", @@ -62,13 +62,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -104,9 +104,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202651474fe73c62d9e0a56c6133f7a0ff1dc1c8cf7a5b03381af2a26553ac9d" +checksum = "d09dbe0e490df5da9d69b36dca48a76635288a82f92eca90024883a56202026d" dependencies = [ "async-trait", "axum-core", @@ -133,13 +133,14 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] name = "axum-core" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77cb22c689c44d4c07b0ab44ebc25d69d8ae601a2f28fb8d672d344178fa17aa" +checksum = "e87c8503f93e6d144ee5690907ba22db7ba79ab001a932ab99034f0fe836b3df" dependencies = [ "async-trait", "bytes", @@ -153,6 +154,7 @@ dependencies = [ "sync_wrapper", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -172,9 +174,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.5" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "c79fed4cdb43e993fcdadc7e58a09fd0e3e649c4436fa11da71c9f1f3ee7feb9" [[package]] name = "base64ct" @@ -286,9 +288,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -310,34 +312,27 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crossbeam-epoch" -version = "0.9.16" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", ] [[package]] name = "crossbeam-queue" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crypto-common" @@ -514,9 +509,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -529,9 +524,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -539,15 +534,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -567,38 +562,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -660,9 +655,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d308f63daf4181410c242d34c11f928dcb3aa105852019e043c9d1f4e4368a" +checksum = "991910e35c615d8cab86b5ab04be67e6ad24d2bf5f4f11fdbbed26da999bbeab" dependencies = [ "bytes", "fnv", @@ -948,7 +943,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.0", + "h2 0.4.1", "http 1.0.0", "http-body 1.0.0", "httparse", @@ -973,9 +968,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d" +checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" dependencies = [ "bytes", "futures-channel", @@ -986,16 +981,14 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", - "tower-service", "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1091,9 +1084,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libm" @@ -1114,9 +1107,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "5f526fdd09d99e19742883e43de41e1aa9e36db0c7ab7f935165d611c5cccc66" dependencies = [ "cc", "libc", @@ -1146,15 +1139,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "mach2" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" -dependencies = [ - "libc", -] - [[package]] name = "matchit" version = "0.7.3" @@ -1173,38 +1157,29 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "metrics" -version = "0.21.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +checksum = "77b9e10a211c839210fd7f99954bda26e5f8e26ec686ad68da6a32df7c80e782" dependencies = [ "ahash", - "metrics-macros", "portable-atomic", ] [[package]] name = "metrics-exporter-prometheus" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" +checksum = "83a4c4718a371ddfb7806378f23617876eea8b82e5ff1324516bcd283249d9ea" dependencies = [ "base64", "hyper 0.14.28", + "hyper-tls", "indexmap 1.9.3", "ipnet", "metrics", @@ -1215,22 +1190,11 @@ dependencies = [ "tracing", ] -[[package]] -name = "metrics-macros" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.41", -] - [[package]] name = "metrics-util" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" +checksum = "2670b8badcc285d486261e2e9f1615b506baff91427b61bd336a472b65bbf5ed" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1403,9 +1367,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -1418,9 +1382,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.61" +version = "0.10.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" +checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -1439,7 +1403,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -1450,9 +1414,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.97" +version = "0.9.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" dependencies = [ "cc", "libc", @@ -1527,7 +1491,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -1565,9 +1529,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" [[package]] name = "portable-atomic" @@ -1593,22 +1557,21 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] [[package]] name = "quanta" -version = "0.11.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +checksum = "9ca0b7bac0b97248c40bb77288fc52029cf1459c0461ea1b05ee32ccf011de2c" dependencies = [ "crossbeam-utils", "libc", - "mach2", "once_cell", "raw-cpuid", "wasi", @@ -1618,9 +1581,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -1657,11 +1620,11 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "10.7.0" +version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.1", ] [[package]] @@ -1826,11 +1789,11 @@ checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1864,29 +1827,29 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -1895,9 +1858,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" dependencies = [ "itoa", "serde", @@ -2268,9 +2231,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.41" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -2306,35 +2269,35 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -2364,9 +2327,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -2389,7 +2352,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -2492,7 +2455,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] @@ -2657,7 +2620,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -2691,7 +2654,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2742,11 +2705,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -2883,9 +2846,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.30" +version = "0.5.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5" +checksum = "b7520bbdec7211caa7c4e682eb1fbe07abe20cee6756b6e00f537c82c11816aa" dependencies = [ "memchr", ] @@ -2902,22 +2865,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.31" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.31" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.48", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1f6a38b6e3ed1..b7cb8fac622c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,8 +12,8 @@ eyre = "0.6.9" futures = { version = "0.3.29" } http = { version = "0.2" } http-body-util = "0.1.0" -metrics = "0.21.1" -metrics-exporter-prometheus = "0.12.1" +metrics = "0.22.0" +metrics-exporter-prometheus = "0.13.0" rdkafka = { version = "0.35.0", features = ["cmake-build", "ssl", "tracing"] } reqwest = { version = "0.11" } regex = "1.10.2" diff --git a/hook-common/src/metrics.rs b/hook-common/src/metrics.rs index 3d9e4c02655a5..1f57c5eec51f9 100644 --- a/hook-common/src/metrics.rs +++ b/hook-common/src/metrics.rs @@ -21,7 +21,10 @@ pub fn setup_metrics_router() -> Router { let recorder_handle = setup_metrics_recorder(); Router::new() - .route("/metrics", get(recorder_handle.render())) + .route( + "/metrics", + get(move || std::future::ready(recorder_handle.render())), + ) .layer(axum::middleware::from_fn(track_metrics)) } @@ -63,8 +66,8 @@ pub async fn track_metrics(req: Request, next: Next) -> impl IntoResponse ("status", status), ]; - metrics::increment_counter!("http_requests_total", &labels); - metrics::histogram!("http_requests_duration_seconds", latency, &labels); + metrics::counter!("http_requests_total", &labels).increment(1); + metrics::histogram!("http_requests_duration_seconds", &labels).record(latency); response } diff --git a/hook-consumer/src/consumer.rs b/hook-consumer/src/consumer.rs index 671c7b94325d3..2114ef80c8bc8 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-consumer/src/consumer.rs @@ -192,7 +192,7 @@ async fn spawn_webhook_job_processing_task( ("target", webhook_job.target()), ]; - metrics::increment_counter!("webhook_jobs_total", &labels); + metrics::counter!("webhook_jobs_total", &labels).increment(1); tokio::spawn(async move { let result = process_webhook_job(client, webhook_job, &retry_policy).await; @@ -247,8 +247,9 @@ async fn process_webhook_job( .await .map_err(|error| ConsumerError::PgJobError(error.to_string()))?; - metrics::increment_counter!("webhook_jobs_completed", &labels); - metrics::histogram!("webhook_jobs_processing_duration_seconds", elapsed, &labels); + metrics::counter!("webhook_jobs_completed", &labels).increment(1); + metrics::histogram!("webhook_jobs_processing_duration_seconds", &labels) + .record(elapsed); Ok(()) } @@ -258,7 +259,7 @@ async fn process_webhook_job( .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; - metrics::increment_counter!("webhook_jobs_failed", &labels); + metrics::counter!("webhook_jobs_failed", &labels).increment(1); Ok(()) } @@ -268,7 +269,7 @@ async fn process_webhook_job( .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; - metrics::increment_counter!("webhook_jobs_failed", &labels); + metrics::counter!("webhook_jobs_failed", &labels).increment(1); Ok(()) } @@ -278,7 +279,7 @@ async fn process_webhook_job( .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; - metrics::increment_counter!("webhook_jobs_failed", &labels); + metrics::counter!("webhook_jobs_failed", &labels).increment(1); Ok(()) } @@ -293,7 +294,7 @@ async fn process_webhook_job( .await { Ok(_) => { - metrics::increment_counter!("webhook_jobs_retried", &labels); + metrics::counter!("webhook_jobs_retried", &labels).increment(1); Ok(()) } @@ -305,7 +306,7 @@ async fn process_webhook_job( .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; - metrics::increment_counter!("webhook_jobs_failed", &labels); + metrics::counter!("webhook_jobs_failed", &labels).increment(1); Ok(()) } @@ -318,7 +319,7 @@ async fn process_webhook_job( .await .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; - metrics::increment_counter!("webhook_jobs_failed", &labels); + metrics::counter!("webhook_jobs_failed", &labels).increment(1); Ok(()) } diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 18a21c9e5d403..228187a735433 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::time::{Duration, Instant}; use async_trait::async_trait; use chrono::{DateTime, Utc}; @@ -350,9 +350,23 @@ impl WebhookCleaner { #[async_trait] impl Cleaner for WebhookCleaner { async fn cleanup(&self) { + let start_time = Instant::now(); + match self.cleanup_impl().await { Ok(stats) => { + metrics::counter!("webhook_cleanup_runs",).increment(1); + if stats.rows_processed > 0 { + let elapsed_time = start_time.elapsed().as_secs_f64(); + metrics::histogram!("webhook_cleanup_duration").record(elapsed_time); + + metrics::counter!("webhook_cleanup_rows_processed",) + .increment(stats.rows_processed); + metrics::counter!("webhook_cleanup_completed_agg_row_count",) + .increment(stats.completed_agg_row_count as u64); + metrics::counter!("webhook_cleanup_failed_agg_row_count",) + .increment(stats.failed_agg_row_count as u64); + debug!( rows_processed = stats.rows_processed, completed_agg_row_count = stats.completed_agg_row_count, @@ -364,6 +378,7 @@ impl Cleaner for WebhookCleaner { } } Err(error) => { + metrics::counter!("webhook_cleanup_failures",).increment(1); error!(error = ?error, "WebhookCleaner::cleanup failed"); } } diff --git a/hook-producer/src/handlers/app.rs b/hook-producer/src/handlers/app.rs index b4d099c96f241..2cafc4a497ae2 100644 --- a/hook-producer/src/handlers/app.rs +++ b/hook-producer/src/handlers/app.rs @@ -1,23 +1,13 @@ use axum::{routing, Router}; -use metrics_exporter_prometheus::PrometheusHandle; -use hook_common::metrics; use hook_common::pgqueue::PgQueue; use super::webhook; -pub fn app(pg_pool: PgQueue, metrics: Option) -> Router { - Router::new() +pub fn add_routes(router: Router, pg_pool: PgQueue) -> Router { + router .route("/", routing::get(index)) - .route( - "/metrics", - routing::get(move || match metrics { - Some(ref recorder_handle) => std::future::ready(recorder_handle.render()), - None => std::future::ready("no metrics recorder installed".to_owned()), - }), - ) .route("/webhook", routing::post(webhook::post).with_state(pg_pool)) - .layer(axum::middleware::from_fn(metrics::track_metrics)) } pub async fn index() -> &'static str { @@ -42,7 +32,7 @@ mod tests { .await .expect("failed to construct pg_queue"); - let app = app(pg_queue, None); + let app = add_routes(Router::new(), pg_queue); let response = app .oneshot(Request::builder().uri("/").body(Body::empty()).unwrap()) diff --git a/hook-producer/src/handlers/mod.rs b/hook-producer/src/handlers/mod.rs index 88f96717c130a..e392f8a9b67f7 100644 --- a/hook-producer/src/handlers/mod.rs +++ b/hook-producer/src/handlers/mod.rs @@ -1,4 +1,4 @@ mod app; mod webhook; -pub use app::app; +pub use app::add_routes; diff --git a/hook-producer/src/handlers/webhook.rs b/hook-producer/src/handlers/webhook.rs index ab1552cf71033..62a4aaa8833c8 100644 --- a/hook-producer/src/handlers/webhook.rs +++ b/hook-producer/src/handlers/webhook.rs @@ -1,3 +1,5 @@ +use std::time::Instant; + use axum::{extract::State, http::StatusCode, Json}; use hook_common::webhook::{WebhookJobMetadata, WebhookJobParameters}; use serde_derive::Deserialize; @@ -61,8 +63,13 @@ pub async fn post( url_hostname.as_str(), ); + let start_time = Instant::now(); + pg_queue.enqueue(job).await.map_err(internal_error)?; + let elapsed_time = start_time.elapsed().as_secs_f64(); + metrics::histogram!("webhook_producer_enqueue").record(elapsed_time); + Ok(Json(WebhookPostResponse { error: None })) } @@ -107,6 +114,7 @@ mod tests { use axum::{ body::Body, http::{self, Request, StatusCode}, + Router, }; use hook_common::pgqueue::PgQueue; use hook_common::webhook::{HttpMethod, WebhookJobParameters}; @@ -115,7 +123,7 @@ mod tests { use std::collections; use tower::ServiceExt; // for `call`, `oneshot`, and `ready` - use crate::handlers::app; + use crate::handlers::app::add_routes; #[sqlx::test(migrations = "../migrations")] async fn webhook_success(db: PgPool) { @@ -123,7 +131,7 @@ mod tests { .await .expect("failed to construct pg_queue"); - let app = app(pg_queue, None); + let app = add_routes(Router::new(), pg_queue); let mut headers = collections::HashMap::new(); headers.insert("Content-Type".to_owned(), "application/json".to_owned()); @@ -167,7 +175,7 @@ mod tests { .await .expect("failed to construct pg_queue"); - let app = app(pg_queue, None); + let app = add_routes(Router::new(), pg_queue); let response = app .oneshot( @@ -206,7 +214,7 @@ mod tests { .await .expect("failed to construct pg_queue"); - let app = app(pg_queue, None); + let app = add_routes(Router::new(), pg_queue); let response = app .oneshot( @@ -229,7 +237,7 @@ mod tests { .await .expect("failed to construct pg_queue"); - let app = app(pg_queue, None); + let app = add_routes(Router::new(), pg_queue); let response = app .oneshot( @@ -252,7 +260,7 @@ mod tests { .await .expect("failed to construct pg_queue"); - let app = app(pg_queue, None); + let app = add_routes(Router::new(), pg_queue); let bytes: Vec = vec![b'a'; 1_000_000 * 2]; let long_string = String::from_utf8_lossy(&bytes); diff --git a/hook-producer/src/main.rs b/hook-producer/src/main.rs index d0190c2a37a54..f078f2fae7781 100644 --- a/hook-producer/src/main.rs +++ b/hook-producer/src/main.rs @@ -3,7 +3,7 @@ use config::Config; use envconfig::Envconfig; use eyre::Result; -use hook_common::metrics; +use hook_common::metrics::setup_metrics_router; use hook_common::pgqueue::PgQueue; mod config; @@ -32,9 +32,8 @@ async fn main() { .await .expect("failed to initialize queue"); - let recorder_handle = metrics::setup_metrics_recorder(); - - let app = handlers::app(pg_queue, Some(recorder_handle)); + let router = setup_metrics_router(); + let app = handlers::add_routes(router, pg_queue); match listen(app, config.bind()).await { Ok(_) => {} From 35d7e331aee28fa5cf2f76196d5664b6d1620492 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 9 Jan 2024 09:33:54 -0700 Subject: [PATCH 166/247] Get total row counts in janitor for metrics and consistency checking (#28) --- hook-janitor/src/webhooks.rs | 114 ++++++++++++++++++++++++++++------- 1 file changed, 92 insertions(+), 22 deletions(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 228187a735433..de02d07a2520e 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -9,7 +9,7 @@ use rdkafka::producer::{FutureProducer, FutureRecord}; use serde_json::error::Error as SerdeError; use sqlx::postgres::{PgPool, PgPoolOptions, Postgres}; use sqlx::types::{chrono, Uuid}; -use sqlx::Transaction; +use sqlx::{Row, Transaction}; use thiserror::Error; use tracing::{debug, error}; @@ -24,6 +24,8 @@ pub enum WebhookCleanerError { PoolCreationError { error: sqlx::Error }, #[error("failed to acquire conn and start txn: {error}")] StartTxnError { error: sqlx::Error }, + #[error("failed to get row count: {error}")] + GetRowCountError { error: sqlx::Error }, #[error("failed to get completed rows: {error}")] GetCompletedRowsError { error: sqlx::Error }, #[error("failed to get failed rows: {error}")] @@ -36,6 +38,10 @@ pub enum WebhookCleanerError { KafkaProduceCanceled, #[error("failed to delete rows: {error}")] DeleteRowsError { error: sqlx::Error }, + #[error("attempted to delete a different number of rows than expected")] + DeleteConsistencyError, + #[error("failed to rollback txn: {error}")] + RollbackTxnError { error: sqlx::Error }, #[error("failed to commit txn: {error}")] CommitTxnError { error: sqlx::Error }, } @@ -125,8 +131,10 @@ struct SerializableTxn<'a>(Transaction<'a, Postgres>); struct CleanupStats { rows_processed: u64, - completed_agg_row_count: usize, - failed_agg_row_count: usize, + completed_row_count: u64, + completed_agg_row_count: u64, + failed_row_count: u64, + failed_agg_row_count: u64, } impl WebhookCleaner { @@ -188,7 +196,32 @@ impl WebhookCleaner { Ok(SerializableTxn(tx)) } - async fn get_completed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { + async fn get_row_count_for_status( + &self, + tx: &mut SerializableTxn<'_>, + status: &str, + ) -> Result { + let base_query = r#" + SELECT count(*) FROM job_queue + WHERE queue = $1 + AND status = $2::job_status; + "#; + + let count: i64 = sqlx::query(base_query) + .bind(&self.queue_name) + .bind(status) + .fetch_one(&mut *tx.0) + .await + .map_err(|e| WebhookCleanerError::GetRowCountError { error: e })? + .get(0); + + Ok(count as u64) + } + + async fn get_completed_agg_rows( + &self, + tx: &mut SerializableTxn<'_>, + ) -> Result> { let base_query = r#" SELECT DATE_TRUNC('hour', last_attempt_finished_at) AS hour, (metadata->>'team_id')::bigint AS team_id, @@ -210,7 +243,7 @@ impl WebhookCleaner { Ok(rows) } - async fn get_failed_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { + async fn get_failed_agg_rows(&self, tx: &mut SerializableTxn<'_>) -> Result> { let base_query = r#" SELECT DATE_TRUNC('hour', last_attempt_finished_at) AS hour, (metadata->>'team_id')::bigint AS team_id, @@ -294,6 +327,14 @@ impl WebhookCleaner { Ok(result.rows_affected()) } + async fn rollback_txn(&self, tx: SerializableTxn<'_>) -> Result<()> { + tx.0.rollback() + .await + .map_err(|e| WebhookCleanerError::RollbackTxnError { error: e })?; + + Ok(()) + } + async fn commit_txn(&self, tx: SerializableTxn<'_>) -> Result<()> { tx.0.commit() .await @@ -315,33 +356,53 @@ impl WebhookCleaner { let mut tx = self.start_serializable_txn().await?; - let completed_agg_row_count = { - let completed_rows = self.get_completed_rows(&mut tx).await?; - let row_count = completed_rows.len(); + let (completed_row_count, completed_agg_row_count) = { + let completed_row_count = self.get_row_count_for_status(&mut tx, "completed").await?; + let completed_agg_rows = self.get_completed_agg_rows(&mut tx).await?; + let agg_row_count = completed_agg_rows.len() as u64; let completed_app_metrics: Vec = - completed_rows.into_iter().map(Into::into).collect(); + completed_agg_rows.into_iter().map(Into::into).collect(); self.send_metrics_to_kafka(completed_app_metrics).await?; - row_count + (completed_row_count, agg_row_count) }; - let failed_agg_row_count = { - let failed_rows = self.get_failed_rows(&mut tx).await?; - let row_count = failed_rows.len(); + let (failed_row_count, failed_agg_row_count) = { + let failed_row_count = self.get_row_count_for_status(&mut tx, "failed").await?; + let failed_agg_rows = self.get_failed_agg_rows(&mut tx).await?; + let agg_row_count = failed_agg_rows.len() as u64; let failed_app_metrics: Vec = - failed_rows.into_iter().map(Into::into).collect(); + failed_agg_rows.into_iter().map(Into::into).collect(); self.send_metrics_to_kafka(failed_app_metrics).await?; - row_count + (failed_row_count, agg_row_count) }; - let mut rows_processed = 0; + let mut rows_deleted = 0; if completed_agg_row_count + failed_agg_row_count != 0 { - rows_processed = self.delete_observed_rows(&mut tx).await?; + rows_deleted = self.delete_observed_rows(&mut tx).await?; + + if rows_deleted != completed_row_count + failed_row_count { + // This should never happen, but if it does, we want to know about it (and abort the + // txn). + error!( + attempted_rows_deleted = rows_deleted, + completed_row_count = completed_row_count, + failed_row_count = failed_row_count, + "WebhookCleaner::cleanup attempted to delete a different number of rows than expected" + ); + + self.rollback_txn(tx).await?; + + return Err(WebhookCleanerError::DeleteConsistencyError); + } + self.commit_txn(tx).await?; } Ok(CleanupStats { - rows_processed, + rows_processed: rows_deleted, + completed_row_count, completed_agg_row_count, + failed_row_count, failed_agg_row_count, }) } @@ -362,14 +423,20 @@ impl Cleaner for WebhookCleaner { metrics::counter!("webhook_cleanup_rows_processed",) .increment(stats.rows_processed); + metrics::counter!("webhook_cleanup_completed_row_count",) + .increment(stats.completed_row_count); metrics::counter!("webhook_cleanup_completed_agg_row_count",) - .increment(stats.completed_agg_row_count as u64); + .increment(stats.completed_agg_row_count); + metrics::counter!("webhook_cleanup_failed_row_count",) + .increment(stats.failed_row_count); metrics::counter!("webhook_cleanup_failed_agg_row_count",) - .increment(stats.failed_agg_row_count as u64); + .increment(stats.failed_agg_row_count); debug!( rows_processed = stats.rows_processed, + completed_row_count = stats.completed_row_count, completed_agg_row_count = stats.completed_agg_row_count, + failed_row_count = stats.failed_row_count, failed_agg_row_count = stats.failed_agg_row_count, "WebhookCleaner::cleanup finished" ); @@ -665,8 +732,11 @@ mod tests { // Important! Serializable txn is started here. let mut tx = webhook_cleaner.start_serializable_txn().await.unwrap(); - webhook_cleaner.get_completed_rows(&mut tx).await.unwrap(); - webhook_cleaner.get_failed_rows(&mut tx).await.unwrap(); + webhook_cleaner + .get_completed_agg_rows(&mut tx) + .await + .unwrap(); + webhook_cleaner.get_failed_agg_rows(&mut tx).await.unwrap(); // All 13 rows in the queue are visible from outside the txn. // The 11 the cleaner will process, plus 1 available and 1 running. From b00ef383563ffe22ec40e5607e57863c65b1560b Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 9 Jan 2024 10:23:40 -0700 Subject: [PATCH 167/247] Login to DockerHub in CI (#74) * Login to DockerHub * Add to rust.yml --- .github/workflows/docker.yml | 8 +++++++- .github/workflows/rust.yml | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 77e731ecebcf2..1fdc6520bd004 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -37,7 +37,13 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io uses: docker/login-action@v2 with: registry: ghcr.io diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 91cf2018ee7e4..de8ec08916e85 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -40,6 +40,12 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Setup end2end dependencies run: | docker compose up -d --wait From 86e6a26ef049f522af5cd0774a92328b8c7161ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Tue, 9 Jan 2024 18:24:11 +0100 Subject: [PATCH 168/247] refactor: Rename producer and consumer images (#29) --- ...ocker-hook-producer.yml => docker-hook-api.yml} | 14 +++++++------- ...er-hook-consumer.yml => docker-hook-worker.yml} | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) rename .github/workflows/{docker-hook-producer.yml => docker-hook-api.yml} (80%) rename .github/workflows/{docker-hook-consumer.yml => docker-hook-worker.yml} (80%) diff --git a/.github/workflows/docker-hook-producer.yml b/.github/workflows/docker-hook-api.yml similarity index 80% rename from .github/workflows/docker-hook-producer.yml rename to .github/workflows/docker-hook-api.yml index ec2594f1f3d46..8c645bc4696fd 100644 --- a/.github/workflows/docker-hook-producer.yml +++ b/.github/workflows/docker-hook-api.yml @@ -1,4 +1,4 @@ -name: Build hook-producer docker image +name: Build hook-api docker image on: workflow_dispatch: @@ -11,7 +11,7 @@ permissions: jobs: build: - name: build and publish hook-producer image + name: build and publish hook-api image runs-on: buildjet-4vcpu-ubuntu-2204-arm steps: @@ -25,7 +25,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/posthog/hook-producer + images: ghcr.io/posthog/hook-api tags: | type=ref,event=pr type=ref,event=branch @@ -44,8 +44,8 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push producer - id: docker_build_hook_producer + - name: Build and push api + id: docker_build_hook_api uses: docker/build-push-action@v4 with: context: ./ @@ -59,5 +59,5 @@ jobs: cache-to: type=gha,mode=max build-args: BIN=hook-producer - - name: Hook-producer image digest - run: echo ${{ steps.docker_build_hook_producer.outputs.digest }} + - name: Hook-api image digest + run: echo ${{ steps.docker_build_hook_api.outputs.digest }} diff --git a/.github/workflows/docker-hook-consumer.yml b/.github/workflows/docker-hook-worker.yml similarity index 80% rename from .github/workflows/docker-hook-consumer.yml rename to .github/workflows/docker-hook-worker.yml index db920744f145c..a5d6c923b57d1 100644 --- a/.github/workflows/docker-hook-consumer.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -1,4 +1,4 @@ -name: Build hook-consumer docker image +name: Build hook-worker docker image on: workflow_dispatch: @@ -11,7 +11,7 @@ permissions: jobs: build: - name: build and publish hook-consumer image + name: build and publish hook-worker image runs-on: buildjet-4vcpu-ubuntu-2204-arm steps: @@ -25,7 +25,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/posthog/hook-consumer + images: ghcr.io/posthog/hook-worker tags: | type=ref,event=pr type=ref,event=branch @@ -44,8 +44,8 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push consumer - id: docker_build_hook_consumer + - name: Build and push worker + id: docker_build_hook_worker uses: docker/build-push-action@v4 with: context: ./ @@ -59,5 +59,5 @@ jobs: cache-to: type=gha,mode=max build-args: BIN=hook-consumer - - name: Hook-consumer image digest - run: echo ${{ steps.docker_build_hook_consumer.outputs.digest }} + - name: Hook-worker image digest + run: echo ${{ steps.docker_build_hook_worker.outputs.digest }} From 29c2415466665b48c12b0dfa09b601b2006ef51f Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 9 Jan 2024 10:35:24 -0700 Subject: [PATCH 169/247] Login to DockerHub in CI (#30) --- .github/workflows/docker-hook-api.yml | 8 +++++++- .github/workflows/docker-hook-janitor.yml | 8 +++++++- .github/workflows/docker-hook-worker.yml | 8 +++++++- .github/workflows/docker-migrator.yml | 8 +++++++- .github/workflows/rust.yml | 6 ++++++ 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index 8c645bc4696fd..abcd9f654a3a5 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -37,7 +37,13 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io uses: docker/login-action@v2 with: registry: ghcr.io diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index b426d51a5f526..1f4cae924ff72 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -37,7 +37,13 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io uses: docker/login-action@v2 with: registry: ghcr.io diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index a5d6c923b57d1..25e4b3a92db8f 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -37,7 +37,13 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io uses: docker/login-action@v2 with: registry: ghcr.io diff --git a/.github/workflows/docker-migrator.yml b/.github/workflows/docker-migrator.yml index 73d7afa20cb62..1ad3893c8386a 100644 --- a/.github/workflows/docker-migrator.yml +++ b/.github/workflows/docker-migrator.yml @@ -37,7 +37,13 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io uses: docker/login-action@v2 with: registry: ghcr.io diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b811c1a52e668..f3aafb00260b4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -43,6 +43,12 @@ jobs: with: toolchain: stable + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Stop/Start stack with Docker Compose shell: bash run: | From 2499ea6a1f63f0584403462869b181364d0a9db6 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 9 Jan 2024 12:26:46 -0700 Subject: [PATCH 170/247] Login to Docker before QEMU (#32) --- .github/workflows/docker-hook-api.yml | 26 +++++++++++------------ .github/workflows/docker-hook-janitor.yml | 26 +++++++++++------------ .github/workflows/docker-hook-worker.yml | 26 +++++++++++------------ .github/workflows/docker-migrator.yml | 26 +++++++++++------------ 4 files changed, 52 insertions(+), 52 deletions(-) diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index abcd9f654a3a5..5ae94f5531036 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -17,6 +17,19 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -37,19 +50,6 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push api id: docker_build_hook_api uses: docker/build-push-action@v4 diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index 1f4cae924ff72..12b9ddeab423a 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -18,6 +18,19 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -37,19 +50,6 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push janitor id: docker_build_hook_janitor uses: docker/build-push-action@v4 diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index 25e4b3a92db8f..052629100ce5f 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -17,6 +17,19 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -37,19 +50,6 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push worker id: docker_build_hook_worker uses: docker/build-push-action@v4 diff --git a/.github/workflows/docker-migrator.yml b/.github/workflows/docker-migrator.yml index 1ad3893c8386a..d239da49afb2d 100644 --- a/.github/workflows/docker-migrator.yml +++ b/.github/workflows/docker-migrator.yml @@ -18,6 +18,19 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -37,19 +50,6 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push migrator id: docker_build_hook_migrator uses: docker/build-push-action@v4 From 9cd11327f899b8c12b6e67a4529110d9600dce22 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 10 Jan 2024 02:53:42 -0700 Subject: [PATCH 171/247] Login to DockerHub (#75) --- .github/workflows/docker.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1fdc6520bd004..a2180db21e4c0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,6 +18,19 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to ghcr.io + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -37,19 +50,6 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push id: docker_build uses: docker/build-push-action@v4 From 46f9f93bb78450a055504e1562934481595f1d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 10 Jan 2024 16:01:29 +0100 Subject: [PATCH 172/247] refactor: Rename producer for api and consumer for worker (#31) Co-authored-by: Brett Hoerner --- .github/workflows/docker-hook-api.yml | 2 +- .github/workflows/docker-hook-worker.yml | 2 +- Cargo.lock | 44 +++++++++---------- Cargo.toml | 2 +- {hook-producer => hook-api}/Cargo.toml | 2 +- {hook-producer => hook-api}/src/config.rs | 0 .../src/handlers/app.rs | 4 +- .../src/handlers/mod.rs | 0 .../src/handlers/webhook.rs | 2 +- {hook-producer => hook-api}/src/main.rs | 4 +- hook-common/src/webhook.rs | 4 +- hook-janitor/src/kafka_producer.rs | 5 +-- {hook-consumer => hook-worker}/Cargo.toml | 2 +- {hook-consumer => hook-worker}/README.md | 2 +- {hook-consumer => hook-worker}/src/config.rs | 4 +- {hook-consumer => hook-worker}/src/error.rs | 4 +- {hook-consumer => hook-worker}/src/lib.rs | 2 +- {hook-consumer => hook-worker}/src/main.rs | 14 +++--- .../consumer.rs => hook-worker/src/worker.rs | 42 +++++++++--------- 19 files changed, 70 insertions(+), 71 deletions(-) rename {hook-producer => hook-api}/Cargo.toml (96%) rename {hook-producer => hook-api}/src/config.rs (100%) rename {hook-producer => hook-api}/src/handlers/app.rs (93%) rename {hook-producer => hook-api}/src/handlers/mod.rs (100%) rename {hook-producer => hook-api}/src/handlers/webhook.rs (99%) rename {hook-producer => hook-api}/src/main.rs (88%) rename {hook-consumer => hook-worker}/Cargo.toml (95%) rename {hook-consumer => hook-worker}/README.md (67%) rename {hook-consumer => hook-worker}/src/config.rs (96%) rename {hook-consumer => hook-worker}/src/error.rs (96%) rename {hook-consumer => hook-worker}/src/lib.rs (63%) rename {hook-consumer => hook-worker}/src/main.rs (78%) rename hook-consumer/src/consumer.rs => hook-worker/src/worker.rs (93%) diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index 5ae94f5531036..6331413e44839 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -63,7 +63,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: BIN=hook-producer + build-args: BIN=hook-api - name: Hook-api image digest run: echo ${{ steps.docker_build_hook_api.outputs.digest }} diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index 052629100ce5f..1f06542c0a0d3 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -63,7 +63,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: BIN=hook-consumer + build-args: BIN=hook-worker - name: Hook-worker image digest run: echo ${{ steps.docker_build_hook_worker.outputs.digest }} diff --git a/Cargo.lock b/Cargo.lock index e4da8161adcc0..2bf16f1654ede 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -755,43 +755,46 @@ dependencies = [ ] [[package]] -name = "hook-common" +name = "hook-api" version = "0.1.0" dependencies = [ - "async-trait", "axum", - "chrono", - "http 0.2.11", + "envconfig", + "eyre", + "hook-common", + "http-body-util", "metrics", "metrics-exporter-prometheus", - "regex", - "reqwest", "serde", "serde_derive", "serde_json", "sqlx", - "thiserror", "tokio", - "uuid", + "tower", + "tracing", + "tracing-subscriber", + "url", ] [[package]] -name = "hook-consumer" +name = "hook-common" version = "0.1.0" dependencies = [ + "async-trait", + "axum", "chrono", - "envconfig", - "futures", - "hook-common", "http 0.2.11", "metrics", + "metrics-exporter-prometheus", + "regex", "reqwest", "serde", "serde_derive", + "serde_json", "sqlx", "thiserror", "tokio", - "url", + "uuid", ] [[package]] @@ -821,24 +824,21 @@ dependencies = [ ] [[package]] -name = "hook-producer" +name = "hook-worker" version = "0.1.0" dependencies = [ - "axum", + "chrono", "envconfig", - "eyre", + "futures", "hook-common", - "http-body-util", + "http 0.2.11", "metrics", - "metrics-exporter-prometheus", + "reqwest", "serde", "serde_derive", - "serde_json", "sqlx", + "thiserror", "tokio", - "tower", - "tracing", - "tracing-subscriber", "url", ] diff --git a/Cargo.toml b/Cargo.toml index b7cb8fac622c1..0b48fb23b015b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] resolver = "2" -members = ["hook-common", "hook-producer", "hook-consumer", "hook-janitor"] +members = ["hook-common", "hook-api", "hook-worker", "hook-janitor"] [workspace.dependencies] async-trait = "0.1.74" diff --git a/hook-producer/Cargo.toml b/hook-api/Cargo.toml similarity index 96% rename from hook-producer/Cargo.toml rename to hook-api/Cargo.toml index f4b116563fd12..96c897cd3ab1d 100644 --- a/hook-producer/Cargo.toml +++ b/hook-api/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "hook-producer" +name = "hook-api" version = "0.1.0" edition = "2021" diff --git a/hook-producer/src/config.rs b/hook-api/src/config.rs similarity index 100% rename from hook-producer/src/config.rs rename to hook-api/src/config.rs diff --git a/hook-producer/src/handlers/app.rs b/hook-api/src/handlers/app.rs similarity index 93% rename from hook-producer/src/handlers/app.rs rename to hook-api/src/handlers/app.rs index 2cafc4a497ae2..73c29e1225842 100644 --- a/hook-producer/src/handlers/app.rs +++ b/hook-api/src/handlers/app.rs @@ -11,7 +11,7 @@ pub fn add_routes(router: Router, pg_pool: PgQueue) -> Router { } pub async fn index() -> &'static str { - "rusty-hook producer" + "rusty-hook api" } #[cfg(test)] @@ -42,6 +42,6 @@ mod tests { assert_eq!(response.status(), StatusCode::OK); let body = response.into_body().collect().await.unwrap().to_bytes(); - assert_eq!(&body[..], b"rusty-hook producer"); + assert_eq!(&body[..], b"rusty-hook api"); } } diff --git a/hook-producer/src/handlers/mod.rs b/hook-api/src/handlers/mod.rs similarity index 100% rename from hook-producer/src/handlers/mod.rs rename to hook-api/src/handlers/mod.rs diff --git a/hook-producer/src/handlers/webhook.rs b/hook-api/src/handlers/webhook.rs similarity index 99% rename from hook-producer/src/handlers/webhook.rs rename to hook-api/src/handlers/webhook.rs index 62a4aaa8833c8..16ebc6dc57179 100644 --- a/hook-producer/src/handlers/webhook.rs +++ b/hook-api/src/handlers/webhook.rs @@ -68,7 +68,7 @@ pub async fn post( pg_queue.enqueue(job).await.map_err(internal_error)?; let elapsed_time = start_time.elapsed().as_secs_f64(); - metrics::histogram!("webhook_producer_enqueue").record(elapsed_time); + metrics::histogram!("webhook_api_enqueue").record(elapsed_time); Ok(Json(WebhookPostResponse { error: None })) } diff --git a/hook-producer/src/main.rs b/hook-api/src/main.rs similarity index 88% rename from hook-producer/src/main.rs rename to hook-api/src/main.rs index f078f2fae7781..dc2739a859131 100644 --- a/hook-producer/src/main.rs +++ b/hook-api/src/main.rs @@ -24,7 +24,7 @@ async fn main() { let config = Config::init_from_env().expect("failed to load configuration from env"); let pg_queue = PgQueue::new( - // TODO: Coupling the queue name to the PgQueue object doesn't seem ideal from the producer + // TODO: Coupling the queue name to the PgQueue object doesn't seem ideal from the api // side, but we don't need more than one queue for now. &config.queue_name, &config.database_url, @@ -37,6 +37,6 @@ async fn main() { match listen(app, config.bind()).await { Ok(_) => {} - Err(e) => tracing::error!("failed to start hook-producer http server, {}", e), + Err(e) => tracing::error!("failed to start hook-api http server, {}", e), } } diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index bb1b5be04390a..475f3decb4b11 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -117,7 +117,7 @@ impl From<&HttpMethod> for http::Method { } } -/// `JobParameters` required for the `WebhookConsumer` to execute a webhook. +/// `JobParameters` required for the `WebhookWorker` to execute a webhook. /// These parameters should match the exported Webhook interface that PostHog plugins. /// implement. See: https://github.com/PostHog/plugin-scaffold/blob/main/src/types.ts#L15. #[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] @@ -128,7 +128,7 @@ pub struct WebhookJobParameters { pub url: String, } -/// `JobMetadata` required for the `WebhookConsumer` to execute a webhook. +/// `JobMetadata` required for the `WebhookWorker` to execute a webhook. /// These should be set if the Webhook is associated with a plugin `composeWebhook` invocation. #[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] pub struct WebhookJobMetadata { diff --git a/hook-janitor/src/kafka_producer.rs b/hook-janitor/src/kafka_producer.rs index 4845e9410df56..1d0144cb75436 100644 --- a/hook-janitor/src/kafka_producer.rs +++ b/hook-janitor/src/kafka_producer.rs @@ -38,8 +38,7 @@ pub async fn create_kafka_producer( }; debug!("rdkafka configuration: {:?}", client_config); - let producer: FutureProducer = - client_config.create_with_context(KafkaContext {})?; + let api: FutureProducer = client_config.create_with_context(KafkaContext {})?; - Ok(producer) + Ok(api) } diff --git a/hook-consumer/Cargo.toml b/hook-worker/Cargo.toml similarity index 95% rename from hook-consumer/Cargo.toml rename to hook-worker/Cargo.toml index fc8ee4a797fac..c84d348f9990b 100644 --- a/hook-consumer/Cargo.toml +++ b/hook-worker/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "hook-consumer" +name = "hook-worker" version = "0.1.0" edition = "2021" diff --git a/hook-consumer/README.md b/hook-worker/README.md similarity index 67% rename from hook-consumer/README.md rename to hook-worker/README.md index 1adab6ea571f4..9b1884aab16b6 100644 --- a/hook-consumer/README.md +++ b/hook-worker/README.md @@ -1,2 +1,2 @@ -# hook-consumer +# hook-worker Consume and process webhook jobs diff --git a/hook-consumer/src/config.rs b/hook-worker/src/config.rs similarity index 96% rename from hook-consumer/src/config.rs rename to hook-worker/src/config.rs index 01f94e76bd428..6f16c893c5282 100644 --- a/hook-consumer/src/config.rs +++ b/hook-worker/src/config.rs @@ -14,8 +14,8 @@ pub struct Config { #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] pub database_url: String, - #[envconfig(default = "consumer")] - pub consumer_name: String, + #[envconfig(default = "worker")] + pub worker_name: String, #[envconfig(default = "default")] pub queue_name: String, diff --git a/hook-consumer/src/error.rs b/hook-worker/src/error.rs similarity index 96% rename from hook-consumer/src/error.rs rename to hook-worker/src/error.rs index b05d476e22849..d0259985c6f6a 100644 --- a/hook-consumer/src/error.rs +++ b/hook-worker/src/error.rs @@ -3,7 +3,7 @@ use std::time; use hook_common::pgqueue; use thiserror::Error; -/// Enumeration of errors related to webhook job processing in the WebhookConsumer. +/// Enumeration of errors related to webhook job processing in the WebhookWorker. #[derive(Error, Debug)] pub enum WebhookError { #[error("{0} is not a valid HttpMethod")] @@ -23,7 +23,7 @@ pub enum WebhookError { /// Enumeration of errors related to initialization and consumption of webhook jobs. #[derive(Error, Debug)] -pub enum ConsumerError { +pub enum WorkerError { #[error("timed out while waiting for jobs to be available")] TimeoutError, #[error("an error occurred in the underlying queue")] diff --git a/hook-consumer/src/lib.rs b/hook-worker/src/lib.rs similarity index 63% rename from hook-consumer/src/lib.rs rename to hook-worker/src/lib.rs index b99481bbc3b58..22823c9a7e5cd 100644 --- a/hook-consumer/src/lib.rs +++ b/hook-worker/src/lib.rs @@ -1,3 +1,3 @@ pub mod config; -pub mod consumer; pub mod error; +pub mod worker; diff --git a/hook-consumer/src/main.rs b/hook-worker/src/main.rs similarity index 78% rename from hook-consumer/src/main.rs rename to hook-worker/src/main.rs index 4182348121bd6..10b34d7b8eb5f 100644 --- a/hook-consumer/src/main.rs +++ b/hook-worker/src/main.rs @@ -4,12 +4,12 @@ use envconfig::Envconfig; use hook_common::{ metrics::serve, metrics::setup_metrics_router, pgqueue::PgQueue, retry::RetryPolicy, }; -use hook_consumer::config::Config; -use hook_consumer::consumer::WebhookConsumer; -use hook_consumer::error::ConsumerError; +use hook_worker::config::Config; +use hook_worker::error::WorkerError; +use hook_worker::worker::WebhookWorker; #[tokio::main] -async fn main() -> Result<(), ConsumerError> { +async fn main() -> Result<(), WorkerError> { let config = Config::init_from_env().expect("Invalid configuration:"); let retry_policy = RetryPolicy::build( @@ -23,8 +23,8 @@ async fn main() -> Result<(), ConsumerError> { .await .expect("failed to initialize queue"); - let consumer = WebhookConsumer::new( - &config.consumer_name, + let worker = WebhookWorker::new( + &config.worker_name, &queue, config.poll_interval.0, config.request_timeout.0, @@ -40,7 +40,7 @@ async fn main() -> Result<(), ConsumerError> { .expect("failed to start serving metrics"); }); - consumer.run(config.transactional).await?; + worker.run(config.transactional).await?; Ok(()) } diff --git a/hook-consumer/src/consumer.rs b/hook-worker/src/worker.rs similarity index 93% rename from hook-consumer/src/consumer.rs rename to hook-worker/src/worker.rs index 2114ef80c8bc8..02ab7e962b71e 100644 --- a/hook-consumer/src/consumer.rs +++ b/hook-worker/src/worker.rs @@ -11,7 +11,7 @@ use http::StatusCode; use reqwest::header; use tokio::sync; -use crate::error::{ConsumerError, WebhookError}; +use crate::error::{WebhookError, WorkerError}; /// A WebhookJob is any `PgQueueJob` with `WebhookJobParameters` and `WebhookJobMetadata`. trait WebhookJob: PgQueueJob + std::marker::Send { @@ -60,9 +60,9 @@ impl WebhookJob for PgJob { } } -/// A consumer to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. -pub struct WebhookConsumer<'p> { - /// An identifier for this consumer. Used to mark jobs we have consumed. +/// A worker to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. +pub struct WebhookWorker<'p> { + /// An identifier for this worker. Used to mark jobs we have consumed. name: String, /// The queue we will be dequeuing jobs from. queue: &'p PgQueue, @@ -76,7 +76,7 @@ pub struct WebhookConsumer<'p> { retry_policy: RetryPolicy, } -impl<'p> WebhookConsumer<'p> { +impl<'p> WebhookWorker<'p> { pub fn new( name: &str, queue: &'p PgQueue, @@ -95,7 +95,7 @@ impl<'p> WebhookConsumer<'p> { .default_headers(headers) .timeout(request_timeout) .build() - .expect("failed to construct reqwest client for webhook consumer"); + .expect("failed to construct reqwest client for webhook worker"); Self { name: name.to_owned(), @@ -110,7 +110,7 @@ impl<'p> WebhookConsumer<'p> { /// Wait until a job becomes available in our queue. async fn wait_for_job<'a>( &self, - ) -> Result, ConsumerError> { + ) -> Result, WorkerError> { let mut interval = tokio::time::interval(self.poll_interval); loop { @@ -125,7 +125,7 @@ impl<'p> WebhookConsumer<'p> { /// Wait until a job becomes available in our queue in transactional mode. async fn wait_for_job_tx<'a>( &self, - ) -> Result, ConsumerError> { + ) -> Result, WorkerError> { let mut interval = tokio::time::interval(self.poll_interval); loop { @@ -137,8 +137,8 @@ impl<'p> WebhookConsumer<'p> { } } - /// Run this consumer to continuously process any jobs that become available. - pub async fn run(&self, transactional: bool) -> Result<(), ConsumerError> { + /// Run this worker to continuously process any jobs that become available. + pub async fn run(&self, transactional: bool) -> Result<(), WorkerError> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); if transactional { @@ -181,7 +181,7 @@ async fn spawn_webhook_job_processing_task( semaphore: Arc, retry_policy: RetryPolicy, webhook_job: W, -) -> tokio::task::JoinHandle> { +) -> tokio::task::JoinHandle> { let permit = semaphore .acquire_owned() .await @@ -219,7 +219,7 @@ async fn process_webhook_job( client: reqwest::Client, webhook_job: W, retry_policy: &RetryPolicy, -) -> Result<(), ConsumerError> { +) -> Result<(), WorkerError> { let parameters = webhook_job.parameters(); let labels = [ @@ -245,7 +245,7 @@ async fn process_webhook_job( webhook_job .complete() .await - .map_err(|error| ConsumerError::PgJobError(error.to_string()))?; + .map_err(|error| WorkerError::PgJobError(error.to_string()))?; metrics::counter!("webhook_jobs_completed", &labels).increment(1); metrics::histogram!("webhook_jobs_processing_duration_seconds", &labels) @@ -257,7 +257,7 @@ async fn process_webhook_job( webhook_job .fail(WebhookJobError::new_parse(&e.to_string())) .await - .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; metrics::counter!("webhook_jobs_failed", &labels).increment(1); @@ -267,7 +267,7 @@ async fn process_webhook_job( webhook_job .fail(WebhookJobError::new_parse(&e)) .await - .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; metrics::counter!("webhook_jobs_failed", &labels).increment(1); @@ -277,7 +277,7 @@ async fn process_webhook_job( webhook_job .fail(WebhookJobError::new_parse(&e.to_string())) .await - .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; metrics::counter!("webhook_jobs_failed", &labels).increment(1); @@ -304,20 +304,20 @@ async fn process_webhook_job( webhook_job .fail(WebhookJobError::from(&error)) .await - .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; metrics::counter!("webhook_jobs_failed", &labels).increment(1); Ok(()) } - Err(job_error) => Err(ConsumerError::PgJobError(job_error.to_string())), + Err(job_error) => Err(WorkerError::PgJobError(job_error.to_string())), } } Err(WebhookError::NonRetryableRetryableRequestError(error)) => { webhook_job .fail(WebhookJobError::from(&error)) .await - .map_err(|job_error| ConsumerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; metrics::counter!("webhook_jobs_failed", &labels).increment(1); @@ -512,7 +512,7 @@ mod tests { ) .await .expect("failed to enqueue job"); - let consumer = WebhookConsumer::new( + let worker = WebhookWorker::new( &worker_id, &queue, time::Duration::from_millis(100), @@ -521,7 +521,7 @@ mod tests { RetryPolicy::default(), ); - let consumed_job = consumer + let consumed_job = worker .wait_for_job() .await .expect("failed to wait and read job"); From d6cc49f4d290a8125a3dc390c3664511c0d68694 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 10 Jan 2024 08:53:09 -0700 Subject: [PATCH 173/247] Sync default ports with charts (#33) --- hook-api/src/config.rs | 2 +- hook-janitor/src/config.rs | 2 +- hook-worker/src/config.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hook-api/src/config.rs b/hook-api/src/config.rs index 8daf04e4ca8b8..3fe88b3e436c0 100644 --- a/hook-api/src/config.rs +++ b/hook-api/src/config.rs @@ -5,7 +5,7 @@ pub struct Config { #[envconfig(from = "BIND_HOST", default = "0.0.0.0")] pub host: String, - #[envconfig(from = "BIND_PORT", default = "8000")] + #[envconfig(from = "BIND_PORT", default = "3300")] pub port: u16, #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] diff --git a/hook-janitor/src/config.rs b/hook-janitor/src/config.rs index 64db0e613f355..852b7cfed14c1 100644 --- a/hook-janitor/src/config.rs +++ b/hook-janitor/src/config.rs @@ -5,7 +5,7 @@ pub struct Config { #[envconfig(from = "BIND_HOST", default = "0.0.0.0")] pub host: String, - #[envconfig(from = "BIND_PORT", default = "8000")] + #[envconfig(from = "BIND_PORT", default = "3302")] pub port: u16, #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] diff --git a/hook-worker/src/config.rs b/hook-worker/src/config.rs index 6f16c893c5282..8b2b4ba49f205 100644 --- a/hook-worker/src/config.rs +++ b/hook-worker/src/config.rs @@ -8,7 +8,7 @@ pub struct Config { #[envconfig(from = "BIND_HOST", default = "0.0.0.0")] pub host: String, - #[envconfig(from = "BIND_PORT", default = "8001")] + #[envconfig(from = "BIND_PORT", default = "3301")] pub port: u16, #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] From 878f11201df2e5777967229f176e971c39d00a9e Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 10 Jan 2024 09:39:01 -0700 Subject: [PATCH 174/247] Log errors in worker (#34) --- Cargo.lock | 2 ++ hook-worker/Cargo.toml | 2 ++ hook-worker/src/main.rs | 2 ++ hook-worker/src/worker.rs | 9 ++++++++- 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 2bf16f1654ede..d242eca5b3925 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -839,6 +839,8 @@ dependencies = [ "sqlx", "thiserror", "tokio", + "tracing", + "tracing-subscriber", "url", ] diff --git a/hook-worker/Cargo.toml b/hook-worker/Cargo.toml index c84d348f9990b..f69489877b4ec 100644 --- a/hook-worker/Cargo.toml +++ b/hook-worker/Cargo.toml @@ -15,5 +15,7 @@ serde = { workspace = true } serde_derive = { workspace = true } sqlx = { workspace = true } thiserror = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } tokio = { workspace = true } url = { version = "2.2" } diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index 10b34d7b8eb5f..fbd4746fd000d 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -10,6 +10,8 @@ use hook_worker::worker::WebhookWorker; #[tokio::main] async fn main() -> Result<(), WorkerError> { + tracing_subscriber::fmt::init(); + let config = Config::init_from_env().expect("Invalid configuration:"); let retry_policy = RetryPolicy::build( diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 02ab7e962b71e..cc5082e35b5ea 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -10,6 +10,7 @@ use hook_common::{ use http::StatusCode; use reqwest::header; use tokio::sync; +use tracing::error; use crate::error::{WebhookError, WorkerError}; @@ -197,7 +198,13 @@ async fn spawn_webhook_job_processing_task( tokio::spawn(async move { let result = process_webhook_job(client, webhook_job, &retry_policy).await; drop(permit); - result + match result { + Ok(_) => Ok(()), + Err(error) => { + error!("failed to process webhook job: {}", error); + Err(error) + } + } }) } From b0966788b0b959f6d286bc9b91d48d23159e925f Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 10 Jan 2024 09:58:01 -0700 Subject: [PATCH 175/247] Log PgJobError String (#35) --- hook-worker/src/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook-worker/src/error.rs b/hook-worker/src/error.rs index d0259985c6f6a..614fe721957e4 100644 --- a/hook-worker/src/error.rs +++ b/hook-worker/src/error.rs @@ -28,6 +28,6 @@ pub enum WorkerError { TimeoutError, #[error("an error occurred in the underlying queue")] QueueError(#[from] pgqueue::PgQueueError), - #[error("an error occurred in the underlying job")] + #[error("an error occurred in the underlying job: {0}")] PgJobError(String), } From 35877d3d9d94738197312525ffc4290d2bb56542 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 10 Jan 2024 10:27:52 -0700 Subject: [PATCH 176/247] Install ca-certificates (#37) --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index 959fd17180fb5..74682f6855330 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,13 @@ COPY . . RUN cargo build --release --bin $BIN FROM debian:bullseye-20230320-slim AS runtime + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + "ca-certificates" \ + && \ + rm -rf /var/lib/apt/lists/* + ARG BIN ENV ENTRYPOINT=/usr/local/bin/$BIN WORKDIR app From 5011f18a44e7ea69f8293152367c0d82e8f92523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 10 Jan 2024 18:34:05 +0100 Subject: [PATCH 177/247] fix: Syntax error in fail method (#38) --- hook-common/src/pgqueue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 35f5b4a9e47fb..d3a409bdbd889 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -172,7 +172,7 @@ UPDATE job_queue SET last_attempt_finished_at = NOW(), - status = 'failed'::job_status + status = 'failed'::job_status, errors = array_append(errors, $3) WHERE queue = $1 From 9b7d313433876ec685b0d6dc1136252ff2af7c18 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 11 Jan 2024 09:42:10 -0700 Subject: [PATCH 178/247] Log productive janitor runs at info level (#40) --- hook-janitor/src/webhooks.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index de02d07a2520e..6f28a153f70a6 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -11,7 +11,7 @@ use sqlx::postgres::{PgPool, PgPoolOptions, Postgres}; use sqlx::types::{chrono, Uuid}; use sqlx::{Row, Transaction}; use thiserror::Error; -use tracing::{debug, error}; +use tracing::{debug, error, info}; use crate::cleanup::Cleaner; use crate::kafka_producer::KafkaContext; @@ -432,7 +432,7 @@ impl Cleaner for WebhookCleaner { metrics::counter!("webhook_cleanup_failed_agg_row_count",) .increment(stats.failed_agg_row_count); - debug!( + info!( rows_processed = stats.rows_processed, completed_row_count = stats.completed_row_count, completed_agg_row_count = stats.completed_agg_row_count, From 677a094f64908ddced814f1a569d2ba1bdaa70e3 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 11 Jan 2024 09:45:47 -0700 Subject: [PATCH 179/247] Drop target from Prom labels (#41) --- hook-worker/src/worker.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index cc5082e35b5ea..8d8c824d7f82b 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -188,10 +188,7 @@ async fn spawn_webhook_job_processing_task( .await .expect("semaphore has been closed"); - let labels = [ - ("queue", webhook_job.queue()), - ("target", webhook_job.target()), - ]; + let labels = [("queue", webhook_job.queue())]; metrics::counter!("webhook_jobs_total", &labels).increment(1); @@ -229,10 +226,7 @@ async fn process_webhook_job( ) -> Result<(), WorkerError> { let parameters = webhook_job.parameters(); - let labels = [ - ("queue", webhook_job.queue()), - ("target", webhook_job.target()), - ]; + let labels = [("queue", webhook_job.queue())]; let now = tokio::time::Instant::now(); From 2437bd55567a43adbd05397919368334f322cf58 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 11 Jan 2024 09:55:10 -0700 Subject: [PATCH 180/247] Add user-agent (#42) --- hook-worker/src/worker.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 8d8c824d7f82b..77c6adb4b13f8 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -94,6 +94,7 @@ impl<'p> WebhookWorker<'p> { let client = reqwest::Client::builder() .default_headers(headers) + .user_agent("PostHog Webhook Worker") .timeout(request_timeout) .build() .expect("failed to construct reqwest client for webhook worker"); From d5dd35d9017bf0f01f65ed0ea38dd54a6faca47a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 11 Jan 2024 17:55:28 +0100 Subject: [PATCH 181/247] feat: Set idle tx timeout in migration (#39) --- migrations/20240110180056_set_idle_in_transaction_timeout.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 migrations/20240110180056_set_idle_in_transaction_timeout.sql diff --git a/migrations/20240110180056_set_idle_in_transaction_timeout.sql b/migrations/20240110180056_set_idle_in_transaction_timeout.sql new file mode 100644 index 0000000000000..e17113125e41b --- /dev/null +++ b/migrations/20240110180056_set_idle_in_transaction_timeout.sql @@ -0,0 +1,2 @@ +-- If running worker in transactional mode, this ensures we clean up any open transactions. +ALTER USER current_user SET idle_in_transaction_session_timeout = '2min'; From 1982b0d8d9b0a45ddc3438c17fab369ad1acae37 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 11 Jan 2024 10:15:58 -0700 Subject: [PATCH 182/247] Fix app_metrics topic, don't encode null error_uuid (#43) --- hook-common/src/kafka_messages/app_metrics.rs | 1 + hook-janitor/src/config.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs index 9acc4112e1113..13f4f2e5000d4 100644 --- a/hook-common/src/kafka_messages/app_metrics.rs +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -62,6 +62,7 @@ pub struct AppMetric { pub successes: u32, pub successes_on_retry: u32, pub failures: u32, + #[serde(skip_serializing_if = "Option::is_none")] pub error_uuid: Option, #[serde( serialize_with = "serialize_error_type", diff --git a/hook-janitor/src/config.rs b/hook-janitor/src/config.rs index 852b7cfed14c1..252a67011460c 100644 --- a/hook-janitor/src/config.rs +++ b/hook-janitor/src/config.rs @@ -44,7 +44,7 @@ pub struct KafkaConfig { #[envconfig(default = "false")] pub kafka_tls: bool, - #[envconfig(default = "app_metrics")] + #[envconfig(default = "clickhouse_app_metrics")] pub app_metrics_topic: String, #[envconfig(default = "plugin_log_entries")] From 51a5c5f0211291075489b36143c3184a1e16d6a1 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 12 Jan 2024 04:03:30 -0700 Subject: [PATCH 183/247] Remove idle_in_transaction_session_timeout migration (#44) --- migrations/20240110180056_set_idle_in_transaction_timeout.sql | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 migrations/20240110180056_set_idle_in_transaction_timeout.sql diff --git a/migrations/20240110180056_set_idle_in_transaction_timeout.sql b/migrations/20240110180056_set_idle_in_transaction_timeout.sql deleted file mode 100644 index e17113125e41b..0000000000000 --- a/migrations/20240110180056_set_idle_in_transaction_timeout.sql +++ /dev/null @@ -1,2 +0,0 @@ --- If running worker in transactional mode, this ensures we clean up any open transactions. -ALTER USER current_user SET idle_in_transaction_session_timeout = '2min'; From 8900f50655a998d859d6d4560d87a00b67e482dd Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 12 Jan 2024 09:14:47 -0700 Subject: [PATCH 184/247] Use signed integers for plugin_id/plugin_config_id (#45) --- hook-common/src/kafka_messages/app_metrics.rs | 2 +- hook-common/src/kafka_messages/plugin_logs.rs | 4 ++-- hook-common/src/pgqueue.rs | 4 ++-- hook-common/src/webhook.rs | 4 ++-- hook-janitor/src/webhooks.rs | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/hook-common/src/kafka_messages/app_metrics.rs b/hook-common/src/kafka_messages/app_metrics.rs index 13f4f2e5000d4..f941f58138b76 100644 --- a/hook-common/src/kafka_messages/app_metrics.rs +++ b/hook-common/src/kafka_messages/app_metrics.rs @@ -51,7 +51,7 @@ pub struct AppMetric { )] pub timestamp: DateTime, pub team_id: u32, - pub plugin_config_id: u32, + pub plugin_config_id: i32, #[serde(skip_serializing_if = "Option::is_none")] pub job_id: Option, #[serde( diff --git a/hook-common/src/kafka_messages/plugin_logs.rs b/hook-common/src/kafka_messages/plugin_logs.rs index e761fa40799ea..fb835804c687c 100644 --- a/hook-common/src/kafka_messages/plugin_logs.rs +++ b/hook-common/src/kafka_messages/plugin_logs.rs @@ -30,8 +30,8 @@ pub struct PluginLogEntry { pub type_: PluginLogEntryType, pub id: Uuid, pub team_id: u32, - pub plugin_id: u32, - pub plugin_config_id: u32, + pub plugin_id: i32, + pub plugin_config_id: i32, #[serde(serialize_with = "serialize_datetime")] pub timestamp: DateTime, #[serde(serialize_with = "serialize_message")] diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index d3a409bdbd889..fa2b5eb0c38be 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -731,8 +731,8 @@ mod tests { #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] struct JobMetadata { team_id: u32, - plugin_config_id: u32, - plugin_id: u32, + plugin_config_id: i32, + plugin_id: i32, } impl Default for JobMetadata { diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index 475f3decb4b11..11e02856703eb 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -133,8 +133,8 @@ pub struct WebhookJobParameters { #[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] pub struct WebhookJobMetadata { pub team_id: u32, - pub plugin_id: u32, - pub plugin_config_id: u32, + pub plugin_id: i32, + pub plugin_config_id: i32, } /// An error originating during a Webhook Job invocation. diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 6f28a153f70a6..0c2941f99650f 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -66,7 +66,7 @@ struct CompletedRow { #[sqlx(try_from = "i64")] team_id: u32, #[sqlx(try_from = "i64")] - plugin_config_id: u32, + plugin_config_id: i32, #[sqlx(try_from = "i64")] successes: u32, } @@ -100,7 +100,7 @@ struct FailedRow { #[sqlx(try_from = "i64")] team_id: u32, #[sqlx(try_from = "i64")] - plugin_config_id: u32, + plugin_config_id: i32, #[sqlx(json)] last_error: WebhookJobError, #[sqlx(try_from = "i64")] From 58d573a2b40db67344b0952aa4811a4d000aba33 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 15 Jan 2024 17:35:59 +0100 Subject: [PATCH 185/247] feat: make sure metrics cover all axum endpoints (#46) --- Cargo.lock | 1 + hook-api/src/main.rs | 6 +++--- hook-common/src/metrics.rs | 6 +++--- hook-janitor/src/handlers/app.rs | 16 ++-------------- hook-janitor/src/main.rs | 5 ++--- hook-worker/Cargo.toml | 1 + hook-worker/src/main.rs | 5 +++-- 7 files changed, 15 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d242eca5b3925..5157ea14869fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -827,6 +827,7 @@ dependencies = [ name = "hook-worker" version = "0.1.0" dependencies = [ + "axum", "chrono", "envconfig", "futures", diff --git a/hook-api/src/main.rs b/hook-api/src/main.rs index dc2739a859131..4fbbdfbb7d4ed 100644 --- a/hook-api/src/main.rs +++ b/hook-api/src/main.rs @@ -3,7 +3,7 @@ use config::Config; use envconfig::Envconfig; use eyre::Result; -use hook_common::metrics::setup_metrics_router; +use hook_common::metrics::setup_metrics_routes; use hook_common::pgqueue::PgQueue; mod config; @@ -32,8 +32,8 @@ async fn main() { .await .expect("failed to initialize queue"); - let router = setup_metrics_router(); - let app = handlers::add_routes(router, pg_queue); + let app = handlers::add_routes(Router::new(), pg_queue); + let app = setup_metrics_routes(app); match listen(app, config.bind()).await { Ok(_) => {} diff --git a/hook-common/src/metrics.rs b/hook-common/src/metrics.rs index 1f57c5eec51f9..0e1ef2db72a33 100644 --- a/hook-common/src/metrics.rs +++ b/hook-common/src/metrics.rs @@ -16,11 +16,11 @@ pub async fn serve(router: Router, bind: &str) -> Result<(), std::io::Error> { Ok(()) } -/// Build a Router for a metrics endpoint. -pub fn setup_metrics_router() -> Router { +/// Add the prometheus endpoint and middleware to a router, should be called last. +pub fn setup_metrics_routes(router: Router) -> Router { let recorder_handle = setup_metrics_recorder(); - Router::new() + router .route( "/metrics", get(move || std::future::ready(recorder_handle.render())), diff --git a/hook-janitor/src/handlers/app.rs b/hook-janitor/src/handlers/app.rs index 279fa0e35923e..cab0e0d27991a 100644 --- a/hook-janitor/src/handlers/app.rs +++ b/hook-janitor/src/handlers/app.rs @@ -1,19 +1,7 @@ use axum::{routing, Router}; -use metrics_exporter_prometheus::PrometheusHandle; -use hook_common::metrics; - -pub fn app(metrics: Option) -> Router { - Router::new() - .route("/", routing::get(index)) - .route( - "/metrics", - routing::get(move || match metrics { - Some(ref recorder_handle) => std::future::ready(recorder_handle.render()), - None => std::future::ready("no metrics recorder installed".to_owned()), - }), - ) - .layer(axum::middleware::from_fn(metrics::track_metrics)) +pub fn app() -> Router { + Router::new().route("/", routing::get(index)) } pub async fn index() -> &'static str { diff --git a/hook-janitor/src/main.rs b/hook-janitor/src/main.rs index 7d7e2230e548e..63068c35262bf 100644 --- a/hook-janitor/src/main.rs +++ b/hook-janitor/src/main.rs @@ -9,7 +9,7 @@ use std::{str::FromStr, time::Duration}; use tokio::sync::Semaphore; use webhooks::WebhookCleaner; -use hook_common::metrics; +use hook_common::metrics::setup_metrics_routes; mod cleanup; mod config; @@ -66,8 +66,7 @@ async fn main() { let cleanup_loop = Box::pin(cleanup_loop(cleaner, config.cleanup_interval_secs)); - let recorder_handle = metrics::setup_metrics_recorder(); - let app = handlers::app(Some(recorder_handle)); + let app = setup_metrics_routes(handlers::app()); let http_server = Box::pin(listen(app, config.bind())); match select(http_server, cleanup_loop).await { diff --git a/hook-worker/Cargo.toml b/hook-worker/Cargo.toml index f69489877b4ec..11da0a80a564c 100644 --- a/hook-worker/Cargo.toml +++ b/hook-worker/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] +axum = { workspace = true } chrono = { workspace = true } envconfig = { workspace = true } futures = "0.3" diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index fbd4746fd000d..cc17169b2bd83 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -1,8 +1,9 @@ //! Consume `PgQueue` jobs to run webhook calls. +use axum::Router; use envconfig::Envconfig; use hook_common::{ - metrics::serve, metrics::setup_metrics_router, pgqueue::PgQueue, retry::RetryPolicy, + metrics::serve, metrics::setup_metrics_routes, pgqueue::PgQueue, retry::RetryPolicy, }; use hook_worker::config::Config; use hook_worker::error::WorkerError; @@ -36,7 +37,7 @@ async fn main() -> Result<(), WorkerError> { let bind = config.bind(); tokio::task::spawn(async move { - let router = setup_metrics_router(); + let router = setup_metrics_routes(Router::new()); serve(router, &bind) .await .expect("failed to start serving metrics"); From 9fa90fdf2358819c6a8e0f7e8a39f2c29c606eb6 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 16 Jan 2024 17:51:40 +0100 Subject: [PATCH 186/247] improve janitor metrics (#48) --- hook-janitor/src/webhooks.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 0c2941f99650f..4248a47281e52 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -412,15 +412,15 @@ impl WebhookCleaner { impl Cleaner for WebhookCleaner { async fn cleanup(&self) { let start_time = Instant::now(); + metrics::counter!("webhook_cleanup_attempts",).increment(1); match self.cleanup_impl().await { Ok(stats) => { - metrics::counter!("webhook_cleanup_runs",).increment(1); + metrics::counter!("webhook_cleanup_success",).increment(1); if stats.rows_processed > 0 { let elapsed_time = start_time.elapsed().as_secs_f64(); metrics::histogram!("webhook_cleanup_duration").record(elapsed_time); - metrics::counter!("webhook_cleanup_rows_processed",) .increment(stats.rows_processed); metrics::counter!("webhook_cleanup_completed_row_count",) From b5029a95b617ad6149514941321824a70f8295c6 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 16 Jan 2024 18:50:43 +0100 Subject: [PATCH 187/247] feat: add readiness and liveness endpoints to all roles (#47) --- Cargo.lock | 37 +++ Cargo.toml | 1 + hook-api/src/handlers/app.rs | 2 + hook-common/Cargo.toml | 2 + hook-common/src/health.rs | 346 +++++++++++++++++++++++++++++ hook-common/src/lib.rs | 1 + hook-janitor/Cargo.toml | 1 + hook-janitor/src/handlers/app.rs | 11 +- hook-janitor/src/kafka_producer.rs | 21 +- hook-janitor/src/main.rs | 25 ++- hook-janitor/src/webhooks.rs | 7 +- hook-worker/Cargo.toml | 1 + hook-worker/src/main.rs | 19 +- hook-worker/src/worker.rs | 17 +- 14 files changed, 477 insertions(+), 14 deletions(-) create mode 100644 hook-common/src/health.rs diff --git a/Cargo.lock b/Cargo.lock index 5157ea14869fa..17b608c879f05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -355,6 +355,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -793,7 +802,9 @@ dependencies = [ "serde_json", "sqlx", "thiserror", + "time", "tokio", + "tracing", "uuid", ] @@ -816,6 +827,7 @@ dependencies = [ "serde_json", "sqlx", "thiserror", + "time", "tokio", "tower", "tracing", @@ -839,6 +851,7 @@ dependencies = [ "serde_derive", "sqlx", "thiserror", + "time", "tokio", "tracing", "tracing-subscriber", @@ -1542,6 +1555,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2313,6 +2332,24 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +dependencies = [ + "deranged", + "powerfmt", + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + [[package]] name = "tinyvec" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index 0b48fb23b015b..b4005931a5bed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ sqlx = { version = "0.7", features = [ "tls-native-tls", "uuid", ] } +time = { version = "0.3.20" } thiserror = { version = "1.0" } tokio = { version = "1.34.0", features = ["full"] } tower = "0.4.13" diff --git a/hook-api/src/handlers/app.rs b/hook-api/src/handlers/app.rs index 73c29e1225842..7b1e840094473 100644 --- a/hook-api/src/handlers/app.rs +++ b/hook-api/src/handlers/app.rs @@ -7,6 +7,8 @@ use super::webhook; pub fn add_routes(router: Router, pg_pool: PgQueue) -> Router { router .route("/", routing::get(index)) + .route("/_readiness", routing::get(index)) + .route("/_liveness", routing::get(index)) // No async loop for now, just check axum health .route("/webhook", routing::post(webhook::post).with_state(pg_pool)) } diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index 00c7bd2296a49..ea7ce2fbb9cac 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -18,8 +18,10 @@ serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } +time = { workspace = true } tokio = { workspace = true } thiserror = { workspace = true } +tracing = { workspace = true } uuid = { workspace = true } [dev-dependencies] diff --git a/hook-common/src/health.rs b/hook-common/src/health.rs new file mode 100644 index 0000000000000..c5c79c904c950 --- /dev/null +++ b/hook-common/src/health.rs @@ -0,0 +1,346 @@ +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use std::collections::HashMap; +use std::ops::Add; +use std::sync::{Arc, RwLock}; + +use time::Duration; +use tokio::sync::mpsc; +use tracing::{info, warn}; + +/// Health reporting for components of the service. +/// +/// FIXME: copied over from capture, make sure to keep in sync until we share the crate +/// +/// The capture server contains several asynchronous loops, and +/// the process can only be trusted with user data if all the +/// loops are properly running and reporting. +/// +/// HealthRegistry allows an arbitrary number of components to +/// be registered and report their health. The process' health +/// status is the combination of these individual health status: +/// - if any component is unhealthy, the process is unhealthy +/// - if all components recently reported healthy, the process is healthy +/// - if a component failed to report healthy for its defined deadline, +/// it is considered unhealthy, and the check fails. +/// +/// Trying to merge the k8s concepts of liveness and readiness in +/// a single state is full of foot-guns, so HealthRegistry does not +/// try to do it. Each probe should have its separate instance of +/// the registry to avoid confusions. + +#[derive(Default, Debug)] +pub struct HealthStatus { + /// The overall status: true of all components are healthy + pub healthy: bool, + /// Current status of each registered component, for display + pub components: HashMap, +} +impl IntoResponse for HealthStatus { + /// Computes the axum status code based on the overall health status, + /// and prints each component status in the body for debugging. + fn into_response(self) -> Response { + let body = format!("{:?}", self); + match self.healthy { + true => (StatusCode::OK, body), + false => (StatusCode::INTERNAL_SERVER_ERROR, body), + } + .into_response() + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum ComponentStatus { + /// Automatically set when a component is newly registered + Starting, + /// Recently reported healthy, will need to report again before the date + HealthyUntil(time::OffsetDateTime), + /// Reported unhealthy + Unhealthy, + /// Automatically set when the HealthyUntil deadline is reached + Stalled, +} +struct HealthMessage { + component: String, + status: ComponentStatus, +} + +pub struct HealthHandle { + component: String, + deadline: Duration, + sender: mpsc::Sender, +} + +impl HealthHandle { + /// Asynchronously report healthy, returns when the message is queued. + /// Must be called more frequently than the configured deadline. + pub async fn report_healthy(&self) { + self.report_status(ComponentStatus::HealthyUntil( + time::OffsetDateTime::now_utc().add(self.deadline), + )) + .await + } + + /// Asynchronously report component status, returns when the message is queued. + pub async fn report_status(&self, status: ComponentStatus) { + let message = HealthMessage { + component: self.component.clone(), + status, + }; + if let Err(err) = self.sender.send(message).await { + warn!("failed to report heath status: {}", err) + } + } + + /// Synchronously report as healthy, returns when the message is queued. + /// Must be called more frequently than the configured deadline. + pub fn report_healthy_blocking(&self) { + self.report_status_blocking(ComponentStatus::HealthyUntil( + time::OffsetDateTime::now_utc().add(self.deadline), + )) + } + + /// Asynchronously report component status, returns when the message is queued. + pub fn report_status_blocking(&self, status: ComponentStatus) { + let message = HealthMessage { + component: self.component.clone(), + status, + }; + if let Err(err) = self.sender.blocking_send(message) { + warn!("failed to report heath status: {}", err) + } + } +} + +#[derive(Clone)] +pub struct HealthRegistry { + name: String, + components: Arc>>, + sender: mpsc::Sender, +} + +impl HealthRegistry { + pub fn new(name: &str) -> Self { + let (tx, mut rx) = mpsc::channel::(16); + let registry = Self { + name: name.to_owned(), + components: Default::default(), + sender: tx, + }; + + let components = registry.components.clone(); + tokio::spawn(async move { + while let Some(message) = rx.recv().await { + if let Ok(mut map) = components.write() { + _ = map.insert(message.component, message.status); + } else { + // Poisoned mutex: Just warn, the probes will fail and the process restart + warn!("poisoned HeathRegistry mutex") + } + } + }); + + registry + } + + /// Registers a new component in the registry. The returned handle should be passed + /// to the component, to allow it to frequently report its health status. + pub async fn register(&self, component: String, deadline: time::Duration) -> HealthHandle { + let handle = HealthHandle { + component, + deadline, + sender: self.sender.clone(), + }; + handle.report_status(ComponentStatus::Starting).await; + handle + } + + /// Returns the overall process status, computed from the status of all the components + /// currently registered. Can be used as an axum handler. + pub fn get_status(&self) -> HealthStatus { + let components = self + .components + .read() + .expect("poisoned HeathRegistry mutex"); + + let result = HealthStatus { + healthy: !components.is_empty(), // unhealthy if no component has registered yet + components: Default::default(), + }; + let now = time::OffsetDateTime::now_utc(); + + let result = components + .iter() + .fold(result, |mut result, (name, status)| { + match status { + ComponentStatus::HealthyUntil(until) => { + if until.gt(&now) { + _ = result.components.insert(name.clone(), status.clone()) + } else { + result.healthy = false; + _ = result + .components + .insert(name.clone(), ComponentStatus::Stalled) + } + } + _ => { + result.healthy = false; + _ = result.components.insert(name.clone(), status.clone()) + } + } + result + }); + match result.healthy { + true => info!("{} health check ok", self.name), + false => warn!("{} health check failed: {:?}", self.name, result.components), + } + result + } +} + +#[cfg(test)] +mod tests { + use crate::health::{ComponentStatus, HealthRegistry, HealthStatus}; + use axum::http::StatusCode; + use axum::response::IntoResponse; + use std::ops::{Add, Sub}; + use time::{Duration, OffsetDateTime}; + + async fn assert_or_retry(check: F) + where + F: Fn() -> bool, + { + assert_or_retry_for_duration(check, Duration::seconds(5)).await + } + + async fn assert_or_retry_for_duration(check: F, timeout: Duration) + where + F: Fn() -> bool, + { + let deadline = OffsetDateTime::now_utc().add(timeout); + while !check() && OffsetDateTime::now_utc().lt(&deadline) { + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert!(check()) + } + #[tokio::test] + async fn defaults_to_unhealthy() { + let registry = HealthRegistry::new("liveness"); + assert!(!registry.get_status().healthy); + } + + #[tokio::test] + async fn one_component() { + let registry = HealthRegistry::new("liveness"); + + // New components are registered in Starting + let handle = registry + .register("one".to_string(), Duration::seconds(30)) + .await; + assert_or_retry(|| registry.get_status().components.len() == 1).await; + let mut status = registry.get_status(); + assert!(!status.healthy); + assert_eq!( + status.components.get("one"), + Some(&ComponentStatus::Starting) + ); + + // Status goes healthy once the component reports + handle.report_healthy().await; + assert_or_retry(|| registry.get_status().healthy).await; + status = registry.get_status(); + assert_eq!(status.components.len(), 1); + + // Status goes unhealthy if the components says so + handle.report_status(ComponentStatus::Unhealthy).await; + assert_or_retry(|| !registry.get_status().healthy).await; + status = registry.get_status(); + assert_eq!(status.components.len(), 1); + assert_eq!( + status.components.get("one"), + Some(&ComponentStatus::Unhealthy) + ); + } + + #[tokio::test] + async fn staleness_check() { + let registry = HealthRegistry::new("liveness"); + let handle = registry + .register("one".to_string(), Duration::seconds(30)) + .await; + + // Status goes healthy once the component reports + handle.report_healthy().await; + assert_or_retry(|| registry.get_status().healthy).await; + let mut status = registry.get_status(); + assert_eq!(status.components.len(), 1); + + // If the component's ping is too old, it is considered stalled and the healthcheck fails + // FIXME: we should mock the time instead + handle + .report_status(ComponentStatus::HealthyUntil( + OffsetDateTime::now_utc().sub(Duration::seconds(1)), + )) + .await; + assert_or_retry(|| !registry.get_status().healthy).await; + status = registry.get_status(); + assert_eq!(status.components.len(), 1); + assert_eq!( + status.components.get("one"), + Some(&ComponentStatus::Stalled) + ); + } + + #[tokio::test] + async fn several_components() { + let registry = HealthRegistry::new("liveness"); + let handle1 = registry + .register("one".to_string(), Duration::seconds(30)) + .await; + let handle2 = registry + .register("two".to_string(), Duration::seconds(30)) + .await; + assert_or_retry(|| registry.get_status().components.len() == 2).await; + + // First component going healthy is not enough + handle1.report_healthy().await; + assert_or_retry(|| { + registry.get_status().components.get("one").unwrap() != &ComponentStatus::Starting + }) + .await; + assert!(!registry.get_status().healthy); + + // Second component going healthy brings the health to green + handle2.report_healthy().await; + assert_or_retry(|| { + registry.get_status().components.get("two").unwrap() != &ComponentStatus::Starting + }) + .await; + assert!(registry.get_status().healthy); + + // First component going unhealthy takes down the health to red + handle1.report_status(ComponentStatus::Unhealthy).await; + assert_or_retry(|| !registry.get_status().healthy).await; + + // First component recovering returns the health to green + handle1.report_healthy().await; + assert_or_retry(|| registry.get_status().healthy).await; + + // Second component going unhealthy takes down the health to red + handle2.report_status(ComponentStatus::Unhealthy).await; + assert_or_retry(|| !registry.get_status().healthy).await; + } + + #[tokio::test] + async fn into_response() { + let nok = HealthStatus::default().into_response(); + assert_eq!(nok.status(), StatusCode::INTERNAL_SERVER_ERROR); + + let ok = HealthStatus { + healthy: true, + components: Default::default(), + } + .into_response(); + assert_eq!(ok.status(), StatusCode::OK); + } +} diff --git a/hook-common/src/lib.rs b/hook-common/src/lib.rs index 8e63ded5a7bf2..7f49049add362 100644 --- a/hook-common/src/lib.rs +++ b/hook-common/src/lib.rs @@ -1,3 +1,4 @@ +pub mod health; pub mod kafka_messages; pub mod metrics; pub mod pgqueue; diff --git a/hook-janitor/Cargo.toml b/hook-janitor/Cargo.toml index f23626bea05a6..96a80ebd9c3d5 100644 --- a/hook-janitor/Cargo.toml +++ b/hook-janitor/Cargo.toml @@ -20,6 +20,7 @@ serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } +time = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tower = { workspace = true } diff --git a/hook-janitor/src/handlers/app.rs b/hook-janitor/src/handlers/app.rs index cab0e0d27991a..507a1cba48d46 100644 --- a/hook-janitor/src/handlers/app.rs +++ b/hook-janitor/src/handlers/app.rs @@ -1,7 +1,12 @@ -use axum::{routing, Router}; +use axum::{routing::get, Router}; +use hook_common::health::HealthRegistry; +use std::future::ready; -pub fn app() -> Router { - Router::new().route("/", routing::get(index)) +pub fn app(liveness: HealthRegistry) -> Router { + Router::new() + .route("/", get(index)) + .route("/_readiness", get(index)) + .route("/_liveness", get(move || ready(liveness.get_status()))) } pub async fn index() -> &'static str { diff --git a/hook-janitor/src/kafka_producer.rs b/hook-janitor/src/kafka_producer.rs index 1d0144cb75436..ba368663072c3 100644 --- a/hook-janitor/src/kafka_producer.rs +++ b/hook-janitor/src/kafka_producer.rs @@ -1,17 +1,27 @@ use crate::config::KafkaConfig; +use hook_common::health::HealthHandle; use rdkafka::error::KafkaError; use rdkafka::producer::FutureProducer; use rdkafka::ClientConfig; use tracing::debug; -// TODO: Take stats recording pieces that we want from `capture-rs`. -pub struct KafkaContext {} +pub struct KafkaContext { + liveness: HealthHandle, +} + +impl rdkafka::ClientContext for KafkaContext { + fn stats(&self, _: rdkafka::Statistics) { + // Signal liveness, as the main rdkafka loop is running and calling us + self.liveness.report_healthy_blocking(); -impl rdkafka::ClientContext for KafkaContext {} + // TODO: Take stats recording pieces that we want from `capture-rs`. + } +} pub async fn create_kafka_producer( config: &KafkaConfig, + liveness: HealthHandle, ) -> Result, KafkaError> { let mut client_config = ClientConfig::new(); client_config @@ -38,7 +48,10 @@ pub async fn create_kafka_producer( }; debug!("rdkafka configuration: {:?}", client_config); - let api: FutureProducer = client_config.create_with_context(KafkaContext {})?; + let api: FutureProducer = + client_config.create_with_context(KafkaContext { liveness })?; + + // TODO: ping the kafka brokers to confirm configuration is OK (copy capture) Ok(api) } diff --git a/hook-janitor/src/main.rs b/hook-janitor/src/main.rs index 63068c35262bf..15d0068ad693a 100644 --- a/hook-janitor/src/main.rs +++ b/hook-janitor/src/main.rs @@ -4,6 +4,7 @@ use config::Config; use envconfig::Envconfig; use eyre::Result; use futures::future::{select, Either}; +use hook_common::health::{HealthHandle, HealthRegistry}; use kafka_producer::create_kafka_producer; use std::{str::FromStr, time::Duration}; use tokio::sync::Semaphore; @@ -25,13 +26,14 @@ async fn listen(app: Router, bind: String) -> Result<()> { Ok(()) } -async fn cleanup_loop(cleaner: Box, interval_secs: u64) { +async fn cleanup_loop(cleaner: Box, interval_secs: u64, liveness: HealthHandle) { let semaphore = Semaphore::new(1); let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); loop { let _permit = semaphore.acquire().await; interval.tick().await; + liveness.report_healthy().await; cleaner.cleanup().await; drop(_permit); } @@ -46,9 +48,14 @@ async fn main() { let mode_name = CleanerModeName::from_str(&config.mode) .unwrap_or_else(|_| panic!("invalid cleaner mode: {}", config.mode)); + let liveness = HealthRegistry::new("liveness"); + let cleaner = match mode_name { CleanerModeName::Webhooks => { - let kafka_producer = create_kafka_producer(&config.kafka) + let kafka_liveness = liveness + .register("rdkafka".to_string(), time::Duration::seconds(30)) + .await; + let kafka_producer = create_kafka_producer(&config.kafka, kafka_liveness) .await .expect("failed to create kafka producer"); @@ -64,9 +71,19 @@ async fn main() { } }; - let cleanup_loop = Box::pin(cleanup_loop(cleaner, config.cleanup_interval_secs)); + let cleanup_liveness = liveness + .register( + "cleanup_loop".to_string(), + time::Duration::seconds(config.cleanup_interval_secs as i64 * 2), + ) + .await; + let cleanup_loop = Box::pin(cleanup_loop( + cleaner, + config.cleanup_interval_secs, + cleanup_liveness, + )); - let app = setup_metrics_routes(handlers::app()); + let app = setup_metrics_routes(handlers::app(liveness)); let http_server = Box::pin(listen(app, config.bind())); match select(http_server, cleanup_loop).await { diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 4248a47281e52..7aa9aa353750f 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -457,6 +457,7 @@ mod tests { use super::*; use crate::config; use crate::kafka_producer::{create_kafka_producer, KafkaContext}; + use hook_common::health::HealthRegistry; use hook_common::kafka_messages::app_metrics::{ Error as WebhookError, ErrorDetails, ErrorType, }; @@ -476,6 +477,10 @@ mod tests { MockCluster<'static, DefaultProducerContext>, FutureProducer, ) { + let registry = HealthRegistry::new("liveness"); + let handle = registry + .register("one".to_string(), time::Duration::seconds(30)) + .await; let cluster = MockCluster::new(1).expect("failed to create mock brokers"); let config = config::KafkaConfig { @@ -491,7 +496,7 @@ mod tests { ( cluster, - create_kafka_producer(&config) + create_kafka_producer(&config, handle) .await .expect("failed to create mocked kafka producer"), ) diff --git a/hook-worker/Cargo.toml b/hook-worker/Cargo.toml index 11da0a80a564c..6ed5796efd1f0 100644 --- a/hook-worker/Cargo.toml +++ b/hook-worker/Cargo.toml @@ -15,6 +15,7 @@ reqwest = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } sqlx = { workspace = true } +time = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index cc17169b2bd83..d036d546a94e8 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -1,7 +1,10 @@ //! Consume `PgQueue` jobs to run webhook calls. +use axum::routing::get; use axum::Router; use envconfig::Envconfig; +use std::future::ready; +use hook_common::health::HealthRegistry; use hook_common::{ metrics::serve, metrics::setup_metrics_routes, pgqueue::PgQueue, retry::RetryPolicy, }; @@ -15,6 +18,11 @@ async fn main() -> Result<(), WorkerError> { let config = Config::init_from_env().expect("Invalid configuration:"); + let liveness = HealthRegistry::new("liveness"); + let worker_liveness = liveness + .register("worker".to_string(), time::Duration::seconds(60)) // TODO: compute the value from worker params + .await; + let retry_policy = RetryPolicy::build( config.retry_policy.backoff_coefficient, config.retry_policy.initial_interval.0, @@ -33,11 +41,16 @@ async fn main() -> Result<(), WorkerError> { config.request_timeout.0, config.max_concurrent_jobs, retry_policy, + worker_liveness, ); + let router = Router::new() + .route("/", get(index)) + .route("/_readiness", get(index)) + .route("/_liveness", get(move || ready(liveness.get_status()))); + let router = setup_metrics_routes(router); let bind = config.bind(); tokio::task::spawn(async move { - let router = setup_metrics_routes(Router::new()); serve(router, &bind) .await .expect("failed to start serving metrics"); @@ -47,3 +60,7 @@ async fn main() -> Result<(), WorkerError> { Ok(()) } + +pub async fn index() -> &'static str { + "rusty-hook worker" +} diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 77c6adb4b13f8..1041422fec17b 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -2,6 +2,7 @@ use std::collections; use std::sync::Arc; use std::time; +use hook_common::health::HealthHandle; use hook_common::{ pgqueue::{Job, PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, retry::RetryPolicy, @@ -75,6 +76,8 @@ pub struct WebhookWorker<'p> { max_concurrent_jobs: usize, /// The retry policy used to calculate retry intervals when a job fails with a retryable error. retry_policy: RetryPolicy, + /// The liveness check handle, to call on a schedule to report healthy + liveness: HealthHandle, } impl<'p> WebhookWorker<'p> { @@ -85,6 +88,7 @@ impl<'p> WebhookWorker<'p> { request_timeout: time::Duration, max_concurrent_jobs: usize, retry_policy: RetryPolicy, + liveness: HealthHandle, ) -> Self { let mut headers = header::HeaderMap::new(); headers.insert( @@ -106,6 +110,7 @@ impl<'p> WebhookWorker<'p> { client, max_concurrent_jobs, retry_policy, + liveness, } } @@ -117,6 +122,7 @@ impl<'p> WebhookWorker<'p> { loop { interval.tick().await; + self.liveness.report_healthy().await; if let Some(job) = self.queue.dequeue(&self.name).await? { return Ok(job); @@ -132,6 +138,7 @@ impl<'p> WebhookWorker<'p> { loop { interval.tick().await; + self.liveness.report_healthy().await; if let Some(job) = self.queue.dequeue_tx(&self.name).await? { return Ok(job); @@ -157,7 +164,6 @@ impl<'p> WebhookWorker<'p> { } else { loop { let webhook_job = self.wait_for_job().await?; - spawn_webhook_job_processing_task( self.client.clone(), semaphore.clone(), @@ -430,6 +436,8 @@ mod tests { // This is due to a long-standing cargo bug that reports imports and helper functions as unused. // See: https://github.com/rust-lang/rust/issues/46379. #[allow(unused_imports)] + use hook_common::health::HealthRegistry; + #[allow(unused_imports)] use hook_common::pgqueue::{JobStatus, NewJob}; #[allow(unused_imports)] use sqlx::PgPool; @@ -502,6 +510,10 @@ mod tests { plugin_id: 2, plugin_config_id: 3, }; + let registry = HealthRegistry::new("liveness"); + let liveness = registry + .register("worker".to_string(), ::time::Duration::seconds(30)) + .await; // enqueue takes ownership of the job enqueued to avoid bugs that can cause duplicate jobs. // Normally, a separate application would be enqueueing jobs for us to consume, so no ownership // conflicts would arise. However, in this test we need to do the enqueueing ourselves. @@ -521,6 +533,7 @@ mod tests { time::Duration::from_millis(5000), 10, RetryPolicy::default(), + liveness, ); let consumed_job = worker @@ -543,6 +556,8 @@ mod tests { .complete() .await .expect("job not successfully completed"); + + assert!(registry.get_status().healthy) } #[sqlx::test(migrations = "../migrations")] From c181c50580625699a077b81fcfada9a7d5c1dd5b Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 16 Jan 2024 12:56:04 -0700 Subject: [PATCH 188/247] Add queue depth gauges (#49) --- hook-janitor/src/webhooks.rs | 49 +++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 7aa9aa353750f..4b4f9a44f3bcc 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -24,6 +24,8 @@ pub enum WebhookCleanerError { PoolCreationError { error: sqlx::Error }, #[error("failed to acquire conn and start txn: {error}")] StartTxnError { error: sqlx::Error }, + #[error("failed to get queue depth: {error}")] + GetQueueDepthError { error: sqlx::Error }, #[error("failed to get row count: {error}")] GetRowCountError { error: sqlx::Error }, #[error("failed to get completed rows: {error}")] @@ -107,6 +109,14 @@ struct FailedRow { failures: u32, } +#[derive(sqlx::FromRow, Debug)] +struct QueueDepth { + oldest_created_at_untried: DateTime, + count_untried: i64, + oldest_created_at_retries: DateTime, + count_retries: i64, +} + impl From for AppMetric { fn from(row: FailedRow) -> Self { AppMetric { @@ -175,6 +185,33 @@ impl WebhookCleaner { }) } + async fn get_queue_depth(&self) -> Result { + let mut conn = self + .pg_pool + .acquire() + .await + .map_err(|e| WebhookCleanerError::StartTxnError { error: e })?; + + let base_query = r#" + SELECT + COALESCE(MIN(CASE WHEN attempt = 0 THEN created_at END), now()) AS oldest_created_at_untried, + SUM(CASE WHEN attempt = 0 THEN 1 ELSE 0 END) AS count_untried, + COALESCE(MIN(CASE WHEN attempt > 0 THEN created_at END), now()) AS oldest_created_at_retries, + SUM(CASE WHEN attempt > 0 THEN 1 ELSE 0 END) AS count_retries + FROM job_queue + WHERE status = 'available' + AND queue = $1; + "#; + + let row = sqlx::query_as::<_, QueueDepth>(base_query) + .bind(&self.queue_name) + .fetch_one(&mut *conn) + .await + .map_err(|e| WebhookCleanerError::GetQueueDepthError { error: e })?; + + Ok(row) + } + async fn start_serializable_txn(&self) -> Result { let mut tx = self .pg_pool @@ -229,7 +266,7 @@ impl WebhookCleaner { count(*) as successes FROM job_queue WHERE status = 'completed' - AND queue = $1 + AND queue = $1 GROUP BY hour, team_id, plugin_config_id ORDER BY hour, team_id, plugin_config_id; "#; @@ -354,6 +391,16 @@ impl WebhookCleaner { // of rows in memory. It seems unlikely we'll need to paginate, but that can be added in the // future if necessary. + let queue_depth = self.get_queue_depth().await?; + metrics::gauge!("queue_depth_oldest_created_at_untried") + .set(queue_depth.oldest_created_at_untried.timestamp() as f64); + metrics::gauge!("queue_depth", &[("status", "untried")]) + .set(queue_depth.count_untried as f64); + metrics::gauge!("queue_depth_oldest_created_at_retries") + .set(queue_depth.oldest_created_at_retries.timestamp() as f64); + metrics::gauge!("queue_depth", &[("status", "retries")]) + .set(queue_depth.count_retries as f64); + let mut tx = self.start_serializable_txn().await?; let (completed_row_count, completed_agg_row_count) = { From 1422683d87708eb5e4261f3ebf92594b92ea8793 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 17 Jan 2024 10:53:51 +0100 Subject: [PATCH 189/247] feat: add webhook_worker_saturation_percent metric for autoscaling (#50) --- hook-worker/src/worker.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 1041422fec17b..7fe6808d02b6f 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -149,9 +149,14 @@ impl<'p> WebhookWorker<'p> { /// Run this worker to continuously process any jobs that become available. pub async fn run(&self, transactional: bool) -> Result<(), WorkerError> { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); + let report_semaphore_utilization = || { + metrics::gauge!("webhook_worker_saturation_percent") + .set(1f64 - semaphore.available_permits() as f64 / self.max_concurrent_jobs as f64); + }; if transactional { loop { + report_semaphore_utilization(); let webhook_job = self.wait_for_job_tx().await?; spawn_webhook_job_processing_task( self.client.clone(), @@ -163,6 +168,7 @@ impl<'p> WebhookWorker<'p> { } } else { loop { + report_semaphore_utilization(); let webhook_job = self.wait_for_job().await?; spawn_webhook_job_processing_task( self.client.clone(), From 23ef5e276d1dc85333500aaa20bafa62a3594f5f Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 17 Jan 2024 12:43:36 +0100 Subject: [PATCH 190/247] fix:fix janitor get_queue_depth when queue is empty (#54) --- hook-janitor/src/webhooks.rs | 46 ++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 4b4f9a44f3bcc..1d536c0e7267b 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -195,9 +195,9 @@ impl WebhookCleaner { let base_query = r#" SELECT COALESCE(MIN(CASE WHEN attempt = 0 THEN created_at END), now()) AS oldest_created_at_untried, - SUM(CASE WHEN attempt = 0 THEN 1 ELSE 0 END) AS count_untried, + COALESCE(SUM(CASE WHEN attempt = 0 THEN 1 ELSE 0 END), 0) AS count_untried, COALESCE(MIN(CASE WHEN attempt > 0 THEN created_at END), now()) AS oldest_created_at_retries, - SUM(CASE WHEN attempt > 0 THEN 1 ELSE 0 END) AS count_retries + COALESCE(SUM(CASE WHEN attempt > 0 THEN 1 ELSE 0 END), 0) AS count_retries FROM job_queue WHERE status = 'available' AND queue = $1; @@ -513,6 +513,7 @@ mod tests { use rdkafka::consumer::{Consumer, StreamConsumer}; use rdkafka::mocking::MockCluster; use rdkafka::producer::{DefaultProducerContext, FutureProducer}; + use rdkafka::types::{RDKafkaApiKey, RDKafkaRespErr}; use rdkafka::{ClientConfig, Message}; use sqlx::{PgPool, Row}; use std::collections::HashMap; @@ -754,6 +755,47 @@ mod tests { check_app_metric_vector_equality(&expected_app_metrics, &received_app_metrics); } + #[sqlx::test(migrations = "../migrations")] + async fn test_cleanup_impl_empty_queue(db: PgPool) { + let (mock_cluster, mock_producer) = create_mock_kafka().await; + mock_cluster + .create_topic(APP_METRICS_TOPIC, 1, 1) + .expect("failed to create mock app_metrics topic"); + + // No payload should be produced to kafka as the queue is empty. + // Set a non-retriable produce error that would bubble-up when cleanup_impl is called. + let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE; 1]; + mock_cluster.request_errors(RDKafkaApiKey::Produce, &err); + + let consumer: StreamConsumer = ClientConfig::new() + .set("bootstrap.servers", mock_cluster.bootstrap_servers()) + .set("group.id", "mock") + .set("auto.offset.reset", "earliest") + .create() + .expect("failed to create mock consumer"); + consumer.subscribe(&[APP_METRICS_TOPIC]).unwrap(); + + let webhook_cleaner = WebhookCleaner::new_from_pool( + &"webhooks", + db, + mock_producer, + APP_METRICS_TOPIC.to_owned(), + ) + .expect("unable to create webhook cleaner"); + + let cleanup_stats = webhook_cleaner + .cleanup_impl() + .await + .expect("webbook cleanup_impl failed"); + + // Reported metrics are all zeroes + assert_eq!(cleanup_stats.rows_processed, 0); + assert_eq!(cleanup_stats.completed_row_count, 0); + assert_eq!(cleanup_stats.completed_agg_row_count, 0); + assert_eq!(cleanup_stats.failed_row_count, 0); + assert_eq!(cleanup_stats.failed_agg_row_count, 0); + } + #[sqlx::test(migrations = "../migrations", fixtures("webhook_cleanup"))] async fn test_serializable_isolation(db: PgPool) { let (_, mock_producer) = create_mock_kafka().await; From 9c9ebd5033ac8dcb1c283a5a31203ea682a8a26f Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 17 Jan 2024 07:56:37 -0700 Subject: [PATCH 191/247] Remove queue filters from janitor (#52) --- hook-janitor/src/config.rs | 3 -- hook-janitor/src/main.rs | 1 - hook-janitor/src/webhooks.rs | 82 ++++++++++++------------------------ 3 files changed, 27 insertions(+), 59 deletions(-) diff --git a/hook-janitor/src/config.rs b/hook-janitor/src/config.rs index 252a67011460c..389de0342e03a 100644 --- a/hook-janitor/src/config.rs +++ b/hook-janitor/src/config.rs @@ -11,9 +11,6 @@ pub struct Config { #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] pub database_url: String, - #[envconfig(default = "default")] - pub queue_name: String, - #[envconfig(default = "30")] pub cleanup_interval_secs: u64, diff --git a/hook-janitor/src/main.rs b/hook-janitor/src/main.rs index 15d0068ad693a..46ee37560bb90 100644 --- a/hook-janitor/src/main.rs +++ b/hook-janitor/src/main.rs @@ -61,7 +61,6 @@ async fn main() { Box::new( WebhookCleaner::new( - &config.queue_name, &config.database_url, kafka_producer, config.kafka.app_metrics_topic.to_owned(), diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 1d536c0e7267b..5ac9d558fa812 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -51,7 +51,6 @@ pub enum WebhookCleanerError { type Result = std::result::Result; pub struct WebhookCleaner { - queue_name: String, pg_pool: PgPool, kafka_producer: FutureProducer, app_metrics_topic: String, @@ -149,19 +148,16 @@ struct CleanupStats { impl WebhookCleaner { pub fn new( - queue_name: &str, database_url: &str, kafka_producer: FutureProducer, app_metrics_topic: String, ) -> Result { - let queue_name = queue_name.to_owned(); let pg_pool = PgPoolOptions::new() .acquire_timeout(Duration::from_secs(10)) .connect_lazy(database_url) .map_err(|error| WebhookCleanerError::PoolCreationError { error })?; Ok(Self { - queue_name, pg_pool, kafka_producer, app_metrics_topic, @@ -170,15 +166,11 @@ impl WebhookCleaner { #[allow(dead_code)] // This is used in tests. pub fn new_from_pool( - queue_name: &str, pg_pool: PgPool, kafka_producer: FutureProducer, app_metrics_topic: String, ) -> Result { - let queue_name = queue_name.to_owned(); - Ok(Self { - queue_name, pg_pool, kafka_producer, app_metrics_topic, @@ -199,12 +191,10 @@ impl WebhookCleaner { COALESCE(MIN(CASE WHEN attempt > 0 THEN created_at END), now()) AS oldest_created_at_retries, COALESCE(SUM(CASE WHEN attempt > 0 THEN 1 ELSE 0 END), 0) AS count_retries FROM job_queue - WHERE status = 'available' - AND queue = $1; + WHERE status = 'available'; "#; let row = sqlx::query_as::<_, QueueDepth>(base_query) - .bind(&self.queue_name) .fetch_one(&mut *conn) .await .map_err(|e| WebhookCleanerError::GetQueueDepthError { error: e })?; @@ -240,12 +230,10 @@ impl WebhookCleaner { ) -> Result { let base_query = r#" SELECT count(*) FROM job_queue - WHERE queue = $1 - AND status = $2::job_status; + WHERE status = $1::job_status; "#; let count: i64 = sqlx::query(base_query) - .bind(&self.queue_name) .bind(status) .fetch_one(&mut *tx.0) .await @@ -266,13 +254,11 @@ impl WebhookCleaner { count(*) as successes FROM job_queue WHERE status = 'completed' - AND queue = $1 GROUP BY hour, team_id, plugin_config_id ORDER BY hour, team_id, plugin_config_id; "#; let rows = sqlx::query_as::<_, CompletedRow>(base_query) - .bind(&self.queue_name) .fetch_all(&mut *tx.0) .await .map_err(|e| WebhookCleanerError::GetCompletedRowsError { error: e })?; @@ -289,13 +275,11 @@ impl WebhookCleaner { count(*) as failures FROM job_queue WHERE status = 'failed' - AND queue = $1 GROUP BY hour, team_id, plugin_config_id, last_error ORDER BY hour, team_id, plugin_config_id, last_error; "#; let rows = sqlx::query_as::<_, FailedRow>(base_query) - .bind(&self.queue_name) .fetch_all(&mut *tx.0) .await .map_err(|e| WebhookCleanerError::GetFailedRowsError { error: e })?; @@ -352,11 +336,9 @@ impl WebhookCleaner { let base_query = r#" DELETE FROM job_queue WHERE status IN ('failed', 'completed') - AND queue = $1; "#; let result = sqlx::query(base_query) - .bind(&self.queue_name) .execute(&mut *tx.0) .await .map_err(|e| WebhookCleanerError::DeleteRowsError { error: e })?; @@ -577,22 +559,17 @@ mod tests { .expect("failed to create mock consumer"); consumer.subscribe(&[APP_METRICS_TOPIC]).unwrap(); - let webhook_cleaner = WebhookCleaner::new_from_pool( - &"webhooks", - db, - mock_producer, - APP_METRICS_TOPIC.to_owned(), - ) - .expect("unable to create webhook cleaner"); + let webhook_cleaner = + WebhookCleaner::new_from_pool(db, mock_producer, APP_METRICS_TOPIC.to_owned()) + .expect("unable to create webhook cleaner"); let cleanup_stats = webhook_cleaner .cleanup_impl() .await .expect("webbook cleanup_impl failed"); - // Rows from other queues and rows that are not 'completed' or 'failed' should not be - // processed. - assert_eq!(cleanup_stats.rows_processed, 11); + // Rows that are not 'completed' or 'failed' should not be processed. + assert_eq!(cleanup_stats.rows_processed, 13); let mut received_app_metrics = Vec::new(); for _ in 0..(cleanup_stats.completed_agg_row_count + cleanup_stats.failed_agg_row_count) { @@ -609,7 +586,7 @@ mod tests { plugin_config_id: 2, job_id: None, category: AppMetricCategory::Webhook, - successes: 2, + successes: 3, successes_on_retry: 0, failures: 0, error_uuid: None, @@ -682,7 +659,7 @@ mod tests { category: AppMetricCategory::Webhook, successes: 0, successes_on_retry: 0, - failures: 2, + failures: 3, error_uuid: Some(Uuid::parse_str("018c8935-d038-714a-957c-0df43d42e377").unwrap()), error_type: Some(ErrorType::TimeoutError), error_details: Some(ErrorDetails { @@ -799,13 +776,9 @@ mod tests { #[sqlx::test(migrations = "../migrations", fixtures("webhook_cleanup"))] async fn test_serializable_isolation(db: PgPool) { let (_, mock_producer) = create_mock_kafka().await; - let webhook_cleaner = WebhookCleaner::new_from_pool( - &"webhooks", - db.clone(), - mock_producer, - APP_METRICS_TOPIC.to_owned(), - ) - .expect("unable to create webhook cleaner"); + let webhook_cleaner = + WebhookCleaner::new_from_pool(db.clone(), mock_producer, APP_METRICS_TOPIC.to_owned()) + .expect("unable to create webhook cleaner"); let queue = PgQueue::new_from_pool("webhooks", db.clone()) .await @@ -813,14 +786,13 @@ mod tests { async fn get_count_from_new_conn(db: &PgPool, status: &str) -> i64 { let mut conn = db.acquire().await.unwrap(); - let count: i64 = sqlx::query( - "SELECT count(*) FROM job_queue WHERE queue = 'webhooks' AND status = $1::job_status", - ) - .bind(&status) - .fetch_one(&mut *conn) - .await - .unwrap() - .get(0); + let count: i64 = + sqlx::query("SELECT count(*) FROM job_queue WHERE status = $1::job_status") + .bind(&status) + .fetch_one(&mut *conn) + .await + .unwrap() + .get(0); count } @@ -832,10 +804,10 @@ mod tests { .unwrap(); webhook_cleaner.get_failed_agg_rows(&mut tx).await.unwrap(); - // All 13 rows in the queue are visible from outside the txn. - // The 11 the cleaner will process, plus 1 available and 1 running. - assert_eq!(get_count_from_new_conn(&db, "completed").await, 5); - assert_eq!(get_count_from_new_conn(&db, "failed").await, 6); + // All 15 rows in the DB are visible from outside the txn. + // The 13 the cleaner will process, plus 1 available and 1 running. + assert_eq!(get_count_from_new_conn(&db, "completed").await, 6); + assert_eq!(get_count_from_new_conn(&db, "failed").await, 7); assert_eq!(get_count_from_new_conn(&db, "available").await, 1); assert_eq!(get_count_from_new_conn(&db, "running").await, 1); @@ -896,15 +868,15 @@ mod tests { } // There are now 2 more completed rows (jobs added above) than before, visible from outside the txn. - assert_eq!(get_count_from_new_conn(&db, "completed").await, 7); + assert_eq!(get_count_from_new_conn(&db, "completed").await, 8); assert_eq!(get_count_from_new_conn(&db, "available").await, 1); let rows_processed = webhook_cleaner.delete_observed_rows(&mut tx).await.unwrap(); - // The 11 rows that were in the queue when the txn started should be deleted. - assert_eq!(rows_processed, 11); + // The 13 rows in the DB when the txn started should be deleted. + assert_eq!(rows_processed, 13); // We haven't committed, so the rows are still visible from outside the txn. - assert_eq!(get_count_from_new_conn(&db, "completed").await, 7); + assert_eq!(get_count_from_new_conn(&db, "completed").await, 8); assert_eq!(get_count_from_new_conn(&db, "available").await, 1); webhook_cleaner.commit_txn(tx).await.unwrap(); From 099053e0b1891be416ea39967589a64d3e4349db Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 17 Jan 2024 08:27:37 -0700 Subject: [PATCH 192/247] Make retry_queue_name truly optional from env to main (#53) --- hook-janitor/src/webhooks.rs | 10 +++------- hook-worker/src/config.rs | 3 +-- hook-worker/src/main.rs | 14 +++++++++----- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 5ac9d558fa812..bc016055d3b3c 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -752,13 +752,9 @@ mod tests { .expect("failed to create mock consumer"); consumer.subscribe(&[APP_METRICS_TOPIC]).unwrap(); - let webhook_cleaner = WebhookCleaner::new_from_pool( - &"webhooks", - db, - mock_producer, - APP_METRICS_TOPIC.to_owned(), - ) - .expect("unable to create webhook cleaner"); + let webhook_cleaner = + WebhookCleaner::new_from_pool(db, mock_producer, APP_METRICS_TOPIC.to_owned()) + .expect("unable to create webhook cleaner"); let cleanup_stats = webhook_cleaner .cleanup_impl() diff --git a/hook-worker/src/config.rs b/hook-worker/src/config.rs index 8b2b4ba49f205..8484671047fc2 100644 --- a/hook-worker/src/config.rs +++ b/hook-worker/src/config.rs @@ -70,6 +70,5 @@ pub struct RetryPolicyConfig { #[envconfig(default = "100000")] pub maximum_interval: EnvMsDuration, - #[envconfig(default = "default")] - pub retry_queue_name: String, + pub retry_queue_name: Option, } diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index d036d546a94e8..bf6b4fd1e1468 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -23,13 +23,17 @@ async fn main() -> Result<(), WorkerError> { .register("worker".to_string(), time::Duration::seconds(60)) // TODO: compute the value from worker params .await; - let retry_policy = RetryPolicy::build( + let mut retry_policy_builder = RetryPolicy::build( config.retry_policy.backoff_coefficient, config.retry_policy.initial_interval.0, ) - .maximum_interval(config.retry_policy.maximum_interval.0) - .queue(&config.retry_policy.retry_queue_name) - .provide(); + .maximum_interval(config.retry_policy.maximum_interval.0); + + retry_policy_builder = match &config.retry_policy.retry_queue_name { + Some(retry_queue_name) => retry_policy_builder.queue(retry_queue_name), + _ => retry_policy_builder, + }; + let queue = PgQueue::new(&config.queue_name, &config.database_url) .await .expect("failed to initialize queue"); @@ -40,7 +44,7 @@ async fn main() -> Result<(), WorkerError> { config.poll_interval.0, config.request_timeout.0, config.max_concurrent_jobs, - retry_policy, + retry_policy_builder.provide(), worker_liveness, ); From 58474c06d22bb5d3a1a2df6289e9cdce847c6b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 17 Jan 2024 20:17:15 +0100 Subject: [PATCH 193/247] refactor: Use new NonEmptyString config type for queue names (#55) --- hook-worker/src/config.rs | 28 ++++++++++++++++++++++++++-- hook-worker/src/main.rs | 9 +++++---- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/hook-worker/src/config.rs b/hook-worker/src/config.rs index 8484671047fc2..74342f71e0030 100644 --- a/hook-worker/src/config.rs +++ b/hook-worker/src/config.rs @@ -18,7 +18,7 @@ pub struct Config { pub worker_name: String, #[envconfig(default = "default")] - pub queue_name: String, + pub queue_name: NonEmptyString, #[envconfig(default = "100")] pub poll_interval: EnvMsDuration, @@ -70,5 +70,29 @@ pub struct RetryPolicyConfig { #[envconfig(default = "100000")] pub maximum_interval: EnvMsDuration, - pub retry_queue_name: Option, + pub retry_queue_name: Option, +} + +#[derive(Debug, Clone)] +pub struct NonEmptyString(pub String); + +impl NonEmptyString { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct StringIsEmptyError; + +impl FromStr for NonEmptyString { + type Err = StringIsEmptyError; + + fn from_str(s: &str) -> Result { + if s.is_empty() { + Err(StringIsEmptyError) + } else { + Ok(NonEmptyString(s.to_owned())) + } + } } diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index bf6b4fd1e1468..bfee9add6271a 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -29,12 +29,13 @@ async fn main() -> Result<(), WorkerError> { ) .maximum_interval(config.retry_policy.maximum_interval.0); - retry_policy_builder = match &config.retry_policy.retry_queue_name { - Some(retry_queue_name) => retry_policy_builder.queue(retry_queue_name), - _ => retry_policy_builder, + retry_policy_builder = if let Some(retry_queue_name) = &config.retry_policy.retry_queue_name { + retry_policy_builder.queue(retry_queue_name.as_str()) + } else { + retry_policy_builder }; - let queue = PgQueue::new(&config.queue_name, &config.database_url) + let queue = PgQueue::new(config.queue_name.as_str(), &config.database_url) .await .expect("failed to initialize queue"); From 499f1432169f0fd8872e16c250c48500c3d10914 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 18 Jan 2024 10:10:33 -0700 Subject: [PATCH 194/247] Log error rather than exiting process on dequeue error (#51) --- hook-worker/src/main.rs | 2 +- hook-worker/src/worker.rs | 35 ++++++++++++++++++++--------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index bfee9add6271a..345fa3dd31c4a 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -61,7 +61,7 @@ async fn main() -> Result<(), WorkerError> { .expect("failed to start serving metrics"); }); - worker.run(config.transactional).await?; + worker.run(config.transactional).await; Ok(()) } diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 7fe6808d02b6f..c526c3f7cce1c 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -115,17 +115,20 @@ impl<'p> WebhookWorker<'p> { } /// Wait until a job becomes available in our queue. - async fn wait_for_job<'a>( - &self, - ) -> Result, WorkerError> { + async fn wait_for_job<'a>(&self) -> PgJob { let mut interval = tokio::time::interval(self.poll_interval); loop { interval.tick().await; self.liveness.report_healthy().await; - if let Some(job) = self.queue.dequeue(&self.name).await? { - return Ok(job); + match self.queue.dequeue(&self.name).await { + Ok(Some(job)) => return job, + Ok(None) => continue, + Err(error) => { + error!("error while trying to dequeue job: {}", error); + continue; + } } } } @@ -133,21 +136,26 @@ impl<'p> WebhookWorker<'p> { /// Wait until a job becomes available in our queue in transactional mode. async fn wait_for_job_tx<'a>( &self, - ) -> Result, WorkerError> { + ) -> PgTransactionJob<'a, WebhookJobParameters, WebhookJobMetadata> { let mut interval = tokio::time::interval(self.poll_interval); loop { interval.tick().await; self.liveness.report_healthy().await; - if let Some(job) = self.queue.dequeue_tx(&self.name).await? { - return Ok(job); + match self.queue.dequeue_tx(&self.name).await { + Ok(Some(job)) => return job, + Ok(None) => continue, + Err(error) => { + error!("error while trying to dequeue_tx job: {}", error); + continue; + } } } } /// Run this worker to continuously process any jobs that become available. - pub async fn run(&self, transactional: bool) -> Result<(), WorkerError> { + pub async fn run(&self, transactional: bool) { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); let report_semaphore_utilization = || { metrics::gauge!("webhook_worker_saturation_percent") @@ -157,7 +165,7 @@ impl<'p> WebhookWorker<'p> { if transactional { loop { report_semaphore_utilization(); - let webhook_job = self.wait_for_job_tx().await?; + let webhook_job = self.wait_for_job_tx().await; spawn_webhook_job_processing_task( self.client.clone(), semaphore.clone(), @@ -169,7 +177,7 @@ impl<'p> WebhookWorker<'p> { } else { loop { report_semaphore_utilization(); - let webhook_job = self.wait_for_job().await?; + let webhook_job = self.wait_for_job().await; spawn_webhook_job_processing_task( self.client.clone(), semaphore.clone(), @@ -542,10 +550,7 @@ mod tests { liveness, ); - let consumed_job = worker - .wait_for_job() - .await - .expect("failed to wait and read job"); + let consumed_job = worker.wait_for_job().await; assert_eq!(consumed_job.job.attempt, 1); assert!(consumed_job.job.attempted_by.contains(&worker_id)); From a4b9943d7f2e9ede94c6089658d4c1b869ca2579 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Fri, 19 Jan 2024 12:28:03 +0100 Subject: [PATCH 195/247] add webhook_cleanup_last_success_timestamp metric for alerting (#59) --- hook-common/src/metrics.rs | 11 ++++++++++- hook-janitor/src/webhooks.rs | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/hook-common/src/metrics.rs b/hook-common/src/metrics.rs index 0e1ef2db72a33..66bcfc95ceeec 100644 --- a/hook-common/src/metrics.rs +++ b/hook-common/src/metrics.rs @@ -1,4 +1,4 @@ -use std::time::Instant; +use std::time::{Instant, SystemTime}; use axum::{ body::Body, extract::MatchedPath, http::Request, middleware::Next, response::IntoResponse, @@ -71,3 +71,12 @@ pub async fn track_metrics(req: Request, next: Next) -> impl IntoResponse response } + +/// Returns the number of seconds since the Unix epoch, to use in prom gauges. +/// Saturates to zero if the system time is set before epoch. +pub fn get_current_timestamp_seconds() -> f64 { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as f64 +} diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index bc016055d3b3c..705e1b394f415 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -17,6 +17,7 @@ use crate::cleanup::Cleaner; use crate::kafka_producer::KafkaContext; use hook_common::kafka_messages::app_metrics::{AppMetric, AppMetricCategory}; +use hook_common::metrics::get_current_timestamp_seconds; #[derive(Error, Debug)] pub enum WebhookCleanerError { @@ -446,6 +447,8 @@ impl Cleaner for WebhookCleaner { match self.cleanup_impl().await { Ok(stats) => { metrics::counter!("webhook_cleanup_success",).increment(1); + metrics::gauge!("webhook_cleanup_last_success_timestamp",) + .set(get_current_timestamp_seconds()); if stats.rows_processed > 0 { let elapsed_time = start_time.elapsed().as_secs_f64(); From e2b5dcb3e02f80f261e2691ce6fff64edc7717bc Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 19 Jan 2024 05:28:51 -0700 Subject: [PATCH 196/247] Change created_at to scheduled_at for metrics (#58) --- hook-janitor/src/webhooks.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 705e1b394f415..9c33c5e487b5c 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -111,9 +111,9 @@ struct FailedRow { #[derive(sqlx::FromRow, Debug)] struct QueueDepth { - oldest_created_at_untried: DateTime, + oldest_scheduled_at_untried: DateTime, count_untried: i64, - oldest_created_at_retries: DateTime, + oldest_scheduled_at_retries: DateTime, count_retries: i64, } @@ -187,9 +187,9 @@ impl WebhookCleaner { let base_query = r#" SELECT - COALESCE(MIN(CASE WHEN attempt = 0 THEN created_at END), now()) AS oldest_created_at_untried, + COALESCE(MIN(CASE WHEN attempt = 0 THEN scheduled_at END), now()) AS oldest_scheduled_at_untried, COALESCE(SUM(CASE WHEN attempt = 0 THEN 1 ELSE 0 END), 0) AS count_untried, - COALESCE(MIN(CASE WHEN attempt > 0 THEN created_at END), now()) AS oldest_created_at_retries, + COALESCE(MIN(CASE WHEN attempt > 0 THEN scheduled_at END), now()) AS oldest_scheduled_at_retries, COALESCE(SUM(CASE WHEN attempt > 0 THEN 1 ELSE 0 END), 0) AS count_retries FROM job_queue WHERE status = 'available'; @@ -374,15 +374,16 @@ impl WebhookCleaner { // of rows in memory. It seems unlikely we'll need to paginate, but that can be added in the // future if necessary. + let untried_status = [("status", "untried")]; + let retries_status = [("status", "retries")]; + let queue_depth = self.get_queue_depth().await?; - metrics::gauge!("queue_depth_oldest_created_at_untried") - .set(queue_depth.oldest_created_at_untried.timestamp() as f64); - metrics::gauge!("queue_depth", &[("status", "untried")]) - .set(queue_depth.count_untried as f64); - metrics::gauge!("queue_depth_oldest_created_at_retries") - .set(queue_depth.oldest_created_at_retries.timestamp() as f64); - metrics::gauge!("queue_depth", &[("status", "retries")]) - .set(queue_depth.count_retries as f64); + metrics::gauge!("queue_depth_oldest_scheduled", &untried_status) + .set(queue_depth.oldest_scheduled_at_untried.timestamp() as f64); + metrics::gauge!("queue_depth", &untried_status).set(queue_depth.count_untried as f64); + metrics::gauge!("queue_depth_oldest_scheduled", &retries_status) + .set(queue_depth.oldest_scheduled_at_retries.timestamp() as f64); + metrics::gauge!("queue_depth", &retries_status).set(queue_depth.count_retries as f64); let mut tx = self.start_serializable_txn().await?; From 6729401db21f737c45bc672af12d57321b8e6b26 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 30 Jan 2024 09:40:34 -0700 Subject: [PATCH 197/247] =?UTF-8?q?Add=20very=20basic=20version=20of=20job?= =?UTF-8?q?=20unstuck-ing=20for=20non-txn=20jobs=20that=20hang=20=E2=80=A6?= =?UTF-8?q?=20(#57)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hook-janitor/src/fixtures/webhook_cleanup.sql | 16 ++++++ hook-janitor/src/webhooks.rs | 55 ++++++++++++++++++- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/hook-janitor/src/fixtures/webhook_cleanup.sql b/hook-janitor/src/fixtures/webhook_cleanup.sql index bddaf269763d5..5dfa8272ef168 100644 --- a/hook-janitor/src/fixtures/webhook_cleanup.sql +++ b/hook-janitor/src/fixtures/webhook_cleanup.sql @@ -2,6 +2,7 @@ INSERT INTO job_queue ( errors, metadata, + attempted_at, last_attempt_finished_at, parameters, queue, @@ -14,6 +15,7 @@ VALUES NULL, '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{}', 'webhooks', 'completed', @@ -24,6 +26,7 @@ VALUES NULL, '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{}', 'webhooks', 'completed', @@ -34,6 +37,7 @@ VALUES NULL, '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 21:01:18.799371+00', + '2023-12-19 21:01:18.799371+00', '{}', 'webhooks', 'completed', @@ -44,6 +48,7 @@ VALUES NULL, '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3}', '2023-12-19 20:01:18.80335+00', + '2023-12-19 20:01:18.80335+00', '{}', 'webhooks', 'completed', @@ -54,6 +59,7 @@ VALUES NULL, '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{}', 'not-webhooks', 'completed', @@ -64,6 +70,7 @@ VALUES NULL, '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{}', 'webhooks', 'completed', @@ -74,6 +81,7 @@ VALUES ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{}', 'webhooks', 'failed', @@ -84,6 +92,7 @@ VALUES ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{}', 'webhooks', 'failed', @@ -94,6 +103,7 @@ VALUES ARRAY ['{"type":"ConnectionError","details":{"error":{"name":"Connection Error"}}}'::jsonb], '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{}', 'webhooks', 'failed', @@ -104,6 +114,7 @@ VALUES ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 21:01:18.799371+00', + '2023-12-19 21:01:18.799371+00', '{}', 'webhooks', 'failed', @@ -114,6 +125,7 @@ VALUES ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{}', 'webhooks', 'failed', @@ -124,6 +136,7 @@ VALUES ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{}', 'not-webhooks', 'failed', @@ -134,6 +147,7 @@ VALUES ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{}', 'webhooks', 'failed', @@ -144,6 +158,7 @@ VALUES NULL, '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', + '2023-12-19 20:01:18.799371+00', '{"body": "hello world", "headers": {}, "method": "POST", "url": "https://myhost/endpoint"}', 'webhooks', 'available', @@ -154,6 +169,7 @@ VALUES NULL, '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', + now() - '1 hour' :: interval, '{}', 'webhooks', 'running', diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 9c33c5e487b5c..ee8ff434c71a3 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -23,8 +23,12 @@ use hook_common::metrics::get_current_timestamp_seconds; pub enum WebhookCleanerError { #[error("failed to create postgres pool: {error}")] PoolCreationError { error: sqlx::Error }, + #[error("failed to acquire conn: {error}")] + AcquireConnError { error: sqlx::Error }, #[error("failed to acquire conn and start txn: {error}")] StartTxnError { error: sqlx::Error }, + #[error("failed to reschedule stuck jobs: {error}")] + RescheduleStuckJobsError { error: sqlx::Error }, #[error("failed to get queue depth: {error}")] GetQueueDepthError { error: sqlx::Error }, #[error("failed to get row count: {error}")] @@ -140,6 +144,7 @@ impl From for AppMetric { struct SerializableTxn<'a>(Transaction<'a, Postgres>); struct CleanupStats { + jobs_unstuck_count: u64, rows_processed: u64, completed_row_count: u64, completed_agg_row_count: u64, @@ -178,12 +183,51 @@ impl WebhookCleaner { }) } + async fn reschedule_stuck_jobs(&self) -> Result { + let mut conn = self + .pg_pool + .acquire() + .await + .map_err(|e| WebhookCleanerError::AcquireConnError { error: e })?; + + // The "non-transactional" worker runs the risk of crashing and leaving jobs permanently in + // the `running` state. This query will reschedule any jobs that have been in the running + // state for more than 2 minutes (which is *much* longer than we expect any Webhook job to + // take). + // + // We don't need to increment the `attempt` counter here because the worker already did that + // when it moved the job into `running`. + // + // If the previous worker was somehow stalled for 2 minutes and completes the task, that + // will mean we sent duplicate Webhooks. Success stats should not be affected, since both + // will update the same job row, which will only be processed once by the janitor. + + let base_query = r#" + UPDATE + job_queue + SET + status = 'available'::job_status, + last_attempt_finished_at = NOW(), + scheduled_at = NOW() + WHERE + status = 'running'::job_status + AND attempted_at < NOW() - INTERVAL '2 minutes' + "#; + + let result = sqlx::query(base_query) + .execute(&mut *conn) + .await + .map_err(|e| WebhookCleanerError::RescheduleStuckJobsError { error: e })?; + + Ok(result.rows_affected()) + } + async fn get_queue_depth(&self) -> Result { let mut conn = self .pg_pool .acquire() .await - .map_err(|e| WebhookCleanerError::StartTxnError { error: e })?; + .map_err(|e| WebhookCleanerError::AcquireConnError { error: e })?; let base_query = r#" SELECT @@ -377,6 +421,8 @@ impl WebhookCleaner { let untried_status = [("status", "untried")]; let retries_status = [("status", "retries")]; + let jobs_unstuck_count = self.reschedule_stuck_jobs().await?; + let queue_depth = self.get_queue_depth().await?; metrics::gauge!("queue_depth_oldest_scheduled", &untried_status) .set(queue_depth.oldest_scheduled_at_untried.timestamp() as f64); @@ -430,6 +476,7 @@ impl WebhookCleaner { } Ok(CleanupStats { + jobs_unstuck_count, rows_processed: rows_deleted, completed_row_count, completed_agg_row_count, @@ -450,6 +497,8 @@ impl Cleaner for WebhookCleaner { metrics::counter!("webhook_cleanup_success",).increment(1); metrics::gauge!("webhook_cleanup_last_success_timestamp",) .set(get_current_timestamp_seconds()); + metrics::counter!("webhook_cleanup_jobs_unstuck") + .increment(stats.jobs_unstuck_count); if stats.rows_processed > 0 { let elapsed_time = start_time.elapsed().as_secs_f64(); @@ -572,6 +621,9 @@ mod tests { .await .expect("webbook cleanup_impl failed"); + // The one 'running' job is transitioned to 'available'. + assert_eq!(cleanup_stats.jobs_unstuck_count, 1); + // Rows that are not 'completed' or 'failed' should not be processed. assert_eq!(cleanup_stats.rows_processed, 13); @@ -766,6 +818,7 @@ mod tests { .expect("webbook cleanup_impl failed"); // Reported metrics are all zeroes + assert_eq!(cleanup_stats.jobs_unstuck_count, 0); assert_eq!(cleanup_stats.rows_processed, 0); assert_eq!(cleanup_stats.completed_row_count, 0); assert_eq!(cleanup_stats.completed_agg_row_count, 0); From da6250b783b28a4570398997df9bfd2645f4cb30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 2 Feb 2024 13:45:41 +0000 Subject: [PATCH 198/247] fix: Use a good index for dequeue (#61) --- migrations/20240202003133_better_dequeue_index.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 migrations/20240202003133_better_dequeue_index.sql diff --git a/migrations/20240202003133_better_dequeue_index.sql b/migrations/20240202003133_better_dequeue_index.sql new file mode 100644 index 0000000000000..a619fb1ac8c9d --- /dev/null +++ b/migrations/20240202003133_better_dequeue_index.sql @@ -0,0 +1,10 @@ +-- Dequeue is not hitting this index, so dropping is safe this time. +DROP INDEX idx_queue_scheduled_at; + +/* +Partial index used for dequeuing from job_queue. + +Dequeue only looks at available jobs so a partial index serves us well. +Moreover, dequeue sorts jobs by attempt and scheduled_at, which matches this index. +*/ +CREATE INDEX idx_queue_dequeue_partial ON job_queue(queue, attempt, scheduled_at) WHERE status = 'available' :: job_status; From 8559b127ef1d05d14185621c11afe1533ec6299b Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 5 Feb 2024 11:24:56 +0100 Subject: [PATCH 199/247] declare a PG application name visible in PG stats (#62) Co-authored-by: Brett Hoerner --- hook-api/src/config.rs | 3 +++ hook-api/src/main.rs | 2 ++ hook-common/src/pgqueue.rs | 16 ++++++++++++---- hook-janitor/src/webhooks.rs | 9 ++++++--- hook-worker/src/config.rs | 3 +++ hook-worker/src/main.rs | 11 ++++++++--- 6 files changed, 34 insertions(+), 10 deletions(-) diff --git a/hook-api/src/config.rs b/hook-api/src/config.rs index 3fe88b3e436c0..55fa404e5149d 100644 --- a/hook-api/src/config.rs +++ b/hook-api/src/config.rs @@ -13,6 +13,9 @@ pub struct Config { #[envconfig(default = "default")] pub queue_name: String, + + #[envconfig(default = "100")] + pub max_pg_connections: u32, } impl Config { diff --git a/hook-api/src/main.rs b/hook-api/src/main.rs index 4fbbdfbb7d4ed..9a9a9fd41c0c2 100644 --- a/hook-api/src/main.rs +++ b/hook-api/src/main.rs @@ -28,6 +28,8 @@ async fn main() { // side, but we don't need more than one queue for now. &config.queue_name, &config.database_url, + config.max_pg_connections, + "hook-api", ) .await .expect("failed to initialize queue"); diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index fa2b5eb0c38be..4dab9183421a4 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -7,7 +7,7 @@ use std::time; use async_trait::async_trait; use chrono; use serde; -use sqlx::postgres::{PgPool, PgPoolOptions}; +use sqlx::postgres::{PgConnectOptions, PgPool, PgPoolOptions}; use thiserror::Error; /// Enumeration of errors for operations with PgQueue. @@ -524,11 +524,19 @@ impl PgQueue { /// /// * `queue_name`: A name for the queue we are going to initialize. /// * `url`: A URL pointing to where the PostgreSQL database is hosted. - pub async fn new(queue_name: &str, url: &str) -> PgQueueResult { + pub async fn new( + queue_name: &str, + url: &str, + max_connections: u32, + app_name: &'static str, + ) -> PgQueueResult { let name = queue_name.to_owned(); + let options = PgConnectOptions::from_str(url) + .map_err(|error| PgQueueError::PoolCreationError { error })? + .application_name(app_name); let pool = PgPoolOptions::new() - .connect_lazy(url) - .map_err(|error| PgQueueError::PoolCreationError { error })?; + .max_connections(max_connections) + .connect_lazy_with(options); Ok(Self { name, pool }) } diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index ee8ff434c71a3..5cdf4318f69b0 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -1,3 +1,4 @@ +use std::str::FromStr; use std::time::{Duration, Instant}; use async_trait::async_trait; @@ -7,7 +8,7 @@ use hook_common::webhook::WebhookJobError; use rdkafka::error::KafkaError; use rdkafka::producer::{FutureProducer, FutureRecord}; use serde_json::error::Error as SerdeError; -use sqlx::postgres::{PgPool, PgPoolOptions, Postgres}; +use sqlx::postgres::{PgConnectOptions, PgPool, PgPoolOptions, Postgres}; use sqlx::types::{chrono, Uuid}; use sqlx::{Row, Transaction}; use thiserror::Error; @@ -158,10 +159,12 @@ impl WebhookCleaner { kafka_producer: FutureProducer, app_metrics_topic: String, ) -> Result { + let options = PgConnectOptions::from_str(database_url) + .map_err(|error| WebhookCleanerError::PoolCreationError { error })? + .application_name("hook-janitor"); let pg_pool = PgPoolOptions::new() .acquire_timeout(Duration::from_secs(10)) - .connect_lazy(database_url) - .map_err(|error| WebhookCleanerError::PoolCreationError { error })?; + .connect_lazy_with(options); Ok(Self { pg_pool, diff --git a/hook-worker/src/config.rs b/hook-worker/src/config.rs index 74342f71e0030..477ff74349242 100644 --- a/hook-worker/src/config.rs +++ b/hook-worker/src/config.rs @@ -29,6 +29,9 @@ pub struct Config { #[envconfig(default = "1024")] pub max_concurrent_jobs: usize, + #[envconfig(default = "100")] + pub max_pg_connections: u32, + #[envconfig(nested = true)] pub retry_policy: RetryPolicyConfig, diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index 345fa3dd31c4a..6cad3fdd01167 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -35,9 +35,14 @@ async fn main() -> Result<(), WorkerError> { retry_policy_builder }; - let queue = PgQueue::new(config.queue_name.as_str(), &config.database_url) - .await - .expect("failed to initialize queue"); + let queue = PgQueue::new( + config.queue_name.as_str(), + &config.database_url, + config.max_pg_connections, + "hook-worker", + ) + .await + .expect("failed to initialize queue"); let worker = WebhookWorker::new( &config.worker_name, From 615c61d9f617028ae6ab1fd3f34d40536f886363 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Mon, 5 Feb 2024 10:31:56 -0700 Subject: [PATCH 200/247] Add proper e2e histrogram based on metadata created_at (#64) --- Cargo.lock | 1 + hook-api/Cargo.toml | 1 + hook-api/src/handlers/webhook.rs | 11 ++++--- hook-common/src/kafka_messages/mod.rs | 9 ++++-- hook-common/src/webhook.rs | 7 +++++ hook-janitor/src/fixtures/webhook_cleanup.sql | 30 +++++++++---------- hook-janitor/src/webhooks.rs | 2 ++ hook-worker/src/worker.rs | 8 +++++ 8 files changed, 47 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17b608c879f05..836810ef1c5e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -768,6 +768,7 @@ name = "hook-api" version = "0.1.0" dependencies = [ "axum", + "chrono", "envconfig", "eyre", "hook-common", diff --git a/hook-api/Cargo.toml b/hook-api/Cargo.toml index 96c897cd3ab1d..5e3530ef15bde 100644 --- a/hook-api/Cargo.toml +++ b/hook-api/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" axum = { workspace = true } envconfig = { workspace = true } eyre = { workspace = true } +chrono = { workspace = true } hook-common = { path = "../hook-common" } http-body-util = { workspace = true } metrics = { workspace = true } diff --git a/hook-api/src/handlers/webhook.rs b/hook-api/src/handlers/webhook.rs index 16ebc6dc57179..3712aa2882d7c 100644 --- a/hook-api/src/handlers/webhook.rs +++ b/hook-api/src/handlers/webhook.rs @@ -1,13 +1,12 @@ use std::time::Instant; use axum::{extract::State, http::StatusCode, Json}; -use hook_common::webhook::{WebhookJobMetadata, WebhookJobParameters}; -use serde_derive::Deserialize; -use url::Url; - use hook_common::pgqueue::{NewJob, PgQueue}; +use hook_common::webhook::{WebhookJobMetadata, WebhookJobParameters}; use serde::Serialize; +use serde_derive::Deserialize; use tracing::{debug, error}; +use url::Url; const MAX_BODY_SIZE: usize = 1_000_000; @@ -116,6 +115,7 @@ mod tests { http::{self, Request, StatusCode}, Router, }; + use chrono::Utc; use hook_common::pgqueue::PgQueue; use hook_common::webhook::{HttpMethod, WebhookJobParameters}; use http_body_util::BodyExt; @@ -153,6 +153,7 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, + created_at: Utc::now(), }, max_attempts: 1, }) @@ -195,6 +196,7 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, + created_at: Utc::now(), }, max_attempts: 1, }) @@ -283,6 +285,7 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, + created_at: Utc::now(), }, max_attempts: 1, }) diff --git a/hook-common/src/kafka_messages/mod.rs b/hook-common/src/kafka_messages/mod.rs index f548563af5ba1..a1de9d5301bce 100644 --- a/hook-common/src/kafka_messages/mod.rs +++ b/hook-common/src/kafka_messages/mod.rs @@ -16,9 +16,12 @@ where D: Deserializer<'de>, { let formatted: String = Deserialize::deserialize(deserializer)?; - let datetime = match NaiveDateTime::parse_from_str(&formatted, "%Y-%m-%d %H:%M:%S") { - Ok(d) => d.and_utc(), - Err(_) => return Err(serde::de::Error::custom("Invalid datetime format")), + let datetime = match DateTime::parse_from_rfc3339(&formatted) { + Ok(d) => d.with_timezone(&Utc), + Err(_) => match NaiveDateTime::parse_from_str(&formatted, "%Y-%m-%d %H:%M:%S") { + Ok(d) => d.and_utc(), + Err(_) => return Err(serde::de::Error::custom("Invalid datetime format")), + }, }; Ok(datetime) diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index 11e02856703eb..4122c20f68527 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -3,9 +3,11 @@ use std::convert::From; use std::fmt; use std::str::FromStr; +use chrono::{DateTime, Utc}; use serde::{de::Visitor, Deserialize, Serialize}; use crate::kafka_messages::app_metrics; +use crate::kafka_messages::{deserialize_datetime, serialize_datetime}; use crate::pgqueue::PgQueueError; /// Supported HTTP methods for webhooks. @@ -135,6 +137,11 @@ pub struct WebhookJobMetadata { pub team_id: u32, pub plugin_id: i32, pub plugin_config_id: i32, + #[serde( + serialize_with = "serialize_datetime", + deserialize_with = "deserialize_datetime" + )] + pub created_at: DateTime, } /// An error originating during a Webhook Job invocation. diff --git a/hook-janitor/src/fixtures/webhook_cleanup.sql b/hook-janitor/src/fixtures/webhook_cleanup.sql index 5dfa8272ef168..5f2f6c11dc67c 100644 --- a/hook-janitor/src/fixtures/webhook_cleanup.sql +++ b/hook-janitor/src/fixtures/webhook_cleanup.sql @@ -13,7 +13,7 @@ VALUES -- team:1, plugin_config:2, completed in hour 20 ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -24,7 +24,7 @@ VALUES -- team:1, plugin_config:2, completed in hour 20 (purposeful duplicate) ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -35,7 +35,7 @@ VALUES -- team:1, plugin_config:2, completed in hour 21 (different hour) ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 21:01:18.799371+00', '2023-12-19 21:01:18.799371+00', '{}', @@ -46,7 +46,7 @@ VALUES -- team:1, plugin_config:3, completed in hour 20 (different plugin_config) ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.80335+00', '2023-12-19 20:01:18.80335+00', '{}', @@ -57,7 +57,7 @@ VALUES -- team:1, plugin_config:2, completed but in a different queue ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -68,7 +68,7 @@ VALUES -- team:2, plugin_config:4, completed in hour 20 (different team) ( NULL, - '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4}', + '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -79,7 +79,7 @@ VALUES -- team:1, plugin_config:2, failed in hour 20 ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -90,7 +90,7 @@ VALUES -- team:1, plugin_config:2, failed in hour 20 (purposeful duplicate) ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -101,7 +101,7 @@ VALUES -- team:1, plugin_config:2, failed in hour 20 (different error) ( ARRAY ['{"type":"ConnectionError","details":{"error":{"name":"Connection Error"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -112,7 +112,7 @@ VALUES -- team:1, plugin_config:2, failed in hour 21 (different hour) ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 21:01:18.799371+00', '2023-12-19 21:01:18.799371+00', '{}', @@ -123,7 +123,7 @@ VALUES -- team:1, plugin_config:3, failed in hour 20 (different plugin_config) ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -134,7 +134,7 @@ VALUES -- team:1, plugin_config:2, failed but in a different queue ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -145,7 +145,7 @@ VALUES -- team:2, plugin_config:4, failed in hour 20 (purposeful duplicate) ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4}', + '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -156,7 +156,7 @@ VALUES -- team:1, plugin_config:2, available ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{"body": "hello world", "headers": {}, "method": "POST", "url": "https://myhost/endpoint"}', @@ -167,7 +167,7 @@ VALUES -- team:1, plugin_config:2, running ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', '2023-12-19 20:01:18.799371+00', now() - '1 hour' :: interval, '{}', diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 5cdf4318f69b0..c1390a7e0e1f7 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -892,6 +892,7 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, + created_at: Utc::now(), }; let new_job = NewJob::new(1, job_metadata, job_parameters, &"target"); queue.enqueue(new_job).await.expect("failed to enqueue job"); @@ -918,6 +919,7 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, + created_at: Utc::now(), }; let new_job = NewJob::new(1, job_metadata, job_parameters, &"target"); queue.enqueue(new_job).await.expect("failed to enqueue job"); diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index c526c3f7cce1c..14edea8b1275c 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -2,6 +2,7 @@ use std::collections; use std::sync::Arc; use std::time; +use chrono::Utc; use hook_common::health::HealthHandle; use hook_common::{ pgqueue::{Job, PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, @@ -264,6 +265,10 @@ async fn process_webhook_job( match send_result { Ok(_) => { + let end_to_end_duration = Utc::now() - webhook_job.metadata().created_at; + metrics::histogram!("webhook_jobs_end_to_end_duration_seconds", &labels) + .record((end_to_end_duration.num_milliseconds() as f64) / 1_000_f64); + webhook_job .complete() .await @@ -450,6 +455,8 @@ mod tests { // This is due to a long-standing cargo bug that reports imports and helper functions as unused. // See: https://github.com/rust-lang/rust/issues/46379. #[allow(unused_imports)] + use chrono::Utc; + #[allow(unused_imports)] use hook_common::health::HealthRegistry; #[allow(unused_imports)] use hook_common::pgqueue::{JobStatus, NewJob}; @@ -523,6 +530,7 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, + created_at: Utc::now(), }; let registry = HealthRegistry::new("liveness"); let liveness = registry From 304852cf2b92626e382a43af5076ca4e9a225729 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Mon, 5 Feb 2024 11:48:49 -0700 Subject: [PATCH 201/247] Revert "Add proper e2e histrogram based on metadata created_at (#64)" (#66) --- Cargo.lock | 1 - hook-api/Cargo.toml | 1 - hook-api/src/handlers/webhook.rs | 11 +++---- hook-common/src/kafka_messages/mod.rs | 9 ++---- hook-common/src/webhook.rs | 7 ----- hook-janitor/src/fixtures/webhook_cleanup.sql | 30 +++++++++---------- hook-janitor/src/webhooks.rs | 2 -- hook-worker/src/worker.rs | 8 ----- 8 files changed, 22 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 836810ef1c5e3..17b608c879f05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -768,7 +768,6 @@ name = "hook-api" version = "0.1.0" dependencies = [ "axum", - "chrono", "envconfig", "eyre", "hook-common", diff --git a/hook-api/Cargo.toml b/hook-api/Cargo.toml index 5e3530ef15bde..96c897cd3ab1d 100644 --- a/hook-api/Cargo.toml +++ b/hook-api/Cargo.toml @@ -9,7 +9,6 @@ edition = "2021" axum = { workspace = true } envconfig = { workspace = true } eyre = { workspace = true } -chrono = { workspace = true } hook-common = { path = "../hook-common" } http-body-util = { workspace = true } metrics = { workspace = true } diff --git a/hook-api/src/handlers/webhook.rs b/hook-api/src/handlers/webhook.rs index 3712aa2882d7c..16ebc6dc57179 100644 --- a/hook-api/src/handlers/webhook.rs +++ b/hook-api/src/handlers/webhook.rs @@ -1,13 +1,14 @@ use std::time::Instant; use axum::{extract::State, http::StatusCode, Json}; -use hook_common::pgqueue::{NewJob, PgQueue}; use hook_common::webhook::{WebhookJobMetadata, WebhookJobParameters}; -use serde::Serialize; use serde_derive::Deserialize; -use tracing::{debug, error}; use url::Url; +use hook_common::pgqueue::{NewJob, PgQueue}; +use serde::Serialize; +use tracing::{debug, error}; + const MAX_BODY_SIZE: usize = 1_000_000; #[derive(Serialize, Deserialize)] @@ -115,7 +116,6 @@ mod tests { http::{self, Request, StatusCode}, Router, }; - use chrono::Utc; use hook_common::pgqueue::PgQueue; use hook_common::webhook::{HttpMethod, WebhookJobParameters}; use http_body_util::BodyExt; @@ -153,7 +153,6 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, - created_at: Utc::now(), }, max_attempts: 1, }) @@ -196,7 +195,6 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, - created_at: Utc::now(), }, max_attempts: 1, }) @@ -285,7 +283,6 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, - created_at: Utc::now(), }, max_attempts: 1, }) diff --git a/hook-common/src/kafka_messages/mod.rs b/hook-common/src/kafka_messages/mod.rs index a1de9d5301bce..f548563af5ba1 100644 --- a/hook-common/src/kafka_messages/mod.rs +++ b/hook-common/src/kafka_messages/mod.rs @@ -16,12 +16,9 @@ where D: Deserializer<'de>, { let formatted: String = Deserialize::deserialize(deserializer)?; - let datetime = match DateTime::parse_from_rfc3339(&formatted) { - Ok(d) => d.with_timezone(&Utc), - Err(_) => match NaiveDateTime::parse_from_str(&formatted, "%Y-%m-%d %H:%M:%S") { - Ok(d) => d.and_utc(), - Err(_) => return Err(serde::de::Error::custom("Invalid datetime format")), - }, + let datetime = match NaiveDateTime::parse_from_str(&formatted, "%Y-%m-%d %H:%M:%S") { + Ok(d) => d.and_utc(), + Err(_) => return Err(serde::de::Error::custom("Invalid datetime format")), }; Ok(datetime) diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index 4122c20f68527..11e02856703eb 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -3,11 +3,9 @@ use std::convert::From; use std::fmt; use std::str::FromStr; -use chrono::{DateTime, Utc}; use serde::{de::Visitor, Deserialize, Serialize}; use crate::kafka_messages::app_metrics; -use crate::kafka_messages::{deserialize_datetime, serialize_datetime}; use crate::pgqueue::PgQueueError; /// Supported HTTP methods for webhooks. @@ -137,11 +135,6 @@ pub struct WebhookJobMetadata { pub team_id: u32, pub plugin_id: i32, pub plugin_config_id: i32, - #[serde( - serialize_with = "serialize_datetime", - deserialize_with = "deserialize_datetime" - )] - pub created_at: DateTime, } /// An error originating during a Webhook Job invocation. diff --git a/hook-janitor/src/fixtures/webhook_cleanup.sql b/hook-janitor/src/fixtures/webhook_cleanup.sql index 5f2f6c11dc67c..5dfa8272ef168 100644 --- a/hook-janitor/src/fixtures/webhook_cleanup.sql +++ b/hook-janitor/src/fixtures/webhook_cleanup.sql @@ -13,7 +13,7 @@ VALUES -- team:1, plugin_config:2, completed in hour 20 ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -24,7 +24,7 @@ VALUES -- team:1, plugin_config:2, completed in hour 20 (purposeful duplicate) ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -35,7 +35,7 @@ VALUES -- team:1, plugin_config:2, completed in hour 21 (different hour) ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 21:01:18.799371+00', '2023-12-19 21:01:18.799371+00', '{}', @@ -46,7 +46,7 @@ VALUES -- team:1, plugin_config:3, completed in hour 20 (different plugin_config) ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3}', '2023-12-19 20:01:18.80335+00', '2023-12-19 20:01:18.80335+00', '{}', @@ -57,7 +57,7 @@ VALUES -- team:1, plugin_config:2, completed but in a different queue ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -68,7 +68,7 @@ VALUES -- team:2, plugin_config:4, completed in hour 20 (different team) ( NULL, - '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -79,7 +79,7 @@ VALUES -- team:1, plugin_config:2, failed in hour 20 ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -90,7 +90,7 @@ VALUES -- team:1, plugin_config:2, failed in hour 20 (purposeful duplicate) ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -101,7 +101,7 @@ VALUES -- team:1, plugin_config:2, failed in hour 20 (different error) ( ARRAY ['{"type":"ConnectionError","details":{"error":{"name":"Connection Error"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -112,7 +112,7 @@ VALUES -- team:1, plugin_config:2, failed in hour 21 (different hour) ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 21:01:18.799371+00', '2023-12-19 21:01:18.799371+00', '{}', @@ -123,7 +123,7 @@ VALUES -- team:1, plugin_config:3, failed in hour 20 (different plugin_config) ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 3}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -134,7 +134,7 @@ VALUES -- team:1, plugin_config:2, failed but in a different queue ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -145,7 +145,7 @@ VALUES -- team:2, plugin_config:4, failed in hour 20 (purposeful duplicate) ( ARRAY ['{"type":"TimeoutError","details":{"error":{"name":"Timeout"}}}'::jsonb], - '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 2, "plugin_id": 99, "plugin_config_id": 4}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{}', @@ -156,7 +156,7 @@ VALUES -- team:1, plugin_config:2, available ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', '2023-12-19 20:01:18.799371+00', '{"body": "hello world", "headers": {}, "method": "POST", "url": "https://myhost/endpoint"}', @@ -167,7 +167,7 @@ VALUES -- team:1, plugin_config:2, running ( NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2, "created_at": "2023-02-05T16:35:06.650Z"}', + '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', '2023-12-19 20:01:18.799371+00', now() - '1 hour' :: interval, '{}', diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index c1390a7e0e1f7..5cdf4318f69b0 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -892,7 +892,6 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, - created_at: Utc::now(), }; let new_job = NewJob::new(1, job_metadata, job_parameters, &"target"); queue.enqueue(new_job).await.expect("failed to enqueue job"); @@ -919,7 +918,6 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, - created_at: Utc::now(), }; let new_job = NewJob::new(1, job_metadata, job_parameters, &"target"); queue.enqueue(new_job).await.expect("failed to enqueue job"); diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 14edea8b1275c..c526c3f7cce1c 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -2,7 +2,6 @@ use std::collections; use std::sync::Arc; use std::time; -use chrono::Utc; use hook_common::health::HealthHandle; use hook_common::{ pgqueue::{Job, PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, @@ -265,10 +264,6 @@ async fn process_webhook_job( match send_result { Ok(_) => { - let end_to_end_duration = Utc::now() - webhook_job.metadata().created_at; - metrics::histogram!("webhook_jobs_end_to_end_duration_seconds", &labels) - .record((end_to_end_duration.num_milliseconds() as f64) / 1_000_f64); - webhook_job .complete() .await @@ -455,8 +450,6 @@ mod tests { // This is due to a long-standing cargo bug that reports imports and helper functions as unused. // See: https://github.com/rust-lang/rust/issues/46379. #[allow(unused_imports)] - use chrono::Utc; - #[allow(unused_imports)] use hook_common::health::HealthRegistry; #[allow(unused_imports)] use hook_common::pgqueue::{JobStatus, NewJob}; @@ -530,7 +523,6 @@ mod tests { team_id: 1, plugin_id: 2, plugin_config_id: 3, - created_at: Utc::now(), }; let registry = HealthRegistry::new("liveness"); let liveness = registry From 54bf761a35333d2ffb915922d2abf52bdac89f3f Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 6 Feb 2024 08:27:34 -0700 Subject: [PATCH 202/247] Dequeue multiple items at a time (#60) --- hook-common/src/pgqueue.rs | 428 ++++++++++++++++++++++++++--------- hook-janitor/src/webhooks.rs | 14 +- hook-worker/src/config.rs | 3 + hook-worker/src/main.rs | 1 + hook-worker/src/worker.rs | 175 +++++++++----- 5 files changed, 453 insertions(+), 168 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 4dab9183421a4..af91fbd1cf7ca 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -1,14 +1,17 @@ //! # PgQueue //! //! A job queue implementation backed by a PostgreSQL table. -use std::str::FromStr; use std::time; +use std::{str::FromStr, sync::Arc}; use async_trait::async_trait; use chrono; use serde; +use sqlx::postgres::any::AnyConnectionBackend; use sqlx::postgres::{PgConnectOptions, PgPool, PgPoolOptions}; use thiserror::Error; +use tokio::sync::Mutex; +use tracing::error; /// Enumeration of errors for operations with PgQueue. /// Errors that can originate from sqlx and are wrapped by us to provide additional context. @@ -24,16 +27,22 @@ pub enum PgQueueError { ParseJobStatusError(String), #[error("{0} is not a valid HttpMethod")] ParseHttpMethodError(String), + #[error("transaction was already closed")] + TransactionAlreadyClosedError, } #[derive(Error, Debug)] pub enum PgJobError { #[error("retry is an invalid state for this PgJob: {error}")] RetryInvalidError { job: T, error: String }, + #[error("connection failed with: {error}")] + ConnectionError { error: sqlx::Error }, #[error("{command} query failed with: {error}")] QueryError { command: String, error: sqlx::Error }, #[error("transaction {command} failed with: {error}")] TransactionError { command: String, error: sqlx::Error }, + #[error("transaction was already closed")] + TransactionAlreadyClosedError, } /// Enumeration of possible statuses for a Job. @@ -217,20 +226,39 @@ pub trait PgQueueJob { #[derive(Debug)] pub struct PgJob { pub job: Job, - pub connection: sqlx::pool::PoolConnection, + pub pool: PgPool, +} + +// Container struct for a batch of PgJobs. +pub struct PgBatch { + pub jobs: Vec>, +} + +impl PgJob { + async fn acquire_conn( + &mut self, + ) -> Result, PgJobError>>> + { + self.pool + .acquire() + .await + .map_err(|error| PgJobError::ConnectionError { error }) + } } #[async_trait] impl PgQueueJob for PgJob { async fn complete(mut self) -> Result>>> { - let completed_job = self - .job - .complete(&mut *self.connection) - .await - .map_err(|error| PgJobError::QueryError { - command: "UPDATE".to_owned(), - error, - })?; + let mut connection = self.acquire_conn().await?; + + let completed_job = + self.job + .complete(&mut *connection) + .await + .map_err(|error| PgJobError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; Ok(completed_job) } @@ -239,9 +267,11 @@ impl PgQueueJob for PgJob { mut self, error: E, ) -> Result, PgJobError>>> { + let mut connection = self.acquire_conn().await?; + let failed_job = self .job - .fail(error, &mut *self.connection) + .fail(error, &mut *connection) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -264,11 +294,13 @@ impl PgQueueJob for PgJob { }); } + let mut connection = self.acquire_conn().await?; + let retried_job = self .job .retryable() .queue(queue) - .retry(error, retry_interval, &mut *self.connection) + .retry(error, retry_interval, &mut *connection) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), @@ -284,7 +316,39 @@ impl PgQueueJob for PgJob { #[derive(Debug)] pub struct PgTransactionJob<'c, J, M> { pub job: Job, - pub transaction: sqlx::Transaction<'c, sqlx::postgres::Postgres>, + + /// The open transaction this job came from. If multiple jobs were queried at once, then this + /// transaction will be shared between them (across async tasks and threads as necessary). See + /// below for more information. + shared_txn: Arc>>>, +} + +// Container struct for a batch of PgTransactionJob. Includes a reference to the shared transaction +// for committing the work when all of the jobs are finished. +pub struct PgTransactionBatch<'c, J, M> { + pub jobs: Vec>, + + /// The open transaction the jobs in the Vec came from. This should be used to commit or + /// rollback when all of the work is finished. + shared_txn: Arc>>>, +} + +impl<'c, J, M> PgTransactionBatch<'_, J, M> { + pub async fn commit(self) -> PgQueueResult<()> { + let mut txn_guard = self.shared_txn.lock().await; + + txn_guard + .as_deref_mut() + .ok_or(PgQueueError::TransactionAlreadyClosedError)? + .commit() + .await + .map_err(|e| PgQueueError::QueryError { + command: "COMMIT".to_owned(), + error: e, + })?; + + Ok(()) + } } #[async_trait] @@ -292,22 +356,20 @@ impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactio async fn complete( mut self, ) -> Result>>> { - let completed_job = self - .job - .complete(&mut *self.transaction) - .await - .map_err(|error| PgJobError::QueryError { - command: "UPDATE".to_owned(), - error, - })?; + let mut txn_guard = self.shared_txn.lock().await; - self.transaction - .commit() - .await - .map_err(|error| PgJobError::TransactionError { - command: "COMMIT".to_owned(), - error, - })?; + let txn_ref = txn_guard + .as_deref_mut() + .ok_or(PgJobError::TransactionAlreadyClosedError)?; + + let completed_job = + self.job + .complete(txn_ref) + .await + .map_err(|error| PgJobError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; Ok(completed_job) } @@ -316,22 +378,20 @@ impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactio mut self, error: S, ) -> Result, PgJobError>>> { - let failed_job = self - .job - .fail(error, &mut *self.transaction) - .await - .map_err(|error| PgJobError::QueryError { - command: "UPDATE".to_owned(), - error, - })?; + let mut txn_guard = self.shared_txn.lock().await; - self.transaction - .commit() - .await - .map_err(|error| PgJobError::TransactionError { - command: "COMMIT".to_owned(), - error, - })?; + let txn_ref = txn_guard + .as_deref_mut() + .ok_or(PgJobError::TransactionAlreadyClosedError)?; + + let failed_job = + self.job + .fail(error, txn_ref) + .await + .map_err(|error| PgJobError::QueryError { + command: "UPDATE".to_owned(), + error, + })?; Ok(failed_job) } @@ -351,25 +411,23 @@ impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactio }); } + let mut txn_guard = self.shared_txn.lock().await; + + let txn_ref = txn_guard + .as_deref_mut() + .ok_or(PgJobError::TransactionAlreadyClosedError)?; + let retried_job = self .job .retryable() .queue(queue) - .retry(error, retry_interval, &mut *self.transaction) + .retry(error, retry_interval, txn_ref) .await .map_err(|error| PgJobError::QueryError { command: "UPDATE".to_owned(), error, })?; - self.transaction - .commit() - .await - .map_err(|error| PgJobError::TransactionError { - command: "COMMIT".to_owned(), - error, - })?; - Ok(retried_job) } } @@ -553,15 +611,16 @@ impl PgQueue { Ok(Self { name, pool }) } - /// Dequeue a `Job` from this `PgQueue`. - /// The `Job` will be updated to `'running'` status, so any other `dequeue` calls will skip it. + /// Dequeue up to `limit` `Job`s from this `PgQueue`. + /// The `Job`s will be updated to `'running'` status, so any other `dequeue` calls will skip it. pub async fn dequeue< J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, M: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, >( &self, attempted_by: &str, - ) -> PgQueueResult>> { + limit: u32, + ) -> PgQueueResult>> { let mut connection = self .pool .acquire() @@ -583,7 +642,7 @@ WITH available_in_queue AS ( ORDER BY attempt, scheduled_at - LIMIT 1 + LIMIT $2 FOR UPDATE SKIP LOCKED ) UPDATE @@ -592,7 +651,7 @@ SET attempted_at = NOW(), status = 'running'::job_status, attempt = attempt + 1, - attempted_by = array_append(attempted_by, $2::text) + attempted_by = array_append(attempted_by, $3::text) FROM available_in_queue WHERE @@ -601,14 +660,29 @@ RETURNING job_queue.* "#; - let query_result: Result, sqlx::Error> = sqlx::query_as(base_query) + let query_result: Result>, sqlx::Error> = sqlx::query_as(base_query) .bind(&self.name) + .bind(limit as i64) .bind(attempted_by) - .fetch_one(&mut *connection) + .fetch_all(&mut *connection) .await; match query_result { - Ok(job) => Ok(Some(PgJob { job, connection })), + Ok(jobs) => { + if jobs.is_empty() { + return Ok(None); + } + + let pg_jobs: Vec> = jobs + .into_iter() + .map(|job| PgJob { + job, + pool: self.pool.clone(), + }) + .collect(); + + Ok(Some(PgBatch { jobs: pg_jobs })) + } // Although connection would be closed once it goes out of scope, sqlx recommends explicitly calling close(). // See: https://docs.rs/sqlx/latest/sqlx/postgres/any/trait.AnyConnectionBackend.html#tymethod.close. @@ -616,6 +690,7 @@ RETURNING let _ = connection.close().await; Ok(None) } + Err(e) => { let _ = connection.close().await; Err(PgQueueError::QueryError { @@ -626,9 +701,10 @@ RETURNING } } - /// Dequeue a `Job` from this `PgQueue` and hold the transaction. - /// Any other `dequeue_tx` calls will skip rows locked, so by holding a transaction we ensure only one worker can dequeue a job. - /// Holding a transaction open can have performance implications, but it means no `'running'` state is required. + /// Dequeue up to `limit` `Job`s from this `PgQueue` and hold the transaction. Any other + /// `dequeue_tx` calls will skip rows locked, so by holding a transaction we ensure only one + /// worker can dequeue a job. Holding a transaction open can have performance implications, but + /// it means no `'running'` state is required. pub async fn dequeue_tx< 'a, J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, @@ -636,7 +712,8 @@ RETURNING >( &self, attempted_by: &str, - ) -> PgQueueResult>> { + limit: u32, + ) -> PgQueueResult>> { let mut tx = self .pool .begin() @@ -658,7 +735,7 @@ WITH available_in_queue AS ( ORDER BY attempt, scheduled_at - LIMIT 1 + LIMIT $2 FOR UPDATE SKIP LOCKED ) UPDATE @@ -667,7 +744,7 @@ SET attempted_at = NOW(), status = 'running'::job_status, attempt = attempt + 1, - attempted_by = array_append(attempted_by, $2::text) + attempted_by = array_append(attempted_by, $3::text) FROM available_in_queue WHERE @@ -676,20 +753,38 @@ RETURNING job_queue.* "#; - let query_result: Result, sqlx::Error> = sqlx::query_as(base_query) + let query_result: Result>, sqlx::Error> = sqlx::query_as(base_query) .bind(&self.name) + .bind(limit as i64) .bind(attempted_by) - .fetch_one(&mut *tx) + .fetch_all(&mut *tx) .await; match query_result { - Ok(job) => Ok(Some(PgTransactionJob { - job, - transaction: tx, - })), + Ok(jobs) => { + if jobs.is_empty() { + return Ok(None); + } + + let shared_txn = Arc::new(Mutex::new(Some(tx))); + + let pg_jobs: Vec> = jobs + .into_iter() + .map(|job| PgTransactionJob { + job, + shared_txn: shared_txn.clone(), + }) + .collect(); + + Ok(Some(PgTransactionBatch { + jobs: pg_jobs, + shared_txn: shared_txn.clone(), + })) + } - // Transaction is rolledback on drop. + // Transaction is rolled back on drop. Err(sqlx::Error::RowNotFound) => Ok(None), + Err(e) => Err(PgQueueError::QueryError { command: "UPDATE".to_owned(), error: e, @@ -736,7 +831,7 @@ mod tests { use super::*; use crate::retry::RetryPolicy; - #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] + #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug, Clone)] struct JobMetadata { team_id: u32, plugin_config_id: i32, @@ -753,7 +848,7 @@ mod tests { } } - #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] + #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug, Clone)] struct JobParameters { method: String, body: String, @@ -795,10 +890,13 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); let pg_job: PgJob = queue - .dequeue(&worker_id) + .dequeue(&worker_id, 1) .await - .expect("failed to dequeue job") - .expect("didn't find a job to dequeue"); + .expect("failed to dequeue jobs") + .expect("didn't find any jobs to dequeue") + .jobs + .pop() + .unwrap(); assert_eq!(pg_job.job.attempt, 1); assert!(pg_job.job.attempted_by.contains(&worker_id)); @@ -816,12 +914,62 @@ mod tests { .await .expect("failed to connect to local test postgresql database"); - let pg_job: Option> = queue - .dequeue(&worker_id) + let pg_jobs: Option> = queue + .dequeue(&worker_id, 1) .await - .expect("failed to dequeue job"); + .expect("failed to dequeue jobs"); + + assert!(pg_jobs.is_none()); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_can_dequeue_multiple_jobs(db: PgPool) { + let job_target = job_target(); + let job_metadata = JobMetadata::default(); + let job_parameters = JobParameters::default(); + let worker_id = worker_id(); - assert!(pg_job.is_none()); + let queue = PgQueue::new_from_pool("test_can_dequeue_multiple_jobs", db) + .await + .expect("failed to connect to local test postgresql database"); + + for _ in 0..5 { + queue + .enqueue(NewJob::new( + 1, + job_metadata.clone(), + job_parameters.clone(), + &job_target, + )) + .await + .expect("failed to enqueue job"); + } + + // Only get 4 jobs, leaving one in the queue. + let limit = 4; + let batch: PgBatch = queue + .dequeue(&worker_id, limit) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + + // Complete those 4. + assert_eq!(batch.jobs.len(), limit as usize); + for job in batch.jobs { + job.complete().await.expect("failed to complete job"); + } + + // Try to get up to 4 jobs, but only 1 remains. + let batch: PgBatch = queue + .dequeue(&worker_id, limit) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + + assert_eq!(batch.jobs.len(), 1); // Only one job should have been left in the queue. + for job in batch.jobs { + job.complete().await.expect("failed to complete job"); + } } #[sqlx::test(migrations = "../migrations")] @@ -830,19 +978,21 @@ mod tests { let job_metadata = JobMetadata::default(); let job_parameters = JobParameters::default(); let worker_id = worker_id(); - let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); let queue = PgQueue::new_from_pool("test_can_dequeue_tx_job", db) .await .expect("failed to connect to local test postgresql database"); + let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); queue.enqueue(new_job).await.expect("failed to enqueue job"); - let tx_job: PgTransactionJob<'_, JobParameters, JobMetadata> = queue - .dequeue_tx(&worker_id) + let mut batch: PgTransactionBatch<'_, JobParameters, JobMetadata> = queue + .dequeue_tx(&worker_id, 1) .await - .expect("failed to dequeue job") - .expect("didn't find a job to dequeue"); + .expect("failed to dequeue jobs") + .expect("didn't find any jobs to dequeue"); + + let tx_job = batch.jobs.pop().unwrap(); assert_eq!(tx_job.job.attempt, 1); assert!(tx_job.job.attempted_by.contains(&worker_id)); @@ -852,6 +1002,65 @@ mod tests { assert_eq!(*tx_job.job.parameters.as_ref(), JobParameters::default()); assert_eq!(tx_job.job.status, JobStatus::Running); assert_eq!(tx_job.job.target, job_target); + + // Transactional jobs must be completed, failed or retried before being dropped. This is + // to prevent logic bugs when using the shared txn. + tx_job.complete().await.expect("failed to complete job"); + + batch.commit().await.expect("failed to commit transaction"); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_can_dequeue_multiple_tx_jobs(db: PgPool) { + let job_target = job_target(); + let job_metadata = JobMetadata::default(); + let job_parameters = JobParameters::default(); + let worker_id = worker_id(); + + let queue = PgQueue::new_from_pool("test_can_dequeue_multiple_tx_jobs", db) + .await + .expect("failed to connect to local test postgresql database"); + + for _ in 0..5 { + queue + .enqueue(NewJob::new( + 1, + job_metadata.clone(), + job_parameters.clone(), + &job_target, + )) + .await + .expect("failed to enqueue job"); + } + + // Only get 4 jobs, leaving one in the queue. + let limit = 4; + let mut batch: PgTransactionBatch<'_, JobParameters, JobMetadata> = queue + .dequeue_tx(&worker_id, limit) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + + assert_eq!(batch.jobs.len(), limit as usize); + + // Complete those 4 and commit. + for job in std::mem::take(&mut batch.jobs) { + job.complete().await.expect("failed to complete job"); + } + batch.commit().await.expect("failed to commit transaction"); + + // Try to get up to 4 jobs, but only 1 remains. + let mut batch: PgTransactionBatch<'_, JobParameters, JobMetadata> = queue + .dequeue_tx(&worker_id, limit) + .await + .expect("failed to dequeue job") + .expect("didn't find a job to dequeue"); + assert_eq!(batch.jobs.len(), 1); // Only one job should have been left in the queue. + + for job in std::mem::take(&mut batch.jobs) { + job.complete().await.expect("failed to complete job"); + } + batch.commit().await.expect("failed to commit transaction"); } #[sqlx::test(migrations = "../migrations")] @@ -861,12 +1070,12 @@ mod tests { .await .expect("failed to connect to local test postgresql database"); - let tx_job: Option> = queue - .dequeue_tx(&worker_id) + let batch: Option> = queue + .dequeue_tx(&worker_id, 1) .await .expect("failed to dequeue job"); - assert!(tx_job.is_none()); + assert!(batch.is_none()); } #[sqlx::test(migrations = "../migrations")] @@ -888,10 +1097,13 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); let job: PgJob = queue - .dequeue(&worker_id) + .dequeue(&worker_id, 1) .await .expect("failed to dequeue job") - .expect("didn't find a job to dequeue"); + .expect("didn't find a job to dequeue") + .jobs + .pop() + .unwrap(); let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); let retry_queue = retry_policy.retry_queue(&job.job.queue).to_owned(); @@ -905,10 +1117,13 @@ mod tests { .expect("failed to retry job"); let retried_job: PgJob = queue - .dequeue(&worker_id) + .dequeue(&worker_id, 1) .await .expect("failed to dequeue job") - .expect("didn't find retried job to dequeue"); + .expect("didn't find retried job to dequeue") + .jobs + .pop() + .unwrap(); assert_eq!(retried_job.job.attempt, 2); assert!(retried_job.job.attempted_by.contains(&worker_id)); @@ -942,10 +1157,13 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); let job: PgJob = queue - .dequeue(&worker_id) + .dequeue(&worker_id, 1) .await .expect("failed to dequeue job") - .expect("didn't find a job to dequeue"); + .expect("didn't find a job to dequeue") + .jobs + .pop() + .unwrap(); let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); let retry_queue = retry_policy.retry_queue(&job.job.queue).to_owned(); @@ -958,8 +1176,8 @@ mod tests { .await .expect("failed to retry job"); - let retried_job_not_found: Option> = queue - .dequeue(&worker_id) + let retried_job_not_found: Option> = queue + .dequeue(&worker_id, 1) .await .expect("failed to dequeue job"); @@ -970,10 +1188,13 @@ mod tests { .expect("failed to connect to retry queue in local test postgresql database"); let retried_job: PgJob = queue - .dequeue(&worker_id) + .dequeue(&worker_id, 1) .await .expect("failed to dequeue job") - .expect("job not found in retry queue"); + .expect("job not found in retry queue") + .jobs + .pop() + .unwrap(); assert_eq!(retried_job.job.attempt, 2); assert!(retried_job.job.attempted_by.contains(&worker_id)); @@ -1004,10 +1225,13 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); let job: PgJob = queue - .dequeue(&worker_id) + .dequeue(&worker_id, 1) .await .expect("failed to dequeue job") - .expect("didn't find a job to dequeue"); + .expect("didn't find a job to dequeue") + .jobs + .pop() + .unwrap(); let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 5cdf4318f69b0..e3b137c343155 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -870,10 +870,13 @@ mod tests { { // The fixtures include an available job, so let's complete it while the txn is open. let webhook_job: PgJob = queue - .dequeue(&"worker_id") + .dequeue(&"worker_id", 1) .await .expect("failed to dequeue job") - .expect("didn't find a job to dequeue"); + .expect("didn't find a job to dequeue") + .jobs + .pop() + .unwrap(); webhook_job .complete() .await @@ -896,10 +899,13 @@ mod tests { let new_job = NewJob::new(1, job_metadata, job_parameters, &"target"); queue.enqueue(new_job).await.expect("failed to enqueue job"); let webhook_job: PgJob = queue - .dequeue(&"worker_id") + .dequeue(&"worker_id", 1) .await .expect("failed to dequeue job") - .expect("didn't find a job to dequeue"); + .expect("didn't find a job to dequeue") + .jobs + .pop() + .unwrap(); webhook_job .complete() .await diff --git a/hook-worker/src/config.rs b/hook-worker/src/config.rs index 477ff74349242..32e49f7e9b49c 100644 --- a/hook-worker/src/config.rs +++ b/hook-worker/src/config.rs @@ -37,6 +37,9 @@ pub struct Config { #[envconfig(default = "true")] pub transactional: bool, + + #[envconfig(default = "1")] + pub dequeue_batch_size: u32, } impl Config { diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index 6cad3fdd01167..fede7d288d95f 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -47,6 +47,7 @@ async fn main() -> Result<(), WorkerError> { let worker = WebhookWorker::new( &config.worker_name, &queue, + config.dequeue_batch_size, config.poll_interval.0, config.request_timeout.0, config.max_concurrent_jobs, diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index c526c3f7cce1c..437a1d352f32c 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -2,7 +2,9 @@ use std::collections; use std::sync::Arc; use std::time; +use futures::future::join_all; use hook_common::health::HealthHandle; +use hook_common::pgqueue::{PgBatch, PgTransactionBatch}; use hook_common::{ pgqueue::{Job, PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, retry::RetryPolicy, @@ -68,6 +70,8 @@ pub struct WebhookWorker<'p> { name: String, /// The queue we will be dequeuing jobs from. queue: &'p PgQueue, + /// The maximum number of jobs to dequeue in one query. + dequeue_batch_size: u32, /// The interval for polling the queue. poll_interval: time::Duration, /// The client used for HTTP requests. @@ -81,9 +85,11 @@ pub struct WebhookWorker<'p> { } impl<'p> WebhookWorker<'p> { + #[allow(clippy::too_many_arguments)] pub fn new( name: &str, queue: &'p PgQueue, + dequeue_batch_size: u32, poll_interval: time::Duration, request_timeout: time::Duration, max_concurrent_jobs: usize, @@ -106,6 +112,7 @@ impl<'p> WebhookWorker<'p> { Self { name: name.to_owned(), queue, + dequeue_batch_size, poll_interval, client, max_concurrent_jobs, @@ -114,16 +121,20 @@ impl<'p> WebhookWorker<'p> { } } - /// Wait until a job becomes available in our queue. - async fn wait_for_job<'a>(&self) -> PgJob { + /// Wait until at least one job becomes available in our queue. + async fn wait_for_jobs<'a>(&self) -> PgBatch { let mut interval = tokio::time::interval(self.poll_interval); loop { interval.tick().await; self.liveness.report_healthy().await; - match self.queue.dequeue(&self.name).await { - Ok(Some(job)) => return job, + match self + .queue + .dequeue(&self.name, self.dequeue_batch_size) + .await + { + Ok(Some(batch)) => return batch, Ok(None) => continue, Err(error) => { error!("error while trying to dequeue job: {}", error); @@ -133,18 +144,22 @@ impl<'p> WebhookWorker<'p> { } } - /// Wait until a job becomes available in our queue in transactional mode. - async fn wait_for_job_tx<'a>( + /// Wait until at least one job becomes available in our queue in transactional mode. + async fn wait_for_jobs_tx<'a>( &self, - ) -> PgTransactionJob<'a, WebhookJobParameters, WebhookJobMetadata> { + ) -> PgTransactionBatch<'a, WebhookJobParameters, WebhookJobMetadata> { let mut interval = tokio::time::interval(self.poll_interval); loop { interval.tick().await; self.liveness.report_healthy().await; - match self.queue.dequeue_tx(&self.name).await { - Ok(Some(job)) => return job, + match self + .queue + .dequeue_tx(&self.name, self.dequeue_batch_size) + .await + { + Ok(Some(batch)) => return batch, Ok(None) => continue, Err(error) => { error!("error while trying to dequeue_tx job: {}", error); @@ -162,70 +177,104 @@ impl<'p> WebhookWorker<'p> { .set(1f64 - semaphore.available_permits() as f64 / self.max_concurrent_jobs as f64); }; + let dequeue_batch_size_histogram = metrics::histogram!("webhook_dequeue_batch_size"); + if transactional { loop { report_semaphore_utilization(); - let webhook_job = self.wait_for_job_tx().await; - spawn_webhook_job_processing_task( - self.client.clone(), - semaphore.clone(), - self.retry_policy.clone(), - webhook_job, - ) - .await; + // TODO: We could grab semaphore permits here using something like: + // `min(semaphore.available_permits(), dequeue_batch_size)` + // And then dequeue only up to that many jobs. We'd then need to hand back the + // difference in permits based on how many jobs were dequeued. + let mut batch = self.wait_for_jobs_tx().await; + dequeue_batch_size_histogram.record(batch.jobs.len() as f64); + + // Get enough permits for the jobs before spawning a task. + let permits = semaphore + .clone() + .acquire_many_owned(batch.jobs.len() as u32) + .await + .expect("semaphore has been closed"); + + let client = self.client.clone(); + let retry_policy = self.retry_policy.clone(); + + tokio::spawn(async move { + let mut futures = Vec::new(); + + // We have to `take` the Vec of jobs from the batch to avoid a borrow checker + // error below when we commit. + for job in std::mem::take(&mut batch.jobs) { + let client = client.clone(); + let retry_policy = retry_policy.clone(); + + let future = + async move { process_webhook_job(client, job, &retry_policy).await }; + + futures.push(future); + } + + let results = join_all(futures).await; + for result in results { + if let Err(e) = result { + error!("error processing webhook job: {}", e); + } + } + + let _ = batch.commit().await.map_err(|e| { + error!("error committing transactional batch: {}", e); + }); + + drop(permits); + }); } } else { loop { report_semaphore_utilization(); - let webhook_job = self.wait_for_job().await; - spawn_webhook_job_processing_task( - self.client.clone(), - semaphore.clone(), - self.retry_policy.clone(), - webhook_job, - ) - .await; + // TODO: We could grab semaphore permits here using something like: + // `min(semaphore.available_permits(), dequeue_batch_size)` + // And then dequeue only up to that many jobs. We'd then need to hand back the + // difference in permits based on how many jobs were dequeued. + let batch = self.wait_for_jobs().await; + dequeue_batch_size_histogram.record(batch.jobs.len() as f64); + + // Get enough permits for the jobs before spawning a task. + let permits = semaphore + .clone() + .acquire_many_owned(batch.jobs.len() as u32) + .await + .expect("semaphore has been closed"); + + let client = self.client.clone(); + let retry_policy = self.retry_policy.clone(); + + tokio::spawn(async move { + let mut futures = Vec::new(); + + for job in batch.jobs { + let client = client.clone(); + let retry_policy = retry_policy.clone(); + + let future = + async move { process_webhook_job(client, job, &retry_policy).await }; + + futures.push(future); + } + + let results = join_all(futures).await; + for result in results { + if let Err(e) = result { + error!("error processing webhook job: {}", e); + } + } + + drop(permits); + }); } } } } -/// Spawn a Tokio task to process a Webhook Job once we successfully acquire a permit. -/// -/// # Arguments -/// -/// * `client`: An HTTP client to execute the webhook job request. -/// * `semaphore`: A semaphore used for rate limiting purposes. This function will panic if this semaphore is closed. -/// * `retry_policy`: The retry policy used to set retry parameters if a job fails and has remaining attempts. -/// * `webhook_job`: The webhook job to process as dequeued from `hook_common::pgqueue::PgQueue`. -async fn spawn_webhook_job_processing_task( - client: reqwest::Client, - semaphore: Arc, - retry_policy: RetryPolicy, - webhook_job: W, -) -> tokio::task::JoinHandle> { - let permit = semaphore - .acquire_owned() - .await - .expect("semaphore has been closed"); - - let labels = [("queue", webhook_job.queue())]; - - metrics::counter!("webhook_jobs_total", &labels).increment(1); - - tokio::spawn(async move { - let result = process_webhook_job(client, webhook_job, &retry_policy).await; - drop(permit); - match result { - Ok(_) => Ok(()), - Err(error) => { - error!("failed to process webhook job: {}", error); - Err(error) - } - } - }) -} - /// Process a webhook job by transitioning it to its appropriate state after its request is sent. /// After we finish, the webhook job will be set as completed (if the request was successful), retryable (if the request /// was unsuccessful but we can still attempt a retry), or failed (if the request was unsuccessful and no more retries @@ -248,6 +297,7 @@ async fn process_webhook_job( let parameters = webhook_job.parameters(); let labels = [("queue", webhook_job.queue())]; + metrics::counter!("webhook_jobs_total", &labels).increment(1); let now = tokio::time::Instant::now(); @@ -543,6 +593,7 @@ mod tests { let worker = WebhookWorker::new( &worker_id, &queue, + 1, time::Duration::from_millis(100), time::Duration::from_millis(5000), 10, @@ -550,7 +601,7 @@ mod tests { liveness, ); - let consumed_job = worker.wait_for_job().await; + let consumed_job = worker.wait_for_jobs().await.jobs.pop().unwrap(); assert_eq!(consumed_job.job.attempt, 1); assert!(consumed_job.job.attempted_by.contains(&worker_id)); From 26672aeb2c8d114e3f82c41916e935a3d753ee8a Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 6 Feb 2024 08:36:10 -0700 Subject: [PATCH 203/247] Remove non-transactional mode (#65) --- hook-common/src/pgqueue.rs | 321 +----------------- hook-janitor/src/fixtures/webhook_cleanup.sql | 11 - hook-janitor/src/webhooks.rs | 78 +---- hook-worker/src/config.rs | 3 - hook-worker/src/main.rs | 2 +- hook-worker/src/worker.rs | 173 +++------- 6 files changed, 77 insertions(+), 511 deletions(-) diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index af91fbd1cf7ca..4a8b489716a7d 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -60,8 +60,6 @@ pub enum JobStatus { Discarded, /// A job that was unsuccessfully completed by a worker. Failed, - /// A job that was picked up by a worker and it's currentlly being run. - Running, } /// Allow casting JobStatus from strings. @@ -73,7 +71,6 @@ impl FromStr for JobStatus { "available" => Ok(JobStatus::Available), "completed" => Ok(JobStatus::Completed), "failed" => Ok(JobStatus::Failed), - "running" => Ok(JobStatus::Running), invalid => Err(PgQueueError::ParseJobStatusError(invalid.to_owned())), } } @@ -222,95 +219,6 @@ pub trait PgQueueJob { ) -> Result>>; } -/// A Job that can be updated in PostgreSQL. -#[derive(Debug)] -pub struct PgJob { - pub job: Job, - pub pool: PgPool, -} - -// Container struct for a batch of PgJobs. -pub struct PgBatch { - pub jobs: Vec>, -} - -impl PgJob { - async fn acquire_conn( - &mut self, - ) -> Result, PgJobError>>> - { - self.pool - .acquire() - .await - .map_err(|error| PgJobError::ConnectionError { error }) - } -} - -#[async_trait] -impl PgQueueJob for PgJob { - async fn complete(mut self) -> Result>>> { - let mut connection = self.acquire_conn().await?; - - let completed_job = - self.job - .complete(&mut *connection) - .await - .map_err(|error| PgJobError::QueryError { - command: "UPDATE".to_owned(), - error, - })?; - - Ok(completed_job) - } - - async fn fail( - mut self, - error: E, - ) -> Result, PgJobError>>> { - let mut connection = self.acquire_conn().await?; - - let failed_job = self - .job - .fail(error, &mut *connection) - .await - .map_err(|error| PgJobError::QueryError { - command: "UPDATE".to_owned(), - error, - })?; - - Ok(failed_job) - } - - async fn retry( - mut self, - error: E, - retry_interval: time::Duration, - queue: &str, - ) -> Result>>> { - if self.job.is_gte_max_attempts() { - return Err(PgJobError::RetryInvalidError { - job: Box::new(self), - error: "Maximum attempts reached".to_owned(), - }); - } - - let mut connection = self.acquire_conn().await?; - - let retried_job = self - .job - .retryable() - .queue(queue) - .retry(error, retry_interval, &mut *connection) - .await - .map_err(|error| PgJobError::QueryError { - command: "UPDATE".to_owned(), - error, - })?; - - Ok(retried_job) - } -} - /// A Job within an open PostgreSQL transaction. /// This implementation allows 'hiding' the job from any other workers running SKIP LOCKED queries. #[derive(Debug)] @@ -611,96 +519,6 @@ impl PgQueue { Ok(Self { name, pool }) } - /// Dequeue up to `limit` `Job`s from this `PgQueue`. - /// The `Job`s will be updated to `'running'` status, so any other `dequeue` calls will skip it. - pub async fn dequeue< - J: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, - M: for<'d> serde::Deserialize<'d> + std::marker::Send + std::marker::Unpin + 'static, - >( - &self, - attempted_by: &str, - limit: u32, - ) -> PgQueueResult>> { - let mut connection = self - .pool - .acquire() - .await - .map_err(|error| PgQueueError::ConnectionError { error })?; - - // The query that follows uses a FOR UPDATE SKIP LOCKED clause. - // For more details on this see: 2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5. - let base_query = r#" -WITH available_in_queue AS ( - SELECT - id - FROM - job_queue - WHERE - status = 'available' - AND scheduled_at <= NOW() - AND queue = $1 - ORDER BY - attempt, - scheduled_at - LIMIT $2 - FOR UPDATE SKIP LOCKED -) -UPDATE - job_queue -SET - attempted_at = NOW(), - status = 'running'::job_status, - attempt = attempt + 1, - attempted_by = array_append(attempted_by, $3::text) -FROM - available_in_queue -WHERE - job_queue.id = available_in_queue.id -RETURNING - job_queue.* - "#; - - let query_result: Result>, sqlx::Error> = sqlx::query_as(base_query) - .bind(&self.name) - .bind(limit as i64) - .bind(attempted_by) - .fetch_all(&mut *connection) - .await; - - match query_result { - Ok(jobs) => { - if jobs.is_empty() { - return Ok(None); - } - - let pg_jobs: Vec> = jobs - .into_iter() - .map(|job| PgJob { - job, - pool: self.pool.clone(), - }) - .collect(); - - Ok(Some(PgBatch { jobs: pg_jobs })) - } - - // Although connection would be closed once it goes out of scope, sqlx recommends explicitly calling close(). - // See: https://docs.rs/sqlx/latest/sqlx/postgres/any/trait.AnyConnectionBackend.html#tymethod.close. - Err(sqlx::Error::RowNotFound) => { - let _ = connection.close().await; - Ok(None) - } - - Err(e) => { - let _ = connection.close().await; - Err(PgQueueError::QueryError { - command: "UPDATE".to_owned(), - error: e, - }) - } - } - } - /// Dequeue up to `limit` `Job`s from this `PgQueue` and hold the transaction. Any other /// `dequeue_tx` calls will skip rows locked, so by holding a transaction we ensure only one /// worker can dequeue a job. Holding a transaction open can have performance implications, but @@ -742,7 +560,6 @@ UPDATE job_queue SET attempted_at = NOW(), - status = 'running'::job_status, attempt = attempt + 1, attempted_by = array_append(attempted_by, $3::text) FROM @@ -875,103 +692,6 @@ mod tests { "https://myhost/endpoint".to_owned() } - #[sqlx::test(migrations = "../migrations")] - async fn test_can_dequeue_job(db: PgPool) { - let job_target = job_target(); - let job_parameters = JobParameters::default(); - let job_metadata = JobMetadata::default(); - let worker_id = worker_id(); - let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); - - let queue = PgQueue::new_from_pool("test_can_dequeue_job", db) - .await - .expect("failed to connect to local test postgresql database"); - - queue.enqueue(new_job).await.expect("failed to enqueue job"); - - let pg_job: PgJob = queue - .dequeue(&worker_id, 1) - .await - .expect("failed to dequeue jobs") - .expect("didn't find any jobs to dequeue") - .jobs - .pop() - .unwrap(); - - assert_eq!(pg_job.job.attempt, 1); - assert!(pg_job.job.attempted_by.contains(&worker_id)); - assert_eq!(pg_job.job.attempted_by.len(), 1); - assert_eq!(pg_job.job.max_attempts, 1); - assert_eq!(*pg_job.job.parameters.as_ref(), JobParameters::default()); - assert_eq!(pg_job.job.status, JobStatus::Running); - assert_eq!(pg_job.job.target, job_target); - } - - #[sqlx::test(migrations = "../migrations")] - async fn test_dequeue_returns_none_on_no_jobs(db: PgPool) { - let worker_id = worker_id(); - let queue = PgQueue::new_from_pool("test_dequeue_returns_none_on_no_jobs", db) - .await - .expect("failed to connect to local test postgresql database"); - - let pg_jobs: Option> = queue - .dequeue(&worker_id, 1) - .await - .expect("failed to dequeue jobs"); - - assert!(pg_jobs.is_none()); - } - - #[sqlx::test(migrations = "../migrations")] - async fn test_can_dequeue_multiple_jobs(db: PgPool) { - let job_target = job_target(); - let job_metadata = JobMetadata::default(); - let job_parameters = JobParameters::default(); - let worker_id = worker_id(); - - let queue = PgQueue::new_from_pool("test_can_dequeue_multiple_jobs", db) - .await - .expect("failed to connect to local test postgresql database"); - - for _ in 0..5 { - queue - .enqueue(NewJob::new( - 1, - job_metadata.clone(), - job_parameters.clone(), - &job_target, - )) - .await - .expect("failed to enqueue job"); - } - - // Only get 4 jobs, leaving one in the queue. - let limit = 4; - let batch: PgBatch = queue - .dequeue(&worker_id, limit) - .await - .expect("failed to dequeue job") - .expect("didn't find a job to dequeue"); - - // Complete those 4. - assert_eq!(batch.jobs.len(), limit as usize); - for job in batch.jobs { - job.complete().await.expect("failed to complete job"); - } - - // Try to get up to 4 jobs, but only 1 remains. - let batch: PgBatch = queue - .dequeue(&worker_id, limit) - .await - .expect("failed to dequeue job") - .expect("didn't find a job to dequeue"); - - assert_eq!(batch.jobs.len(), 1); // Only one job should have been left in the queue. - for job in batch.jobs { - job.complete().await.expect("failed to complete job"); - } - } - #[sqlx::test(migrations = "../migrations")] async fn test_can_dequeue_tx_job(db: PgPool) { let job_target = job_target(); @@ -1000,7 +720,6 @@ mod tests { assert_eq!(tx_job.job.max_attempts, 1); assert_eq!(*tx_job.job.metadata.as_ref(), JobMetadata::default()); assert_eq!(*tx_job.job.parameters.as_ref(), JobParameters::default()); - assert_eq!(tx_job.job.status, JobStatus::Running); assert_eq!(tx_job.job.target, job_target); // Transactional jobs must be completed, failed or retried before being dropped. This is @@ -1096,14 +815,12 @@ mod tests { .expect("failed to connect to local test postgresql database"); queue.enqueue(new_job).await.expect("failed to enqueue job"); - let job: PgJob = queue - .dequeue(&worker_id, 1) + let mut batch: PgTransactionBatch<'_, JobParameters, JobMetadata> = queue + .dequeue_tx(&worker_id, 1) .await .expect("failed to dequeue job") - .expect("didn't find a job to dequeue") - .jobs - .pop() - .unwrap(); + .expect("didn't find a job to dequeue"); + let job = batch.jobs.pop().unwrap(); let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); let retry_queue = retry_policy.retry_queue(&job.job.queue).to_owned(); @@ -1115,9 +832,10 @@ mod tests { ) .await .expect("failed to retry job"); + batch.commit().await.expect("failed to commit transaction"); - let retried_job: PgJob = queue - .dequeue(&worker_id, 1) + let retried_job: PgTransactionJob = queue + .dequeue_tx(&worker_id, 1) .await .expect("failed to dequeue job") .expect("didn't find retried job to dequeue") @@ -1133,7 +851,6 @@ mod tests { *retried_job.job.parameters.as_ref(), JobParameters::default() ); - assert_eq!(retried_job.job.status, JobStatus::Running); assert_eq!(retried_job.job.target, job_target); } @@ -1156,14 +873,12 @@ mod tests { .expect("failed to connect to queue in local test postgresql database"); queue.enqueue(new_job).await.expect("failed to enqueue job"); - let job: PgJob = queue - .dequeue(&worker_id, 1) + let mut batch: PgTransactionBatch = queue + .dequeue_tx(&worker_id, 1) .await .expect("failed to dequeue job") - .expect("didn't find a job to dequeue") - .jobs - .pop() - .unwrap(); + .expect("didn't find a job to dequeue"); + let job = batch.jobs.pop().unwrap(); let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); let retry_queue = retry_policy.retry_queue(&job.job.queue).to_owned(); @@ -1175,9 +890,10 @@ mod tests { ) .await .expect("failed to retry job"); + batch.commit().await.expect("failed to commit transaction"); - let retried_job_not_found: Option> = queue - .dequeue(&worker_id, 1) + let retried_job_not_found: Option> = queue + .dequeue_tx(&worker_id, 1) .await .expect("failed to dequeue job"); @@ -1187,8 +903,8 @@ mod tests { .await .expect("failed to connect to retry queue in local test postgresql database"); - let retried_job: PgJob = queue - .dequeue(&worker_id, 1) + let retried_job: PgTransactionJob = queue + .dequeue_tx(&worker_id, 1) .await .expect("failed to dequeue job") .expect("job not found in retry queue") @@ -1204,7 +920,6 @@ mod tests { *retried_job.job.parameters.as_ref(), JobParameters::default() ); - assert_eq!(retried_job.job.status, JobStatus::Running); assert_eq!(retried_job.job.target, job_target); } @@ -1224,8 +939,8 @@ mod tests { queue.enqueue(new_job).await.expect("failed to enqueue job"); - let job: PgJob = queue - .dequeue(&worker_id, 1) + let job: PgTransactionJob = queue + .dequeue_tx(&worker_id, 1) .await .expect("failed to dequeue job") .expect("didn't find a job to dequeue") diff --git a/hook-janitor/src/fixtures/webhook_cleanup.sql b/hook-janitor/src/fixtures/webhook_cleanup.sql index 5dfa8272ef168..e0b9a7a9ea4d8 100644 --- a/hook-janitor/src/fixtures/webhook_cleanup.sql +++ b/hook-janitor/src/fixtures/webhook_cleanup.sql @@ -163,15 +163,4 @@ VALUES 'webhooks', 'available', 'https://myhost/endpoint' - ), - -- team:1, plugin_config:2, running - ( - NULL, - '{"team_id": 1, "plugin_id": 99, "plugin_config_id": 2}', - '2023-12-19 20:01:18.799371+00', - now() - '1 hour' :: interval, - '{}', - 'webhooks', - 'running', - 'https://myhost/endpoint' ); \ No newline at end of file diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index e3b137c343155..7f7fadd408565 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -28,8 +28,6 @@ pub enum WebhookCleanerError { AcquireConnError { error: sqlx::Error }, #[error("failed to acquire conn and start txn: {error}")] StartTxnError { error: sqlx::Error }, - #[error("failed to reschedule stuck jobs: {error}")] - RescheduleStuckJobsError { error: sqlx::Error }, #[error("failed to get queue depth: {error}")] GetQueueDepthError { error: sqlx::Error }, #[error("failed to get row count: {error}")] @@ -145,7 +143,6 @@ impl From for AppMetric { struct SerializableTxn<'a>(Transaction<'a, Postgres>); struct CleanupStats { - jobs_unstuck_count: u64, rows_processed: u64, completed_row_count: u64, completed_agg_row_count: u64, @@ -186,45 +183,6 @@ impl WebhookCleaner { }) } - async fn reschedule_stuck_jobs(&self) -> Result { - let mut conn = self - .pg_pool - .acquire() - .await - .map_err(|e| WebhookCleanerError::AcquireConnError { error: e })?; - - // The "non-transactional" worker runs the risk of crashing and leaving jobs permanently in - // the `running` state. This query will reschedule any jobs that have been in the running - // state for more than 2 minutes (which is *much* longer than we expect any Webhook job to - // take). - // - // We don't need to increment the `attempt` counter here because the worker already did that - // when it moved the job into `running`. - // - // If the previous worker was somehow stalled for 2 minutes and completes the task, that - // will mean we sent duplicate Webhooks. Success stats should not be affected, since both - // will update the same job row, which will only be processed once by the janitor. - - let base_query = r#" - UPDATE - job_queue - SET - status = 'available'::job_status, - last_attempt_finished_at = NOW(), - scheduled_at = NOW() - WHERE - status = 'running'::job_status - AND attempted_at < NOW() - INTERVAL '2 minutes' - "#; - - let result = sqlx::query(base_query) - .execute(&mut *conn) - .await - .map_err(|e| WebhookCleanerError::RescheduleStuckJobsError { error: e })?; - - Ok(result.rows_affected()) - } - async fn get_queue_depth(&self) -> Result { let mut conn = self .pg_pool @@ -424,8 +382,6 @@ impl WebhookCleaner { let untried_status = [("status", "untried")]; let retries_status = [("status", "retries")]; - let jobs_unstuck_count = self.reschedule_stuck_jobs().await?; - let queue_depth = self.get_queue_depth().await?; metrics::gauge!("queue_depth_oldest_scheduled", &untried_status) .set(queue_depth.oldest_scheduled_at_untried.timestamp() as f64); @@ -479,7 +435,6 @@ impl WebhookCleaner { } Ok(CleanupStats { - jobs_unstuck_count, rows_processed: rows_deleted, completed_row_count, completed_agg_row_count, @@ -500,8 +455,6 @@ impl Cleaner for WebhookCleaner { metrics::counter!("webhook_cleanup_success",).increment(1); metrics::gauge!("webhook_cleanup_last_success_timestamp",) .set(get_current_timestamp_seconds()); - metrics::counter!("webhook_cleanup_jobs_unstuck") - .increment(stats.jobs_unstuck_count); if stats.rows_processed > 0 { let elapsed_time = start_time.elapsed().as_secs_f64(); @@ -546,7 +499,8 @@ mod tests { use hook_common::kafka_messages::app_metrics::{ Error as WebhookError, ErrorDetails, ErrorType, }; - use hook_common::pgqueue::{NewJob, PgJob, PgQueue, PgQueueJob}; + use hook_common::pgqueue::PgQueueJob; + use hook_common::pgqueue::{NewJob, PgQueue, PgTransactionBatch}; use hook_common::webhook::{HttpMethod, WebhookJobMetadata, WebhookJobParameters}; use rdkafka::consumer::{Consumer, StreamConsumer}; use rdkafka::mocking::MockCluster; @@ -624,9 +578,6 @@ mod tests { .await .expect("webbook cleanup_impl failed"); - // The one 'running' job is transitioned to 'available'. - assert_eq!(cleanup_stats.jobs_unstuck_count, 1); - // Rows that are not 'completed' or 'failed' should not be processed. assert_eq!(cleanup_stats.rows_processed, 13); @@ -821,7 +772,6 @@ mod tests { .expect("webbook cleanup_impl failed"); // Reported metrics are all zeroes - assert_eq!(cleanup_stats.jobs_unstuck_count, 0); assert_eq!(cleanup_stats.rows_processed, 0); assert_eq!(cleanup_stats.completed_row_count, 0); assert_eq!(cleanup_stats.completed_agg_row_count, 0); @@ -865,22 +815,20 @@ mod tests { assert_eq!(get_count_from_new_conn(&db, "completed").await, 6); assert_eq!(get_count_from_new_conn(&db, "failed").await, 7); assert_eq!(get_count_from_new_conn(&db, "available").await, 1); - assert_eq!(get_count_from_new_conn(&db, "running").await, 1); { // The fixtures include an available job, so let's complete it while the txn is open. - let webhook_job: PgJob = queue - .dequeue(&"worker_id", 1) + let mut batch: PgTransactionBatch<'_, WebhookJobParameters, WebhookJobMetadata> = queue + .dequeue_tx(&"worker_id", 1) .await .expect("failed to dequeue job") - .expect("didn't find a job to dequeue") - .jobs - .pop() - .unwrap(); + .expect("didn't find a job to dequeue"); + let webhook_job = batch.jobs.pop().unwrap(); webhook_job .complete() .await .expect("failed to complete job"); + batch.commit().await.expect("failed to commit batch"); } { @@ -898,18 +846,17 @@ mod tests { }; let new_job = NewJob::new(1, job_metadata, job_parameters, &"target"); queue.enqueue(new_job).await.expect("failed to enqueue job"); - let webhook_job: PgJob = queue - .dequeue(&"worker_id", 1) + let mut batch: PgTransactionBatch<'_, WebhookJobParameters, WebhookJobMetadata> = queue + .dequeue_tx(&"worker_id", 1) .await .expect("failed to dequeue job") - .expect("didn't find a job to dequeue") - .jobs - .pop() - .unwrap(); + .expect("didn't find a job to dequeue"); + let webhook_job = batch.jobs.pop().unwrap(); webhook_job .complete() .await .expect("failed to complete job"); + batch.commit().await.expect("failed to commit batch"); } { @@ -950,6 +897,5 @@ mod tests { assert_eq!(get_count_from_new_conn(&db, "completed").await, 2); assert_eq!(get_count_from_new_conn(&db, "failed").await, 0); assert_eq!(get_count_from_new_conn(&db, "available").await, 1); - assert_eq!(get_count_from_new_conn(&db, "running").await, 1); } } diff --git a/hook-worker/src/config.rs b/hook-worker/src/config.rs index 32e49f7e9b49c..ceb690f38846e 100644 --- a/hook-worker/src/config.rs +++ b/hook-worker/src/config.rs @@ -35,9 +35,6 @@ pub struct Config { #[envconfig(nested = true)] pub retry_policy: RetryPolicyConfig, - #[envconfig(default = "true")] - pub transactional: bool, - #[envconfig(default = "1")] pub dequeue_batch_size: u32, } diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index fede7d288d95f..2997dfc65ff50 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -67,7 +67,7 @@ async fn main() -> Result<(), WorkerError> { .expect("failed to start serving metrics"); }); - worker.run(config.transactional).await; + worker.run().await; Ok(()) } diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 437a1d352f32c..b83c909cb698e 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -4,9 +4,9 @@ use std::time; use futures::future::join_all; use hook_common::health::HealthHandle; -use hook_common::pgqueue::{PgBatch, PgTransactionBatch}; +use hook_common::pgqueue::PgTransactionBatch; use hook_common::{ - pgqueue::{Job, PgJob, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, + pgqueue::{Job, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, retry::RetryPolicy, webhook::{HttpMethod, WebhookJobError, WebhookJobMetadata, WebhookJobParameters}, }; @@ -50,20 +50,6 @@ impl WebhookJob for PgTransactionJob<'_, WebhookJobParameters, WebhookJobMetadat } } -impl WebhookJob for PgJob { - fn parameters(&self) -> &WebhookJobParameters { - &self.job.parameters - } - - fn metadata(&self) -> &WebhookJobMetadata { - &self.job.metadata - } - - fn job(&self) -> &Job { - &self.job - } -} - /// A worker to poll `PgQueue` and spawn tasks to process webhooks when a job becomes available. pub struct WebhookWorker<'p> { /// An identifier for this worker. Used to mark jobs we have consumed. @@ -121,29 +107,6 @@ impl<'p> WebhookWorker<'p> { } } - /// Wait until at least one job becomes available in our queue. - async fn wait_for_jobs<'a>(&self) -> PgBatch { - let mut interval = tokio::time::interval(self.poll_interval); - - loop { - interval.tick().await; - self.liveness.report_healthy().await; - - match self - .queue - .dequeue(&self.name, self.dequeue_batch_size) - .await - { - Ok(Some(batch)) => return batch, - Ok(None) => continue, - Err(error) => { - error!("error while trying to dequeue job: {}", error); - continue; - } - } - } - } - /// Wait until at least one job becomes available in our queue in transactional mode. async fn wait_for_jobs_tx<'a>( &self, @@ -170,7 +133,7 @@ impl<'p> WebhookWorker<'p> { } /// Run this worker to continuously process any jobs that become available. - pub async fn run(&self, transactional: bool) { + pub async fn run(&self) { let semaphore = Arc::new(sync::Semaphore::new(self.max_concurrent_jobs)); let report_semaphore_utilization = || { metrics::gauge!("webhook_worker_saturation_percent") @@ -179,98 +142,53 @@ impl<'p> WebhookWorker<'p> { let dequeue_batch_size_histogram = metrics::histogram!("webhook_dequeue_batch_size"); - if transactional { - loop { - report_semaphore_utilization(); - // TODO: We could grab semaphore permits here using something like: - // `min(semaphore.available_permits(), dequeue_batch_size)` - // And then dequeue only up to that many jobs. We'd then need to hand back the - // difference in permits based on how many jobs were dequeued. - let mut batch = self.wait_for_jobs_tx().await; - dequeue_batch_size_histogram.record(batch.jobs.len() as f64); - - // Get enough permits for the jobs before spawning a task. - let permits = semaphore - .clone() - .acquire_many_owned(batch.jobs.len() as u32) - .await - .expect("semaphore has been closed"); - - let client = self.client.clone(); - let retry_policy = self.retry_policy.clone(); - - tokio::spawn(async move { - let mut futures = Vec::new(); - - // We have to `take` the Vec of jobs from the batch to avoid a borrow checker - // error below when we commit. - for job in std::mem::take(&mut batch.jobs) { - let client = client.clone(); - let retry_policy = retry_policy.clone(); - - let future = - async move { process_webhook_job(client, job, &retry_policy).await }; - - futures.push(future); - } + loop { + report_semaphore_utilization(); + // TODO: We could grab semaphore permits here using something like: + // `min(semaphore.available_permits(), dequeue_batch_size)` + // And then dequeue only up to that many jobs. We'd then need to hand back the + // difference in permits based on how many jobs were dequeued. + let mut batch = self.wait_for_jobs_tx().await; + dequeue_batch_size_histogram.record(batch.jobs.len() as f64); + + // Get enough permits for the jobs before spawning a task. + let permits = semaphore + .clone() + .acquire_many_owned(batch.jobs.len() as u32) + .await + .expect("semaphore has been closed"); - let results = join_all(futures).await; - for result in results { - if let Err(e) = result { - error!("error processing webhook job: {}", e); - } - } + let client = self.client.clone(); + let retry_policy = self.retry_policy.clone(); - let _ = batch.commit().await.map_err(|e| { - error!("error committing transactional batch: {}", e); - }); + tokio::spawn(async move { + let mut futures = Vec::new(); - drop(permits); - }); - } - } else { - loop { - report_semaphore_utilization(); - // TODO: We could grab semaphore permits here using something like: - // `min(semaphore.available_permits(), dequeue_batch_size)` - // And then dequeue only up to that many jobs. We'd then need to hand back the - // difference in permits based on how many jobs were dequeued. - let batch = self.wait_for_jobs().await; - dequeue_batch_size_histogram.record(batch.jobs.len() as f64); - - // Get enough permits for the jobs before spawning a task. - let permits = semaphore - .clone() - .acquire_many_owned(batch.jobs.len() as u32) - .await - .expect("semaphore has been closed"); - - let client = self.client.clone(); - let retry_policy = self.retry_policy.clone(); - - tokio::spawn(async move { - let mut futures = Vec::new(); - - for job in batch.jobs { - let client = client.clone(); - let retry_policy = retry_policy.clone(); - - let future = - async move { process_webhook_job(client, job, &retry_policy).await }; - - futures.push(future); - } + // We have to `take` the Vec of jobs from the batch to avoid a borrow checker + // error below when we commit. + for job in std::mem::take(&mut batch.jobs) { + let client = client.clone(); + let retry_policy = retry_policy.clone(); + + let future = + async move { process_webhook_job(client, job, &retry_policy).await }; - let results = join_all(futures).await; - for result in results { - if let Err(e) = result { - error!("error processing webhook job: {}", e); - } + futures.push(future); + } + + let results = join_all(futures).await; + for result in results { + if let Err(e) = result { + error!("error processing webhook job: {}", e); } + } - drop(permits); + let _ = batch.commit().await.map_err(|e| { + error!("error committing transactional batch: {}", e); }); - } + + drop(permits); + }); } } } @@ -601,7 +519,8 @@ mod tests { liveness, ); - let consumed_job = worker.wait_for_jobs().await.jobs.pop().unwrap(); + let mut batch = worker.wait_for_jobs_tx().await; + let consumed_job = batch.jobs.pop().unwrap(); assert_eq!(consumed_job.job.attempt, 1); assert!(consumed_job.job.attempted_by.contains(&worker_id)); @@ -611,13 +530,13 @@ mod tests { *consumed_job.job.parameters.as_ref(), webhook_job_parameters ); - assert_eq!(consumed_job.job.status, JobStatus::Running); assert_eq!(consumed_job.job.target, webhook_job_parameters.url); consumed_job .complete() .await .expect("job not successfully completed"); + batch.commit().await.expect("failed to commit batch"); assert!(registry.get_status().healthy) } From bf7cccf26e42c421c2b984f22ba5737024e35e49 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 6 Feb 2024 10:32:13 -0700 Subject: [PATCH 204/247] Sync workflow style/naming --- .github/workflows/docker-capture.yml | 12 ++++++------ .github/workflows/docker-hook-api.yml | 2 +- .github/workflows/docker-hook-janitor.yml | 2 +- .github/workflows/docker-hook-worker.yml | 2 +- .github/workflows/rust.yml | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-capture.yml index 5bc5100b79e5f..4b4c1cd2a98eb 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-capture.yml @@ -1,4 +1,4 @@ -name: Build docker image +name: Build capture docker image on: workflow_dispatch: @@ -37,7 +37,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/${{ github.repository }} + images: ghcr.io/posthog/capture tags: | type=ref,event=pr type=ref,event=branch @@ -49,8 +49,8 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Build and push - id: docker_build + - name: Build and push capture + id: docker_build_capture uses: docker/build-push-action@v4 with: context: ./ @@ -64,5 +64,5 @@ jobs: cache-to: type=gha,mode=max build-args: RUST_BACKTRACE=1 BIN=capture-server - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} + - name: Capture image digest + run: echo ${{ steps.docker_build_capture.outputs.digest }} diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index 2cafd628a8765..0057d6cece253 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -62,7 +62,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: BIN=hook-api + build-args: RUST_BACKTRACE=1 BIN=hook-api - name: Hook-api image digest run: echo ${{ steps.docker_build_hook_api.outputs.digest }} diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index fc662bd8f38b1..32903baa44465 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -62,7 +62,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: BIN=hook-janitor + build-args: RUST_BACKTRACE=1 BIN=hook-janitor - name: Hook-janitor image digest run: echo ${{ steps.docker_build_hook_janitor.outputs.digest }} diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index 77db4a7c18beb..05a1054f73805 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -62,7 +62,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: BIN=hook-worker + build-args: RUST_BACKTRACE=1 BIN=hook-worker - name: Hook-worker image digest run: echo ${{ steps.docker_build_hook_worker.outputs.digest }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ea3be4ec632e9..7ee5eb83a4c90 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -61,8 +61,8 @@ jobs: with: path: | ~/.cargo/registry - ~/.cargo/git - target + ~/.cargo/git + target key: ${ runner.os }-cargo-debug-${{ hashFiles('**/Cargo.lock') }} - name: Run cargo test From 5c64ef0b73a88836946f7d315825d7739485a035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Tue, 6 Feb 2024 18:13:27 +0100 Subject: [PATCH 205/247] fix: Bump failure counter on job error (#36) --- hook-api/src/handlers/app.rs | 4 +- hook-api/src/handlers/webhook.rs | 20 ++--- hook-common/src/pgqueue.rs | 124 +++++++++++++++---------------- hook-common/src/webhook.rs | 6 +- hook-janitor/src/webhooks.rs | 4 +- hook-worker/src/error.rs | 8 +- hook-worker/src/worker.rs | 52 ++++++++----- 7 files changed, 110 insertions(+), 108 deletions(-) diff --git a/hook-api/src/handlers/app.rs b/hook-api/src/handlers/app.rs index 7b1e840094473..f8d4b24695b99 100644 --- a/hook-api/src/handlers/app.rs +++ b/hook-api/src/handlers/app.rs @@ -30,9 +30,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn index(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", db) - .await - .expect("failed to construct pg_queue"); + let pg_queue = PgQueue::new_from_pool("test_index", db).await; let app = add_routes(Router::new(), pg_queue); diff --git a/hook-api/src/handlers/webhook.rs b/hook-api/src/handlers/webhook.rs index 16ebc6dc57179..e50b8b0241a51 100644 --- a/hook-api/src/handlers/webhook.rs +++ b/hook-api/src/handlers/webhook.rs @@ -127,9 +127,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn webhook_success(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", db) - .await - .expect("failed to construct pg_queue"); + let pg_queue = PgQueue::new_from_pool("test_index", db).await; let app = add_routes(Router::new(), pg_queue); @@ -171,9 +169,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn webhook_bad_url(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", db) - .await - .expect("failed to construct pg_queue"); + let pg_queue = PgQueue::new_from_pool("test_index", db).await; let app = add_routes(Router::new(), pg_queue); @@ -210,9 +206,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn webhook_payload_missing_fields(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", db) - .await - .expect("failed to construct pg_queue"); + let pg_queue = PgQueue::new_from_pool("test_index", db).await; let app = add_routes(Router::new(), pg_queue); @@ -233,9 +227,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn webhook_payload_not_json(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", db) - .await - .expect("failed to construct pg_queue"); + let pg_queue = PgQueue::new_from_pool("test_index", db).await; let app = add_routes(Router::new(), pg_queue); @@ -256,9 +248,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn webhook_payload_body_too_large(db: PgPool) { - let pg_queue = PgQueue::new_from_pool("test_index", db) - .await - .expect("failed to construct pg_queue"); + let pg_queue = PgQueue::new_from_pool("test_index", db).await; let app = add_routes(Router::new(), pg_queue); diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index 4a8b489716a7d..f155506123b35 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -13,16 +13,9 @@ use thiserror::Error; use tokio::sync::Mutex; use tracing::error; -/// Enumeration of errors for operations with PgQueue. -/// Errors that can originate from sqlx and are wrapped by us to provide additional context. +/// Enumeration of parsing errors in PgQueue. #[derive(Error, Debug)] -pub enum PgQueueError { - #[error("pool creation failed with: {error}")] - PoolCreationError { error: sqlx::Error }, - #[error("connection failed with: {error}")] - ConnectionError { error: sqlx::Error }, - #[error("{command} query failed with: {error}")] - QueryError { command: String, error: sqlx::Error }, +pub enum ParseError { #[error("{0} is not a valid JobStatus")] ParseJobStatusError(String), #[error("{0} is not a valid HttpMethod")] @@ -31,10 +24,12 @@ pub enum PgQueueError { TransactionAlreadyClosedError, } +/// Enumeration of database-related errors in PgQueue. +/// Errors that can originate from sqlx and are wrapped by us to provide additional context. #[derive(Error, Debug)] -pub enum PgJobError { - #[error("retry is an invalid state for this PgJob: {error}")] - RetryInvalidError { job: T, error: String }, +pub enum DatabaseError { + #[error("pool creation failed with: {error}")] + PoolCreationError { error: sqlx::Error }, #[error("connection failed with: {error}")] ConnectionError { error: sqlx::Error }, #[error("{command} query failed with: {error}")] @@ -45,6 +40,25 @@ pub enum PgJobError { TransactionAlreadyClosedError, } +/// An error that occurs when a job cannot be retried. +/// Returns the underlying job so that a client can fail it. +#[derive(Error, Debug)] +#[error("retry is an invalid state for this job: {error}")] +pub struct RetryInvalidError { + pub job: T, + pub error: String, +} + +/// Enumeration of errors that can occur when retrying a job. +/// They are in a separate enum a failed retry could be returning the underlying job. +#[derive(Error, Debug)] +pub enum RetryError { + #[error(transparent)] + DatabaseError(#[from] DatabaseError), + #[error(transparent)] + RetryInvalidError(#[from] RetryInvalidError), +} + /// Enumeration of possible statuses for a Job. #[derive(Debug, PartialEq, sqlx::Type)] #[sqlx(type_name = "job_status")] @@ -64,14 +78,14 @@ pub enum JobStatus { /// Allow casting JobStatus from strings. impl FromStr for JobStatus { - type Err = PgQueueError; + type Err = ParseError; fn from_str(s: &str) -> Result { match s { "available" => Ok(JobStatus::Available), "completed" => Ok(JobStatus::Completed), "failed" => Ok(JobStatus::Failed), - invalid => Err(PgQueueError::ParseJobStatusError(invalid.to_owned())), + invalid => Err(ParseError::ParseJobStatusError(invalid.to_owned())), } } } @@ -204,19 +218,19 @@ RETURNING #[async_trait] pub trait PgQueueJob { - async fn complete(mut self) -> Result>>; + async fn complete(mut self) -> Result; async fn fail( mut self, error: E, - ) -> Result, PgJobError>>; + ) -> Result, DatabaseError>; async fn retry( mut self, error: E, retry_interval: time::Duration, queue: &str, - ) -> Result>>; + ) -> Result>>; } /// A Job within an open PostgreSQL transaction. @@ -247,10 +261,10 @@ impl<'c, J, M> PgTransactionBatch<'_, J, M> { txn_guard .as_deref_mut() - .ok_or(PgQueueError::TransactionAlreadyClosedError)? + .ok_or(DatabaseError::TransactionAlreadyClosedError)? .commit() .await - .map_err(|e| PgQueueError::QueryError { + .map_err(|e| DatabaseError::QueryError { command: "COMMIT".to_owned(), error: e, })?; @@ -261,20 +275,18 @@ impl<'c, J, M> PgTransactionBatch<'_, J, M> { #[async_trait] impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactionJob<'c, J, M> { - async fn complete( - mut self, - ) -> Result>>> { + async fn complete(mut self) -> Result { let mut txn_guard = self.shared_txn.lock().await; let txn_ref = txn_guard .as_deref_mut() - .ok_or(PgJobError::TransactionAlreadyClosedError)?; + .ok_or(DatabaseError::TransactionAlreadyClosedError)?; let completed_job = self.job .complete(txn_ref) .await - .map_err(|error| PgJobError::QueryError { + .map_err(|error| DatabaseError::QueryError { command: "UPDATE".to_owned(), error, })?; @@ -285,18 +297,18 @@ impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactio async fn fail( mut self, error: S, - ) -> Result, PgJobError>>> { + ) -> Result, DatabaseError> { let mut txn_guard = self.shared_txn.lock().await; let txn_ref = txn_guard .as_deref_mut() - .ok_or(PgJobError::TransactionAlreadyClosedError)?; + .ok_or(DatabaseError::TransactionAlreadyClosedError)?; let failed_job = self.job .fail(error, txn_ref) .await - .map_err(|error| PgJobError::QueryError { + .map_err(|error| DatabaseError::QueryError { command: "UPDATE".to_owned(), error, })?; @@ -309,21 +321,21 @@ impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactio error: E, retry_interval: time::Duration, queue: &str, - ) -> Result>>> { + ) -> Result>>> { // Ideally, the transition to RetryableJob should be fallible. // But taking ownership of self when we return this error makes things difficult. if self.job.is_gte_max_attempts() { - return Err(PgJobError::RetryInvalidError { + return Err(RetryError::from(RetryInvalidError { job: Box::new(self), error: "Maximum attempts reached".to_owned(), - }); + })); } let mut txn_guard = self.shared_txn.lock().await; let txn_ref = txn_guard .as_deref_mut() - .ok_or(PgJobError::TransactionAlreadyClosedError)?; + .ok_or(DatabaseError::TransactionAlreadyClosedError)?; let retried_job = self .job @@ -331,7 +343,7 @@ impl<'c, J: std::marker::Send, M: std::marker::Send> PgQueueJob for PgTransactio .queue(queue) .retry(error, retry_interval, txn_ref) .await - .map_err(|error| PgJobError::QueryError { + .map_err(|error| DatabaseError::QueryError { command: "UPDATE".to_owned(), error, })?; @@ -481,7 +493,7 @@ pub struct PgQueue { pool: PgPool, } -pub type PgQueueResult = std::result::Result; +pub type PgQueueResult = std::result::Result; impl PgQueue { /// Initialize a new PgQueue backed by table in PostgreSQL by intializing a connection pool to the database in `url`. @@ -498,7 +510,7 @@ impl PgQueue { ) -> PgQueueResult { let name = queue_name.to_owned(); let options = PgConnectOptions::from_str(url) - .map_err(|error| PgQueueError::PoolCreationError { error })? + .map_err(|error| DatabaseError::PoolCreationError { error })? .application_name(app_name); let pool = PgPoolOptions::new() .max_connections(max_connections) @@ -513,14 +525,14 @@ impl PgQueue { /// /// * `queue_name`: A name for the queue we are going to initialize. /// * `pool`: A database connection pool to be used by this queue. - pub async fn new_from_pool(queue_name: &str, pool: PgPool) -> PgQueueResult { + pub async fn new_from_pool(queue_name: &str, pool: PgPool) -> PgQueue { let name = queue_name.to_owned(); - Ok(Self { name, pool }) + Self { name, pool } } - /// Dequeue up to `limit` `Job`s from this `PgQueue` and hold the transaction. Any other - /// `dequeue_tx` calls will skip rows locked, so by holding a transaction we ensure only one + /// Dequeue up to `limit` `Job`s from this `PgQueue` and hold the transaction. + /// Any other `dequeue_tx` calls will skip rows locked, so by holding a transaction we ensure only one /// worker can dequeue a job. Holding a transaction open can have performance implications, but /// it means no `'running'` state is required. pub async fn dequeue_tx< @@ -536,7 +548,7 @@ impl PgQueue { .pool .begin() .await - .map_err(|error| PgQueueError::ConnectionError { error })?; + .map_err(|error| DatabaseError::ConnectionError { error })?; // The query that follows uses a FOR UPDATE SKIP LOCKED clause. // For more details on this see: 2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5. @@ -601,8 +613,7 @@ RETURNING // Transaction is rolled back on drop. Err(sqlx::Error::RowNotFound) => Ok(None), - - Err(e) => Err(PgQueueError::QueryError { + Err(e) => Err(DatabaseError::QueryError { command: "UPDATE".to_owned(), error: e, }), @@ -634,7 +645,7 @@ VALUES .bind(&job.target) .execute(&self.pool) .await - .map_err(|error| PgQueueError::QueryError { + .map_err(|error| DatabaseError::QueryError { command: "INSERT".to_owned(), error, })?; @@ -699,9 +710,7 @@ mod tests { let job_parameters = JobParameters::default(); let worker_id = worker_id(); - let queue = PgQueue::new_from_pool("test_can_dequeue_tx_job", db) - .await - .expect("failed to connect to local test postgresql database"); + let queue = PgQueue::new_from_pool("test_can_dequeue_tx_job", db).await; let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); queue.enqueue(new_job).await.expect("failed to enqueue job"); @@ -736,9 +745,7 @@ mod tests { let job_parameters = JobParameters::default(); let worker_id = worker_id(); - let queue = PgQueue::new_from_pool("test_can_dequeue_multiple_tx_jobs", db) - .await - .expect("failed to connect to local test postgresql database"); + let queue = PgQueue::new_from_pool("test_can_dequeue_multiple_tx_jobs", db).await; for _ in 0..5 { queue @@ -785,9 +792,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn test_dequeue_tx_returns_none_on_no_jobs(db: PgPool) { let worker_id = worker_id(); - let queue = PgQueue::new_from_pool("test_dequeue_tx_returns_none_on_no_jobs", db) - .await - .expect("failed to connect to local test postgresql database"); + let queue = PgQueue::new_from_pool("test_dequeue_tx_returns_none_on_no_jobs", db).await; let batch: Option> = queue .dequeue_tx(&worker_id, 1) @@ -810,9 +815,7 @@ mod tests { .queue(&queue_name) .provide(); - let queue = PgQueue::new_from_pool(&queue_name, db) - .await - .expect("failed to connect to local test postgresql database"); + let queue = PgQueue::new_from_pool(&queue_name, db).await; queue.enqueue(new_job).await.expect("failed to enqueue job"); let mut batch: PgTransactionBatch<'_, JobParameters, JobMetadata> = queue @@ -868,9 +871,7 @@ mod tests { .queue(&retry_queue_name) .provide(); - let queue = PgQueue::new_from_pool(&queue_name, db.clone()) - .await - .expect("failed to connect to queue in local test postgresql database"); + let queue = PgQueue::new_from_pool(&queue_name, db.clone()).await; queue.enqueue(new_job).await.expect("failed to enqueue job"); let mut batch: PgTransactionBatch = queue @@ -899,9 +900,7 @@ mod tests { assert!(retried_job_not_found.is_none()); - let queue = PgQueue::new_from_pool(&retry_queue_name, db) - .await - .expect("failed to connect to retry queue in local test postgresql database"); + let queue = PgQueue::new_from_pool(&retry_queue_name, db).await; let retried_job: PgTransactionJob = queue .dequeue_tx(&worker_id, 1) @@ -933,9 +932,8 @@ mod tests { let new_job = NewJob::new(1, job_metadata, job_parameters, &job_target); let retry_policy = RetryPolicy::build(0, time::Duration::from_secs(0)).provide(); - let queue = PgQueue::new_from_pool("test_cannot_retry_job_without_remaining_attempts", db) - .await - .expect("failed to connect to local test postgresql database"); + let queue = + PgQueue::new_from_pool("test_cannot_retry_job_without_remaining_attempts", db).await; queue.enqueue(new_job).await.expect("failed to enqueue job"); diff --git a/hook-common/src/webhook.rs b/hook-common/src/webhook.rs index 11e02856703eb..5286629978931 100644 --- a/hook-common/src/webhook.rs +++ b/hook-common/src/webhook.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use serde::{de::Visitor, Deserialize, Serialize}; use crate::kafka_messages::app_metrics; -use crate::pgqueue::PgQueueError; +use crate::pgqueue::ParseError; /// Supported HTTP methods for webhooks. #[derive(Debug, PartialEq, Clone, Copy)] @@ -20,7 +20,7 @@ pub enum HttpMethod { /// Allow casting `HttpMethod` from strings. impl FromStr for HttpMethod { - type Err = PgQueueError; + type Err = ParseError; fn from_str(s: &str) -> Result { match s.to_ascii_uppercase().as_ref() { @@ -29,7 +29,7 @@ impl FromStr for HttpMethod { "PATCH" => Ok(HttpMethod::PATCH), "POST" => Ok(HttpMethod::POST), "PUT" => Ok(HttpMethod::PUT), - invalid => Err(PgQueueError::ParseHttpMethodError(invalid.to_owned())), + invalid => Err(ParseError::ParseHttpMethodError(invalid.to_owned())), } } } diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 7f7fadd408565..0e3900c045d84 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -786,9 +786,7 @@ mod tests { WebhookCleaner::new_from_pool(db.clone(), mock_producer, APP_METRICS_TOPIC.to_owned()) .expect("unable to create webhook cleaner"); - let queue = PgQueue::new_from_pool("webhooks", db.clone()) - .await - .expect("failed to connect to local test postgresql database"); + let queue = PgQueue::new_from_pool("webhooks", db.clone()).await; async fn get_count_from_new_conn(db: &PgPool, status: &str) -> i64 { let mut conn = db.acquire().await.unwrap(); diff --git a/hook-worker/src/error.rs b/hook-worker/src/error.rs index 614fe721957e4..914ffb1b2e2ee 100644 --- a/hook-worker/src/error.rs +++ b/hook-worker/src/error.rs @@ -24,10 +24,10 @@ pub enum WebhookError { /// Enumeration of errors related to initialization and consumption of webhook jobs. #[derive(Error, Debug)] pub enum WorkerError { + #[error("a database error occurred when executing a job")] + DatabaseError(#[from] pgqueue::DatabaseError), + #[error("a parsing error occurred in the underlying queue")] + QueueParseError(#[from] pgqueue::ParseError), #[error("timed out while waiting for jobs to be available")] TimeoutError, - #[error("an error occurred in the underlying queue")] - QueueError(#[from] pgqueue::PgQueueError), - #[error("an error occurred in the underlying job: {0}")] - PgJobError(String), } diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index b83c909cb698e..484edf7ff36a0 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -6,7 +6,9 @@ use futures::future::join_all; use hook_common::health::HealthHandle; use hook_common::pgqueue::PgTransactionBatch; use hook_common::{ - pgqueue::{Job, PgJobError, PgQueue, PgQueueError, PgQueueJob, PgTransactionJob}, + pgqueue::{ + DatabaseError, Job, PgQueue, PgQueueJob, PgTransactionJob, RetryError, RetryInvalidError, + }, retry::RetryPolicy, webhook::{HttpMethod, WebhookJobError, WebhookJobMetadata, WebhookJobParameters}, }; @@ -232,10 +234,10 @@ async fn process_webhook_job( match send_result { Ok(_) => { - webhook_job - .complete() - .await - .map_err(|error| WorkerError::PgJobError(error.to_string()))?; + webhook_job.complete().await.map_err(|error| { + metrics::counter!("webhook_jobs_database_error", &labels).increment(1); + error + })?; metrics::counter!("webhook_jobs_completed", &labels).increment(1); metrics::histogram!("webhook_jobs_processing_duration_seconds", &labels) @@ -247,7 +249,10 @@ async fn process_webhook_job( webhook_job .fail(WebhookJobError::new_parse(&e.to_string())) .await - .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| { + metrics::counter!("webhook_jobs_database_error", &labels).increment(1); + job_error + })?; metrics::counter!("webhook_jobs_failed", &labels).increment(1); @@ -257,7 +262,10 @@ async fn process_webhook_job( webhook_job .fail(WebhookJobError::new_parse(&e)) .await - .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| { + metrics::counter!("webhook_jobs_database_error", &labels).increment(1); + job_error + })?; metrics::counter!("webhook_jobs_failed", &labels).increment(1); @@ -267,7 +275,10 @@ async fn process_webhook_job( webhook_job .fail(WebhookJobError::new_parse(&e.to_string())) .await - .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| { + metrics::counter!("webhook_jobs_database_error", &labels).increment(1); + job_error + })?; metrics::counter!("webhook_jobs_failed", &labels).increment(1); @@ -288,26 +299,35 @@ async fn process_webhook_job( Ok(()) } - Err(PgJobError::RetryInvalidError { + Err(RetryError::RetryInvalidError(RetryInvalidError { job: webhook_job, .. - }) => { + })) => { webhook_job .fail(WebhookJobError::from(&error)) .await - .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| { + metrics::counter!("webhook_jobs_database_error", &labels).increment(1); + job_error + })?; metrics::counter!("webhook_jobs_failed", &labels).increment(1); Ok(()) } - Err(job_error) => Err(WorkerError::PgJobError(job_error.to_string())), + Err(RetryError::DatabaseError(job_error)) => { + metrics::counter!("webhook_jobs_database_error", &labels).increment(1); + Err(WorkerError::from(job_error)) + } } } Err(WebhookError::NonRetryableRetryableRequestError(error)) => { webhook_job .fail(WebhookJobError::from(&error)) .await - .map_err(|job_error| WorkerError::PgJobError(job_error.to_string()))?; + .map_err(|job_error| { + metrics::counter!("webhook_jobs_database_error", &labels).increment(1); + job_error + })?; metrics::counter!("webhook_jobs_failed", &labels).increment(1); @@ -436,7 +456,7 @@ mod tests { max_attempts: i32, job_parameters: WebhookJobParameters, job_metadata: WebhookJobMetadata, - ) -> Result<(), PgQueueError> { + ) -> Result<(), DatabaseError> { let job_target = job_parameters.url.to_owned(); let new_job = NewJob::new(max_attempts, job_metadata, job_parameters, &job_target); queue.enqueue(new_job).await?; @@ -477,9 +497,7 @@ mod tests { async fn test_wait_for_job(db: PgPool) { let worker_id = worker_id(); let queue_name = "test_wait_for_job".to_string(); - let queue = PgQueue::new_from_pool(&queue_name, db) - .await - .expect("failed to connect to PG"); + let queue = PgQueue::new_from_pool(&queue_name, db).await; let webhook_job_parameters = WebhookJobParameters { body: "a webhook job body. much wow.".to_owned(), From fca80b06bc7b28c0a18005fec6eaefa827cbe0ee Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 6 Feb 2024 10:43:31 -0700 Subject: [PATCH 206/247] Use a single image type, namespace images under hog-rs --- .github/workflows/docker-capture.yml | 4 ++-- .github/workflows/docker-hook-api.yml | 2 +- .github/workflows/docker-hook-janitor.yml | 2 +- .github/workflows/docker-hook-worker.yml | 2 +- .github/workflows/docker-migrator.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-capture.yml index 4b4c1cd2a98eb..c810709a88ae9 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-capture.yml @@ -12,7 +12,7 @@ permissions: jobs: build: name: build and publish capture image - runs-on: buildjet-8vcpu-ubuntu-2204-arm + runs-on: buildjet-4vcpu-ubuntu-2204-arm steps: - name: Check Out Repo uses: actions/checkout@v3 @@ -37,7 +37,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/posthog/capture + images: ghcr.io/posthog/hog-rs/capture tags: | type=ref,event=pr type=ref,event=branch diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index 0057d6cece253..eba5b31124d92 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -37,7 +37,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/posthog/hook-api + images: ghcr.io/posthog/hog-rs/hook-api tags: | type=ref,event=pr type=ref,event=branch diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index 32903baa44465..5a362aa8757a2 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -37,7 +37,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/posthog/hook-janitor + images: ghcr.io/posthog/hog-rs/hook-janitor tags: | type=ref,event=pr type=ref,event=branch diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index 05a1054f73805..808bcd47cac97 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -37,7 +37,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/posthog/hook-worker + images: ghcr.io/posthog/hog-rs/hook-worker tags: | type=ref,event=pr type=ref,event=branch diff --git a/.github/workflows/docker-migrator.yml b/.github/workflows/docker-migrator.yml index c186aabee3dd3..a7880d1ec8faa 100644 --- a/.github/workflows/docker-migrator.yml +++ b/.github/workflows/docker-migrator.yml @@ -37,7 +37,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/posthog/hook-migrator + images: ghcr.io/posthog/hog-rs/hook-migrator tags: | type=ref,event=pr type=ref,event=branch From 58d777ff9c7bb196350b6dad53711eaefc2a4654 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 7 Feb 2024 07:50:09 -0700 Subject: [PATCH 207/247] Use Depot for build-push-action (#6) --- .github/workflows/docker-capture.yml | 5 ++++- .github/workflows/docker-hook-api.yml | 5 ++++- .github/workflows/docker-hook-janitor.yml | 5 ++++- .github/workflows/docker-hook-worker.yml | 5 ++++- .github/workflows/docker-migrator.yml | 5 ++++- depot.json | 1 + 6 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 depot.json diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-capture.yml index c810709a88ae9..d158b39e7d7f9 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-capture.yml @@ -17,6 +17,9 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + - name: Set up Depot CLI + uses: depot/setup-action@v1 + - name: Login to DockerHub uses: docker/login-action@v2 with: @@ -51,7 +54,7 @@ jobs: - name: Build and push capture id: docker_build_capture - uses: docker/build-push-action@v4 + uses: depot/build-push-action@v4 with: context: ./ file: ./Dockerfile diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index eba5b31124d92..efc5736f71e67 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -17,6 +17,9 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + - name: Set up Depot CLI + uses: depot/setup-action@v1 + - name: Login to DockerHub uses: docker/login-action@v2 with: @@ -51,7 +54,7 @@ jobs: - name: Build and push api id: docker_build_hook_api - uses: docker/build-push-action@v4 + uses: depot/build-push-action@v4 with: context: ./ file: ./Dockerfile diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index 5a362aa8757a2..ee4420516f83a 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -17,6 +17,9 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + - name: Set up Depot CLI + uses: depot/setup-action@v1 + - name: Login to DockerHub uses: docker/login-action@v2 with: @@ -51,7 +54,7 @@ jobs: - name: Build and push janitor id: docker_build_hook_janitor - uses: docker/build-push-action@v4 + uses: depot/build-push-action@v4 with: context: ./ file: ./Dockerfile diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index 808bcd47cac97..940a36c477e12 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -17,6 +17,9 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + - name: Set up Depot CLI + uses: depot/setup-action@v1 + - name: Login to DockerHub uses: docker/login-action@v2 with: @@ -51,7 +54,7 @@ jobs: - name: Build and push worker id: docker_build_hook_worker - uses: docker/build-push-action@v4 + uses: depot/build-push-action@v4 with: context: ./ file: ./Dockerfile diff --git a/.github/workflows/docker-migrator.yml b/.github/workflows/docker-migrator.yml index a7880d1ec8faa..91e34a42e106f 100644 --- a/.github/workflows/docker-migrator.yml +++ b/.github/workflows/docker-migrator.yml @@ -17,6 +17,9 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + - name: Set up Depot CLI + uses: depot/setup-action@v1 + - name: Login to DockerHub uses: docker/login-action@v2 with: @@ -51,7 +54,7 @@ jobs: - name: Build and push migrator id: docker_build_hook_migrator - uses: docker/build-push-action@v4 + uses: depot/build-push-action@v4 with: context: ./ file: ./Dockerfile.migrate diff --git a/depot.json b/depot.json new file mode 100644 index 0000000000000..ea625b42fe589 --- /dev/null +++ b/depot.json @@ -0,0 +1 @@ +{"id":"zcszdgwzsw"} From 017ae0397e7e6954924201e84e9ad2dd581bb46f Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 7 Feb 2024 08:03:52 -0700 Subject: [PATCH 208/247] Fix depot build-push-action version and drop invalid builder argument (#7) --- .github/workflows/docker-capture.yml | 3 +-- .github/workflows/docker-hook-api.yml | 3 +-- .github/workflows/docker-hook-janitor.yml | 3 +-- .github/workflows/docker-hook-worker.yml | 3 +-- .github/workflows/docker-migrator.yml | 3 +-- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-capture.yml index d158b39e7d7f9..c6d1865222233 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-capture.yml @@ -54,11 +54,10 @@ jobs: - name: Build and push capture id: docker_build_capture - uses: depot/build-push-action@v4 + uses: depot/build-push-action@v1 with: context: ./ file: ./Dockerfile - builder: ${{ steps.buildx.outputs.name }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index efc5736f71e67..0bf097c4e68e7 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -54,11 +54,10 @@ jobs: - name: Build and push api id: docker_build_hook_api - uses: depot/build-push-action@v4 + uses: depot/build-push-action@v1 with: context: ./ file: ./Dockerfile - builder: ${{ steps.buildx.outputs.name }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index ee4420516f83a..66b0eef7ba41b 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -54,11 +54,10 @@ jobs: - name: Build and push janitor id: docker_build_hook_janitor - uses: depot/build-push-action@v4 + uses: depot/build-push-action@v1 with: context: ./ file: ./Dockerfile - builder: ${{ steps.buildx.outputs.name }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index 940a36c477e12..bb4998f2bbfc4 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -54,11 +54,10 @@ jobs: - name: Build and push worker id: docker_build_hook_worker - uses: depot/build-push-action@v4 + uses: depot/build-push-action@v1 with: context: ./ file: ./Dockerfile - builder: ${{ steps.buildx.outputs.name }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docker-migrator.yml b/.github/workflows/docker-migrator.yml index 91e34a42e106f..b6bfad6d15e0f 100644 --- a/.github/workflows/docker-migrator.yml +++ b/.github/workflows/docker-migrator.yml @@ -54,11 +54,10 @@ jobs: - name: Build and push migrator id: docker_build_hook_migrator - uses: depot/build-push-action@v4 + uses: depot/build-push-action@v1 with: context: ./ file: ./Dockerfile.migrate - builder: ${{ steps.buildx.outputs.name }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 49fa92b7e9f1a4330d5f109679e484238cc8b072 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 7 Feb 2024 08:19:03 -0700 Subject: [PATCH 209/247] Configure Actions permissions for build runs (#8) --- .github/workflows/docker-capture.yml | 5 +++++ .github/workflows/docker-hook-api.yml | 5 +++++ .github/workflows/docker-hook-janitor.yml | 5 +++++ .github/workflows/docker-hook-worker.yml | 5 +++++ .github/workflows/docker-migrator.yml | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-capture.yml index c6d1865222233..4426828e8b49a 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-capture.yml @@ -13,6 +13,11 @@ jobs: build: name: build and publish capture image runs-on: buildjet-4vcpu-ubuntu-2204-arm + permissions: + id-token: write # allow issuing OIDC tokens for this workflow run + contents: read # allow reading the repo contents + packages: write # allow push to ghcr.io + steps: - name: Check Out Repo uses: actions/checkout@v3 diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index 0bf097c4e68e7..4df1eb6b0ad95 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -13,6 +13,11 @@ jobs: build: name: build and publish hook-api image runs-on: buildjet-4vcpu-ubuntu-2204-arm + permissions: + id-token: write # allow issuing OIDC tokens for this workflow run + contents: read # allow reading the repo contents + packages: write # allow push to ghcr.io + steps: - name: Check Out Repo uses: actions/checkout@v3 diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index 66b0eef7ba41b..0894d4548d658 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -13,6 +13,11 @@ jobs: build: name: build and publish hook-janitor image runs-on: buildjet-4vcpu-ubuntu-2204-arm + permissions: + id-token: write # allow issuing OIDC tokens for this workflow run + contents: read # allow reading the repo contents + packages: write # allow push to ghcr.io + steps: - name: Check Out Repo uses: actions/checkout@v3 diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index bb4998f2bbfc4..2568f5eec9859 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -13,6 +13,11 @@ jobs: build: name: build and publish hook-worker image runs-on: buildjet-4vcpu-ubuntu-2204-arm + permissions: + id-token: write # allow issuing OIDC tokens for this workflow run + contents: read # allow reading the repo contents + packages: write # allow push to ghcr.io + steps: - name: Check Out Repo uses: actions/checkout@v3 diff --git a/.github/workflows/docker-migrator.yml b/.github/workflows/docker-migrator.yml index b6bfad6d15e0f..250fe9b34ccd4 100644 --- a/.github/workflows/docker-migrator.yml +++ b/.github/workflows/docker-migrator.yml @@ -13,6 +13,11 @@ jobs: build: name: build and publish hook-migrator image runs-on: buildjet-4vcpu-ubuntu-2204-arm + permissions: + id-token: write # allow issuing OIDC tokens for this workflow run + contents: read # allow reading the repo contents + packages: write # allow push to ghcr.io + steps: - name: Check Out Repo uses: actions/checkout@v3 From a42a056a952f22e7dccdff0bffd108421ee578cc Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 7 Feb 2024 08:44:14 -0700 Subject: [PATCH 210/247] Actions docker build: change build-args to a newline delimited string (#9) --- .github/workflows/docker-capture.yml | 4 +++- .github/workflows/docker-hook-api.yml | 4 +++- .github/workflows/docker-hook-janitor.yml | 4 +++- .github/workflows/docker-hook-worker.yml | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-capture.yml index 4426828e8b49a..75a94e5ee5d97 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-capture.yml @@ -69,7 +69,9 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: RUST_BACKTRACE=1 BIN=capture-server + build-args: | + "RUST_BACKTRACE=1" + "BIN=capture-server" - name: Capture image digest run: echo ${{ steps.docker_build_capture.outputs.digest }} diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index 4df1eb6b0ad95..57e603cfc9e3c 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -69,7 +69,9 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: RUST_BACKTRACE=1 BIN=hook-api + build-args: | + "RUST_BACKTRACE=1" + "BIN=hook-api" - name: Hook-api image digest run: echo ${{ steps.docker_build_hook_api.outputs.digest }} diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index 0894d4548d658..98fc7371b0f8c 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -69,7 +69,9 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: RUST_BACKTRACE=1 BIN=hook-janitor + build-args: | + "RUST_BACKTRACE=1" + "BIN=hook-janitor" - name: Hook-janitor image digest run: echo ${{ steps.docker_build_hook_janitor.outputs.digest }} diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index 2568f5eec9859..9604d0871f16a 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -69,7 +69,9 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: RUST_BACKTRACE=1 BIN=hook-worker + build-args: | + "RUST_BACKTRACE=1" + "BIN=hook-worker" - name: Hook-worker image digest run: echo ${{ steps.docker_build_hook_worker.outputs.digest }} From 6770bed595c592298d3a183e8a5f78c26bac3707 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 7 Feb 2024 09:47:36 -0700 Subject: [PATCH 211/247] Temporarily switch to a single build-args argument (#10) --- .github/workflows/docker-capture.yml | 4 +--- .github/workflows/docker-hook-api.yml | 4 +--- .github/workflows/docker-hook-janitor.yml | 4 +--- .github/workflows/docker-hook-worker.yml | 4 +--- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-capture.yml index 75a94e5ee5d97..468c5c2738331 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-capture.yml @@ -69,9 +69,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: | - "RUST_BACKTRACE=1" - "BIN=capture-server" + build-args: BIN=capture-server - name: Capture image digest run: echo ${{ steps.docker_build_capture.outputs.digest }} diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index 57e603cfc9e3c..ef22c69487c8a 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -69,9 +69,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: | - "RUST_BACKTRACE=1" - "BIN=hook-api" + build-args: BIN=hook-api - name: Hook-api image digest run: echo ${{ steps.docker_build_hook_api.outputs.digest }} diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index 98fc7371b0f8c..54129c65f072c 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -69,9 +69,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: | - "RUST_BACKTRACE=1" - "BIN=hook-janitor" + build-args: BIN=hook-janitor - name: Hook-janitor image digest run: echo ${{ steps.docker_build_hook_janitor.outputs.digest }} diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index 9604d0871f16a..1c3e17e2ae18a 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -69,9 +69,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: | - "RUST_BACKTRACE=1" - "BIN=hook-worker" + build-args: BIN=hook-worker - name: Hook-worker image digest run: echo ${{ steps.docker_build_hook_worker.outputs.digest }} From cb635eb6b88557e5fac079ac49fc4afd34390c03 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 7 Feb 2024 12:21:44 -0700 Subject: [PATCH 212/247] Fix Docker entrypoint (#11) --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7bbfda6fe2342..f34d1eeb31d6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,10 +29,9 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* ARG BIN -ENV ENTRYPOINT=/usr/local/bin/$BIN WORKDIR /app USER nobody COPY --from=builder /app/target/release/$BIN /usr/local/bin -ENTRYPOINT [ $ENTRYPOINT ] +ENTRYPOINT ["/bin/sh", "-c", "/usr/local/bin/$BIN"] From f9626e94dd9fe3c265e48e69aa2958d766e3b4f1 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Wed, 7 Feb 2024 12:37:10 -0700 Subject: [PATCH 213/247] Add ENV for BIN (#12) --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index f34d1eeb31d6e..67aea7f210f44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* ARG BIN +ENV BIN=$BIN WORKDIR /app USER nobody From c856730f94d71e45612894e3c3d4c39d7929b517 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 8 Feb 2024 08:09:02 -0700 Subject: [PATCH 214/247] Add histogram for insertion time to completion time (#5) * Add histogram for insertion time to completion time * Handle feedback * Oops, spurious import --- hook-worker/src/worker.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 484edf7ff36a0..441e1ee609de8 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -2,6 +2,7 @@ use std::collections; use std::sync::Arc; use std::time; +use chrono::Utc; use futures::future::join_all; use hook_common::health::HealthHandle; use hook_common::pgqueue::PgTransactionBatch; @@ -234,11 +235,24 @@ async fn process_webhook_job( match send_result { Ok(_) => { + let created_at = webhook_job.job().created_at; + let retries = webhook_job.job().attempt - 1; + let labels_with_retries = [ + ("queue", webhook_job.queue()), + ("retries", retries.to_string()), + ]; + webhook_job.complete().await.map_err(|error| { metrics::counter!("webhook_jobs_database_error", &labels).increment(1); error })?; + let insert_to_complete_duration = Utc::now() - created_at; + metrics::histogram!( + "webhook_jobs_insert_to_complete_duration_seconds", + &labels_with_retries + ) + .record((insert_to_complete_duration.num_milliseconds() as f64) / 1_000_f64); metrics::counter!("webhook_jobs_completed", &labels).increment(1); metrics::histogram!("webhook_jobs_processing_duration_seconds", &labels) .record(elapsed); From 4214362ef0203ff82cd9d8aea5d77c59ec41b4d3 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Thu, 8 Feb 2024 08:23:50 -0700 Subject: [PATCH 215/247] Fix clippy complaint (#13) --- hook-common/src/kafka_messages/plugin_logs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook-common/src/kafka_messages/plugin_logs.rs b/hook-common/src/kafka_messages/plugin_logs.rs index fb835804c687c..5a852e6aa3221 100644 --- a/hook-common/src/kafka_messages/plugin_logs.rs +++ b/hook-common/src/kafka_messages/plugin_logs.rs @@ -65,7 +65,7 @@ where serializer.serialize_str(type_str) } -fn serialize_message(msg: &String, serializer: S) -> Result +fn serialize_message(msg: &str, serializer: S) -> Result where S: Serializer, { From 3bcbb239dc5851aeb1ec0ec3b256d16984bd1a6e Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 9 Feb 2024 13:25:01 -0700 Subject: [PATCH 216/247] Drop debug symbols from release (#15) --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index db01334cdd8c3..3aeacb8bcf0a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,8 +10,8 @@ members = [ "hook-janitor", ] -[profile.release] -debug = 2 # https://www.polarsignals.com/docs/rust +# [profile.release] +# debug = 2 # https://www.polarsignals.com/docs/rust [workspace.dependencies] anyhow = "1.0" From e3047a6ff7cbca2e0dc745e412ae6a178c62e1eb Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 12 Feb 2024 18:58:39 +0100 Subject: [PATCH 217/247] replace vestigial unwraps with proper error reporting (#14) --- capture/src/capture.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/capture/src/capture.rs b/capture/src/capture.rs index d7303fd20f892..7f90d575fcf1c 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -75,10 +75,16 @@ pub async fn event( "application/x-www-form-urlencoded" => { tracing::Span::current().record("content_type", "application/x-www-form-urlencoded"); - let input: EventFormData = serde_urlencoded::from_bytes(body.deref()).unwrap(); + let input: EventFormData = serde_urlencoded::from_bytes(body.deref()).map_err(|e| { + tracing::error!("failed to decode body: {}", e); + CaptureError::RequestDecodingError(String::from("invalid form data")) + })?; let payload = base64::engine::general_purpose::STANDARD .decode(input.data) - .unwrap(); + .map_err(|e| { + tracing::error!("failed to decode form data: {}", e); + CaptureError::RequestDecodingError(String::from("missing data field")) + })?; RawEvent::from_bytes(payload.into()) } ct => { From 1ac549d27d68f577bb982b30f78459b96b1111a0 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Wed, 21 Feb 2024 15:50:32 +0000 Subject: [PATCH 218/247] Replace DOCKERHUB_USERNAME secret with plaintext (#18) This isn't a secret and it causes every instance of "posthog" in the action logs to be replaced by "***" because it's the value of a secret --- .github/workflows/docker-capture.yml | 2 +- .github/workflows/docker-hook-api.yml | 2 +- .github/workflows/docker-hook-janitor.yml | 2 +- .github/workflows/docker-hook-worker.yml | 2 +- .github/workflows/docker-migrator.yml | 2 +- .github/workflows/rust.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-capture.yml index 468c5c2738331..30232bc5c1a3c 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-capture.yml @@ -28,7 +28,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: posthog password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to ghcr.io diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index ef22c69487c8a..2d9384529fb0a 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -28,7 +28,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: posthog password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to ghcr.io diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index 54129c65f072c..31a4e2d86c5cf 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -28,7 +28,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: posthog password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to ghcr.io diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index 1c3e17e2ae18a..c3ba8ce888452 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -28,7 +28,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: posthog password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to ghcr.io diff --git a/.github/workflows/docker-migrator.yml b/.github/workflows/docker-migrator.yml index 250fe9b34ccd4..479d6f9126255 100644 --- a/.github/workflows/docker-migrator.yml +++ b/.github/workflows/docker-migrator.yml @@ -28,7 +28,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: posthog password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to ghcr.io diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7ee5eb83a4c90..72dfd0bc5093f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -43,7 +43,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} + username: posthog password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Setup dependencies From 92018ab4a97221db4a074c0d234c5a821aad362e Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 23 Feb 2024 07:47:38 -0700 Subject: [PATCH 219/247] Use Depot runners (#16) --- .github/workflows/docker-capture.yml | 2 +- .github/workflows/docker-hook-api.yml | 2 +- .github/workflows/docker-hook-janitor.yml | 2 +- .github/workflows/docker-hook-worker.yml | 2 +- .github/workflows/docker-migrator.yml | 2 +- .github/workflows/rust.yml | 8 ++++---- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-capture.yml index 30232bc5c1a3c..d0efac36b0852 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-capture.yml @@ -12,7 +12,7 @@ permissions: jobs: build: name: build and publish capture image - runs-on: buildjet-4vcpu-ubuntu-2204-arm + runs-on: depot-ubuntu-22.04-4 permissions: id-token: write # allow issuing OIDC tokens for this workflow run contents: read # allow reading the repo contents diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml index 2d9384529fb0a..e6f83f1b6b3fb 100644 --- a/.github/workflows/docker-hook-api.yml +++ b/.github/workflows/docker-hook-api.yml @@ -12,7 +12,7 @@ permissions: jobs: build: name: build and publish hook-api image - runs-on: buildjet-4vcpu-ubuntu-2204-arm + runs-on: depot-ubuntu-22.04-4 permissions: id-token: write # allow issuing OIDC tokens for this workflow run contents: read # allow reading the repo contents diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml index 31a4e2d86c5cf..33706d0987637 100644 --- a/.github/workflows/docker-hook-janitor.yml +++ b/.github/workflows/docker-hook-janitor.yml @@ -12,7 +12,7 @@ permissions: jobs: build: name: build and publish hook-janitor image - runs-on: buildjet-4vcpu-ubuntu-2204-arm + runs-on: depot-ubuntu-22.04-4 permissions: id-token: write # allow issuing OIDC tokens for this workflow run contents: read # allow reading the repo contents diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml index c3ba8ce888452..dc5ca53abef88 100644 --- a/.github/workflows/docker-hook-worker.yml +++ b/.github/workflows/docker-hook-worker.yml @@ -12,7 +12,7 @@ permissions: jobs: build: name: build and publish hook-worker image - runs-on: buildjet-4vcpu-ubuntu-2204-arm + runs-on: depot-ubuntu-22.04-4 permissions: id-token: write # allow issuing OIDC tokens for this workflow run contents: read # allow reading the repo contents diff --git a/.github/workflows/docker-migrator.yml b/.github/workflows/docker-migrator.yml index 479d6f9126255..906a022391b58 100644 --- a/.github/workflows/docker-migrator.yml +++ b/.github/workflows/docker-migrator.yml @@ -12,7 +12,7 @@ permissions: jobs: build: name: build and publish hook-migrator image - runs-on: buildjet-4vcpu-ubuntu-2204-arm + runs-on: depot-ubuntu-22.04-4 permissions: id-token: write # allow issuing OIDC tokens for this workflow run contents: read # allow reading the repo contents diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 72dfd0bc5093f..30f17341d5f08 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -12,7 +12,7 @@ env: jobs: build: - runs-on: buildjet-4vcpu-ubuntu-2204 + runs-on: depot-ubuntu-22.04-4 steps: - uses: actions/checkout@v3 @@ -34,7 +34,7 @@ jobs: run: cargo build --all --locked --release && find target/release/ -maxdepth 1 -executable -type f | xargs strip test: - runs-on: buildjet-4vcpu-ubuntu-2204 + runs-on: depot-ubuntu-22.04-4 timeout-minutes: 10 steps: @@ -72,7 +72,7 @@ jobs: run: cargo check --all-features clippy: - runs-on: buildjet-4vcpu-ubuntu-2204 + runs-on: depot-ubuntu-22.04-4 steps: - uses: actions/checkout@v3 @@ -95,7 +95,7 @@ jobs: run: cargo clippy -- -D warnings format: - runs-on: buildjet-4vcpu-ubuntu-2204 + runs-on: depot-ubuntu-22.04-4 steps: - uses: actions/checkout@v3 From 56d4615b3aa11ba4c7ad74c8c61ef786912e58c3 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 15 Apr 2024 16:20:52 +0200 Subject: [PATCH 220/247] cleanup and upgrade deps (#20) --- Cargo.lock | 545 ++++++++++++++++++--------------- Cargo.toml | 19 +- capture-server/Cargo.toml | 14 +- capture-server/src/main.rs | 5 +- capture-server/tests/common.rs | 17 +- capture-server/tests/events.rs | 10 +- capture/Cargo.toml | 9 +- capture/src/capture.rs | 3 +- capture/src/prometheus.rs | 3 +- capture/src/server.rs | 18 +- hook-api/Cargo.toml | 1 - hook-common/Cargo.toml | 2 - hook-janitor/Cargo.toml | 6 - hook-worker/Cargo.toml | 4 +- 14 files changed, 339 insertions(+), 317 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f611e201f067..71cbc8d51a5d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,28 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "async-trait" version = "0.1.77" @@ -140,11 +162,7 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", - "tokio", + "sync_wrapper 0.1.2", "tower", "tower-layer", "tower-service", @@ -152,15 +170,16 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1236b4b292f6c4d6dc34604bb5120d85c3fe1d1aa596bd5cc52ca054d13e7b9e" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", "axum-core 0.4.3", + "axum-macros", "bytes", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "http-body-util", "hyper 1.1.0", @@ -176,7 +195,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.1", "tokio", "tower", "tower-layer", @@ -186,11 +205,11 @@ dependencies = [ [[package]] name = "axum-client-ip" -version = "0.4.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ef117890a418b7832678d9ea1e1c08456dd7b2fd1dadb9676cd6f0fe7eb4b21" +checksum = "72188bed20deb981f3a4a9fe674e5980fd9e9c2bd880baa94715ad5d60d64c67" dependencies = [ - "axum 0.6.20", + "axum 0.7.5", "forwarded-header-value", "serde", ] @@ -221,30 +240,41 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower-layer", "tower-service", "tracing", ] [[package]] -name = "axum-test-helper" -version = "0.2.0" +name = "axum-macros" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d349b3174ceac58442ea1f768233c817e59447c0343be2584fca9f0ed71d3a" +checksum = "00c055ee2d014ae5981ce1016374e8213682aa14d9bf40e48ab48b5f3ef20eaa" dependencies = [ - "axum 0.6.20", + "heck", + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "axum-test-helper" +version = "0.4.0" +source = "git+https://github.com/orphan-rs/axum-test-helper.git#8ca0aedaad5a6bdf351c34d5b80593ae1b7d2f3f" +dependencies = [ + "axum 0.7.5", "bytes", - "http 0.2.11", - "http-body 0.4.6", - "hyper 0.14.28", - "reqwest", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.1.0", + "reqwest 0.11.24", "serde", "tokio", "tower", @@ -272,6 +302,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + [[package]] name = "base64ct" version = "1.6.0" @@ -327,22 +363,19 @@ dependencies = [ "anyhow", "assert-json-diff", "async-trait", - "axum 0.6.20", + "axum 0.7.5", "axum-client-ip", "axum-test-helper", - "base64", + "base64 0.22.0", "bytes", - "dashmap", "envconfig", "flate2", "governor", "metrics", "metrics-exporter-prometheus", - "mockall", "rand", "rdkafka", "redis", - "redis-test", "serde", "serde_json", "serde_urlencoded", @@ -350,9 +383,7 @@ dependencies = [ "time", "tokio", "tower-http", - "tower_governor", "tracing", - "tracing-subscriber", "uuid", ] @@ -362,7 +393,6 @@ version = "0.1.0" dependencies = [ "anyhow", "assert-json-diff", - "axum 0.7.4", "capture", "envconfig", "futures", @@ -372,9 +402,8 @@ dependencies = [ "opentelemetry_sdk", "rand", "rdkafka", - "reqwest", + "reqwest 0.12.3", "serde_json", - "time", "tokio", "tracing", "tracing-opentelemetry", @@ -571,12 +600,6 @@ dependencies = [ "serde", ] -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - [[package]] name = "digest" version = "0.10.7" @@ -595,12 +618,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "downcast" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" - [[package]] name = "either" version = "1.9.0" @@ -704,15 +721,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "float-cmp" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" -dependencies = [ - "num-traits", -] - [[package]] name = "flume" version = "0.11.0" @@ -764,12 +772,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "fragile" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" - [[package]] name = "futures" version = "0.3.30" @@ -957,7 +959,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 1.0.0", + "http 1.1.0", "indexmap 2.2.2", "slab", "tokio", @@ -1051,13 +1053,12 @@ dependencies = [ name = "hook-api" version = "0.1.0" dependencies = [ - "axum 0.7.4", + "axum 0.7.5", "envconfig", "eyre", "hook-common", "http-body-util", "metrics", - "metrics-exporter-prometheus", "serde", "serde_derive", "serde_json", @@ -1074,15 +1075,13 @@ name = "hook-common" version = "0.1.0" dependencies = [ "async-trait", - "axum 0.7.4", + "axum 0.7.5", "chrono", - "http 0.2.11", + "http 1.1.0", "metrics", "metrics-exporter-prometheus", - "regex", - "reqwest", + "reqwest 0.12.3", "serde", - "serde_derive", "serde_json", "sqlx", "thiserror", @@ -1097,42 +1096,34 @@ name = "hook-janitor" version = "0.1.0" dependencies = [ "async-trait", - "axum 0.7.4", + "axum 0.7.5", "envconfig", "eyre", "futures", "hook-common", - "http-body-util", "metrics", - "metrics-exporter-prometheus", "rdkafka", - "serde", - "serde_derive", "serde_json", "sqlx", "thiserror", "time", "tokio", - "tower", "tracing", "tracing-subscriber", - "url", ] [[package]] name = "hook-worker" version = "0.1.0" dependencies = [ - "axum 0.7.4", + "axum 0.7.5", "chrono", "envconfig", "futures", "hook-common", - "http 0.2.11", + "http 1.1.0", "metrics", - "reqwest", - "serde", - "serde_derive", + "reqwest 0.12.3", "sqlx", "thiserror", "time", @@ -1155,9 +1146,9 @@ dependencies = [ [[package]] name = "http" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -1182,7 +1173,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http 1.0.0", + "http 1.1.0", ] [[package]] @@ -1193,17 +1184,11 @@ checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" dependencies = [ "bytes", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "pin-project-lite", ] -[[package]] -name = "http-range-header" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" - [[package]] name = "httparse" version = "1.8.0" @@ -1250,13 +1235,28 @@ dependencies = [ "futures-channel", "futures-util", "h2 0.4.2", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "httparse", "httpdate", "itoa", "pin-project-lite", "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.11", + "hyper 0.14.28", + "rustls", + "tokio", + "tokio-rustls", ] [[package]] @@ -1273,15 +1273,18 @@ dependencies = [ [[package]] name = "hyper-tls" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", - "hyper 0.14.28", + "http-body-util", + "hyper 1.1.0", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", ] [[package]] @@ -1291,13 +1294,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ "bytes", + "futures-channel", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "hyper 1.1.0", "pin-project-lite", "socket2 0.5.5", "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -1365,15 +1372,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.1" @@ -1516,14 +1514,16 @@ dependencies = [ [[package]] name = "metrics-exporter-prometheus" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a4c4718a371ddfb7806378f23617876eea8b82e5ff1324516bcd283249d9ea" +checksum = "5d58e362dc7206e9456ddbcdbd53c71ba441020e62104703075a69151e38d85f" dependencies = [ - "base64", - "hyper 0.14.28", + "base64 0.22.0", + "http-body-util", + "hyper 1.1.0", "hyper-tls", - "indexmap 1.9.3", + "hyper-util", + "indexmap 2.2.2", "ipnet", "metrics", "metrics-util", @@ -1590,33 +1590,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "mockall" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" -dependencies = [ - "cfg-if", - "downcast", - "fragile", - "lazy_static", - "mockall_derive", - "predicates", - "predicates-tree", -] - -[[package]] -name = "mockall_derive" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "native-tls" version = "0.2.11" @@ -1674,12 +1647,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1836,13 +1803,12 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +checksum = "900d57987be3f2aeb70d385fff9b27fb74c5723cc9a52d904d4f9c807a0667bf" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.2.2", "js-sys", "once_cell", "pin-project-lite", @@ -1852,9 +1818,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930" +checksum = "1a016b8d9495c639af2145ac22387dcb88e44118e45320d9238fbf4e7889abcb" dependencies = [ "async-trait", "futures-core", @@ -1871,9 +1837,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1" +checksum = "3a8fddc9b68f5b80dae9d6f510b88e02396f006ad48cac349411fbecc80caae4" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1883,18 +1849,15 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" -dependencies = [ - "opentelemetry", -] +checksum = "f9ab5bd6c42fb9349dcf28af2ba9a0667f697f9bdcca045d39f2cec5543e2910" [[package]] name = "opentelemetry_sdk" -version = "0.21.2" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" +checksum = "9e90c7113be649e31e9a0f8b5ee24ed7a16923b322c3c5ab6367469c049d6b7e" dependencies = [ "async-trait", "crossbeam-channel", @@ -2048,36 +2011,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "predicates" -version = "2.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" -dependencies = [ - "difflib", - "float-cmp", - "itertools 0.10.5", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" - -[[package]] -name = "predicates-tree" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" -dependencies = [ - "predicates-core", - "termtree", -] - [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -2099,9 +2032,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.9" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" dependencies = [ "bytes", "prost-derive", @@ -2109,15 +2042,15 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.48", ] [[package]] @@ -2266,15 +2199,6 @@ dependencies = [ "url", ] -[[package]] -name = "redis-test" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba266ca48ae66978bf439fd2ac0d7a36a8635823754e2bc73afaf9d2fc25272" -dependencies = [ - "redis", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -2334,7 +2258,7 @@ version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -2343,24 +2267,24 @@ dependencies = [ "http 0.2.11", "http-body 0.4.6", "hyper 0.14.28", - "hyper-tls", + "hyper-rustls", "ipnet", "js-sys", "log", "mime", "mime_guess", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", - "tokio-native-tls", + "tokio-rustls", "tokio-util", "tower-service", "url", @@ -2368,7 +2292,65 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg", + "webpki-roots", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6cc1e89e689536eb5aeede61520e874df5a4707df811cd5da4aa5fbb2aae19" +dependencies = [ + "base64 0.22.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.2", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.1.0", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 2.1.2", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.52.0", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted", + "windows-sys 0.52.0", ] [[package]] @@ -2410,13 +2392,51 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", +] + +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.0", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", ] [[package]] @@ -2446,6 +2466,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -2650,7 +2680,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" dependencies = [ - "itertools 0.12.1", + "itertools", "nom", "unicode_categories", ] @@ -2758,7 +2788,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" dependencies = [ "atoi", - "base64", + "base64 0.21.7", "bitflags 2.4.2", "byteorder", "bytes", @@ -2802,7 +2832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" dependencies = [ "atoi", - "base64", + "base64 0.21.7", "bitflags 2.4.2", "byteorder", "chrono", @@ -2906,6 +2936,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + [[package]] name = "system-configuration" version = "0.5.1" @@ -2939,12 +2975,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "termtree" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" - [[package]] name = "thiserror" version = "1.0.56" @@ -3071,6 +3101,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -3115,16 +3155,15 @@ dependencies = [ [[package]] name = "tonic" -version = "0.9.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" dependencies = [ + "async-stream", "async-trait", "axum 0.6.20", - "base64", + "base64 0.21.7", "bytes", - "futures-core", - "futures-util", "h2 0.3.24", "http 0.2.11", "http-body 0.4.6", @@ -3163,17 +3202,15 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "bitflags 2.4.2", "bytes", - "futures-core", - "futures-util", - "http 0.2.11", - "http-body 0.4.6", - "http-range-header", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", "pin-project-lite", "tower-layer", "tower-service", @@ -3192,26 +3229,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" -[[package]] -name = "tower_governor" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6be418f6d18863291f0a7fa1da1de71495a19a54b5fb44969136f731a47e86" -dependencies = [ - "axum 0.6.20", - "forwarded-header-value", - "futures", - "futures-core", - "governor", - "http 0.2.11", - "pin-project", - "thiserror", - "tokio", - "tower", - "tower-layer", - "tracing", -] - [[package]] name = "tracing" version = "0.1.40" @@ -3258,9 +3275,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" +checksum = "a9be14ba1bbe4ab79e9229f7f89fab8d120b865859f10527f31c033e599d2284" dependencies = [ "js-sys", "once_cell", @@ -3346,6 +3363,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.0" @@ -3504,14 +3527,20 @@ dependencies = [ [[package]] name = "web-time" -version = "0.2.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + [[package]] name = "whoami" version = "1.4.1" @@ -3700,6 +3729,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "zerocopy" version = "0.7.32" diff --git a/Cargo.toml b/Cargo.toml index 3aeacb8bcf0a5..f77c1fbd85ad2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,16 +10,13 @@ members = [ "hook-janitor", ] -# [profile.release] -# debug = 2 # https://www.polarsignals.com/docs/rust - [workspace.dependencies] anyhow = "1.0" assert-json-diff = "2.0.2" async-trait = "0.1.74" -axum = { version = "0.7.1", features = ["http2"] } -axum-client-ip = "0.4.1" -base64 = "0.21.1" +axum = { version = "0.7.5", features = ["http2", "macros"] } +axum-client-ip = "0.6.0" +base64 = "0.22.0" bytes = "1" chrono = { version = "0.4" } envconfig = "0.10.0" @@ -27,14 +24,13 @@ eyre = "0.6.9" flate2 = "1.0" futures = { version = "0.3.29" } governor = { version = "0.5.1", features = ["dashmap"] } -http = { version = "0.2" } +http = { version = "1.1.0" } http-body-util = "0.1.0" metrics = "0.22.0" -metrics-exporter-prometheus = "0.13.0" +metrics-exporter-prometheus = "0.14.0" rand = "0.8.5" rdkafka = { version = "0.36.0", features = ["cmake-build", "ssl", "tracing"] } -regex = "1.10.2" -reqwest = { version = "0.11" } +reqwest = { version = "0.12.3" } serde = { version = "1.0", features = ["derive"] } serde_derive = { version = "1.0" } serde_json = { version = "1.0" } @@ -57,8 +53,7 @@ time = { version = "0.3.20", features = [ ] } tokio = { version = "1.34.0", features = ["full"] } tower = "0.4.13" -tower_governor = "0.0.4" -tower-http = { version = "0.4.0", features = ["cors", "trace"] } +tower-http = { version = "0.5.2", features = ["cors", "trace"] } tracing = "0.1.40" tracing-subscriber = "0.3.18" url = { version = "2.5.0 " } diff --git a/capture-server/Cargo.toml b/capture-server/Cargo.toml index e8aa5595486a6..ae06664ec96f9 100644 --- a/capture-server/Cargo.toml +++ b/capture-server/Cargo.toml @@ -4,16 +4,14 @@ version = "0.1.0" edition = "2021" [dependencies] -axum = { workspace = true } capture = { path = "../capture" } envconfig = { workspace = true } -opentelemetry = { version = "0.21.0", features = ["trace"]} -opentelemetry-otlp = "0.14.0" -opentelemetry_sdk = { version = "0.21.0", features = ["trace", "rt-tokio"] } -time = { workspace = true } +opentelemetry = { version = "0.22.0", features = ["trace"]} +opentelemetry-otlp = "0.15.0" +opentelemetry_sdk = { version = "0.22.1", features = ["trace", "rt-tokio"] } tokio = { workspace = true } tracing = { workspace = true } -tracing-opentelemetry = "0.22.0" +tracing-opentelemetry = "0.23.0" tracing-subscriber = { workspace = true, features = ["env-filter"] } [dev-dependencies] @@ -23,5 +21,5 @@ futures = "0.3.29" once_cell = "1.18.0" rand = { workspace = true } rdkafka = { workspace = true } -reqwest = "0.11.22" -serde_json = { workspace = true } \ No newline at end of file +reqwest = { workspace = true } +serde_json = { workspace = true } diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index 402fc3245d883..97967ed5f0ae7 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -1,4 +1,3 @@ -use std::net::TcpListener; use std::time::Duration; use envconfig::Envconfig; @@ -76,6 +75,8 @@ async fn main() { .init(); // Open the TCP port and start the server - let listener = TcpListener::bind(config.address).unwrap(); + let listener = tokio::net::TcpListener::bind(config.address) + .await + .expect("could not bind port"); serve(config, listener, shutdown()).await } diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index fa8688156e650..e33ef4c20f561 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] use std::default::Default; -use std::net::{SocketAddr, TcpListener}; +use std::net::SocketAddr; use std::num::NonZeroU32; use std::str::FromStr; use std::string::ToString; @@ -17,6 +17,7 @@ use rdkafka::config::{ClientConfig, FromClientConfig}; use rdkafka::consumer::{BaseConsumer, Consumer}; use rdkafka::util::Timeout; use rdkafka::{Message, TopicPartitionList}; +use tokio::net::TcpListener; use tokio::sync::Notify; use tokio::time::timeout; use tracing::{debug, warn}; @@ -59,20 +60,20 @@ pub struct ServerHandle { } impl ServerHandle { - pub fn for_topic(topic: &EphemeralTopic) -> Self { + pub async fn for_topic(topic: &EphemeralTopic) -> Self { let mut config = DEFAULT_CONFIG.clone(); config.kafka.kafka_topic = topic.topic_name().to_string(); - Self::for_config(config) + Self::for_config(config).await } - pub fn for_config(config: Config) -> Self { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + pub async fn for_config(config: Config) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let notify = Arc::new(Notify::new()); let shutdown = notify.clone(); - tokio::spawn( - async move { serve(config, listener, async { notify.notified().await }).await }, - ); + tokio::spawn(async move { + serve(config, listener, async move { notify.notified().await }).await + }); Self { addr, shutdown } } diff --git a/capture-server/tests/events.rs b/capture-server/tests/events.rs index 56fcdf79bf15d..8a4220b8e3920 100644 --- a/capture-server/tests/events.rs +++ b/capture-server/tests/events.rs @@ -14,7 +14,7 @@ async fn it_captures_one_event() -> Result<()> { let token = random_string("token", 16); let distinct_id = random_string("id", 16); let topic = EphemeralTopic::new().await; - let server = ServerHandle::for_topic(&topic); + let server = ServerHandle::for_topic(&topic).await; let event = json!({ "token": token, @@ -44,7 +44,7 @@ async fn it_captures_a_batch() -> Result<()> { let distinct_id2 = random_string("id", 16); let topic = EphemeralTopic::new().await; - let server = ServerHandle::for_topic(&topic); + let server = ServerHandle::for_topic(&topic).await; let event = json!([{ "token": token, @@ -90,7 +90,7 @@ async fn it_overflows_events_on_burst() -> Result<()> { config.overflow_burst_limit = NonZeroU32::new(2).unwrap(); config.overflow_per_second_limit = NonZeroU32::new(1).unwrap(); - let server = ServerHandle::for_config(config); + let server = ServerHandle::for_config(config).await; let event = json!([{ "token": token, @@ -139,7 +139,7 @@ async fn it_does_not_overflow_team_with_different_ids() -> Result<()> { config.overflow_burst_limit = NonZeroU32::new(1).unwrap(); config.overflow_per_second_limit = NonZeroU32::new(1).unwrap(); - let server = ServerHandle::for_config(config); + let server = ServerHandle::for_config(config).await; let event = json!([{ "token": token, @@ -176,7 +176,7 @@ async fn it_trims_distinct_id() -> Result<()> { let (trimmed_distinct_id2, _) = distinct_id2.split_at(200); // works because ascii chars let topic = EphemeralTopic::new().await; - let server = ServerHandle::for_topic(&topic); + let server = ServerHandle::for_topic(&topic).await; let event = json!([{ "token": token, diff --git a/capture/Cargo.toml b/capture/Cargo.toml index bd8f79f57ff72..9ec0f97fb3ddb 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -6,15 +6,13 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axum = { version = "0.6.15" } # TODO: Bring up to date with the workspace. +axum = { workspace = true } axum-client-ip = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } governor = { workspace = true } -tower_governor = { workspace = true } time = { workspace = true } tower-http = { workspace = true } bytes = { workspace = true } @@ -35,10 +33,7 @@ redis = { version = "0.23.3", features = [ "cluster-async", ] } envconfig = { workspace = true } -dashmap = "5.5.3" [dev-dependencies] assert-json-diff = { workspace = true } -axum-test-helper = "0.2.0" -mockall = "0.11.2" -redis-test = "0.2.3" +axum-test-helper = { git = "https://github.com/orphan-rs/axum-test-helper.git" } # TODO: remove, directly use reqwest like capture-server tests diff --git a/capture/src/capture.rs b/capture/src/capture.rs index 7f90d575fcf1c..622bc750f90c7 100644 --- a/capture/src/capture.rs +++ b/capture/src/capture.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use bytes::Bytes; -use axum::Json; +use axum::{debug_handler, Json}; // TODO: stream this instead use axum::extract::{Query, State}; use axum::http::{HeaderMap, Method}; @@ -38,6 +38,7 @@ use crate::{ compression ) )] +#[debug_handler] pub async fn event( state: State, InsecureClientIp(ip): InsecureClientIp, diff --git a/capture/src/prometheus.rs b/capture/src/prometheus.rs index 6f5dc12af28e4..b4e19974ab51f 100644 --- a/capture/src/prometheus.rs +++ b/capture/src/prometheus.rs @@ -2,6 +2,7 @@ use std::time::Instant; +use axum::body::Body; use axum::{extract::MatchedPath, http::Request, middleware::Next, response::IntoResponse}; use metrics::counter; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; @@ -38,7 +39,7 @@ pub fn setup_metrics_recorder() -> PrometheusHandle { /// Middleware to record some common HTTP metrics /// Generic over B to allow for arbitrary body types (eg Vec, Streams, a deserialized thing, etc) /// Someday tower-http might provide a metrics middleware: https://github.com/tower-rs/tower-http/issues/57 -pub async fn track_metrics(req: Request, next: Next) -> impl IntoResponse { +pub async fn track_metrics(req: Request, next: Next) -> impl IntoResponse { let start = Instant::now(); let path = if let Some(matched_path) = req.extensions().get::() { diff --git a/capture/src/server.rs b/capture/src/server.rs index 22a1f3bc0bf04..2fc88c60687e6 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -1,8 +1,9 @@ use std::future::Future; -use std::net::{SocketAddr, TcpListener}; +use std::net::SocketAddr; use std::sync::Arc; use time::Duration; +use tokio::net::TcpListener; use crate::config::Config; use crate::health::{ComponentStatus, HealthRegistry}; @@ -15,7 +16,7 @@ use crate::sinks::print::PrintSink; pub async fn serve(config: Config, listener: TcpListener, shutdown: F) where - F: Future, + F: Future + Send + 'static, { let liveness = HealthRegistry::new("liveness"); @@ -80,10 +81,11 @@ where // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` tracing::info!("listening on {:?}", listener.local_addr().unwrap()); - axum::Server::from_tcp(listener) - .unwrap() - .serve(app.into_make_service_with_connect_info::()) - .with_graceful_shutdown(shutdown) - .await - .unwrap() + axum::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(shutdown) + .await + .unwrap() } diff --git a/hook-api/Cargo.toml b/hook-api/Cargo.toml index 96c897cd3ab1d..a596e87076b18 100644 --- a/hook-api/Cargo.toml +++ b/hook-api/Cargo.toml @@ -12,7 +12,6 @@ eyre = { workspace = true } hook-common = { path = "../hook-common" } http-body-util = { workspace = true } metrics = { workspace = true } -metrics-exporter-prometheus = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index ea7ce2fbb9cac..8ccf8dd5b2ebb 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -13,9 +13,7 @@ http = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } reqwest = { workspace = true } -regex = { workspace = true } serde = { workspace = true } -serde_derive = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } time = { workspace = true } diff --git a/hook-janitor/Cargo.toml b/hook-janitor/Cargo.toml index 96a80ebd9c3d5..a29a80c1c5256 100644 --- a/hook-janitor/Cargo.toml +++ b/hook-janitor/Cargo.toml @@ -12,18 +12,12 @@ envconfig = { workspace = true } eyre = { workspace = true } futures = { workspace = true } hook-common = { path = "../hook-common" } -http-body-util = { workspace = true } metrics = { workspace = true } -metrics-exporter-prometheus = { workspace = true } rdkafka = { workspace = true } -serde = { workspace = true } -serde_derive = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } time = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } -tower = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } -url = { workspace = true } diff --git a/hook-worker/Cargo.toml b/hook-worker/Cargo.toml index 6ed5796efd1f0..5d6874a8af4cd 100644 --- a/hook-worker/Cargo.toml +++ b/hook-worker/Cargo.toml @@ -9,11 +9,9 @@ chrono = { workspace = true } envconfig = { workspace = true } futures = "0.3" hook-common = { path = "../hook-common" } -http = { version = "0.2" } +http = { workspace = true } metrics = { workspace = true } reqwest = { workspace = true } -serde = { workspace = true } -serde_derive = { workspace = true } sqlx = { workspace = true } time = { workspace = true } thiserror = { workspace = true } From 591d765691c28861c6adfccc8606fa61afca1c7d Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 22 Apr 2024 15:38:45 +0200 Subject: [PATCH 221/247] capture: make otel service name configurable, will use deploy name (#22) --- capture-server/src/main.rs | 14 ++++++++++---- capture-server/tests/common.rs | 1 + capture/src/config.rs | 3 +++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/capture-server/src/main.rs b/capture-server/src/main.rs index 97967ed5f0ae7..12b91941c7f6c 100644 --- a/capture-server/src/main.rs +++ b/capture-server/src/main.rs @@ -1,7 +1,7 @@ use std::time::Duration; use envconfig::Envconfig; -use opentelemetry::KeyValue; +use opentelemetry::{KeyValue, Value}; use opentelemetry_otlp::WithExportConfig; use opentelemetry_sdk::trace::{BatchConfig, RandomIdGenerator, Sampler, Tracer}; use opentelemetry_sdk::{runtime, Resource}; @@ -31,7 +31,7 @@ async fn shutdown() { tracing::info!("Shutting down gracefully..."); } -fn init_tracer(sink_url: &str, sampling_rate: f64) -> Tracer { +fn init_tracer(sink_url: &str, sampling_rate: f64, service_name: &str) -> Tracer { opentelemetry_otlp::new_pipeline() .tracing() .with_trace_config( @@ -42,7 +42,7 @@ fn init_tracer(sink_url: &str, sampling_rate: f64) -> Tracer { .with_id_generator(RandomIdGenerator::default()) .with_resource(Resource::new(vec![KeyValue::new( "service.name", - "capture", + Value::from(service_name.to_string()), )])), ) .with_batch_config(BatchConfig::default()) @@ -67,7 +67,13 @@ async fn main() { let otel_layer = config .otel_url .clone() - .map(|url| OpenTelemetryLayer::new(init_tracer(&url, config.otel_sampling_rate))) + .map(|url| { + OpenTelemetryLayer::new(init_tracer( + &url, + config.otel_sampling_rate, + &config.otel_service_name, + )) + }) .with_filter(LevelFilter::from_level(Level::INFO)); tracing_subscriber::registry() .with(log_layer) diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index e33ef4c20f561..5ee2caa5ca7a9 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -43,6 +43,7 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { }, otel_url: None, otel_sampling_rate: 0.0, + otel_service_name: "capture-testing".to_string(), export_prometheus: false, }); diff --git a/capture/src/config.rs b/capture/src/config.rs index 0c6ab1ce9eb62..a4bd8f2cfd5cc 100644 --- a/capture/src/config.rs +++ b/capture/src/config.rs @@ -27,6 +27,9 @@ pub struct Config { #[envconfig(default = "1.0")] pub otel_sampling_rate: f64, + #[envconfig(default = "capture")] + pub otel_service_name: String, + #[envconfig(default = "true")] pub export_prometheus: bool, } From 4db3670622086546ffc9b37fcf1bca25490cb0c9 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 22 Apr 2024 17:32:53 +0200 Subject: [PATCH 222/247] capture: add support for the /batch request shape (#21) --- Cargo.toml | 2 +- capture/src/api.rs | 32 ++-- capture/src/lib.rs | 4 +- capture/src/router.rs | 40 +++- capture/src/sinks/kafka.rs | 6 +- capture/src/sinks/mod.rs | 3 +- capture/src/sinks/print.rs | 3 +- capture/src/{capture.rs => v0_endpoint.rs} | 160 ++++------------ capture/src/{event.rs => v0_request.rs} | 210 +++++++++++++++++---- capture/tests/django_compat.rs | 19 +- capture/tests/requests_dump.jsonl | 2 + 11 files changed, 276 insertions(+), 205 deletions(-) rename capture/src/{capture.rs => v0_endpoint.rs} (57%) rename capture/src/{event.rs => v0_request.rs} (60%) diff --git a/Cargo.toml b/Cargo.toml index f77c1fbd85ad2..ef70e645d4e13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ anyhow = "1.0" assert-json-diff = "2.0.2" async-trait = "0.1.74" -axum = { version = "0.7.5", features = ["http2", "macros"] } +axum = { version = "0.7.5", features = ["http2", "macros", "matched-path"] } axum-client-ip = "0.6.0" base64 = "0.22.0" bytes = "1" diff --git a/capture/src/api.rs b/capture/src/api.rs index b27b1a9630c45..0938ced399773 100644 --- a/capture/src/api.rs +++ b/capture/src/api.rs @@ -1,19 +1,11 @@ -use crate::token::InvalidTokenReason; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::collections::HashMap; use thiserror::Error; +use time::OffsetDateTime; +use uuid::Uuid; -#[derive(Debug, Deserialize, Serialize)] -pub struct CaptureRequest { - #[serde(alias = "$token", alias = "api_key")] - pub token: String, - - pub event: String, - pub properties: HashMap, -} +use crate::token::InvalidTokenReason; #[derive(Debug, PartialEq, Eq, Deserialize, Serialize)] pub enum CaptureResponseCode { @@ -84,3 +76,21 @@ impl IntoResponse for CaptureError { .into_response() } } + +#[derive(Clone, Default, Debug, Serialize, Eq, PartialEq)] +pub struct ProcessedEvent { + pub uuid: Uuid, + pub distinct_id: String, + pub ip: String, + pub data: String, + pub now: String, + #[serde(with = "time::serde::rfc3339::option")] + pub sent_at: Option, + pub token: String, +} + +impl ProcessedEvent { + pub fn key(&self) -> String { + format!("{}:{}", self.token, self.distinct_id) + } +} diff --git a/capture/src/lib.rs b/capture/src/lib.rs index 058e994186edb..176fc6f09963e 100644 --- a/capture/src/lib.rs +++ b/capture/src/lib.rs @@ -1,7 +1,5 @@ pub mod api; -pub mod capture; pub mod config; -pub mod event; pub mod health; pub mod limiters; pub mod prometheus; @@ -12,3 +10,5 @@ pub mod sinks; pub mod time; pub mod token; pub mod utils; +pub mod v0_endpoint; +pub mod v0_request; diff --git a/capture/src/router.rs b/capture/src/router.rs index d02e63faaad5d..85475ceb2a990 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -10,7 +10,9 @@ use tower_http::cors::{AllowHeaders, AllowOrigin, CorsLayer}; use tower_http::trace::TraceLayer; use crate::health::HealthRegistry; -use crate::{capture, limiters::billing::BillingLimiter, redis::Client, sinks, time::TimeSource}; +use crate::{ + limiters::billing::BillingLimiter, redis::Client, sinks, time::TimeSource, v0_endpoint, +}; use crate::prometheus::{setup_metrics_recorder, track_metrics}; @@ -58,17 +60,41 @@ pub fn router< .route("/", get(index)) .route("/_readiness", get(index)) .route("/_liveness", get(move || ready(liveness.get_status()))) + .route( + "/e", + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), + ) + .route( + "/e/", + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), + ) + .route( + "/batch", + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), + ) + .route( + "/batch/", + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), + ) .route( "/i/v0/e", - post(capture::event) - .get(capture::event) - .options(capture::options), + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), ) .route( "/i/v0/e/", - post(capture::event) - .get(capture::event) - .options(capture::options), + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), ) .layer(TraceLayer::new_for_http()) .layer(cors) diff --git a/capture/src/sinks/kafka.rs b/capture/src/sinks/kafka.rs index 4a2bd94f7e30f..4a48b6e4d5335 100644 --- a/capture/src/sinks/kafka.rs +++ b/capture/src/sinks/kafka.rs @@ -10,9 +10,8 @@ use tokio::task::JoinSet; use tracing::log::{debug, error, info}; use tracing::{info_span, instrument, Instrument}; -use crate::api::CaptureError; +use crate::api::{CaptureError, ProcessedEvent}; use crate::config::KafkaConfig; -use crate::event::ProcessedEvent; use crate::health::HealthHandle; use crate::limiters::overflow::OverflowLimiter; use crate::prometheus::report_dropped_events; @@ -259,9 +258,8 @@ impl Event for KafkaSink { #[cfg(test)] mod tests { - use crate::api::CaptureError; + use crate::api::{CaptureError, ProcessedEvent}; use crate::config; - use crate::event::ProcessedEvent; use crate::health::HealthRegistry; use crate::limiters::overflow::OverflowLimiter; use crate::sinks::kafka::KafkaSink; diff --git a/capture/src/sinks/mod.rs b/capture/src/sinks/mod.rs index 0747f0e222a79..bedbcbc8df69d 100644 --- a/capture/src/sinks/mod.rs +++ b/capture/src/sinks/mod.rs @@ -1,7 +1,6 @@ use async_trait::async_trait; -use crate::api::CaptureError; -use crate::event::ProcessedEvent; +use crate::api::{CaptureError, ProcessedEvent}; pub mod kafka; pub mod print; diff --git a/capture/src/sinks/print.rs b/capture/src/sinks/print.rs index 5e71899ce68c5..7845a3d039b56 100644 --- a/capture/src/sinks/print.rs +++ b/capture/src/sinks/print.rs @@ -2,8 +2,7 @@ use async_trait::async_trait; use metrics::{counter, histogram}; use tracing::log::info; -use crate::api::CaptureError; -use crate::event::ProcessedEvent; +use crate::api::{CaptureError, ProcessedEvent}; use crate::sinks::Event; pub struct PrintSink {} diff --git a/capture/src/capture.rs b/capture/src/v0_endpoint.rs similarity index 57% rename from capture/src/capture.rs rename to capture/src/v0_endpoint.rs index 622bc750f90c7..3862995f2f4f8 100644 --- a/capture/src/capture.rs +++ b/capture/src/v0_endpoint.rs @@ -1,41 +1,45 @@ -use std::collections::HashSet; use std::ops::Deref; use std::sync::Arc; -use bytes::Bytes; - use axum::{debug_handler, Json}; +use bytes::Bytes; // TODO: stream this instead -use axum::extract::{Query, State}; +use axum::extract::{MatchedPath, Query, State}; use axum::http::{HeaderMap, Method}; use axum_client_ip::InsecureClientIp; use base64::Engine; use metrics::counter; - -use time::OffsetDateTime; use tracing::instrument; -use crate::event::{Compression, ProcessingContext}; use crate::limiters::billing::QuotaResource; use crate::prometheus::report_dropped_events; -use crate::token::validate_token; +use crate::v0_request::{Compression, ProcessingContext, RawRequest}; use crate::{ - api::{CaptureError, CaptureResponse, CaptureResponseCode}, - event::{EventFormData, EventQuery, ProcessedEvent, RawEvent}, + api::{CaptureError, CaptureResponse, CaptureResponseCode, ProcessedEvent}, router, sinks, utils::uuid_v7, + v0_request::{EventFormData, EventQuery, RawEvent}, }; +/// Flexible endpoint that targets wide compatibility with the wide range of requests +/// currently processed by posthog-events (analytics events capture). Replay is out +/// of scope and should be processed on a separate endpoint. +/// +/// Because it must accommodate several shapes, it is inefficient in places. A v1 +/// endpoint should be created, that only accepts the BatchedRequest payload shape. + #[instrument( skip_all, fields( + path, token, batch_size, user_agent, content_encoding, content_type, version, - compression + compression, + is_historical ) )] #[debug_handler] @@ -45,11 +49,9 @@ pub async fn event( meta: Query, headers: HeaderMap, method: Method, + path: MatchedPath, body: Bytes, ) -> Result, CaptureError> { - // content-type - // user-agent - let user_agent = headers .get("user-agent") .map_or("unknown", |v| v.to_str().unwrap_or("unknown")); @@ -68,8 +70,9 @@ pub async fn event( tracing::Span::current().record("version", meta.lib_version.clone()); tracing::Span::current().record("compression", comp.as_str()); tracing::Span::current().record("method", method.as_str()); + tracing::Span::current().record("path", path.as_str().trim_end_matches('/')); - let events = match headers + let request = match headers .get("content-type") .map_or("", |v| v.to_str().unwrap_or("")) { @@ -86,41 +89,36 @@ pub async fn event( tracing::error!("failed to decode form data: {}", e); CaptureError::RequestDecodingError(String::from("missing data field")) })?; - RawEvent::from_bytes(payload.into()) + RawRequest::from_bytes(payload.into()) } ct => { tracing::Span::current().record("content_type", ct); - RawEvent::from_bytes(body) + RawRequest::from_bytes(body) } }?; + let sent_at = request.sent_at().or(meta.sent_at()); + let token = match request.extract_and_verify_token() { + Ok(token) => token, + Err(err) => { + report_dropped_events("token_shape_invalid", request.events().len() as u64); + return Err(err); + } + }; + let is_historical = request.is_historical(); // TODO: use to write to historical topic + let events = request.events(); // Takes ownership of request + + tracing::Span::current().record("token", &token); + tracing::Span::current().record("is_historical", is_historical); tracing::Span::current().record("batch_size", events.len()); if events.is_empty() { return Err(CaptureError::EmptyBatch); } - let token = extract_and_verify_token(&events).map_err(|err| { - report_dropped_events("token_shape_invalid", events.len() as u64); - err - })?; - - tracing::Span::current().record("token", &token); - counter!("capture_events_received_total").increment(events.len() as u64); - let sent_at = meta.sent_at.and_then(|value| { - let value_nanos: i128 = i128::from(value) * 1_000_000; // Assuming the value is in milliseconds, latest posthog-js releases - if let Ok(sent_at) = OffsetDateTime::from_unix_timestamp_nanos(value_nanos) { - if sent_at.year() > 2020 { - // Could be lower if the input is in seconds - return Some(sent_at); - } - } - None - }); - let context = ProcessingContext { lib_version: meta.lib_version.clone(), sent_at, @@ -192,28 +190,6 @@ pub fn process_single_event( }) } -#[instrument(skip_all, fields(events = events.len()))] -pub fn extract_and_verify_token(events: &[RawEvent]) -> Result { - let distinct_tokens: HashSet> = HashSet::from_iter( - events - .iter() - .map(RawEvent::extract_token) - .filter(Option::is_some), - ); - - return match distinct_tokens.len() { - 0 => Err(CaptureError::NoTokenError), - 1 => match distinct_tokens.iter().last() { - Some(Some(token)) => { - validate_token(token)?; - Ok(token.clone()) - } - _ => Err(CaptureError::NoTokenError), - }, - _ => Err(CaptureError::MultipleTokensError), - }; -} - #[instrument(skip_all, fields(events = events.len()))] pub async fn process_events<'a>( sink: Arc, @@ -233,73 +209,3 @@ pub async fn process_events<'a>( sink.send_batch(events).await } } - -#[cfg(test)] -mod tests { - use crate::capture::extract_and_verify_token; - use crate::event::RawEvent; - use serde_json::json; - use std::collections::HashMap; - - #[tokio::test] - async fn all_events_have_same_token() { - let events = vec![ - RawEvent { - token: Some(String::from("hello")), - distinct_id: Some(json!("testing")), - uuid: None, - event: String::new(), - properties: HashMap::new(), - timestamp: None, - offset: None, - set: Default::default(), - set_once: Default::default(), - }, - RawEvent { - token: None, - distinct_id: Some(json!("testing")), - uuid: None, - event: String::new(), - properties: HashMap::from([(String::from("token"), json!("hello"))]), - timestamp: None, - offset: None, - set: Default::default(), - set_once: Default::default(), - }, - ]; - - let processed = extract_and_verify_token(&events); - assert_eq!(processed.is_ok(), true, "{:?}", processed); - } - - #[tokio::test] - async fn all_events_have_different_token() { - let events = vec![ - RawEvent { - token: Some(String::from("hello")), - distinct_id: Some(json!("testing")), - uuid: None, - event: String::new(), - properties: HashMap::new(), - timestamp: None, - offset: None, - set: Default::default(), - set_once: Default::default(), - }, - RawEvent { - token: None, - distinct_id: Some(json!("testing")), - uuid: None, - event: String::new(), - properties: HashMap::from([(String::from("token"), json!("goodbye"))]), - timestamp: None, - offset: None, - set: Default::default(), - set_once: Default::default(), - }, - ]; - - let processed = extract_and_verify_token(&events); - assert_eq!(processed.is_err(), true); - } -} diff --git a/capture/src/event.rs b/capture/src/v0_request.rs similarity index 60% rename from capture/src/event.rs rename to capture/src/v0_request.rs index ea71a3f276704..3d0052e0c072c 100644 --- a/capture/src/event.rs +++ b/capture/src/v0_request.rs @@ -1,15 +1,17 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io::prelude::*; use bytes::{Buf, Bytes}; use flate2::read::GzDecoder; use serde::{Deserialize, Serialize}; use serde_json::Value; +use time::format_description::well_known::Iso8601; use time::OffsetDateTime; use tracing::instrument; use uuid::Uuid; use crate::api::CaptureError; +use crate::token::validate_token; #[derive(Deserialize, Default)] pub enum Compression { @@ -28,7 +30,25 @@ pub struct EventQuery { pub lib_version: Option, #[serde(alias = "_")] - pub sent_at: Option, + sent_at: Option, +} + +impl EventQuery { + /// Returns the parsed value of the sent_at timestamp if present in the query params. + /// We only support the format sent by recent posthog-js versions, in milliseconds integer. + /// Values in seconds integer (older SDKs will be ignored). + pub fn sent_at(&self) -> Option { + if let Some(value) = self.sent_at { + let value_nanos: i128 = i128::from(value) * 1_000_000; // Assuming the value is in milliseconds, latest posthog-js releases + if let Ok(sent_at) = OffsetDateTime::from_unix_timestamp_nanos(value_nanos) { + if sent_at.year() > 2020 { + // Could be lower if the input is in seconds + return Some(sent_at); + } + } + } + None + } } #[derive(Debug, Deserialize)] @@ -64,30 +84,32 @@ static GZIP_MAGIC_NUMBERS: [u8; 3] = [0x1f, 0x8b, 8]; #[derive(Deserialize)] #[serde(untagged)] -enum RawRequest { - /// Batch of events - Batch(Vec), - /// Single event +pub enum RawRequest { + /// Array of events (posthog-js) + Array(Vec), + /// Batched events (/batch) + Batch(BatchedRequest), + /// Single event (/capture) One(Box), } -impl RawRequest { - pub fn events(self) -> Vec { - match self { - RawRequest::Batch(events) => events, - RawRequest::One(event) => vec![*event], - } - } +#[derive(Deserialize)] +pub struct BatchedRequest { + #[serde(alias = "api_key")] + pub token: String, + pub historical_migration: Option, + pub sent_at: Option, + pub batch: Vec, } -impl RawEvent { - /// Takes a request payload and tries to decompress and unmarshall it into events. +impl RawRequest { + /// Takes a request payload and tries to decompress and unmarshall it. /// While posthog-js sends a compression query param, a sizable portion of requests /// fail due to it being missing when the body is compressed. /// Instead of trusting the parameter, we peek at the payload's first three bytes to /// detect gzip, fallback to uncompressed utf8 otherwise. #[instrument(skip_all)] - pub fn from_bytes(bytes: Bytes) -> Result, CaptureError> { + pub fn from_bytes(bytes: Bytes) -> Result { tracing::debug!(len = bytes.len(), "decoding new event"); let payload = if bytes.starts_with(&GZIP_MAGIC_NUMBERS) { @@ -106,9 +128,66 @@ impl RawEvent { }; tracing::debug!(json = payload, "decoded event data"); - Ok(serde_json::from_str::(&payload)?.events()) + Ok(serde_json::from_str::(&payload)?) } + pub fn events(self) -> Vec { + match self { + RawRequest::Array(events) => events, + RawRequest::One(event) => vec![*event], + RawRequest::Batch(req) => req.batch, + } + } + + pub fn extract_and_verify_token(&self) -> Result { + let token = match self { + RawRequest::Batch(req) => req.token.to_string(), + RawRequest::One(event) => event.extract_token().ok_or(CaptureError::NoTokenError)?, + RawRequest::Array(events) => extract_token(events)?, + }; + validate_token(&token)?; + Ok(token) + } + + pub fn is_historical(&self) -> bool { + match self { + RawRequest::Batch(req) => req.historical_migration.unwrap_or_default(), + _ => false, + } + } + + pub fn sent_at(&self) -> Option { + if let RawRequest::Batch(req) = &self { + if let Some(value) = &req.sent_at { + if let Ok(parsed) = OffsetDateTime::parse(value, &Iso8601::DEFAULT) { + return Some(parsed); + } + } + } + None + } +} + +#[instrument(skip_all, fields(events = events.len()))] +pub fn extract_token(events: &[RawEvent]) -> Result { + let distinct_tokens: HashSet> = HashSet::from_iter( + events + .iter() + .map(RawEvent::extract_token) + .filter(Option::is_some), + ); + + return match distinct_tokens.len() { + 0 => Err(CaptureError::NoTokenError), + 1 => match distinct_tokens.iter().last() { + Some(Some(token)) => Ok(token.clone()), + _ => Err(CaptureError::NoTokenError), + }, + _ => Err(CaptureError::MultipleTokensError), + }; +} + +impl RawEvent { pub fn extract_token(&self) -> Option { match &self.token { Some(value) => Some(value.clone()), @@ -154,26 +233,9 @@ pub struct ProcessingContext { pub client_ip: String, } -#[derive(Clone, Default, Debug, Serialize, Eq, PartialEq)] -pub struct ProcessedEvent { - pub uuid: Uuid, - pub distinct_id: String, - pub ip: String, - pub data: String, - pub now: String, - #[serde(with = "time::serde::rfc3339::option")] - pub sent_at: Option, - pub token: String, -} - -impl ProcessedEvent { - pub fn key(&self) -> String { - format!("{}:{}", self.token, self.distinct_id) - } -} - #[cfg(test)] mod tests { + use crate::token::InvalidTokenReason; use base64::Engine as _; use bytes::Bytes; use rand::distributions::Alphanumeric; @@ -181,7 +243,7 @@ mod tests { use serde_json::json; use super::CaptureError; - use super::RawEvent; + use super::RawRequest; #[test] fn decode_uncompressed_raw_event() { @@ -192,7 +254,9 @@ mod tests { .expect("payload is not base64"), ); - let events = RawEvent::from_bytes(compressed_bytes).expect("failed to parse"); + let events = RawRequest::from_bytes(compressed_bytes) + .expect("failed to parse") + .events(); assert_eq!(1, events.len()); assert_eq!(Some("my_token1".to_string()), events[0].extract_token()); assert_eq!("my_event1".to_string(), events[0].event); @@ -212,7 +276,9 @@ mod tests { .expect("payload is not base64"), ); - let events = RawEvent::from_bytes(compressed_bytes).expect("failed to parse"); + let events = RawRequest::from_bytes(compressed_bytes) + .expect("failed to parse") + .events(); assert_eq!(1, events.len()); assert_eq!(Some("my_token2".to_string()), events[0].extract_token()); assert_eq!("my_event2".to_string(), events[0].event); @@ -227,7 +293,9 @@ mod tests { #[test] fn extract_distinct_id() { let parse_and_extract = |input: &'static str| -> Result { - let parsed = RawEvent::from_bytes(input.into()).expect("failed to parse"); + let parsed = RawRequest::from_bytes(input.into()) + .expect("failed to parse") + .events(); parsed[0].extract_distinct_id() }; // Return MissingDistinctId if not found @@ -288,10 +356,72 @@ mod tests { "distinct_id": distinct_id }]); - let parsed = RawEvent::from_bytes(input.to_string().into()).expect("failed to parse"); + let parsed = RawRequest::from_bytes(input.to_string().into()) + .expect("failed to parse") + .events(); assert_eq!( parsed[0].extract_distinct_id().expect("failed to extract"), expected_distinct_id ); } + + #[test] + fn extract_and_verify_token() { + let parse_and_extract = |input: &'static str| -> Result { + RawRequest::from_bytes(input.into()) + .expect("failed to parse") + .extract_and_verify_token() + }; + + let assert_extracted_token = |input: &'static str, expected: &str| { + let id = parse_and_extract(input).expect("failed to extract"); + assert_eq!(id, expected); + }; + + // Return NoTokenError if not found + assert!(matches!( + parse_and_extract(r#"{"event": "e"}"#), + Err(CaptureError::NoTokenError) + )); + + // Return TokenValidationError if token empty + assert!(matches!( + parse_and_extract(r#"{"api_key": "", "batch":[{"event": "e"}]}"#), + Err(CaptureError::TokenValidationError( + InvalidTokenReason::Empty + )) + )); + + // Return TokenValidationError if personal apikey + assert!(matches!( + parse_and_extract(r#"[{"event": "e", "token": "phx_hellothere"}]"#), + Err(CaptureError::TokenValidationError( + InvalidTokenReason::PersonalApiKey + )) + )); + + // Return MultipleTokensError if tokens don't match in array + assert!(matches!( + parse_and_extract( + r#"[{"event": "e", "token": "token1"},{"event": "e", "token": "token2"}]"# + ), + Err(CaptureError::MultipleTokensError) + )); + + // Return token from array if consistent + assert_extracted_token( + r#"[{"event":"e","token":"token1"},{"event":"e","token":"token1"}]"#, + "token1", + ); + + // Return token from batch if present + assert_extracted_token( + r#"{"batch":[{"event":"e","token":"token1"}],"api_key":"batched"}"#, + "batched", + ); + + // Return token from single event if present + assert_extracted_token(r#"{"event":"e","$token":"single_token"}"#, "single_token"); + assert_extracted_token(r#"{"event":"e","api_key":"single_token"}"#, "single_token"); + } } diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index 5d778997a89a9..c7ec0ad8d2770 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -4,8 +4,7 @@ use axum::http::StatusCode; use axum_test_helper::TestClient; use base64::engine::general_purpose; use base64::Engine; -use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode}; -use capture::event::ProcessedEvent; +use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode, ProcessedEvent}; use capture::health::HealthRegistry; use capture::limiters::billing::BillingLimiter; use capture::redis::MockRedisClient; @@ -88,11 +87,6 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { continue; } let case: RequestDump = serde_json::from_str(&line_contents)?; - if !case.path.starts_with("/e/") { - println!("Skipping {} test case", &case.path); - continue; - } - let raw_body = general_purpose::STANDARD.decode(&case.body)?; assert_eq!( case.method, "POST", @@ -117,7 +111,7 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { ); let client = TestClient::new(app); - let mut req = client.post(&format!("/i/v0{}", case.path)).body(raw_body); + let mut req = client.post(&case.path).body(raw_body); if !case.content_encoding.is_empty() { req = req.header("Content-encoding", case.content_encoding); } @@ -164,8 +158,15 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { if let Some(expected_data) = expected.get_mut("data") { // Data is a serialized JSON map. Unmarshall both and compare them, // instead of expecting the serialized bytes to be equal - let expected_props: Value = + let mut expected_props: Value = serde_json::from_str(expected_data.as_str().expect("not str"))?; + if let Some(object) = expected_props.as_object_mut() { + // toplevel fields added by posthog-node that plugin-server will ignore anyway + object.remove("type"); + object.remove("library"); + object.remove("library_version"); + } + let found_props: Value = serde_json::from_str(&message.data)?; let match_config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict); diff --git a/capture/tests/requests_dump.jsonl b/capture/tests/requests_dump.jsonl index ec0f4df482afb..36cf8ade36439 100644 --- a/capture/tests/requests_dump.jsonl +++ b/capture/tests/requests_dump.jsonl @@ -13,3 +13,5 @@ ### Compression query param mismatch, to confirm gzip autodetection {"path":"/e/?compression=gzip-js&ip=1&_=1694769302319&ver=1.78.5","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-09-15T09:15:02.321230+00:00","body":"ZGF0YT1leUoxZFdsa0lqb2lNREU0WVRrNE1XWXROR0l5WlMwM1ltUXpMV0ppTXpBdE5qWXhOalkxTjJNek9HWmhJaXdpWlhabGJuUWlPaUlrYjNCMFgybHVJaXdpY0hKdmNHVnlkR2xsY3lJNmV5SWtiM01pT2lKTllXTWdUMU1nV0NJc0lpUnZjMTkyWlhKemFXOXVJam9pTVRBdU1UVXVNQ0lzSWlSaWNtOTNjMlZ5SWpvaVJtbHlaV1p2ZUNJc0lpUmtaWFpwWTJWZmRIbHdaU0k2SWtSbGMydDBiM0FpTENJa1kzVnljbVZ1ZEY5MWNtd2lPaUpvZEhSd09pOHZiRzlqWVd4b2IzTjBPamd3TURBdklpd2lKR2h2YzNRaU9pSnNiMk5oYkdodmMzUTZPREF3TUNJc0lpUndZWFJvYm1GdFpTSTZJaThpTENJa1luSnZkM05sY2w5MlpYSnphVzl1SWpveE1UY3NJaVJpY205M2MyVnlYMnhoYm1kMVlXZGxJam9pWlc0dFZWTWlMQ0lrYzJOeVpXVnVYMmhsYVdkb2RDSTZNVEExTWl3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRZeU1Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPVEV4TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFMU5EZ3NJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpjNExqVWlMQ0lrYVc1elpYSjBYMmxrSWpvaU1XVTRaV1p5WkdSbE1HSTBNV2RtYkNJc0lpUjBhVzFsSWpveE5qazBOelk1TXpBeUxqTXhPQ3dpWkdsemRHbHVZM1JmYVdRaU9pSXdNVGhoT1RneFppMDBZakprTFRjNFlXVXRPVFEzWWkxaVpXUmlZV0V3TW1Fd1pqUWlMQ0lrWkdWMmFXTmxYMmxrSWpvaU1ERTRZVGs0TVdZdE5HSXlaQzAzT0dGbExUazBOMkl0WW1Wa1ltRmhNREpoTUdZMElpd2lKSEpsWm1WeWNtVnlJam9pSkdScGNtVmpkQ0lzSWlSeVpXWmxjbkpwYm1kZlpHOXRZV2x1SWpvaUpHUnBjbVZqZENJc0luUnZhMlZ1SWpvaWNHaGpYM0ZuVlVad05YZDZNa0pxUXpSU2MwWnFUVWR3VVROUVIwUnljMmsyVVRCRFJ6QkRVRFJPUVdGak1Fa2lMQ0lrYzJWemMybHZibDlwWkNJNklqQXhPR0U1T0RGbUxUUmlNbVV0TjJKa015MWlZak13TFRZMk1UZGlaalE0T0RnMk9TSXNJaVIzYVc1a2IzZGZhV1FpT2lJd01UaGhPVGd4WmkwMFlqSmxMVGRpWkRNdFltSXpNQzAyTmpFNE1UQm1aV1EyTldZaWZTd2lkR2x0WlhOMFlXMXdJam9pTWpBeU15MHdPUzB4TlZRd09Ub3hOVG93TWk0ek1UaGFJbjAlM0Q=","output":[{"uuid":"018a981f-4b2e-7bd3-bb30-6616657c38fa","distinct_id":"018a981f-4b2d-78ae-947b-bedbaa02a0f4","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b2e-7bd3-bb30-6616657c38fa\", \"event\": \"$opt_in\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"1e8efrdde0b41gfl\", \"$time\": 1694769302.318, \"distinct_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"timestamp\": \"2023-09-15T09:15:02.318Z\"}","now":"2023-09-15T09:15:02.321230+00:00","sent_at":"2023-09-15T09:15:02.319000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} {"path":"/e/?ip=1&_=1694769329412&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:29.500667+00:00","body":"H4sIAAAAAAAAA+0Ya2/bNvCvGF4/bEPo6EVSyjBgbdq0HdauaZp2Q1EIFElZqmlRkehXiwL7Lftp+yU7yo7jV2KkHYoikz/Y1t3xXryX7u3H7miUi+5R13FDFoVuihhLMaJp6qAoTQUKUyzdNMTUpWn3oCvHsjBAfo+NjOasNKNKArisdCkrk8u6e/Sxe0/DT/cZ453fzzp/ABoA8VhWda4LQLhOz8U9x8KTSk9qWQHwJK9kqqcWKOQ45zI2s1IC4qGsB0aXFsFHVQXi41GlAJEZUx4dHipQQ2W6Nkeh4ziHoEati8OlOUHiCURDJlEU0AQlUiSMOR5z0uA7xk0+lq9Y8nMta6vcS8l1JfKiX1txlinIWRdgESUzWcGGVrvbyFsx+MobrktXwIoV/RHrW86yQOdn9kjNKymLOJN5PwN9XAd7V9BJLkwGQOI5ABznclLqyiyJI9ddBV9S4yAEsMoTkDORiZUCD6tX1KNhD1t4XoBeJp7HiB6LQTRVxThKs5QNLN7k1g8uAWtJ5HukF/pgkMhrkxd8ca58c3/w9NXs1D9+Tn3fO784nRZPSOmfvA5f/ypo+ScJ6CPxfDB9w1fufz0s9/h1ZL33GcLyOhZyqGMI4PeSg8dSpmoJDPuVHpVNNC9Rl8pIlIaJhyAYHCSwFCjyQ0dKzJgIrC911WdF/oGZuS+vTmHM56dYiAXyMHexxBgLLKwmRW1Ywe3V74zs7ifQaiXrYvAxS5QUMZgONxfXuYDDl/rPYztOJWuIU8UgqI/evgPUKiwu2UxpJqylIKCSTA2tBiAVDOMq54NMg3Nt6g9Zrhph9nbYGJ6s/KXIWjE+uAGfsTpOcqUgveISAn2JgBTikNYQ9vGohu8eeFyMIHhYwdTM5ByUg+heZv8NNFusFokdVxJEznbw2UGwxWTDiVs8tvH3oJhJILGV7Z6A0gbxs4RaBwgN3rTRsYJtautl2Wtc3xQ9uVIsLJWSQ6Czl/mxC0rGi0pUl+DSgy4zpopjwQxD9q9NiKZC1aiSitlLbbREhtlQLUwW8yxXkDjA2z7pdKHBXFZs5NSG/sncxM7cxE8Ha6JFPgZeXLEavAl6dX+DlCqgrtZxDHIQ14WxbQOCb65eQwoHr6G7Wa8N4Sq/SfY+mUuPVdqGKjC+ArEqZ6gGf3NwG+CacF1igTIvhJzaDF/XGG9ojDc0HqlrNU5YdaPGFr+tsYJqezunXX9jN4hfSq7NrBGNEMSiLmws1ahWUH8q1LSYow4Neh6Bj4cjSjD8lNOfOrvodZrW0hx1XEp6XhRFIaWRF5GQ2hM3xrPlsm51sGF1sM9qm4OIleX1EbpF8WVuPgObHzDL/zqBWxRfZuGC3bVybmbv72OfofkcAtBUQS7Mf8BZalviJS3UEDltvhq6z3DonG/T7SutN1zk72GRaDG7SuIMimlTROy4tKnxOmOYujYYg4lGD8B6CMqMxxf985MSTz54D94fBy/rk/fPHpen/ovHD6F6k1Pn+LFz/CJ4fp9x52kz1y06z9acIxFNhI+SxHcQIS5N0iAMQxLZQxOoOXqy90zoOqkUBKd2apinGBiAqWedsWPiZ4hGMkQMRKFAklAmJOC+szbxl9Dq7CR5B8b9RcM+aZpZO+lvT/ozf0qJZGR4cZF7Fw7bPekHoF476beTfjvp327S/+abBqiYm2bEmzIoKdUvJURJpvs9roedf/76u/NiPoPN/wPuie6vdxpCd3aa1IfUjkQAWR0wJKgXcQqFxhG2wtzR3VLbbPY0G2UmtFJBmuVB0r8g/o5mQ3uRD8Np22zaZtM2m6+6Vvq6u521ndPmmunb2/RsvhI2r1f/s01PCHsbHyqi5wY+xgGJ9m16fEJ7Dmx6KMYhjvx20dMuetpFz3+66HED2uyct8fvgMCw43soYiRCHBNCscvtlvaOLnoam9qpe9fUTWbKxxNhxnjsupo0I/L21N2ueNqpu5262xXP1orHDXaveBLHCRH1KQOZAchkhIVuwjwvsKl92WMuRrKadUBUqaQdudtOc4c7TdgfTcbD0Jtlk1mWarKj04Q91ydtp2k7TdtpbttpmlJqA24ALQAwj5pCdNpAwbvNUsO+wf24LL+NqlB97BueZkrWXH5/VX57q0XyoLOKWBQPW8N+gFf8znmlOoeds8sXw1VSWyRsixlKCM9haV8S50Kfrqcnj9g8tkmYwqNLAwBTjhNbRVhqGlcgL8hsHItRtcgOl0KB+uZ77Gq79Kj/6d2/ToySO28rAAA=","output":[{"uuid":"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0ovdk9xlnv9fhfak\", \"$time\": 1694769326.837, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"attr__data-attr\": \"persons-related-flags-tab\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Feature flags\"}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 74.26666259765625px; --lemon-tabs-slider-offset: 176.29998779296875px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2572}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-aafa-79e8-abf4-4e68eb64c30f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aafa-79e8-abf4-4e68eb64c30f\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"y3x76ea6mqqi2q0a\", \"$time\": 1694769326.842, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 2567}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af3d-79d4-be4a-d729c776e0da","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af3d-79d4-be4a-d729c776e0da\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ltw7rl4fhi4bgq63\", \"$time\": 1694769327.934, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"\"}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 86.23332214355469px; --lemon-tabs-slider-offset: 367.0999755859375px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1475}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af46-7832-9a69-c566751c6625","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af46-7832-9a69-c566751c6625\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"6yl35wdtv5v11o6c\", \"$time\": 1694769327.942, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 1467}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-b008-737a-bb40-6a6a81ba2244","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-b008-737a-bb40-6a6a81ba2244\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"8guwvm82yhwyhfo6\", \"$time\": 1694769328.136, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"EventsQuery\", \"select\": [\"*\", \"event\", \"person\", \"coalesce(properties.$current_url, properties.$screen_name) -- Url / Screen\", \"properties.$lib\", \"timestamp\"], \"personId\": \"018a981f-4c9a-0000-68ff-417481f7c5b5\", \"after\": \"-24h\"}, \"duration\": 171, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1273}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +### nodejs, default params +{"path":"/batch/","method":"POST","content_encoding":"","content_type":"application/json","ip":"127.0.0.1","now":"2024-04-17T14:40:56.918578+00:00","body":"eyJhcGlfa2V5IjoicGhjX05WWk95WWI4aFgyR0RzODBtUWQwNzJPVDBXVXJBYmRRb200WGVDVjVOZmgiLCJiYXRjaCI6W3siZGlzdGluY3RfaWQiOiJpZDEiLCJldmVudCI6InRoaXMgZXZlbnQiLCJwcm9wZXJ0aWVzIjp7IiRsaWIiOiJwb3N0aG9nLW5vZGUiLCIkbGliX3ZlcnNpb24iOiI0LjAuMCIsIiRnZW9pcF9kaXNhYmxlIjp0cnVlfSwidHlwZSI6ImNhcHR1cmUiLCJsaWJyYXJ5IjoicG9zdGhvZy1ub2RlIiwibGlicmFyeV92ZXJzaW9uIjoiNC4wLjAiLCJ0aW1lc3RhbXAiOiIyMDI0LTA0LTE3VDE0OjQwOjU2LjkwMFoiLCJ1dWlkIjoiMDE4ZWVjODAtZjA0NC03YzJkLWI5NjYtMzVlZmM4ZWQ2MWQ1In0seyJkaXN0aW5jdF9pZCI6ImRpc3RpbmN0X2lkX29mX3RoZV91c2VyIiwiZXZlbnQiOiJ1c2VyIHNpZ25lZCB1cCIsInByb3BlcnRpZXMiOnsibG9naW5fdHlwZSI6ImVtYWlsIiwiaXNfZnJlZV90cmlhbCI6dHJ1ZSwiJGxpYiI6InBvc3Rob2ctbm9kZSIsIiRsaWJfdmVyc2lvbiI6IjQuMC4wIiwiJGdlb2lwX2Rpc2FibGUiOnRydWV9LCJ0eXBlIjoiY2FwdHVyZSIsImxpYnJhcnkiOiJwb3N0aG9nLW5vZGUiLCJsaWJyYXJ5X3ZlcnNpb24iOiI0LjAuMCIsInRpbWVzdGFtcCI6IjIwMjQtMDQtMTdUMTQ6NDA6NTYuOTAwWiIsInV1aWQiOiIwMThlZWM4MC1mMDQ0LTdjMmQtYjk2Ni0zNWYwNTA1OWYzNzUifV0sInNlbnRfYXQiOiIyMDI0LTA0LTE3VDE0OjQwOjU2LjkwMFoifQ==","output":[{"uuid":"018eec80-f044-7c2d-b966-35efc8ed61d5","distinct_id":"id1","ip":"127.0.0.1","site_url":"http://127.0.0.1:8000","data":"{\"distinct_id\": \"id1\", \"event\": \"this event\", \"properties\": {\"$lib\": \"posthog-node\", \"$lib_version\": \"4.0.0\", \"$geoip_disable\": true}, \"type\": \"capture\", \"library\": \"posthog-node\", \"library_version\": \"4.0.0\", \"timestamp\": \"2024-04-17T14:40:56.900Z\", \"uuid\": \"018eec80-f044-7c2d-b966-35efc8ed61d5\"}","now":"2024-04-17T14:40:56.918578+00:00","sent_at":"2024-04-17T14:40:56.900000+00:00","token":"phc_NVZOyYb8hX2GDs80mQd072OT0WUrAbdQom4XeCV5Nfh"},{"uuid":"018eec80-f044-7c2d-b966-35f05059f375","distinct_id":"distinct_id_of_the_user","ip":"127.0.0.1","site_url":"http://127.0.0.1:8000","data":"{\"distinct_id\": \"distinct_id_of_the_user\", \"event\": \"user signed up\", \"properties\": {\"login_type\": \"email\", \"is_free_trial\": true, \"$lib\": \"posthog-node\", \"$lib_version\": \"4.0.0\", \"$geoip_disable\": true}, \"type\": \"capture\", \"library\": \"posthog-node\", \"library_version\": \"4.0.0\", \"timestamp\": \"2024-04-17T14:40:56.900Z\", \"uuid\": \"018eec80-f044-7c2d-b966-35f05059f375\"}","now":"2024-04-17T14:40:56.918578+00:00","sent_at":"2024-04-17T14:40:56.900000+00:00","token":"phc_NVZOyYb8hX2GDs80mQd072OT0WUrAbdQom4XeCV5Nfh"}]} \ No newline at end of file From 0d48419bdd584d424ba5f9241b9242628520e3e4 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 24 Apr 2024 17:30:11 +0200 Subject: [PATCH 223/247] common: refactor health into its own lib crate (#23) --- Cargo.lock | 17 +- Cargo.toml | 7 +- capture/Cargo.toml | 33 +- capture/src/lib.rs | 1 - capture/src/router.rs | 2 +- capture/src/server.rs | 3 +- capture/src/sinks/kafka.rs | 4 +- capture/tests/django_compat.rs | 2 +- common/README.md | 8 + common/health/Cargo.toml | 12 + .../src/health.rs => common/health/src/lib.rs | 8 +- hook-common/Cargo.toml | 2 +- hook-common/src/health.rs | 346 ------------------ hook-common/src/lib.rs | 1 - hook-janitor/Cargo.toml | 3 +- hook-janitor/src/handlers/app.rs | 2 +- hook-janitor/src/kafka_producer.rs | 2 +- hook-janitor/src/main.rs | 2 +- hook-janitor/src/webhooks.rs | 2 +- hook-worker/Cargo.toml | 5 +- hook-worker/src/main.rs | 2 +- hook-worker/src/worker.rs | 4 +- 22 files changed, 79 insertions(+), 389 deletions(-) create mode 100644 common/README.md create mode 100644 common/health/Cargo.toml rename capture/src/health.rs => common/health/src/lib.rs (98%) delete mode 100644 hook-common/src/health.rs diff --git a/Cargo.lock b/Cargo.lock index 71cbc8d51a5d1..82c787aded2cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,6 +371,7 @@ dependencies = [ "envconfig", "flate2", "governor", + "health", "metrics", "metrics-exporter-prometheus", "rand", @@ -450,9 +451,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "futures-core", @@ -1001,6 +1002,16 @@ dependencies = [ "hashbrown 0.14.3", ] +[[package]] +name = "health" +version = "0.1.0" +dependencies = [ + "axum 0.7.5", + "time", + "tokio", + "tracing", +] + [[package]] name = "heck" version = "0.4.1" @@ -1100,6 +1111,7 @@ dependencies = [ "envconfig", "eyre", "futures", + "health", "hook-common", "metrics", "rdkafka", @@ -1120,6 +1132,7 @@ dependencies = [ "chrono", "envconfig", "futures", + "health", "hook-common", "http 1.1.0", "metrics", diff --git a/Cargo.toml b/Cargo.toml index ef70e645d4e13..cb4aa8e58e88d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,10 +4,11 @@ resolver = "2" members = [ "capture", "capture-server", - "hook-common", + "common/health", "hook-api", - "hook-worker", + "hook-common", "hook-janitor", + "hook-worker", ] [workspace.dependencies] @@ -44,13 +45,13 @@ sqlx = { version = "0.7", features = [ "tls-native-tls", "uuid", ] } -thiserror = { version = "1.0" } time = { version = "0.3.20", features = [ "formatting", "macros", "parsing", "serde", ] } +thiserror = { version = "1.0" } tokio = { version = "1.34.0", features = ["full"] } tower = "0.4.13" tower-http = { version = "0.5.2", features = ["cors", "trace"] } diff --git a/capture/Cargo.toml b/capture/Cargo.toml index 9ec0f97fb3ddb..02ada312932c1 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -6,33 +6,34 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } axum = { workspace = true } axum-client-ip = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -governor = { workspace = true } -time = { workspace = true } -tower-http = { workspace = true } +base64 = { workspace = true } bytes = { workspace = true } -anyhow = { workspace = true } +envconfig = { workspace = true } flate2 = { workspace = true } -base64 = { workspace = true } -uuid = { workspace = true } -async-trait = { workspace = true } -serde_urlencoded = { workspace = true } -rand = { workspace = true } -rdkafka = { workspace = true } +governor = { workspace = true } +health = { path = "../common/health" } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } -thiserror = { workspace = true } +rand = { workspace = true } +rdkafka = { workspace = true } redis = { version = "0.23.3", features = [ "tokio-comp", "cluster", "cluster-async", ] } -envconfig = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +serde_urlencoded = { workspace = true } +thiserror = { workspace = true } +time = { workspace = true } +tokio = { workspace = true } +tower-http = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } [dev-dependencies] assert-json-diff = { workspace = true } diff --git a/capture/src/lib.rs b/capture/src/lib.rs index 176fc6f09963e..d5d47dd9ea421 100644 --- a/capture/src/lib.rs +++ b/capture/src/lib.rs @@ -1,6 +1,5 @@ pub mod api; pub mod config; -pub mod health; pub mod limiters; pub mod prometheus; pub mod redis; diff --git a/capture/src/router.rs b/capture/src/router.rs index 85475ceb2a990..0710302549def 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -6,10 +6,10 @@ use axum::{ routing::{get, post}, Router, }; +use health::HealthRegistry; use tower_http::cors::{AllowHeaders, AllowOrigin, CorsLayer}; use tower_http::trace::TraceLayer; -use crate::health::HealthRegistry; use crate::{ limiters::billing::BillingLimiter, redis::Client, sinks, time::TimeSource, v0_endpoint, }; diff --git a/capture/src/server.rs b/capture/src/server.rs index 2fc88c60687e6..07049874edef9 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -2,11 +2,12 @@ use std::future::Future; use std::net::SocketAddr; use std::sync::Arc; +use health::{ComponentStatus, HealthRegistry}; use time::Duration; use tokio::net::TcpListener; use crate::config::Config; -use crate::health::{ComponentStatus, HealthRegistry}; + use crate::limiters::billing::BillingLimiter; use crate::limiters::overflow::OverflowLimiter; use crate::redis::RedisClient; diff --git a/capture/src/sinks/kafka.rs b/capture/src/sinks/kafka.rs index 4a48b6e4d5335..8b286e84d8838 100644 --- a/capture/src/sinks/kafka.rs +++ b/capture/src/sinks/kafka.rs @@ -1,6 +1,7 @@ use std::time::Duration; use async_trait::async_trait; +use health::HealthHandle; use metrics::{counter, gauge, histogram}; use rdkafka::error::{KafkaError, RDKafkaErrorCode}; use rdkafka::producer::{DeliveryFuture, FutureProducer, FutureRecord, Producer}; @@ -12,7 +13,6 @@ use tracing::{info_span, instrument, Instrument}; use crate::api::{CaptureError, ProcessedEvent}; use crate::config::KafkaConfig; -use crate::health::HealthHandle; use crate::limiters::overflow::OverflowLimiter; use crate::prometheus::report_dropped_events; use crate::sinks::Event; @@ -260,11 +260,11 @@ impl Event for KafkaSink { mod tests { use crate::api::{CaptureError, ProcessedEvent}; use crate::config; - use crate::health::HealthRegistry; use crate::limiters::overflow::OverflowLimiter; use crate::sinks::kafka::KafkaSink; use crate::sinks::Event; use crate::utils::uuid_v7; + use health::HealthRegistry; use rand::distributions::Alphanumeric; use rand::Rng; use rdkafka::mocking::MockCluster; diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index c7ec0ad8d2770..9ef4e391b5934 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -5,12 +5,12 @@ use axum_test_helper::TestClient; use base64::engine::general_purpose; use base64::Engine; use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode, ProcessedEvent}; -use capture::health::HealthRegistry; use capture::limiters::billing::BillingLimiter; use capture::redis::MockRedisClient; use capture::router::router; use capture::sinks::Event; use capture::time::TimeSource; +use health::HealthRegistry; use serde::Deserialize; use serde_json::{json, Value}; use std::fs::File; diff --git a/common/README.md b/common/README.md new file mode 100644 index 0000000000000..0e490c70a6245 --- /dev/null +++ b/common/README.md @@ -0,0 +1,8 @@ +# Common crates for the hog-rs services + +This folder holds internal crates for code reuse between services in the monorepo. To keep maintenance costs low, +these crates should ideally: + +- Cover a small feature scope and use as little dependencies as possible +- Only use `{ workspace = true }` dependencies, instead of pinning versions that could diverge from the workspace +- Have adequate test coverage and documentation diff --git a/common/health/Cargo.toml b/common/health/Cargo.toml new file mode 100644 index 0000000000000..9bbadc151f9db --- /dev/null +++ b/common/health/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "health" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +axum = { workspace = true } +time = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } diff --git a/capture/src/health.rs b/common/health/src/lib.rs similarity index 98% rename from capture/src/health.rs rename to common/health/src/lib.rs index dcddbe477e7cc..5d42bafa8ff05 100644 --- a/capture/src/health.rs +++ b/common/health/src/lib.rs @@ -1,9 +1,9 @@ -use axum::http::StatusCode; -use axum::response::{IntoResponse, Response}; use std::collections::HashMap; use std::ops::Add; use std::sync::{Arc, RwLock}; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; use time::Duration; use tokio::sync::mpsc; use tracing::{info, warn}; @@ -98,7 +98,7 @@ impl HealthHandle { )) } - /// Asynchronously report component status, returns when the message is queued. + /// Synchronously report component status, returns when the message is queued. pub fn report_status_blocking(&self, status: ComponentStatus) { let message = HealthMessage { component: self.component.clone(), @@ -198,7 +198,7 @@ impl HealthRegistry { #[cfg(test)] mod tests { - use crate::health::{ComponentStatus, HealthRegistry, HealthStatus}; + use crate::{ComponentStatus, HealthRegistry, HealthStatus}; use axum::http::StatusCode; use axum::response::IntoResponse; use std::ops::{Add, Sub}; diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index 8ccf8dd5b2ebb..af31e052f1351 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -16,9 +16,9 @@ reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } +thiserror = { workspace = true } time = { workspace = true } tokio = { workspace = true } -thiserror = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } diff --git a/hook-common/src/health.rs b/hook-common/src/health.rs deleted file mode 100644 index c5c79c904c950..0000000000000 --- a/hook-common/src/health.rs +++ /dev/null @@ -1,346 +0,0 @@ -use axum::http::StatusCode; -use axum::response::{IntoResponse, Response}; -use std::collections::HashMap; -use std::ops::Add; -use std::sync::{Arc, RwLock}; - -use time::Duration; -use tokio::sync::mpsc; -use tracing::{info, warn}; - -/// Health reporting for components of the service. -/// -/// FIXME: copied over from capture, make sure to keep in sync until we share the crate -/// -/// The capture server contains several asynchronous loops, and -/// the process can only be trusted with user data if all the -/// loops are properly running and reporting. -/// -/// HealthRegistry allows an arbitrary number of components to -/// be registered and report their health. The process' health -/// status is the combination of these individual health status: -/// - if any component is unhealthy, the process is unhealthy -/// - if all components recently reported healthy, the process is healthy -/// - if a component failed to report healthy for its defined deadline, -/// it is considered unhealthy, and the check fails. -/// -/// Trying to merge the k8s concepts of liveness and readiness in -/// a single state is full of foot-guns, so HealthRegistry does not -/// try to do it. Each probe should have its separate instance of -/// the registry to avoid confusions. - -#[derive(Default, Debug)] -pub struct HealthStatus { - /// The overall status: true of all components are healthy - pub healthy: bool, - /// Current status of each registered component, for display - pub components: HashMap, -} -impl IntoResponse for HealthStatus { - /// Computes the axum status code based on the overall health status, - /// and prints each component status in the body for debugging. - fn into_response(self) -> Response { - let body = format!("{:?}", self); - match self.healthy { - true => (StatusCode::OK, body), - false => (StatusCode::INTERNAL_SERVER_ERROR, body), - } - .into_response() - } -} - -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum ComponentStatus { - /// Automatically set when a component is newly registered - Starting, - /// Recently reported healthy, will need to report again before the date - HealthyUntil(time::OffsetDateTime), - /// Reported unhealthy - Unhealthy, - /// Automatically set when the HealthyUntil deadline is reached - Stalled, -} -struct HealthMessage { - component: String, - status: ComponentStatus, -} - -pub struct HealthHandle { - component: String, - deadline: Duration, - sender: mpsc::Sender, -} - -impl HealthHandle { - /// Asynchronously report healthy, returns when the message is queued. - /// Must be called more frequently than the configured deadline. - pub async fn report_healthy(&self) { - self.report_status(ComponentStatus::HealthyUntil( - time::OffsetDateTime::now_utc().add(self.deadline), - )) - .await - } - - /// Asynchronously report component status, returns when the message is queued. - pub async fn report_status(&self, status: ComponentStatus) { - let message = HealthMessage { - component: self.component.clone(), - status, - }; - if let Err(err) = self.sender.send(message).await { - warn!("failed to report heath status: {}", err) - } - } - - /// Synchronously report as healthy, returns when the message is queued. - /// Must be called more frequently than the configured deadline. - pub fn report_healthy_blocking(&self) { - self.report_status_blocking(ComponentStatus::HealthyUntil( - time::OffsetDateTime::now_utc().add(self.deadline), - )) - } - - /// Asynchronously report component status, returns when the message is queued. - pub fn report_status_blocking(&self, status: ComponentStatus) { - let message = HealthMessage { - component: self.component.clone(), - status, - }; - if let Err(err) = self.sender.blocking_send(message) { - warn!("failed to report heath status: {}", err) - } - } -} - -#[derive(Clone)] -pub struct HealthRegistry { - name: String, - components: Arc>>, - sender: mpsc::Sender, -} - -impl HealthRegistry { - pub fn new(name: &str) -> Self { - let (tx, mut rx) = mpsc::channel::(16); - let registry = Self { - name: name.to_owned(), - components: Default::default(), - sender: tx, - }; - - let components = registry.components.clone(); - tokio::spawn(async move { - while let Some(message) = rx.recv().await { - if let Ok(mut map) = components.write() { - _ = map.insert(message.component, message.status); - } else { - // Poisoned mutex: Just warn, the probes will fail and the process restart - warn!("poisoned HeathRegistry mutex") - } - } - }); - - registry - } - - /// Registers a new component in the registry. The returned handle should be passed - /// to the component, to allow it to frequently report its health status. - pub async fn register(&self, component: String, deadline: time::Duration) -> HealthHandle { - let handle = HealthHandle { - component, - deadline, - sender: self.sender.clone(), - }; - handle.report_status(ComponentStatus::Starting).await; - handle - } - - /// Returns the overall process status, computed from the status of all the components - /// currently registered. Can be used as an axum handler. - pub fn get_status(&self) -> HealthStatus { - let components = self - .components - .read() - .expect("poisoned HeathRegistry mutex"); - - let result = HealthStatus { - healthy: !components.is_empty(), // unhealthy if no component has registered yet - components: Default::default(), - }; - let now = time::OffsetDateTime::now_utc(); - - let result = components - .iter() - .fold(result, |mut result, (name, status)| { - match status { - ComponentStatus::HealthyUntil(until) => { - if until.gt(&now) { - _ = result.components.insert(name.clone(), status.clone()) - } else { - result.healthy = false; - _ = result - .components - .insert(name.clone(), ComponentStatus::Stalled) - } - } - _ => { - result.healthy = false; - _ = result.components.insert(name.clone(), status.clone()) - } - } - result - }); - match result.healthy { - true => info!("{} health check ok", self.name), - false => warn!("{} health check failed: {:?}", self.name, result.components), - } - result - } -} - -#[cfg(test)] -mod tests { - use crate::health::{ComponentStatus, HealthRegistry, HealthStatus}; - use axum::http::StatusCode; - use axum::response::IntoResponse; - use std::ops::{Add, Sub}; - use time::{Duration, OffsetDateTime}; - - async fn assert_or_retry(check: F) - where - F: Fn() -> bool, - { - assert_or_retry_for_duration(check, Duration::seconds(5)).await - } - - async fn assert_or_retry_for_duration(check: F, timeout: Duration) - where - F: Fn() -> bool, - { - let deadline = OffsetDateTime::now_utc().add(timeout); - while !check() && OffsetDateTime::now_utc().lt(&deadline) { - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } - assert!(check()) - } - #[tokio::test] - async fn defaults_to_unhealthy() { - let registry = HealthRegistry::new("liveness"); - assert!(!registry.get_status().healthy); - } - - #[tokio::test] - async fn one_component() { - let registry = HealthRegistry::new("liveness"); - - // New components are registered in Starting - let handle = registry - .register("one".to_string(), Duration::seconds(30)) - .await; - assert_or_retry(|| registry.get_status().components.len() == 1).await; - let mut status = registry.get_status(); - assert!(!status.healthy); - assert_eq!( - status.components.get("one"), - Some(&ComponentStatus::Starting) - ); - - // Status goes healthy once the component reports - handle.report_healthy().await; - assert_or_retry(|| registry.get_status().healthy).await; - status = registry.get_status(); - assert_eq!(status.components.len(), 1); - - // Status goes unhealthy if the components says so - handle.report_status(ComponentStatus::Unhealthy).await; - assert_or_retry(|| !registry.get_status().healthy).await; - status = registry.get_status(); - assert_eq!(status.components.len(), 1); - assert_eq!( - status.components.get("one"), - Some(&ComponentStatus::Unhealthy) - ); - } - - #[tokio::test] - async fn staleness_check() { - let registry = HealthRegistry::new("liveness"); - let handle = registry - .register("one".to_string(), Duration::seconds(30)) - .await; - - // Status goes healthy once the component reports - handle.report_healthy().await; - assert_or_retry(|| registry.get_status().healthy).await; - let mut status = registry.get_status(); - assert_eq!(status.components.len(), 1); - - // If the component's ping is too old, it is considered stalled and the healthcheck fails - // FIXME: we should mock the time instead - handle - .report_status(ComponentStatus::HealthyUntil( - OffsetDateTime::now_utc().sub(Duration::seconds(1)), - )) - .await; - assert_or_retry(|| !registry.get_status().healthy).await; - status = registry.get_status(); - assert_eq!(status.components.len(), 1); - assert_eq!( - status.components.get("one"), - Some(&ComponentStatus::Stalled) - ); - } - - #[tokio::test] - async fn several_components() { - let registry = HealthRegistry::new("liveness"); - let handle1 = registry - .register("one".to_string(), Duration::seconds(30)) - .await; - let handle2 = registry - .register("two".to_string(), Duration::seconds(30)) - .await; - assert_or_retry(|| registry.get_status().components.len() == 2).await; - - // First component going healthy is not enough - handle1.report_healthy().await; - assert_or_retry(|| { - registry.get_status().components.get("one").unwrap() != &ComponentStatus::Starting - }) - .await; - assert!(!registry.get_status().healthy); - - // Second component going healthy brings the health to green - handle2.report_healthy().await; - assert_or_retry(|| { - registry.get_status().components.get("two").unwrap() != &ComponentStatus::Starting - }) - .await; - assert!(registry.get_status().healthy); - - // First component going unhealthy takes down the health to red - handle1.report_status(ComponentStatus::Unhealthy).await; - assert_or_retry(|| !registry.get_status().healthy).await; - - // First component recovering returns the health to green - handle1.report_healthy().await; - assert_or_retry(|| registry.get_status().healthy).await; - - // Second component going unhealthy takes down the health to red - handle2.report_status(ComponentStatus::Unhealthy).await; - assert_or_retry(|| !registry.get_status().healthy).await; - } - - #[tokio::test] - async fn into_response() { - let nok = HealthStatus::default().into_response(); - assert_eq!(nok.status(), StatusCode::INTERNAL_SERVER_ERROR); - - let ok = HealthStatus { - healthy: true, - components: Default::default(), - } - .into_response(); - assert_eq!(ok.status(), StatusCode::OK); - } -} diff --git a/hook-common/src/lib.rs b/hook-common/src/lib.rs index 7f49049add362..8e63ded5a7bf2 100644 --- a/hook-common/src/lib.rs +++ b/hook-common/src/lib.rs @@ -1,4 +1,3 @@ -pub mod health; pub mod kafka_messages; pub mod metrics; pub mod pgqueue; diff --git a/hook-janitor/Cargo.toml b/hook-janitor/Cargo.toml index a29a80c1c5256..654798d95f148 100644 --- a/hook-janitor/Cargo.toml +++ b/hook-janitor/Cargo.toml @@ -11,13 +11,14 @@ axum = { workspace = true } envconfig = { workspace = true } eyre = { workspace = true } futures = { workspace = true } +health = { path = "../common/health" } hook-common = { path = "../hook-common" } metrics = { workspace = true } rdkafka = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } -time = { workspace = true } thiserror = { workspace = true } +time = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/hook-janitor/src/handlers/app.rs b/hook-janitor/src/handlers/app.rs index 507a1cba48d46..65692b14592a2 100644 --- a/hook-janitor/src/handlers/app.rs +++ b/hook-janitor/src/handlers/app.rs @@ -1,5 +1,5 @@ use axum::{routing::get, Router}; -use hook_common::health::HealthRegistry; +use health::HealthRegistry; use std::future::ready; pub fn app(liveness: HealthRegistry) -> Router { diff --git a/hook-janitor/src/kafka_producer.rs b/hook-janitor/src/kafka_producer.rs index ba368663072c3..92608bcb999c8 100644 --- a/hook-janitor/src/kafka_producer.rs +++ b/hook-janitor/src/kafka_producer.rs @@ -1,6 +1,6 @@ use crate::config::KafkaConfig; -use hook_common::health::HealthHandle; +use health::HealthHandle; use rdkafka::error::KafkaError; use rdkafka::producer::FutureProducer; use rdkafka::ClientConfig; diff --git a/hook-janitor/src/main.rs b/hook-janitor/src/main.rs index 46ee37560bb90..325aa098ed6fe 100644 --- a/hook-janitor/src/main.rs +++ b/hook-janitor/src/main.rs @@ -4,7 +4,7 @@ use config::Config; use envconfig::Envconfig; use eyre::Result; use futures::future::{select, Either}; -use hook_common::health::{HealthHandle, HealthRegistry}; +use health::{HealthHandle, HealthRegistry}; use kafka_producer::create_kafka_producer; use std::{str::FromStr, time::Duration}; use tokio::sync::Semaphore; diff --git a/hook-janitor/src/webhooks.rs b/hook-janitor/src/webhooks.rs index 0e3900c045d84..c1dfbba51aa35 100644 --- a/hook-janitor/src/webhooks.rs +++ b/hook-janitor/src/webhooks.rs @@ -495,7 +495,7 @@ mod tests { use super::*; use crate::config; use crate::kafka_producer::{create_kafka_producer, KafkaContext}; - use hook_common::health::HealthRegistry; + use health::HealthRegistry; use hook_common::kafka_messages::app_metrics::{ Error as WebhookError, ErrorDetails, ErrorType, }; diff --git a/hook-worker/Cargo.toml b/hook-worker/Cargo.toml index 5d6874a8af4cd..83ea923b1b9b3 100644 --- a/hook-worker/Cargo.toml +++ b/hook-worker/Cargo.toml @@ -8,14 +8,15 @@ axum = { workspace = true } chrono = { workspace = true } envconfig = { workspace = true } futures = "0.3" +health = { path = "../common/health" } hook-common = { path = "../hook-common" } http = { workspace = true } metrics = { workspace = true } reqwest = { workspace = true } sqlx = { workspace = true } -time = { workspace = true } thiserror = { workspace = true } +time = { workspace = true } +tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } -tokio = { workspace = true } url = { version = "2.2" } diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index 2997dfc65ff50..8a6eeb37435ab 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -4,7 +4,7 @@ use axum::Router; use envconfig::Envconfig; use std::future::ready; -use hook_common::health::HealthRegistry; +use health::HealthRegistry; use hook_common::{ metrics::serve, metrics::setup_metrics_routes, pgqueue::PgQueue, retry::RetryPolicy, }; diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 441e1ee609de8..5965d26a0801d 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -4,7 +4,7 @@ use std::time; use chrono::Utc; use futures::future::join_all; -use hook_common::health::HealthHandle; +use health::HealthHandle; use hook_common::pgqueue::PgTransactionBatch; use hook_common::{ pgqueue::{ @@ -452,7 +452,7 @@ mod tests { // This is due to a long-standing cargo bug that reports imports and helper functions as unused. // See: https://github.com/rust-lang/rust/issues/46379. #[allow(unused_imports)] - use hook_common::health::HealthRegistry; + use health::HealthRegistry; #[allow(unused_imports)] use hook_common::pgqueue::{JobStatus, NewJob}; #[allow(unused_imports)] From 55292bd6287a53a053f7bf4f80d775c5cd1580ec Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 24 Apr 2024 17:30:40 +0200 Subject: [PATCH 224/247] capture: don't allow events submitted with an empty distinct_id (#25) --- capture/src/api.rs | 3 +++ capture/src/v0_request.rs | 16 +++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/capture/src/api.rs b/capture/src/api.rs index 0938ced399773..91ed578f24864 100644 --- a/capture/src/api.rs +++ b/capture/src/api.rs @@ -28,6 +28,8 @@ pub enum CaptureError { EmptyBatch, #[error("event submitted with an empty event name")] MissingEventName, + #[error("event submitted with an empty distinct_id")] + EmptyDistinctId, #[error("event submitted without a distinct_id")] MissingDistinctId, @@ -59,6 +61,7 @@ impl IntoResponse for CaptureError { | CaptureError::RequestParsingError(_) | CaptureError::EmptyBatch | CaptureError::MissingEventName + | CaptureError::EmptyDistinctId | CaptureError::MissingDistinctId | CaptureError::EventTooBig | CaptureError::NonRetryableSinkError => (StatusCode::BAD_REQUEST, self.to_string()), diff --git a/capture/src/v0_request.rs b/capture/src/v0_request.rs index 3d0052e0c072c..a4e1042f49c78 100644 --- a/capture/src/v0_request.rs +++ b/capture/src/v0_request.rs @@ -217,10 +217,11 @@ impl RawEvent { .as_str() .map(|s| s.to_owned()) .unwrap_or_else(|| value.to_string()); - Ok(match distinct_id.len() { - 0..=200 => distinct_id, - _ => distinct_id.chars().take(200).collect(), - }) + match distinct_id.len() { + 0 => Err(CaptureError::EmptyDistinctId), + 1..=200 => Ok(distinct_id), + _ => Ok(distinct_id.chars().take(200).collect()), + } } } @@ -303,11 +304,16 @@ mod tests { parse_and_extract(r#"{"event": "e"}"#), Err(CaptureError::MissingDistinctId) )); - // Return MissingDistinctId if null, breaking compat with capture-py + // Return MissingDistinctId if null assert!(matches!( parse_and_extract(r#"{"event": "e", "distinct_id": null}"#), Err(CaptureError::MissingDistinctId) )); + // Return EmptyDistinctId if empty string + assert!(matches!( + parse_and_extract(r#"{"event": "e", "distinct_id": ""}"#), + Err(CaptureError::EmptyDistinctId) + )); let assert_extracted_id = |input: &'static str, expected: &str| { let id = parse_and_extract(input).expect("failed to extract"); From 60debad5f115bf5fd9a549b919b3f505ffee0d14 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 24 Apr 2024 17:39:18 +0200 Subject: [PATCH 225/247] chore: improve linting (#26) --- .github/workflows/rust.yml | 24 ++++++++++++------------ Cargo.toml | 15 +++++++++++++++ capture-server/Cargo.toml | 3 +++ capture/Cargo.toml | 3 ++- capture/src/sinks/kafka.rs | 4 ++-- common/health/Cargo.toml | 3 ++- hook-api/Cargo.toml | 3 ++- hook-common/Cargo.toml | 3 ++- hook-common/src/pgqueue.rs | 14 ++++++++------ hook-janitor/Cargo.toml | 3 ++- hook-worker/Cargo.toml | 3 +++ 11 files changed, 53 insertions(+), 25 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 30f17341d5f08..ed5acb2ed6a90 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -5,7 +5,6 @@ on: push: branches: [main] pull_request: - branches: [main] env: CARGO_TERM_COLOR: always @@ -71,7 +70,7 @@ jobs: - name: Run cargo check run: cargo check --all-features - clippy: + linting: runs-on: depot-ubuntu-22.04-4 steps: @@ -81,7 +80,7 @@ jobs: uses: dtolnay/rust-toolchain@master with: toolchain: stable - components: clippy + components: clippy,rustfmt - uses: actions/cache@v3 with: @@ -94,17 +93,18 @@ jobs: - name: Run clippy run: cargo clippy -- -D warnings - format: - runs-on: depot-ubuntu-22.04-4 + - name: Check format + run: cargo fmt -- --check + shear: + runs-on: depot-ubuntu-22.04-4 steps: - uses: actions/checkout@v3 - - name: Install latest rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: rustfmt + - name: Install cargo-binstall + uses: cargo-bins/cargo-binstall@main - - name: Format - run: cargo fmt -- --check + - name: Install cargo-shear + run: cargo binstall --no-confirm cargo-shear + + - run: cargo shear diff --git a/Cargo.toml b/Cargo.toml index cb4aa8e58e88d..d34cd0ae39231 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,21 @@ members = [ "hook-worker", ] +[workspace.lints.rust] +# See https://doc.rust-lang.org/stable/rustc/lints/listing/allowed-by-default.html +unsafe_code = "forbid" # forbid cannot be ignored with an annotation +unstable_features = "forbid" +macro_use_extern_crate = "forbid" +let_underscore_drop = "deny" +non_ascii_idents = "deny" +trivial_casts = "deny" +trivial_numeric_casts = "deny" +unit_bindings = "deny" + +[workspace.lints.clippy] +# See https://rust-lang.github.io/rust-clippy/, we might want to add more +enum_glob_use = "deny" + [workspace.dependencies] anyhow = "1.0" assert-json-diff = "2.0.2" diff --git a/capture-server/Cargo.toml b/capture-server/Cargo.toml index ae06664ec96f9..39ee742d2b048 100644 --- a/capture-server/Cargo.toml +++ b/capture-server/Cargo.toml @@ -3,6 +3,9 @@ name = "capture-server" version = "0.1.0" edition = "2021" +[lints] +workspace = true + [dependencies] capture = { path = "../capture" } envconfig = { workspace = true } diff --git a/capture/Cargo.toml b/capture/Cargo.toml index 02ada312932c1..4e35d10be7b85 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -3,7 +3,8 @@ name = "capture" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lints] +workspace = true [dependencies] anyhow = { workspace = true } diff --git a/capture/src/sinks/kafka.rs b/capture/src/sinks/kafka.rs index 8b286e84d8838..b7af44993bc13 100644 --- a/capture/src/sinks/kafka.rs +++ b/capture/src/sinks/kafka.rs @@ -118,10 +118,10 @@ impl KafkaSink { client_config.create_with_context(KafkaContext { liveness })?; // Ping the cluster to make sure we can reach brokers, fail after 10 seconds - _ = producer.client().fetch_metadata( + drop(producer.client().fetch_metadata( Some("__consumer_offsets"), Timeout::After(Duration::new(10, 0)), - )?; + )?); info!("connected to Kafka brokers"); Ok(KafkaSink { diff --git a/common/health/Cargo.toml b/common/health/Cargo.toml index 9bbadc151f9db..c38e704bd7ce3 100644 --- a/common/health/Cargo.toml +++ b/common/health/Cargo.toml @@ -3,7 +3,8 @@ name = "health" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lints] +workspace = true [dependencies] axum = { workspace = true } diff --git a/hook-api/Cargo.toml b/hook-api/Cargo.toml index a596e87076b18..c3528d23da5d2 100644 --- a/hook-api/Cargo.toml +++ b/hook-api/Cargo.toml @@ -3,7 +3,8 @@ name = "hook-api" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lints] +workspace = true [dependencies] axum = { workspace = true } diff --git a/hook-common/Cargo.toml b/hook-common/Cargo.toml index af31e052f1351..58232a80fe17d 100644 --- a/hook-common/Cargo.toml +++ b/hook-common/Cargo.toml @@ -3,7 +3,8 @@ name = "hook-common" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lints] +workspace = true [dependencies] async-trait = { workspace = true } diff --git a/hook-common/src/pgqueue.rs b/hook-common/src/pgqueue.rs index f155506123b35..5d8a14485697e 100644 --- a/hook-common/src/pgqueue.rs +++ b/hook-common/src/pgqueue.rs @@ -827,14 +827,15 @@ mod tests { let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); let retry_queue = retry_policy.retry_queue(&job.job.queue).to_owned(); - let _ = job - .retry( + drop( + job.retry( "a very reasonable failure reason", retry_interval, &retry_queue, ) .await - .expect("failed to retry job"); + .expect("failed to retry job"), + ); batch.commit().await.expect("failed to commit transaction"); let retried_job: PgTransactionJob = queue @@ -883,14 +884,15 @@ mod tests { let retry_interval = retry_policy.retry_interval(job.job.attempt as u32, None); let retry_queue = retry_policy.retry_queue(&job.job.queue).to_owned(); - let _ = job - .retry( + drop( + job.retry( "a very reasonable failure reason", retry_interval, &retry_queue, ) .await - .expect("failed to retry job"); + .expect("failed to retry job"), + ); batch.commit().await.expect("failed to commit transaction"); let retried_job_not_found: Option> = queue diff --git a/hook-janitor/Cargo.toml b/hook-janitor/Cargo.toml index 654798d95f148..741918e79385a 100644 --- a/hook-janitor/Cargo.toml +++ b/hook-janitor/Cargo.toml @@ -3,7 +3,8 @@ name = "hook-janitor" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lints] +workspace = true [dependencies] async-trait = { workspace = true } diff --git a/hook-worker/Cargo.toml b/hook-worker/Cargo.toml index 83ea923b1b9b3..79416f9004a10 100644 --- a/hook-worker/Cargo.toml +++ b/hook-worker/Cargo.toml @@ -3,6 +3,9 @@ name = "hook-worker" version = "0.1.0" edition = "2021" +[lints] +workspace = true + [dependencies] axum = { workspace = true } chrono = { workspace = true } From 26a67e9c586606a5a2b5f9ec6a9221e4fbd4bd7d Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Thu, 25 Apr 2024 16:42:17 +0200 Subject: [PATCH 226/247] capture: set kafka partitioner to murmur2_random (#27) --- capture/src/sinks/kafka.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/capture/src/sinks/kafka.rs b/capture/src/sinks/kafka.rs index b7af44993bc13..8ffb2d6d7f898 100644 --- a/capture/src/sinks/kafka.rs +++ b/capture/src/sinks/kafka.rs @@ -96,6 +96,7 @@ impl KafkaSink { client_config .set("bootstrap.servers", &config.kafka_hosts) .set("statistics.interval.ms", "10000") + .set("partitioner", "murmur2_random") // Compatibility with python-kafka .set("linger.ms", config.kafka_producer_linger_ms.to_string()) .set( "message.timeout.ms", From 9f5b72db5f8960c90a1ffbc2baa344a058d05c8c Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 30 Apr 2024 12:24:13 +0200 Subject: [PATCH 227/247] capture: don't serialize sent_at if empty (#28) --- capture/src/api.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/capture/src/api.rs b/capture/src/api.rs index 91ed578f24864..8e595fcfeb0a0 100644 --- a/capture/src/api.rs +++ b/capture/src/api.rs @@ -87,7 +87,10 @@ pub struct ProcessedEvent { pub ip: String, pub data: String, pub now: String, - #[serde(with = "time::serde::rfc3339::option")] + #[serde( + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] pub sent_at: Option, pub token: String, } From bca9f8457eb1a8a3689cb9ad72cf102701528238 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 30 Apr 2024 12:24:25 +0200 Subject: [PATCH 228/247] feat(capture): send historical_migration batches to separate topic (#30) --- capture-server/tests/common.rs | 6 +- capture-server/tests/events.rs | 112 ++++++++++++++++++++++++++---- capture/src/api.rs | 9 ++- capture/src/config.rs | 3 + capture/src/sinks/kafka.rs | 48 +++++++------ capture/src/v0_endpoint.rs | 15 ++-- capture/src/v0_request.rs | 3 +- capture/tests/django_compat.rs | 23 ++++-- capture/tests/requests_dump.jsonl | 8 ++- 9 files changed, 179 insertions(+), 48 deletions(-) diff --git a/capture-server/tests/common.rs b/capture-server/tests/common.rs index 5ee2caa5ca7a9..788e6e28240c7 100644 --- a/capture-server/tests/common.rs +++ b/capture-server/tests/common.rs @@ -39,6 +39,7 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { kafka_compression_codec: "none".to_string(), kafka_hosts: "kafka:9092".to_string(), kafka_topic: "events_plugin_ingestion".to_string(), + kafka_historical_topic: "events_plugin_ingestion_historical".to_string(), kafka_tls: false, }, otel_url: None, @@ -61,9 +62,10 @@ pub struct ServerHandle { } impl ServerHandle { - pub async fn for_topic(topic: &EphemeralTopic) -> Self { + pub async fn for_topics(main: &EphemeralTopic, historical: &EphemeralTopic) -> Self { let mut config = DEFAULT_CONFIG.clone(); - config.kafka.kafka_topic = topic.topic_name().to_string(); + config.kafka.kafka_topic = main.topic_name().to_string(); + config.kafka.kafka_historical_topic = historical.topic_name().to_string(); Self::for_config(config).await } pub async fn for_config(config: Config) -> Self { diff --git a/capture-server/tests/events.rs b/capture-server/tests/events.rs index 8a4220b8e3920..111b02c7f2cb1 100644 --- a/capture-server/tests/events.rs +++ b/capture-server/tests/events.rs @@ -13,8 +13,10 @@ async fn it_captures_one_event() -> Result<()> { setup_tracing(); let token = random_string("token", 16); let distinct_id = random_string("id", 16); - let topic = EphemeralTopic::new().await; - let server = ServerHandle::for_topic(&topic).await; + + let main_topic = EphemeralTopic::new().await; + let histo_topic = EphemeralTopic::new().await; + let server = ServerHandle::for_topics(&main_topic, &histo_topic).await; let event = json!({ "token": token, @@ -24,7 +26,7 @@ async fn it_captures_one_event() -> Result<()> { let res = server.capture_events(event.to_string()).await; assert_eq!(StatusCode::OK, res.status()); - let event = topic.next_event()?; + let event = main_topic.next_event()?; assert_json_include!( actual: event, expected: json!({ @@ -37,14 +39,15 @@ async fn it_captures_one_event() -> Result<()> { } #[tokio::test] -async fn it_captures_a_batch() -> Result<()> { +async fn it_captures_a_posthogjs_array() -> Result<()> { setup_tracing(); let token = random_string("token", 16); let distinct_id1 = random_string("id", 16); let distinct_id2 = random_string("id", 16); - let topic = EphemeralTopic::new().await; - let server = ServerHandle::for_topic(&topic).await; + let main_topic = EphemeralTopic::new().await; + let histo_topic = EphemeralTopic::new().await; + let server = ServerHandle::for_topics(&main_topic, &histo_topic).await; let event = json!([{ "token": token, @@ -59,14 +62,98 @@ async fn it_captures_a_batch() -> Result<()> { assert_eq!(StatusCode::OK, res.status()); assert_json_include!( - actual: topic.next_event()?, + actual: main_topic.next_event()?, expected: json!({ "token": token, "distinct_id": distinct_id1 }) ); assert_json_include!( - actual: topic.next_event()?, + actual: main_topic.next_event()?, + expected: json!({ + "token": token, + "distinct_id": distinct_id2 + }) + ); + + Ok(()) +} + +#[tokio::test] +async fn it_captures_a_batch() -> Result<()> { + setup_tracing(); + let token = random_string("token", 16); + let distinct_id1 = random_string("id", 16); + let distinct_id2 = random_string("id", 16); + + let main_topic = EphemeralTopic::new().await; + let histo_topic = EphemeralTopic::new().await; + let server = ServerHandle::for_topics(&main_topic, &histo_topic).await; + + let event = json!({ + "token": token, + "batch": [{ + "event": "event1", + "distinct_id": distinct_id1 + },{ + "event": "event2", + "distinct_id": distinct_id2 + }] + }); + let res = server.capture_events(event.to_string()).await; + assert_eq!(StatusCode::OK, res.status()); + + assert_json_include!( + actual: main_topic.next_event()?, + expected: json!({ + "token": token, + "distinct_id": distinct_id1 + }) + ); + assert_json_include!( + actual: main_topic.next_event()?, + expected: json!({ + "token": token, + "distinct_id": distinct_id2 + }) + ); + + Ok(()) +} +#[tokio::test] +async fn it_captures_a_historical_batch() -> Result<()> { + setup_tracing(); + let token = random_string("token", 16); + let distinct_id1 = random_string("id", 16); + let distinct_id2 = random_string("id", 16); + + let main_topic = EphemeralTopic::new().await; + let histo_topic = EphemeralTopic::new().await; + let server = ServerHandle::for_topics(&main_topic, &histo_topic).await; + + let event = json!({ + "token": token, + "historical_migration": true, + "batch": [{ + "event": "event1", + "distinct_id": distinct_id1 + },{ + "event": "event2", + "distinct_id": distinct_id2 + }] + }); + let res = server.capture_events(event.to_string()).await; + assert_eq!(StatusCode::OK, res.status()); + + assert_json_include!( + actual: histo_topic.next_event()?, + expected: json!({ + "token": token, + "distinct_id": distinct_id1 + }) + ); + assert_json_include!( + actual: histo_topic.next_event()?, expected: json!({ "token": token, "distinct_id": distinct_id2 @@ -175,8 +262,9 @@ async fn it_trims_distinct_id() -> Result<()> { let distinct_id2 = random_string("id", 222); let (trimmed_distinct_id2, _) = distinct_id2.split_at(200); // works because ascii chars - let topic = EphemeralTopic::new().await; - let server = ServerHandle::for_topic(&topic).await; + let main_topic = EphemeralTopic::new().await; + let histo_topic = EphemeralTopic::new().await; + let server = ServerHandle::for_topics(&main_topic, &histo_topic).await; let event = json!([{ "token": token, @@ -191,14 +279,14 @@ async fn it_trims_distinct_id() -> Result<()> { assert_eq!(StatusCode::OK, res.status()); assert_json_include!( - actual: topic.next_event()?, + actual: main_topic.next_event()?, expected: json!({ "token": token, "distinct_id": distinct_id1 }) ); assert_json_include!( - actual: topic.next_event()?, + actual: main_topic.next_event()?, expected: json!({ "token": token, "distinct_id": trimmed_distinct_id2 diff --git a/capture/src/api.rs b/capture/src/api.rs index 8e595fcfeb0a0..97d84857075e4 100644 --- a/capture/src/api.rs +++ b/capture/src/api.rs @@ -80,8 +80,15 @@ impl IntoResponse for CaptureError { } } -#[derive(Clone, Default, Debug, Serialize, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum DataType { + AnalyticsMain, + AnalyticsHistorical, +} +#[derive(Clone, Debug, Serialize, Eq, PartialEq)] pub struct ProcessedEvent { + #[serde(skip_serializing)] + pub data_type: DataType, pub uuid: Uuid, pub distinct_id: String, pub ip: String, diff --git a/capture/src/config.rs b/capture/src/config.rs index a4bd8f2cfd5cc..07b7f89496d44 100644 --- a/capture/src/config.rs +++ b/capture/src/config.rs @@ -45,7 +45,10 @@ pub struct KafkaConfig { #[envconfig(default = "none")] pub kafka_compression_codec: String, // none, gzip, snappy, lz4, zstd pub kafka_hosts: String, + #[envconfig(default = "events_plugin_ingestion")] pub kafka_topic: String, + #[envconfig(default = "events_plugin_ingestion_historical")] + pub kafka_historical_topic: String, #[envconfig(default = "false")] pub kafka_tls: bool, } diff --git a/capture/src/sinks/kafka.rs b/capture/src/sinks/kafka.rs index 8ffb2d6d7f898..945e581183d37 100644 --- a/capture/src/sinks/kafka.rs +++ b/capture/src/sinks/kafka.rs @@ -11,7 +11,7 @@ use tokio::task::JoinSet; use tracing::log::{debug, error, info}; use tracing::{info_span, instrument, Instrument}; -use crate::api::{CaptureError, ProcessedEvent}; +use crate::api::{CaptureError, DataType, ProcessedEvent}; use crate::config::KafkaConfig; use crate::limiters::overflow::OverflowLimiter; use crate::prometheus::report_dropped_events; @@ -80,8 +80,9 @@ impl rdkafka::ClientContext for KafkaContext { #[derive(Clone)] pub struct KafkaSink { producer: FutureProducer, - topic: String, partition: OverflowLimiter, + main_topic: String, + historical_topic: String, } impl KafkaSink { @@ -128,7 +129,8 @@ impl KafkaSink { Ok(KafkaSink { producer, partition, - topic: config.kafka_topic, + main_topic: config.kafka_topic, + historical_topic: config.kafka_historical_topic, }) } @@ -137,22 +139,27 @@ impl KafkaSink { self.producer.flush(Duration::new(30, 0)) } - async fn kafka_send( - producer: FutureProducer, - topic: String, - event: ProcessedEvent, - limited: bool, - ) -> Result { + async fn kafka_send(&self, event: ProcessedEvent) -> Result { let payload = serde_json::to_string(&event).map_err(|e| { error!("failed to serialize event: {}", e); CaptureError::NonRetryableSinkError })?; - let key = event.key(); - let partition_key = if limited { None } else { Some(key.as_str()) }; + let event_key = event.key(); + let (topic, partition_key): (&str, Option<&str>) = match &event.data_type { + DataType::AnalyticsHistorical => (&self.historical_topic, Some(event_key.as_str())), // We never trigger overflow on historical events + DataType::AnalyticsMain => { + // TODO: deprecate capture-led overflow or move logic in handler + if self.partition.is_limited(&event_key) { + (&self.main_topic, None) // Analytics overflow goes to the main topic without locality + } else { + (&self.main_topic, Some(event_key.as_str())) + } + } + }; - match producer.send_result(FutureRecord { - topic: topic.as_str(), + match self.producer.send_result(FutureRecord { + topic, payload: Some(&payload), partition: None, key: partition_key, @@ -206,9 +213,7 @@ impl KafkaSink { impl Event for KafkaSink { #[instrument(skip_all)] async fn send(&self, event: ProcessedEvent) -> Result<(), CaptureError> { - let limited = self.partition.is_limited(&event.key()); - let ack = - Self::kafka_send(self.producer.clone(), self.topic.clone(), event, limited).await?; + let ack = self.kafka_send(event).await?; histogram!("capture_event_batch_size").record(1.0); Self::process_ack(ack) .instrument(info_span!("ack_wait_one")) @@ -220,12 +225,8 @@ impl Event for KafkaSink { let mut set = JoinSet::new(); let batch_size = events.len(); for event in events { - let producer = self.producer.clone(); - let topic = self.topic.clone(); - let limited = self.partition.is_limited(&event.key()); - // We await kafka_send to get events in the producer queue sequentially - let ack = Self::kafka_send(producer, topic, event, limited).await?; + let ack = self.kafka_send(event).await?; // Then stash the returned DeliveryFuture, waiting concurrently for the write ACKs from brokers. set.spawn(Self::process_ack(ack)); @@ -259,7 +260,7 @@ impl Event for KafkaSink { #[cfg(test)] mod tests { - use crate::api::{CaptureError, ProcessedEvent}; + use crate::api::{CaptureError, DataType, ProcessedEvent}; use crate::config; use crate::limiters::overflow::OverflowLimiter; use crate::sinks::kafka::KafkaSink; @@ -292,6 +293,7 @@ mod tests { kafka_compression_codec: "none".to_string(), kafka_hosts: cluster.bootstrap_servers(), kafka_topic: "events_plugin_ingestion".to_string(), + kafka_historical_topic: "events_plugin_ingestion_historical".to_string(), kafka_tls: false, }; let sink = KafkaSink::new(config, handle, limiter).expect("failed to create sink"); @@ -305,6 +307,7 @@ mod tests { let (cluster, sink) = start_on_mocked_sink().await; let event: ProcessedEvent = ProcessedEvent { + data_type: DataType::AnalyticsMain, uuid: uuid_v7(), distinct_id: "id1".to_string(), ip: "".to_string(), @@ -336,6 +339,7 @@ mod tests { .map(char::from) .collect(); let big_event: ProcessedEvent = ProcessedEvent { + data_type: DataType::AnalyticsMain, uuid: uuid_v7(), distinct_id: "id1".to_string(), ip: "".to_string(), diff --git a/capture/src/v0_endpoint.rs b/capture/src/v0_endpoint.rs index 3862995f2f4f8..3849e29328efa 100644 --- a/capture/src/v0_endpoint.rs +++ b/capture/src/v0_endpoint.rs @@ -15,7 +15,7 @@ use crate::limiters::billing::QuotaResource; use crate::prometheus::report_dropped_events; use crate::v0_request::{Compression, ProcessingContext, RawRequest}; use crate::{ - api::{CaptureError, CaptureResponse, CaptureResponseCode, ProcessedEvent}, + api::{CaptureError, CaptureResponse, CaptureResponseCode, DataType, ProcessedEvent}, router, sinks, utils::uuid_v7, v0_request::{EventFormData, EventQuery, RawEvent}, @@ -39,7 +39,7 @@ use crate::{ content_type, version, compression, - is_historical + historical_migration ) )] #[debug_handler] @@ -106,11 +106,11 @@ pub async fn event( return Err(err); } }; - let is_historical = request.is_historical(); // TODO: use to write to historical topic + let historical_migration = request.historical_migration(); let events = request.events(); // Takes ownership of request tracing::Span::current().record("token", &token); - tracing::Span::current().record("is_historical", is_historical); + tracing::Span::current().record("historical_migration", historical_migration); tracing::Span::current().record("batch_size", events.len()); if events.is_empty() { @@ -125,6 +125,7 @@ pub async fn event( token, now: state.timesource.current_time(), client_ip: ip.to_string(), + historical_migration, }; let billing_limited = state @@ -174,12 +175,18 @@ pub fn process_single_event( return Err(CaptureError::MissingEventName); } + let data_type = match context.historical_migration { + true => DataType::AnalyticsHistorical, + false => DataType::AnalyticsMain, + }; + let data = serde_json::to_string(&event).map_err(|e| { tracing::error!("failed to encode data field: {}", e); CaptureError::NonRetryableSinkError })?; Ok(ProcessedEvent { + data_type, uuid: event.uuid.unwrap_or_else(uuid_v7), distinct_id: event.extract_distinct_id()?, ip: context.client_ip.clone(), diff --git a/capture/src/v0_request.rs b/capture/src/v0_request.rs index a4e1042f49c78..c0d5f36d3577f 100644 --- a/capture/src/v0_request.rs +++ b/capture/src/v0_request.rs @@ -149,7 +149,7 @@ impl RawRequest { Ok(token) } - pub fn is_historical(&self) -> bool { + pub fn historical_migration(&self) -> bool { match self { RawRequest::Batch(req) => req.historical_migration.unwrap_or_default(), _ => false, @@ -232,6 +232,7 @@ pub struct ProcessingContext { pub token: String, pub now: String, pub client_ip: String, + pub historical_migration: bool, } #[cfg(test)] diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index 9ef4e391b5934..d1d313cd5e112 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -4,7 +4,7 @@ use axum::http::StatusCode; use axum_test_helper::TestClient; use base64::engine::general_purpose; use base64::Engine; -use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode, ProcessedEvent}; +use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode, DataType, ProcessedEvent}; use capture::limiters::billing::BillingLimiter; use capture::redis::MockRedisClient; use capture::router::router; @@ -29,6 +29,8 @@ struct RequestDump { now: String, body: String, output: Vec, + #[serde(default)] // default = false + historical_migration: bool, } static REQUESTS_DUMP_FILE_NAME: &str = "tests/requests_dump.jsonl"; @@ -146,14 +148,27 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { for (event_number, (message, expected)) in sink.events().iter().zip(case.output.iter()).enumerate() { + // Ensure the data type matches + if case.historical_migration { + assert_eq!(DataType::AnalyticsHistorical, message.data_type); + } else { + assert_eq!(DataType::AnalyticsMain, message.data_type); + } + // Normalizing the expected event to align with known django->rust inconsistencies let mut expected = expected.clone(); if let Some(value) = expected.get_mut("sent_at") { // Default ISO format is different between python and rust, both are valid // Parse and re-print the value before comparison - let sent_at = - OffsetDateTime::parse(value.as_str().expect("empty"), &Iso8601::DEFAULT)?; - *value = Value::String(sent_at.format(&Rfc3339)?) + let raw_value = value.as_str().expect("sent_at field is not a string"); + if raw_value.is_empty() { + *value = Value::Null + } else { + let sent_at = + OffsetDateTime::parse(value.as_str().expect("empty"), &Iso8601::DEFAULT) + .expect("failed to parse expected sent_at"); + *value = Value::String(sent_at.format(&Rfc3339)?) + } } if let Some(expected_data) = expected.get_mut("data") { // Data is a serialized JSON map. Unmarshall both and compare them, diff --git a/capture/tests/requests_dump.jsonl b/capture/tests/requests_dump.jsonl index 36cf8ade36439..4b59c3bc971b3 100644 --- a/capture/tests/requests_dump.jsonl +++ b/capture/tests/requests_dump.jsonl @@ -1,3 +1,4 @@ +### posthog-js {"path":"/e/?ip=1&_=1694769302325&ver=1.78.5","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-09-15T09:15:02.328551+00:00","body":"ZGF0YT1leUoxZFdsa0lqb2lNREU0WVRrNE1XWXROR0l6TlMwM1l6WTJMVGd5TldVdE9HSXdaV1ZoWlRZMU56RTBJaXdpWlhabGJuUWlPaUlrYVdSbGJuUnBabmtpTENKd2NtOXdaWEowYVdWeklqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR1JsZG1salpWOTBlWEJsSWpvaVJHVnphM1J2Y0NJc0lpUmpkWEp5Wlc1MFgzVnliQ0k2SW1oMGRIQTZMeTlzYjJOaGJHaHZjM1E2T0RBd01DOGlMQ0lrYUc5emRDSTZJbXh2WTJGc2FHOXpkRG80TURBd0lpd2lKSEJoZEdodVlXMWxJam9pTHlJc0lpUmljbTkzYzJWeVgzWmxjbk5wYjI0aU9qRXhOeXdpSkdKeWIzZHpaWEpmYkdGdVozVmhaMlVpT2lKbGJpMVZVeUlzSWlSelkzSmxaVzVmYUdWcFoyaDBJam94TURVeUxDSWtjMk55WldWdVgzZHBaSFJvSWpveE5qSXdMQ0lrZG1sbGQzQnZjblJmYUdWcFoyaDBJam81TVRFc0lpUjJhV1YzY0c5eWRGOTNhV1IwYUNJNk1UVTBPQ3dpSkd4cFlpSTZJbmRsWWlJc0lpUnNhV0pmZG1WeWMybHZiaUk2SWpFdU56Z3VOU0lzSWlScGJuTmxjblJmYVdRaU9pSTBNSEIwTVhWamNHczNORFpwYkdWd0lpd2lKSFJwYldVaU9qRTJPVFEzTmprek1ESXVNekkxTENKa2FYTjBhVzVqZEY5cFpDSTZJbkJYUVd0SlZIbFJNME5PTnpNek1sVnhVWGh1U0Rad00wWldPRlpLWkRkd1dUWTBOMFZrVG10NFYyTWlMQ0lrWkdWMmFXTmxYMmxrSWpvaU1ERTRZVGs0TVdZdE5HSXlaQzAzT0dGbExUazBOMkl0WW1Wa1ltRmhNREpoTUdZMElpd2lKSFZ6WlhKZmFXUWlPaUp3VjBGclNWUjVVVE5EVGpjek16SlZjVkY0YmtnMmNETkdWamhXU21RM2NGazJORGRGWkU1cmVGZGpJaXdpSkhKbFptVnljbVZ5SWpvaUpHUnBjbVZqZENJc0lpUnlaV1psY25KcGJtZGZaRzl0WVdsdUlqb2lKR1JwY21WamRDSXNJaVJoYm05dVgyUnBjM1JwYm1OMFgybGtJam9pTURFNFlUazRNV1l0TkdJeVpDMDNPR0ZsTFRrME4ySXRZbVZrWW1GaE1ESmhNR1kwSWl3aWRHOXJaVzRpT2lKd2FHTmZjV2RWUm5BMWQzb3lRbXBETkZKelJtcE5SM0JSTTFCSFJISnphVFpSTUVOSE1FTlFORTVCWVdNd1NTSXNJaVJ6WlhOemFXOXVYMmxrSWpvaU1ERTRZVGs0TVdZdE5HSXlaUzAzWW1RekxXSmlNekF0TmpZeE4ySm1ORGc0T0RZNUlpd2lKSGRwYm1SdmQxOXBaQ0k2SWpBeE9HRTVPREZtTFRSaU1tVXROMkprTXkxaVlqTXdMVFkyTVRneE1HWmxaRFkxWmlKOUxDSWtjMlYwSWpwN2ZTd2lKSE5sZEY5dmJtTmxJanA3ZlN3aWRHbHRaWE4wWVcxd0lqb2lNakF5TXkwd09TMHhOVlF3T1RveE5Ub3dNaTR6TWpWYUluMCUzRA==","output":[{"uuid":"018a981f-4b35-7c66-825e-8b0eeae65714","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b35-7c66-825e-8b0eeae65714\", \"event\": \"$identify\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"40pt1ucpk746ilep\", \"$time\": 1694769302.325, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$anon_distinct_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"$set\": {}, \"$set_once\": {}, \"timestamp\": \"2023-09-15T09:15:02.325Z\"}","now":"2023-09-15T09:15:02.328551+00:00","sent_at":"2023-09-15T09:15:02.325000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} {"path":"/e/?ip=1&_=1694769302319&ver=1.78.5","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-09-15T09:15:02.322717+00:00","body":"ZGF0YT1leUoxZFdsa0lqb2lNREU0WVRrNE1XWXROR0l5WmkwM09URmhMVGxsWWpFdE5qZGhaVFZpT1dWalpqZzBJaXdpWlhabGJuUWlPaUlrY0dGblpYWnBaWGNpTENKd2NtOXdaWEowYVdWeklqcDdJaVJ2Y3lJNklrMWhZeUJQVXlCWUlpd2lKRzl6WDNabGNuTnBiMjRpT2lJeE1DNHhOUzR3SWl3aUpHSnliM2R6WlhJaU9pSkdhWEpsWm05NElpd2lKR1JsZG1salpWOTBlWEJsSWpvaVJHVnphM1J2Y0NJc0lpUmpkWEp5Wlc1MFgzVnliQ0k2SW1oMGRIQTZMeTlzYjJOaGJHaHZjM1E2T0RBd01DOGlMQ0lrYUc5emRDSTZJbXh2WTJGc2FHOXpkRG80TURBd0lpd2lKSEJoZEdodVlXMWxJam9pTHlJc0lpUmljbTkzYzJWeVgzWmxjbk5wYjI0aU9qRXhOeXdpSkdKeWIzZHpaWEpmYkdGdVozVmhaMlVpT2lKbGJpMVZVeUlzSWlSelkzSmxaVzVmYUdWcFoyaDBJam94TURVeUxDSWtjMk55WldWdVgzZHBaSFJvSWpveE5qSXdMQ0lrZG1sbGQzQnZjblJmYUdWcFoyaDBJam81TVRFc0lpUjJhV1YzY0c5eWRGOTNhV1IwYUNJNk1UVTBPQ3dpSkd4cFlpSTZJbmRsWWlJc0lpUnNhV0pmZG1WeWMybHZiaUk2SWpFdU56Z3VOU0lzSWlScGJuTmxjblJmYVdRaU9pSnpjSEozT0RVM2JHVnVOM0ZxTXpSMklpd2lKSFJwYldVaU9qRTJPVFEzTmprek1ESXVNekU1TENKa2FYTjBhVzVqZEY5cFpDSTZJakF4T0dFNU9ERm1MVFJpTW1RdE56aGhaUzA1TkRkaUxXSmxaR0poWVRBeVlUQm1OQ0lzSWlSa1pYWnBZMlZmYVdRaU9pSXdNVGhoT1RneFppMDBZakprTFRjNFlXVXRPVFEzWWkxaVpXUmlZV0V3TW1Fd1pqUWlMQ0lrY21WbVpYSnlaWElpT2lJa1pHbHlaV04wSWl3aUpISmxabVZ5Y21sdVoxOWtiMjFoYVc0aU9pSWtaR2x5WldOMElpd2lkR2wwYkdVaU9pSlFiM04wU0c5bklpd2lkRzlyWlc0aU9pSndhR05mY1dkVlJuQTFkM295UW1wRE5GSnpSbXBOUjNCUk0xQkhSSEp6YVRaUk1FTkhNRU5RTkU1QllXTXdTU0lzSWlSelpYTnphVzl1WDJsa0lqb2lNREU0WVRrNE1XWXROR0l5WlMwM1ltUXpMV0ppTXpBdE5qWXhOMkptTkRnNE9EWTVJaXdpSkhkcGJtUnZkMTlwWkNJNklqQXhPR0U1T0RGbUxUUmlNbVV0TjJKa015MWlZak13TFRZMk1UZ3hNR1psWkRZMVppSjlMQ0owYVcxbGMzUmhiWEFpT2lJeU1ESXpMVEE1TFRFMVZEQTVPakUxT2pBeUxqTXhPVm9pZlElM0QlM0Q=","output":[{"uuid":"018a981f-4b2f-791a-9eb1-67ae5b9ecf84","distinct_id":"018a981f-4b2d-78ae-947b-bedbaa02a0f4","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b2f-791a-9eb1-67ae5b9ecf84\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"sprw857len7qj34v\", \"$time\": 1694769302.319, \"distinct_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"title\": \"PostHog\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"timestamp\": \"2023-09-15T09:15:02.319Z\"}","now":"2023-09-15T09:15:02.322717+00:00","sent_at":"2023-09-15T09:15:02.319000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} {"path":"/e/?ip=1&_=1694769302319&ver=1.78.5","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-09-15T09:15:02.321230+00:00","body":"ZGF0YT1leUoxZFdsa0lqb2lNREU0WVRrNE1XWXROR0l5WlMwM1ltUXpMV0ppTXpBdE5qWXhOalkxTjJNek9HWmhJaXdpWlhabGJuUWlPaUlrYjNCMFgybHVJaXdpY0hKdmNHVnlkR2xsY3lJNmV5SWtiM01pT2lKTllXTWdUMU1nV0NJc0lpUnZjMTkyWlhKemFXOXVJam9pTVRBdU1UVXVNQ0lzSWlSaWNtOTNjMlZ5SWpvaVJtbHlaV1p2ZUNJc0lpUmtaWFpwWTJWZmRIbHdaU0k2SWtSbGMydDBiM0FpTENJa1kzVnljbVZ1ZEY5MWNtd2lPaUpvZEhSd09pOHZiRzlqWVd4b2IzTjBPamd3TURBdklpd2lKR2h2YzNRaU9pSnNiMk5oYkdodmMzUTZPREF3TUNJc0lpUndZWFJvYm1GdFpTSTZJaThpTENJa1luSnZkM05sY2w5MlpYSnphVzl1SWpveE1UY3NJaVJpY205M2MyVnlYMnhoYm1kMVlXZGxJam9pWlc0dFZWTWlMQ0lrYzJOeVpXVnVYMmhsYVdkb2RDSTZNVEExTWl3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRZeU1Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPVEV4TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFMU5EZ3NJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpjNExqVWlMQ0lrYVc1elpYSjBYMmxrSWpvaU1XVTRaV1p5WkdSbE1HSTBNV2RtYkNJc0lpUjBhVzFsSWpveE5qazBOelk1TXpBeUxqTXhPQ3dpWkdsemRHbHVZM1JmYVdRaU9pSXdNVGhoT1RneFppMDBZakprTFRjNFlXVXRPVFEzWWkxaVpXUmlZV0V3TW1Fd1pqUWlMQ0lrWkdWMmFXTmxYMmxrSWpvaU1ERTRZVGs0TVdZdE5HSXlaQzAzT0dGbExUazBOMkl0WW1Wa1ltRmhNREpoTUdZMElpd2lKSEpsWm1WeWNtVnlJam9pSkdScGNtVmpkQ0lzSWlSeVpXWmxjbkpwYm1kZlpHOXRZV2x1SWpvaUpHUnBjbVZqZENJc0luUnZhMlZ1SWpvaWNHaGpYM0ZuVlVad05YZDZNa0pxUXpSU2MwWnFUVWR3VVROUVIwUnljMmsyVVRCRFJ6QkRVRFJPUVdGak1Fa2lMQ0lrYzJWemMybHZibDlwWkNJNklqQXhPR0U1T0RGbUxUUmlNbVV0TjJKa015MWlZak13TFRZMk1UZGlaalE0T0RnMk9TSXNJaVIzYVc1a2IzZGZhV1FpT2lJd01UaGhPVGd4WmkwMFlqSmxMVGRpWkRNdFltSXpNQzAyTmpFNE1UQm1aV1EyTldZaWZTd2lkR2x0WlhOMFlXMXdJam9pTWpBeU15MHdPUzB4TlZRd09Ub3hOVG93TWk0ek1UaGFJbjAlM0Q=","output":[{"uuid":"018a981f-4b2e-7bd3-bb30-6616657c38fa","distinct_id":"018a981f-4b2d-78ae-947b-bedbaa02a0f4","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b2e-7bd3-bb30-6616657c38fa\", \"event\": \"$opt_in\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"1e8efrdde0b41gfl\", \"$time\": 1694769302.318, \"distinct_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"timestamp\": \"2023-09-15T09:15:02.318Z\"}","now":"2023-09-15T09:15:02.321230+00:00","sent_at":"2023-09-15T09:15:02.319000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} @@ -10,8 +11,11 @@ {"path":"/e/?compression=gzip-js&ip=1&_=1694769323385&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:23.391445+00:00","body":"H4sIAAAAAAAAA+0a7W7bNvBVDCPYrzLV90f+temSdli7ZF3bDUUhUCRlMZZFRaLiOEWAPcsebU+yOzm2ZdlOlq7YklV/bOnueHe8b5P++HlY15IPD4aGGdAwMBMSuokgvkfhKTE54dQVruXFjhGYwydDcSFyDeR7tNaK0ULXpQBwUapClFqKanjwebin4Gv4mrLBT28HvwIaANGFKCupckCYxr7p7hsIj0s1rUQJwCNZikRdIpCLC8lEpGeFAMQLUY21KhDB6rIE8VFdZoBItS4Onj7NQI0sVZU+CAzDeApqVCqvkByBQLdOgIiC6jSnE+Tepr9RZqWpafotcEbzUU1HuErk5N1bXFKxUog8SoUcpSDLNFxrBZ1KrlMAepYBwAsppoUq9ZI4NM02eEHtOgGAMxmDnKmIUQq8tM237wf7LsJlDnrpaO6/sjqvrMC6HBX2lDsK8VriHk0vdHwvtC1z34H9cFlpmbObZcWHZ+NXv8xO7cM3vm1b785PL/OXXmEfvQ/e/8D94jfP8b/nb8aXH1jLNesR48QWJ35ABQFBMYkFjyk1LGokDq6p0XhfIExWERcTFUFsnQkGBktoVglgOCpVXTSBtkQtlBEkCWKLgJ8Nwl3BSWgHhhAupdxBU6pyRHN5RfXclKtVrsvmq2jgcmK5zISwd13uctQkrzTNGXp+a9ANr0GrVkJEYGMaZ4JHsHVwXFRJDosX+lOm5YWIEkEb4iSjI9jNx0+AasOigs4yRTnuFASUgmYT1ACkwsZYJtk4VWBczMoJlVkjDL1DL+AN5S9FVhll41vwKa2iWGaZzEEqxPkSAdnBIOMg6qO6gs99sDivIXhoTrOZlgyUg+BeJuYtNBusKlFhREelAJGzLXy2EGww6Rhxg8cmfg/qjAASLDp7HKoOxM8SigbgCqyJ0dHCNmVvUZEa0zf1SLRqBVJlYgJ06MzPQ1AyuikyVQEmBc0yWsGeADvU4lITLhJaZ8gfIno6BP9TrcsoauhgWZto0JA8GeY6jVgqM0gnkIhvKrnRa65BhKtg8aFKoapgXZszTWGDWO/YDfz6yR0q/gi5lz+vtQYPREzlGgv/ho5bqdbUhIK4pqbVkUwb+27Ru62LzNHgLWHrb4Ro7D60nHXhkLi6ruALsgfKaweb1FlG5oW3g4GMIBL2s2XHoMugRdt+XumxDl3XYh230mEdvtRglwcXcE41JfgISAi/mkgtJmRFdVvIdHyRyQ693V1gdFbU2f0kcHmx7tq3UBqfU9hGlcET2nlXqO2k/KoK3Cn364jbKWadvdNhb9/FPiXz4QOgSSZwnMIviIZsU+KCdoAkzUdD9wX7m/NtenypVMcjGzHUYRErjmk756FTKKFNJOKM1NX49spiXsMWtRrD7mHWSFl0Pnp3VLjTK+v52aHzc3V09vq4OLVPjl9AzfZOjcNj4/DEefOMMuNVM8zd9JuN6Qbm4ZjbJI5tg3ie6ceJEwSBF+Kiqcy5mt65JjCNRHDPTXBWUElSCRwXQ7Nx6MYI7nkO8allkjBwLeJYTJghCwKDee0RfG1YgHkEBo4HNYqvStDfG8Vb9I96FJ9NEq8Yu06dz/K0DrC3bI7irg950c/i/Szez+L3nMXbbHByTdWU3GyTyFyXiojLJq9GC3BTUtbcWYqqgF/9K6s/qrYB0B1tIyY+Z4zQWEAeOWYchB71HYYpvGgbYEZVcrDOAOud4IOphNEzVwN0LJapQVVPJvNRuu8l/3Uvqa9c93xmqTgurSvbG23vJcC6byV9K+lbyf1aybwARphP0QTY2FYAmYRarWDAelkxI16XTYwtMSWkcjSlZY5YpmqssAAHkaqcv+MaRK6YIP8YHDenzEewHzw1iiogAN8B8HE1I2PXbxhYbvkWCeKQkzhJOHNZ4Bjcav+GKWD36Ia+2TyAZuNObR1YM31GZ0Z+LtEp25oNqNd3m77b9N3mft3mwVd1UFFqdOjiEH7w5+9/DE4gVF6qUafmN4d4GzWfWwnUfBGTIAxDyGEWJhYNE9dZq/kP9er4Wy37emzqfKbklTOK87HVRHKn7Nv7Norsy35f9vuy/6/eHXeuR0/mf1cZfDc4nkdyf0+665509c+e2+5JV1T/5J60exUV9tek/TVpf036dY4Y/B3Dpu0T3wQOsQEHDJ7J7CRgie1ZODk87AOGb/VPip4T5GcTd6Snl5ZIzs52TJphP2n2k2Y/af6PDxi6Y+yukwbPuf70FzjrKd2kLgAA","output":[{"uuid":"018a981f-95fe-76af-9f1d-da5e526b4081","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-95fe-76af-9f1d-da5e526b4081\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0rsqs282xgp3wd4o\", \"$time\": 1694769321.47, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"classes\": [\"text-default\", \"grow\"], \"attr__class\": \"text-default grow\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Cohorts\", \"attr__href\": \"/cohorts\"}, {\"tag_name\": \"span\", \"classes\": [\"LemonButton__content\"], \"attr__class\": \"LemonButton__content\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"a\", \"$el_text\": \"Cohorts\", \"classes\": [\"Link\", \"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-stealth\", \"LemonButton--full-width\", \"LemonButton--has-icon\"], \"attr__class\": \"Link LemonButton LemonButton--tertiary LemonButton--status-stealth LemonButton--full-width LemonButton--has-icon\", \"attr__href\": \"/cohorts\", \"attr__data-attr\": \"menu-item-cohorts\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"nth_child\": 13, \"nth_of_type\": 10}, {\"tag_name\": \"ul\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider__content\"], \"attr__class\": \"SideBar__slider__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider\"], \"attr__class\": \"SideBar__slider\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1913}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-9664-7a21-9852-42ce19c880c6","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9664-7a21-9852-42ce19c880c6\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ymf6pk54unynhu8h\", \"$time\": 1694769321.573, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"show-product-intro-existing-products\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1810}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-966b-7dcc-abee-f41b896a74cc","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-966b-7dcc-abee-f41b896a74cc\", \"event\": \"recording viewed with no playtime summary\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"uz55qy2obbr2z36g\", \"$time\": 1694769321.58, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"viewed_time_ms\": 3288, \"play_time_ms\": 0, \"recording_duration_ms\": 0, \"rrweb_warning_count\": 0, \"error_count_during_recording_playback\": 0, \"engagement_score\": 0, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1803}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-966e-7272-8b9d-bffdc5c840d2","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-966e-7272-8b9d-bffdc5c840d2\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"5w3t82ytjay0nqiw\", \"$time\": 1694769321.582, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Cohorts \\u2022 PostHog\"}, \"offset\": 1801}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-9d2f-72eb-8999-bec9f2a9f542","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9d2f-72eb-8999-bec9f2a9f542\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/cohorts\", \"$host\": \"localhost:8000\", \"$pathname\": \"/cohorts\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"tk1tnyoiz4gbnk2t\", \"$time\": 1694769323.311, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"a\", \"$el_text\": \"Persons & Groups\", \"classes\": [\"Link\", \"LemonButton\", \"LemonButton--tertiary\", \"LemonButton--status-stealth\", \"LemonButton--full-width\", \"LemonButton--has-icon\"], \"attr__class\": \"Link LemonButton LemonButton--tertiary LemonButton--status-stealth LemonButton--full-width LemonButton--has-icon\", \"attr__href\": \"/persons\", \"attr__data-attr\": \"menu-item-persons\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"nth_child\": 12, \"nth_of_type\": 9}, {\"tag_name\": \"ul\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider__content\"], \"attr__class\": \"SideBar__slider__content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__slider\"], \"attr__class\": \"SideBar__slider\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 71}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-9d37-712e-b09d-61c3f8cf3624","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9d37-712e-b09d-61c3f8cf3624\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"648njm5gtwx2efjj\", \"$time\": 1694769323.319, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Persons & Groups \\u2022 PostHog\"}, \"offset\": 64}","now":"2023-09-15T09:15:23.391445+00:00","sent_at":"2023-09-15T09:15:23.385000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} {"path":"/e/?compression=gzip-js&ip=1&_=1694769326396&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:26.539412+00:00","body":"H4sIAAAAAAAAA+1b6Y7cNhJ+lUavf5pt3ccsFtjEWTtZbLx2HCe7CAKBIqmWptWiRqT6GMPAPss+Wp4kH9Xn9JV08scz0PyYaRWLVcW6PrKp+enjsG0LPrwZWnZE48jOSMxTn4R2FJHIDmwS2yLOeGA5GQ+Hz4diJioN9rtWNMsBk9O6FFpwjNSNrEWjC6GGNx+HzyT+DL+lbPDv94P/YBiEZCYaVcgKA7Y1sv2RZehpI+dKNCC+KhqRyYUhcjErmEj0shYY+EqoiZa1GWBt08CCpG1KDORa1zcvXpSS0TKXSt9ElmW9gBlKVsqwGyL4HjKYgZrqvKJTI32ff23MzlLbDvfIJa3GLR2bWaIiH96bKYo1QlRJLopxDl225Ts76rzgOgcxcCwQZ4WY17LRW+bYtvfJG27fi0AuixR65iI1WvCw775RGI18Qy8q2KWTLoTMX97Hd9mcRsEsX9Buni7MGu0g9sIgdh135Hn+8yEvlC4qtp5X//jF5Jvvl+/cl29C13U+3L1bVF8Htfvqh+iHf/Kw/m/ghf/gbyaLH9lebB5mjZc6nIQRFQSaUpIKnlJqOdTKPDOnNd77A8oKlXAxlQmS61YweCyjpRIQOG5kW3eZth3aGCNIFqUOQaAtwn3BSexGlhA+pdwzPpHNmFbFPdUrX+5m+T5bzaKRz4njM9sXvu9z36Q3HK1pxUzoT2bd8BOsoq0GsdZtIxL4mKal4AmWjsglquCYvLGfMl3MRJIJ2jFnJR1jNT/9jKF9WlLTZSkpNyuFgkbQcmosgFYsjJUFm+QSzjWVOaVF2Skz0aEzPBn9W5WqpGxyYTynKkmLsiwqaEWibwdQHgwlh7RPWoXfI3ict0geWtFyqQsG45Dd28q8wHMkSgllUjppBFQuT8g5wXAk5MCJRzKOx5+h0QiwmK7zjKPtIH+2VOMALuFNkx270a7jmYSbFJVJ5LerrvFGIqwIDW+bdUbZFhqAlhNh5tc5S+7GH17V/vze+fL2pfedenX77ev6nfv29Vco5+Cd9fK19fKt9+YLyqxvuoayXvJRgQkSptwlaepaJAjsMM28KIqC2Eyawyg5/805kW1lggd+ZkyWWaYECseJPf/T82MkoA7KIPRcfIpslwQ8tmI3tjKfGZUbJHiYrygJ5PxnCAcvfm+rugIyfrfMRw0rhW1Zd7RN/ZyJtuTdgg5gxRsFLhbUw0oPKz2sXAcrD+JiNnGKcKryVNIG4TmMWyNUDdDZufcx4Ywd+u5pnAk8ElKHosjdmNhxFIcuY2EURns4A3/JhsOLA9PYBB/MC50PKjkwETT9aKDa6ZQCo3vweTrgM79vZpbj5WFqTUMadUhxDD79maYHnx58rgefVSdNTEElU4ixnRiVZKza0SB623qTzSlnO9KglpM5bSozymRrWjXoUCmb1bOZYwZ3Qoz8FIFbcVZjrGdqVqbAgNiB+LhQ7dzpKQhI6HKH0CDIiJ2ljLkOjzPL2T891Vi9CUOPWk8ItSZK6ftFfNuWY1nPQ3oGtfojU49aPWpdjVqfPTrAxEKbgG6+JBz88r//D94iVb6W4wPsOH0i8lIXwj2LpB6Ex17scztFKcfuHnasOur6ONTjxxPCD+Vk97f3AQu1JSZjPT2BH/7IcYMeP3r86PHjWvzY9cnNicVGlRqvdZ4f3uimXbtx1XzW7mOt0nJqMvhgugeZiGMu4e9j2Rj97BFrH5PQtE5iUhDiPBM6DloCE8TJ4tTlQRp5qKS988xe4fSQ9IQgqaqXE2RZE+Tav7tP85OQFEQ9JPWQ1EPS9bdAXf/ctLbO9V1jE3vNwnCV3TdlJpgfhzByjU9DVcOlz4dU6yZJONWUmI/bIwJZL5Rsv4NTRHfvB1U6T1helCgeyDdPMltbsdKXaLEw6f/ddiZy8YFuXswgiJVUQQsMG/4LNVV9T1OVJFBCmKy0wQZk38q+jhUTz/BdNupAeVlc0v1bOrcua2R3Wtsn0aag8FuJ+OBwdTPs8nU7Ck5ArFiYEn9osXtgcXe+27O4Lc9anNLmosVm/NjiEu32Oqedj9gF9VvNSi871YQgGZFTMEERVaIBNaTDmJtBEI3sGD8AhsjxHDuuF38dnOJf7TluBhYYLuRvl63Ggr1FYlf1YJHdnuXSIk3NEVrX5xPyiOPPefU9lvglNfLPKTzi+HMrXIs7q+ey+MNEPRKfk9W+A9SsROqv/sBZ5bHGDe/AsHS/Or4/4NCV3A7dGykPXHRYbIciUsnNdfC6ZnM0z65nmO3RocUPBWOXdSAYS3xMu/jQOreJT3HVjvuIGG/skJRRygRzvDgwG6Incynxl9UOCH3rb2vP7+FXv7s/3t1nuWqDOfazy1DPZq0pmhO7+xgq+919v7vvd/dP9sJiQdFSmr+vv84a4R88VncXl+8xgjg+iTZhhGuM2PFIikIiLp7jLLSF45sb0eMXu8xWdpAJzfLHe5/RQ8+10MNEOrsNdDtO51447869R9AT+/0bXj309NBzNfSYMHTvcpmLCNSQabGGdVeV5kb5sweoPazxXP/Tz78Ca5NvR7g4AAA=","output":[{"uuid":"018a981f-9db5-7188-8161-91e9fd602fd7","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-9db5-7188-8161-91e9fd602fd7\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/persons\", \"$host\": \"localhost:8000\", \"$pathname\": \"/persons\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"c5yz9qfwa86vhxab\", \"$time\": 1694769323.445, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"PersonsNode\"}, \"duration\": 102, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2945}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a25d-743f-a813-6d909390f5c9","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a25d-743f-a813-6d909390f5c9\", \"event\": \"$feature_flag_called\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"i100qaub5hceuld4\", \"$time\": 1694769324.637, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$feature_flag\": \"cs-dashboards\", \"$feature_flag_response\": false, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1753}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a264-7a2a-9439-198973cc7878","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a264-7a2a-9439-198973cc7878\", \"event\": \"recording viewed with no playtime summary\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"wzrv024h7b0m7a8c\", \"$time\": 1694769324.645, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"viewed_time_ms\": 1295, \"play_time_ms\": 0, \"recording_duration_ms\": 0, \"rrweb_warning_count\": 0, \"error_count_during_recording_playback\": 0, \"engagement_score\": 0, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1745}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a266-73d2-a66f-1fbcc32d9f02","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a266-73d2-a66f-1fbcc32d9f02\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ksstzx9julgopw7a\", \"$time\": 1694769324.647, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"Persons \\u2022 PostHog\"}, \"offset\": 1743}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a4b3-7b40-b430-9495d1bbed93","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a4b3-7b40-b430-9495d1bbed93\", \"event\": \"person viewed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"s2fzjz6c7t0ekgtm\", \"$time\": 1694769325.236, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"properties_count\": 18, \"has_email\": true, \"has_name\": false, \"custom_properties_count\": 4, \"posthog_properties_count\": 14, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1154}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a676-7722-bece-2f9b3d6b84ce","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a676-7722-bece-2f9b3d6b84ce\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"npyk617r6ht5qzbh\", \"$time\": 1694769325.686, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"attr__data-attr\": \"person-session-recordings-tab\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Recordings\"}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 3, \"nth_of_type\": 3}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 68.19999694824219px; --lemon-tabs-slider-offset: 0px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 704}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a67b-7a6f-9637-bcaacec2496c","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a67b-7a6f-9637-bcaacec2496c\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"fhsu6w5edy7tvvuy\", \"$time\": 1694769325.691, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 699}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-a783-7924-b938-3a789f71e25a","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-a783-7924-b938-3a789f71e25a\", \"event\": \"recording list fetched\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"fcebvj6tugbw47wk\", \"$time\": 1694769325.955, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"load_time\": 145, \"listing_version\": \"3\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 435}","now":"2023-09-15T09:15:26.539412+00:00","sent_at":"2023-09-15T09:15:26.396000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} {"path":"/e/?compression=gzip-js&ip=1&_=1694769329412&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:29.500667+00:00","body":"H4sIAAAAAAAAA+0Ya2/bNvCvGF4/bEPo6EVSyjBgbdq0HdauaZp2Q1EIFElZqmlRkehXiwL7Lftp+yU7yo7jV2KkHYoikz/Y1t3xXryX7u3H7miUi+5R13FDFoVuihhLMaJp6qAoTQUKUyzdNMTUpWn3oCvHsjBAfo+NjOasNKNKArisdCkrk8u6e/Sxe0/DT/cZ453fzzp/ABoA8VhWda4LQLhOz8U9x8KTSk9qWQHwJK9kqqcWKOQ45zI2s1IC4qGsB0aXFsFHVQXi41GlAJEZUx4dHipQQ2W6Nkeh4ziHoEati8OlOUHiCURDJlEU0AQlUiSMOR5z0uA7xk0+lq9Y8nMta6vcS8l1JfKiX1txlinIWRdgESUzWcGGVrvbyFsx+MobrktXwIoV/RHrW86yQOdn9kjNKymLOJN5PwN9XAd7V9BJLkwGQOI5ABznclLqyiyJI9ddBV9S4yAEsMoTkDORiZUCD6tX1KNhD1t4XoBeJp7HiB6LQTRVxThKs5QNLN7k1g8uAWtJ5HukF/pgkMhrkxd8ca58c3/w9NXs1D9+Tn3fO784nRZPSOmfvA5f/ypo+ScJ6CPxfDB9w1fufz0s9/h1ZL33GcLyOhZyqGMI4PeSg8dSpmoJDPuVHpVNNC9Rl8pIlIaJhyAYHCSwFCjyQ0dKzJgIrC911WdF/oGZuS+vTmHM56dYiAXyMHexxBgLLKwmRW1Ywe3V74zs7ifQaiXrYvAxS5QUMZgONxfXuYDDl/rPYztOJWuIU8UgqI/evgPUKiwu2UxpJqylIKCSTA2tBiAVDOMq54NMg3Nt6g9Zrhph9nbYGJ6s/KXIWjE+uAGfsTpOcqUgveISAn2JgBTikNYQ9vGohu8eeFyMIHhYwdTM5ByUg+heZv8NNFusFokdVxJEznbw2UGwxWTDiVs8tvH3oJhJILGV7Z6A0gbxs4RaBwgN3rTRsYJtautl2Wtc3xQ9uVIsLJWSQ6Czl/mxC0rGi0pUl+DSgy4zpopjwQxD9q9NiKZC1aiSitlLbbREhtlQLUwW8yxXkDjA2z7pdKHBXFZs5NSG/sncxM7cxE8Ha6JFPgZeXLEavAl6dX+DlCqgrtZxDHIQ14WxbQOCb65eQwoHr6G7Wa8N4Sq/SfY+mUuPVdqGKjC+ArEqZ6gGf3NwG+CacF1igTIvhJzaDF/XGG9ojDc0HqlrNU5YdaPGFr+tsYJqezunXX9jN4hfSq7NrBGNEMSiLmws1ahWUH8q1LSYow4Neh6Bj4cjSjD8lNOfOrvodZrW0hx1XEp6XhRFIaWRF5GQ2hM3xrPlsm51sGF1sM9qm4OIleX1EbpF8WVuPgObHzDL/zqBWxRfZuGC3bVybmbv72OfofkcAtBUQS7Mf8BZalviJS3UEDltvhq6z3DonG/T7SutN1zk72GRaDG7SuIMimlTROy4tKnxOmOYujYYg4lGD8B6CMqMxxf985MSTz54D94fBy/rk/fPHpen/ovHD6F6k1Pn+LFz/CJ4fp9x52kz1y06z9acIxFNhI+SxHcQIS5N0iAMQxLZQxOoOXqy90zoOqkUBKd2apinGBiAqWedsWPiZ4hGMkQMRKFAklAmJOC+szbxl9Dq7CR5B8b9RcM+aZpZO+lvT/ozf0qJZGR4cZF7Fw7bPekHoF476beTfjvp327S/+abBqiYm2bEmzIoKdUvJURJpvs9roedf/76u/NiPoPN/wPuie6vdxpCd3aa1IfUjkQAWR0wJKgXcQqFxhG2wtzR3VLbbPY0G2UmtFJBmuVB0r8g/o5mQ3uRD8Np22zaZtM2m6+6Vvq6u521ndPmmunb2/RsvhI2r1f/s01PCHsbHyqi5wY+xgGJ9m16fEJ7Dmx6KMYhjvx20dMuetpFz3+66HED2uyct8fvgMCw43soYiRCHBNCscvtlvaOLnoam9qpe9fUTWbKxxNhxnjsupo0I/L21N2ueNqpu5262xXP1orHDXaveBLHCRH1KQOZAchkhIVuwjwvsKl92WMuRrKadUBUqaQdudtOc4c7TdgfTcbD0Jtlk1mWarKj04Q91ydtp2k7TdtpbttpmlJqA24ALQAwj5pCdNpAwbvNUsO+wf24LL+NqlB97BueZkrWXH5/VX57q0XyoLOKWBQPW8N+gFf8znmlOoeds8sXw1VSWyRsixlKCM9haV8S50Kfrqcnj9g8tkmYwqNLAwBTjhNbRVhqGlcgL8hsHItRtcgOl0KB+uZ77Gq79Kj/6d2/ToySO28rAAA=","output":[{"uuid":"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0ovdk9xlnv9fhfak\", \"$time\": 1694769326.837, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"attr__data-attr\": \"persons-related-flags-tab\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Feature flags\"}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 74.26666259765625px; --lemon-tabs-slider-offset: 176.29998779296875px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2572}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-aafa-79e8-abf4-4e68eb64c30f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aafa-79e8-abf4-4e68eb64c30f\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"y3x76ea6mqqi2q0a\", \"$time\": 1694769326.842, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 2567}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af3d-79d4-be4a-d729c776e0da","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af3d-79d4-be4a-d729c776e0da\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ltw7rl4fhi4bgq63\", \"$time\": 1694769327.934, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"\"}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 86.23332214355469px; --lemon-tabs-slider-offset: 367.0999755859375px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1475}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af46-7832-9a69-c566751c6625","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af46-7832-9a69-c566751c6625\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"6yl35wdtv5v11o6c\", \"$time\": 1694769327.942, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 1467}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-b008-737a-bb40-6a6a81ba2244","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-b008-737a-bb40-6a6a81ba2244\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"8guwvm82yhwyhfo6\", \"$time\": 1694769328.136, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"EventsQuery\", \"select\": [\"*\", \"event\", \"person\", \"coalesce(properties.$current_url, properties.$screen_name) -- Url / Screen\", \"properties.$lib\", \"timestamp\"], \"personId\": \"018a981f-4c9a-0000-68ff-417481f7c5b5\", \"after\": \"-24h\"}, \"duration\": 171, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1273}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} -### Compression query param mismatch, to confirm gzip autodetection +### posthog-js with compression query param mismatch, to confirm gzip autodetection {"path":"/e/?compression=gzip-js&ip=1&_=1694769302319&ver=1.78.5","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-09-15T09:15:02.321230+00:00","body":"ZGF0YT1leUoxZFdsa0lqb2lNREU0WVRrNE1XWXROR0l5WlMwM1ltUXpMV0ppTXpBdE5qWXhOalkxTjJNek9HWmhJaXdpWlhabGJuUWlPaUlrYjNCMFgybHVJaXdpY0hKdmNHVnlkR2xsY3lJNmV5SWtiM01pT2lKTllXTWdUMU1nV0NJc0lpUnZjMTkyWlhKemFXOXVJam9pTVRBdU1UVXVNQ0lzSWlSaWNtOTNjMlZ5SWpvaVJtbHlaV1p2ZUNJc0lpUmtaWFpwWTJWZmRIbHdaU0k2SWtSbGMydDBiM0FpTENJa1kzVnljbVZ1ZEY5MWNtd2lPaUpvZEhSd09pOHZiRzlqWVd4b2IzTjBPamd3TURBdklpd2lKR2h2YzNRaU9pSnNiMk5oYkdodmMzUTZPREF3TUNJc0lpUndZWFJvYm1GdFpTSTZJaThpTENJa1luSnZkM05sY2w5MlpYSnphVzl1SWpveE1UY3NJaVJpY205M2MyVnlYMnhoYm1kMVlXZGxJam9pWlc0dFZWTWlMQ0lrYzJOeVpXVnVYMmhsYVdkb2RDSTZNVEExTWl3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRZeU1Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPVEV4TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFMU5EZ3NJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpjNExqVWlMQ0lrYVc1elpYSjBYMmxrSWpvaU1XVTRaV1p5WkdSbE1HSTBNV2RtYkNJc0lpUjBhVzFsSWpveE5qazBOelk1TXpBeUxqTXhPQ3dpWkdsemRHbHVZM1JmYVdRaU9pSXdNVGhoT1RneFppMDBZakprTFRjNFlXVXRPVFEzWWkxaVpXUmlZV0V3TW1Fd1pqUWlMQ0lrWkdWMmFXTmxYMmxrSWpvaU1ERTRZVGs0TVdZdE5HSXlaQzAzT0dGbExUazBOMkl0WW1Wa1ltRmhNREpoTUdZMElpd2lKSEpsWm1WeWNtVnlJam9pSkdScGNtVmpkQ0lzSWlSeVpXWmxjbkpwYm1kZlpHOXRZV2x1SWpvaUpHUnBjbVZqZENJc0luUnZhMlZ1SWpvaWNHaGpYM0ZuVlVad05YZDZNa0pxUXpSU2MwWnFUVWR3VVROUVIwUnljMmsyVVRCRFJ6QkRVRFJPUVdGak1Fa2lMQ0lrYzJWemMybHZibDlwWkNJNklqQXhPR0U1T0RGbUxUUmlNbVV0TjJKa015MWlZak13TFRZMk1UZGlaalE0T0RnMk9TSXNJaVIzYVc1a2IzZGZhV1FpT2lJd01UaGhPVGd4WmkwMFlqSmxMVGRpWkRNdFltSXpNQzAyTmpFNE1UQm1aV1EyTldZaWZTd2lkR2x0WlhOMFlXMXdJam9pTWpBeU15MHdPUzB4TlZRd09Ub3hOVG93TWk0ek1UaGFJbjAlM0Q=","output":[{"uuid":"018a981f-4b2e-7bd3-bb30-6616657c38fa","distinct_id":"018a981f-4b2d-78ae-947b-bedbaa02a0f4","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b2e-7bd3-bb30-6616657c38fa\", \"event\": \"$opt_in\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"1e8efrdde0b41gfl\", \"$time\": 1694769302.318, \"distinct_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"timestamp\": \"2023-09-15T09:15:02.318Z\"}","now":"2023-09-15T09:15:02.321230+00:00","sent_at":"2023-09-15T09:15:02.319000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} {"path":"/e/?ip=1&_=1694769329412&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:29.500667+00:00","body":"H4sIAAAAAAAAA+0Ya2/bNvCvGF4/bEPo6EVSyjBgbdq0HdauaZp2Q1EIFElZqmlRkehXiwL7Lftp+yU7yo7jV2KkHYoikz/Y1t3xXryX7u3H7miUi+5R13FDFoVuihhLMaJp6qAoTQUKUyzdNMTUpWn3oCvHsjBAfo+NjOasNKNKArisdCkrk8u6e/Sxe0/DT/cZ453fzzp/ABoA8VhWda4LQLhOz8U9x8KTSk9qWQHwJK9kqqcWKOQ45zI2s1IC4qGsB0aXFsFHVQXi41GlAJEZUx4dHipQQ2W6Nkeh4ziHoEati8OlOUHiCURDJlEU0AQlUiSMOR5z0uA7xk0+lq9Y8nMta6vcS8l1JfKiX1txlinIWRdgESUzWcGGVrvbyFsx+MobrktXwIoV/RHrW86yQOdn9kjNKymLOJN5PwN9XAd7V9BJLkwGQOI5ABznclLqyiyJI9ddBV9S4yAEsMoTkDORiZUCD6tX1KNhD1t4XoBeJp7HiB6LQTRVxThKs5QNLN7k1g8uAWtJ5HukF/pgkMhrkxd8ca58c3/w9NXs1D9+Tn3fO784nRZPSOmfvA5f/ypo+ScJ6CPxfDB9w1fufz0s9/h1ZL33GcLyOhZyqGMI4PeSg8dSpmoJDPuVHpVNNC9Rl8pIlIaJhyAYHCSwFCjyQ0dKzJgIrC911WdF/oGZuS+vTmHM56dYiAXyMHexxBgLLKwmRW1Ywe3V74zs7ifQaiXrYvAxS5QUMZgONxfXuYDDl/rPYztOJWuIU8UgqI/evgPUKiwu2UxpJqylIKCSTA2tBiAVDOMq54NMg3Nt6g9Zrhph9nbYGJ6s/KXIWjE+uAGfsTpOcqUgveISAn2JgBTikNYQ9vGohu8eeFyMIHhYwdTM5ByUg+heZv8NNFusFokdVxJEznbw2UGwxWTDiVs8tvH3oJhJILGV7Z6A0gbxs4RaBwgN3rTRsYJtautl2Wtc3xQ9uVIsLJWSQ6Czl/mxC0rGi0pUl+DSgy4zpopjwQxD9q9NiKZC1aiSitlLbbREhtlQLUwW8yxXkDjA2z7pdKHBXFZs5NSG/sncxM7cxE8Ha6JFPgZeXLEavAl6dX+DlCqgrtZxDHIQ14WxbQOCb65eQwoHr6G7Wa8N4Sq/SfY+mUuPVdqGKjC+ArEqZ6gGf3NwG+CacF1igTIvhJzaDF/XGG9ojDc0HqlrNU5YdaPGFr+tsYJqezunXX9jN4hfSq7NrBGNEMSiLmws1ahWUH8q1LSYow4Neh6Bj4cjSjD8lNOfOrvodZrW0hx1XEp6XhRFIaWRF5GQ2hM3xrPlsm51sGF1sM9qm4OIleX1EbpF8WVuPgObHzDL/zqBWxRfZuGC3bVybmbv72OfofkcAtBUQS7Mf8BZalviJS3UEDltvhq6z3DonG/T7SutN1zk72GRaDG7SuIMimlTROy4tKnxOmOYujYYg4lGD8B6CMqMxxf985MSTz54D94fBy/rk/fPHpen/ovHD6F6k1Pn+LFz/CJ4fp9x52kz1y06z9acIxFNhI+SxHcQIS5N0iAMQxLZQxOoOXqy90zoOqkUBKd2apinGBiAqWedsWPiZ4hGMkQMRKFAklAmJOC+szbxl9Dq7CR5B8b9RcM+aZpZO+lvT/ozf0qJZGR4cZF7Fw7bPekHoF476beTfjvp327S/+abBqiYm2bEmzIoKdUvJURJpvs9roedf/76u/NiPoPN/wPuie6vdxpCd3aa1IfUjkQAWR0wJKgXcQqFxhG2wtzR3VLbbPY0G2UmtFJBmuVB0r8g/o5mQ3uRD8Np22zaZtM2m6+6Vvq6u521ndPmmunb2/RsvhI2r1f/s01PCHsbHyqi5wY+xgGJ9m16fEJ7Dmx6KMYhjvx20dMuetpFz3+66HED2uyct8fvgMCw43soYiRCHBNCscvtlvaOLnoam9qpe9fUTWbKxxNhxnjsupo0I/L21N2ueNqpu5262xXP1orHDXaveBLHCRH1KQOZAchkhIVuwjwvsKl92WMuRrKadUBUqaQdudtOc4c7TdgfTcbD0Jtlk1mWarKj04Q91ydtp2k7TdtpbttpmlJqA24ALQAwj5pCdNpAwbvNUsO+wf24LL+NqlB97BueZkrWXH5/VX57q0XyoLOKWBQPW8N+gFf8znmlOoeds8sXw1VSWyRsixlKCM9haV8S50Kfrqcnj9g8tkmYwqNLAwBTjhNbRVhqGlcgL8hsHItRtcgOl0KB+uZ77Gq79Kj/6d2/ToySO28rAAA=","output":[{"uuid":"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0ovdk9xlnv9fhfak\", \"$time\": 1694769326.837, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"attr__data-attr\": \"persons-related-flags-tab\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Feature flags\"}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 74.26666259765625px; --lemon-tabs-slider-offset: 176.29998779296875px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2572}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-aafa-79e8-abf4-4e68eb64c30f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aafa-79e8-abf4-4e68eb64c30f\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"y3x76ea6mqqi2q0a\", \"$time\": 1694769326.842, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 2567}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af3d-79d4-be4a-d729c776e0da","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af3d-79d4-be4a-d729c776e0da\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ltw7rl4fhi4bgq63\", \"$time\": 1694769327.934, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"\"}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 86.23332214355469px; --lemon-tabs-slider-offset: 367.0999755859375px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1475}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af46-7832-9a69-c566751c6625","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af46-7832-9a69-c566751c6625\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"6yl35wdtv5v11o6c\", \"$time\": 1694769327.942, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 1467}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-b008-737a-bb40-6a6a81ba2244","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-b008-737a-bb40-6a6a81ba2244\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"8guwvm82yhwyhfo6\", \"$time\": 1694769328.136, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"EventsQuery\", \"select\": [\"*\", \"event\", \"person\", \"coalesce(properties.$current_url, properties.$screen_name) -- Url / Screen\", \"properties.$lib\", \"timestamp\"], \"personId\": \"018a981f-4c9a-0000-68ff-417481f7c5b5\", \"after\": \"-24h\"}, \"duration\": 171, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1273}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} ### nodejs, default params -{"path":"/batch/","method":"POST","content_encoding":"","content_type":"application/json","ip":"127.0.0.1","now":"2024-04-17T14:40:56.918578+00:00","body":"eyJhcGlfa2V5IjoicGhjX05WWk95WWI4aFgyR0RzODBtUWQwNzJPVDBXVXJBYmRRb200WGVDVjVOZmgiLCJiYXRjaCI6W3siZGlzdGluY3RfaWQiOiJpZDEiLCJldmVudCI6InRoaXMgZXZlbnQiLCJwcm9wZXJ0aWVzIjp7IiRsaWIiOiJwb3N0aG9nLW5vZGUiLCIkbGliX3ZlcnNpb24iOiI0LjAuMCIsIiRnZW9pcF9kaXNhYmxlIjp0cnVlfSwidHlwZSI6ImNhcHR1cmUiLCJsaWJyYXJ5IjoicG9zdGhvZy1ub2RlIiwibGlicmFyeV92ZXJzaW9uIjoiNC4wLjAiLCJ0aW1lc3RhbXAiOiIyMDI0LTA0LTE3VDE0OjQwOjU2LjkwMFoiLCJ1dWlkIjoiMDE4ZWVjODAtZjA0NC03YzJkLWI5NjYtMzVlZmM4ZWQ2MWQ1In0seyJkaXN0aW5jdF9pZCI6ImRpc3RpbmN0X2lkX29mX3RoZV91c2VyIiwiZXZlbnQiOiJ1c2VyIHNpZ25lZCB1cCIsInByb3BlcnRpZXMiOnsibG9naW5fdHlwZSI6ImVtYWlsIiwiaXNfZnJlZV90cmlhbCI6dHJ1ZSwiJGxpYiI6InBvc3Rob2ctbm9kZSIsIiRsaWJfdmVyc2lvbiI6IjQuMC4wIiwiJGdlb2lwX2Rpc2FibGUiOnRydWV9LCJ0eXBlIjoiY2FwdHVyZSIsImxpYnJhcnkiOiJwb3N0aG9nLW5vZGUiLCJsaWJyYXJ5X3ZlcnNpb24iOiI0LjAuMCIsInRpbWVzdGFtcCI6IjIwMjQtMDQtMTdUMTQ6NDA6NTYuOTAwWiIsInV1aWQiOiIwMThlZWM4MC1mMDQ0LTdjMmQtYjk2Ni0zNWYwNTA1OWYzNzUifV0sInNlbnRfYXQiOiIyMDI0LTA0LTE3VDE0OjQwOjU2LjkwMFoifQ==","output":[{"uuid":"018eec80-f044-7c2d-b966-35efc8ed61d5","distinct_id":"id1","ip":"127.0.0.1","site_url":"http://127.0.0.1:8000","data":"{\"distinct_id\": \"id1\", \"event\": \"this event\", \"properties\": {\"$lib\": \"posthog-node\", \"$lib_version\": \"4.0.0\", \"$geoip_disable\": true}, \"type\": \"capture\", \"library\": \"posthog-node\", \"library_version\": \"4.0.0\", \"timestamp\": \"2024-04-17T14:40:56.900Z\", \"uuid\": \"018eec80-f044-7c2d-b966-35efc8ed61d5\"}","now":"2024-04-17T14:40:56.918578+00:00","sent_at":"2024-04-17T14:40:56.900000+00:00","token":"phc_NVZOyYb8hX2GDs80mQd072OT0WUrAbdQom4XeCV5Nfh"},{"uuid":"018eec80-f044-7c2d-b966-35f05059f375","distinct_id":"distinct_id_of_the_user","ip":"127.0.0.1","site_url":"http://127.0.0.1:8000","data":"{\"distinct_id\": \"distinct_id_of_the_user\", \"event\": \"user signed up\", \"properties\": {\"login_type\": \"email\", \"is_free_trial\": true, \"$lib\": \"posthog-node\", \"$lib_version\": \"4.0.0\", \"$geoip_disable\": true}, \"type\": \"capture\", \"library\": \"posthog-node\", \"library_version\": \"4.0.0\", \"timestamp\": \"2024-04-17T14:40:56.900Z\", \"uuid\": \"018eec80-f044-7c2d-b966-35f05059f375\"}","now":"2024-04-17T14:40:56.918578+00:00","sent_at":"2024-04-17T14:40:56.900000+00:00","token":"phc_NVZOyYb8hX2GDs80mQd072OT0WUrAbdQom4XeCV5Nfh"}]} \ No newline at end of file +{"path":"/batch/","method":"POST","content_encoding":"","content_type":"application/json","ip":"127.0.0.1","now":"2024-04-17T14:40:56.918578+00:00","body":"eyJhcGlfa2V5IjoicGhjX05WWk95WWI4aFgyR0RzODBtUWQwNzJPVDBXVXJBYmRRb200WGVDVjVOZmgiLCJiYXRjaCI6W3siZGlzdGluY3RfaWQiOiJpZDEiLCJldmVudCI6InRoaXMgZXZlbnQiLCJwcm9wZXJ0aWVzIjp7IiRsaWIiOiJwb3N0aG9nLW5vZGUiLCIkbGliX3ZlcnNpb24iOiI0LjAuMCIsIiRnZW9pcF9kaXNhYmxlIjp0cnVlfSwidHlwZSI6ImNhcHR1cmUiLCJsaWJyYXJ5IjoicG9zdGhvZy1ub2RlIiwibGlicmFyeV92ZXJzaW9uIjoiNC4wLjAiLCJ0aW1lc3RhbXAiOiIyMDI0LTA0LTE3VDE0OjQwOjU2LjkwMFoiLCJ1dWlkIjoiMDE4ZWVjODAtZjA0NC03YzJkLWI5NjYtMzVlZmM4ZWQ2MWQ1In0seyJkaXN0aW5jdF9pZCI6ImRpc3RpbmN0X2lkX29mX3RoZV91c2VyIiwiZXZlbnQiOiJ1c2VyIHNpZ25lZCB1cCIsInByb3BlcnRpZXMiOnsibG9naW5fdHlwZSI6ImVtYWlsIiwiaXNfZnJlZV90cmlhbCI6dHJ1ZSwiJGxpYiI6InBvc3Rob2ctbm9kZSIsIiRsaWJfdmVyc2lvbiI6IjQuMC4wIiwiJGdlb2lwX2Rpc2FibGUiOnRydWV9LCJ0eXBlIjoiY2FwdHVyZSIsImxpYnJhcnkiOiJwb3N0aG9nLW5vZGUiLCJsaWJyYXJ5X3ZlcnNpb24iOiI0LjAuMCIsInRpbWVzdGFtcCI6IjIwMjQtMDQtMTdUMTQ6NDA6NTYuOTAwWiIsInV1aWQiOiIwMThlZWM4MC1mMDQ0LTdjMmQtYjk2Ni0zNWYwNTA1OWYzNzUifV0sInNlbnRfYXQiOiIyMDI0LTA0LTE3VDE0OjQwOjU2LjkwMFoifQ==","output":[{"uuid":"018eec80-f044-7c2d-b966-35efc8ed61d5","distinct_id":"id1","ip":"127.0.0.1","site_url":"http://127.0.0.1:8000","data":"{\"distinct_id\": \"id1\", \"event\": \"this event\", \"properties\": {\"$lib\": \"posthog-node\", \"$lib_version\": \"4.0.0\", \"$geoip_disable\": true}, \"type\": \"capture\", \"library\": \"posthog-node\", \"library_version\": \"4.0.0\", \"timestamp\": \"2024-04-17T14:40:56.900Z\", \"uuid\": \"018eec80-f044-7c2d-b966-35efc8ed61d5\"}","now":"2024-04-17T14:40:56.918578+00:00","sent_at":"2024-04-17T14:40:56.900000+00:00","token":"phc_NVZOyYb8hX2GDs80mQd072OT0WUrAbdQom4XeCV5Nfh"},{"uuid":"018eec80-f044-7c2d-b966-35f05059f375","distinct_id":"distinct_id_of_the_user","ip":"127.0.0.1","site_url":"http://127.0.0.1:8000","data":"{\"distinct_id\": \"distinct_id_of_the_user\", \"event\": \"user signed up\", \"properties\": {\"login_type\": \"email\", \"is_free_trial\": true, \"$lib\": \"posthog-node\", \"$lib_version\": \"4.0.0\", \"$geoip_disable\": true}, \"type\": \"capture\", \"library\": \"posthog-node\", \"library_version\": \"4.0.0\", \"timestamp\": \"2024-04-17T14:40:56.900Z\", \"uuid\": \"018eec80-f044-7c2d-b966-35f05059f375\"}","now":"2024-04-17T14:40:56.918578+00:00","sent_at":"2024-04-17T14:40:56.900000+00:00","token":"phc_NVZOyYb8hX2GDs80mQd072OT0WUrAbdQom4XeCV5Nfh"}]} +### batch from temporal batch export +{"historical_migration":true,"path":"/batch","method":"POST","content_encoding":"","content_type":"application/json","ip":"127.0.0.1","now":"2024-04-29T14:49:29.553700+00:00","body":"eyJhcGlfa2V5IjogImFiY2RlZjEyMzQ1NiIsImhpc3RvcmljYWxfbWlncmF0aW9uIjp0cnVlLCJiYXRjaCI6IFt7InV1aWQiOiJmNGI3NmJhNy1lMGZmLTRkYTctOTBkNS02ZTkwZjY4ZWRjMmUiLCJkaXN0aW5jdF9pZCI6ImJhYTkyOGNiLTU4YjUtNGY0Zi04ZWQ0LWJkYzMzMzQ4YmRjYyIsInRpbWVzdGFtcCI6IjIwMjMtMDQtMjRUMDY6MzQ6MDArMDA6MDAiLCJldmVudCI6InRlc3Qtbm8tcHJvcC0wIiwicHJvcGVydGllcyI6eyIkZ2VvaXBfZGlzYWJsZSI6dHJ1ZX19LHsidXVpZCI6ImFhNDZlODNmLWJmNzctNDJjNy04OGExLTczMGE5MmFlMjZhNyIsImRpc3RpbmN0X2lkIjoiMzhlMWM3MTAtYTAzNC00MjAzLTg2NzUtZTVkNDgxY2FjOTIzIiwidGltZXN0YW1wIjoiMjAyMy0wNC0yMVQyMzo1MzowMCswMDowMCIsImV2ZW50IjoidGVzdC0xIiwicHJvcGVydGllcyI6eyIkYnJvd3NlciI6IkNocm9tZSIsIiRvcyI6Ik1hYyBPUyBYIiwiJGdlb2lwX2Rpc2FibGUiOnRydWV9fSx7InV1aWQiOiJiNWY1ZGFhOS00YmRlLTQ2ZDAtYjk2YS1kM2VkZmI4N2ViODEiLCJkaXN0aW5jdF9pZCI6ImZhMWI1NjFhLTAxNGMtNDMzNS1hM2VmLTk4YzA2ZTRmZDdlOCIsInRpbWVzdGFtcCI6IjIwMjMtMDQtMjJUMTU6MDY6MDArMDA6MDAiLCJldmVudCI6InRlc3QtMCIsInByb3BlcnRpZXMiOnsiJGJyb3dzZXIiOiJDaHJvbWUiLCIkb3MiOiJNYWMgT1MgWCIsIiRnZW9pcF9kaXNhYmxlIjp0cnVlfX0seyJ1dWlkIjoiYjQyOTM1ZTUtZDIwNi00ODRiLTk1OTEtODcxZjI2MmYzMGRkIiwiZGlzdGluY3RfaWQiOiIxMDk5OGJmZS03M2Y4LTQ3MDktYTBkNi0yMDI3MTk1MjA2MDYiLCJ0aW1lc3RhbXAiOiIyMDIzLTA0LTI0VDEzOjQ2OjAwKzAwOjAwIiwiZXZlbnQiOiJ0ZXN0LTIiLCJwcm9wZXJ0aWVzIjp7IiRicm93c2VyIjoiQ2hyb21lIiwiJG9zIjoiTWFjIE9TIFgiLCIkZ2VvaXBfZGlzYWJsZSI6dHJ1ZX19XX0=","output":[{"uuid":"f4b76ba7-e0ff-4da7-90d5-6e90f68edc2e","distinct_id":"baa928cb-58b5-4f4f-8ed4-bdc33348bdcc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"f4b76ba7-e0ff-4da7-90d5-6e90f68edc2e\", \"distinct_id\": \"baa928cb-58b5-4f4f-8ed4-bdc33348bdcc\", \"timestamp\": \"2023-04-24T06:34:00+00:00\", \"event\": \"test-no-prop-0\", \"properties\": {\"$geoip_disable\": true}}","now":"2024-04-29T14:49:29.553700+00:00","sent_at":"","token":"abcdef123456"},{"uuid":"aa46e83f-bf77-42c7-88a1-730a92ae26a7","distinct_id":"38e1c710-a034-4203-8675-e5d481cac923","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"aa46e83f-bf77-42c7-88a1-730a92ae26a7\", \"distinct_id\": \"38e1c710-a034-4203-8675-e5d481cac923\", \"timestamp\": \"2023-04-21T23:53:00+00:00\", \"event\": \"test-1\", \"properties\": {\"$browser\": \"Chrome\", \"$os\": \"Mac OS X\", \"$geoip_disable\": true}}","now":"2024-04-29T14:49:29.553700+00:00","sent_at":"","token":"abcdef123456"},{"uuid":"b5f5daa9-4bde-46d0-b96a-d3edfb87eb81","distinct_id":"fa1b561a-014c-4335-a3ef-98c06e4fd7e8","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"b5f5daa9-4bde-46d0-b96a-d3edfb87eb81\", \"distinct_id\": \"fa1b561a-014c-4335-a3ef-98c06e4fd7e8\", \"timestamp\": \"2023-04-22T15:06:00+00:00\", \"event\": \"test-0\", \"properties\": {\"$browser\": \"Chrome\", \"$os\": \"Mac OS X\", \"$geoip_disable\": true}}","now":"2024-04-29T14:49:29.553700+00:00","sent_at":"","token":"abcdef123456"},{"uuid":"b42935e5-d206-484b-9591-871f262f30dd","distinct_id":"10998bfe-73f8-4709-a0d6-202719520606","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"b42935e5-d206-484b-9591-871f262f30dd\", \"distinct_id\": \"10998bfe-73f8-4709-a0d6-202719520606\", \"timestamp\": \"2023-04-24T13:46:00+00:00\", \"event\": \"test-2\", \"properties\": {\"$browser\": \"Chrome\", \"$os\": \"Mac OS X\", \"$geoip_disable\": true}}","now":"2024-04-29T14:49:29.553700+00:00","sent_at":"","token":"abcdef123456"}]} +### end \ No newline at end of file From db332e59f2c9cb75b2fb91d1d842df3203cab6a4 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Thu, 2 May 2024 12:04:53 +0200 Subject: [PATCH 229/247] fix capture CI (#32) --- Cargo.lock | 2 +- capture/Cargo.toml | 2 +- capture/tests/django_compat.rs | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82c787aded2cc..9d7fc97883afd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -267,7 +267,7 @@ dependencies = [ [[package]] name = "axum-test-helper" version = "0.4.0" -source = "git+https://github.com/orphan-rs/axum-test-helper.git#8ca0aedaad5a6bdf351c34d5b80593ae1b7d2f3f" +source = "git+https://github.com/posthog/axum-test-helper.git#002d45d8bbbac04e6a474e9a850b7f023a87d32f" dependencies = [ "axum 0.7.5", "bytes", diff --git a/capture/Cargo.toml b/capture/Cargo.toml index 4e35d10be7b85..ae5ad9a3bd127 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -38,4 +38,4 @@ uuid = { workspace = true } [dev-dependencies] assert-json-diff = { workspace = true } -axum-test-helper = { git = "https://github.com/orphan-rs/axum-test-helper.git" } # TODO: remove, directly use reqwest like capture-server tests +axum-test-helper = { git = "https://github.com/posthog/axum-test-helper.git" } # TODO: remove, directly use reqwest like capture-server tests diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index d1d313cd5e112..87b0a1b269256 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -201,6 +201,11 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { if let Some(object) = expected.as_object_mut() { // site_url is unused in the pipeline now, let's drop it object.remove("site_url"); + + // Remove sent_at field if empty: Rust will skip marshalling it + if let Some(None) = object.get("sent_at").map(|v| v.as_str()) { + object.remove("sent_at"); + } } let match_config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict); @@ -209,7 +214,9 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { { println!( "record mismatch at line {}, event {}: {}", - line_number, event_number, e + line_number + 1, + event_number, + e ); mismatches += 1; } From c6d5c67d901248d0b8019973201ada1db704c587 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Thu, 2 May 2024 12:11:41 +0200 Subject: [PATCH 230/247] ci: move cargo check to linting job (#33) --- .github/workflows/rust.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ed5acb2ed6a90..a525f3bf942c1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -67,9 +67,6 @@ jobs: - name: Run cargo test run: cargo test --all-features - - name: Run cargo check - run: cargo check --all-features - linting: runs-on: depot-ubuntu-22.04-4 @@ -89,12 +86,15 @@ jobs: ~/.cargo/git target key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }} - + + - name: Check format + run: cargo fmt -- --check + - name: Run clippy run: cargo clippy -- -D warnings - - name: Check format - run: cargo fmt -- --check + - name: Run cargo check + run: cargo check --all-features shear: runs-on: depot-ubuntu-22.04-4 From f7e02cc92c68de49227e6b5217a1d120a70dfc5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 3 May 2024 10:34:51 +0200 Subject: [PATCH 231/247] feat: Track response in error details (#29) Co-authored-by: Brett Hoerner --- Cargo.lock | 2 + Cargo.toml | 2 +- hook-worker/src/error.rs | 111 ++++++++++++++++++++++-- hook-worker/src/lib.rs | 1 + hook-worker/src/util.rs | 35 ++++++++ hook-worker/src/worker.rs | 175 +++++++++++++++++++++++++++----------- 6 files changed, 270 insertions(+), 56 deletions(-) create mode 100644 hook-worker/src/util.rs diff --git a/Cargo.lock b/Cargo.lock index 9d7fc97883afd..7cc582ee6696c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2343,10 +2343,12 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "winreg 0.52.0", ] diff --git a/Cargo.toml b/Cargo.toml index d34cd0ae39231..d1f75274aa629 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ metrics = "0.22.0" metrics-exporter-prometheus = "0.14.0" rand = "0.8.5" rdkafka = { version = "0.36.0", features = ["cmake-build", "ssl", "tracing"] } -reqwest = { version = "0.12.3" } +reqwest = { version = "0.12.3", features = ["stream"] } serde = { version = "1.0", features = ["derive"] } serde_derive = { version = "1.0" } serde_json = { version = "1.0" } diff --git a/hook-worker/src/error.rs b/hook-worker/src/error.rs index 914ffb1b2e2ee..48468bc65f544 100644 --- a/hook-worker/src/error.rs +++ b/hook-worker/src/error.rs @@ -1,24 +1,125 @@ +use std::fmt; use std::time; -use hook_common::pgqueue; +use hook_common::{pgqueue, webhook::WebhookJobError}; use thiserror::Error; -/// Enumeration of errors related to webhook job processing in the WebhookWorker. +/// Enumeration of error classes handled by `WebhookWorker`. #[derive(Error, Debug)] pub enum WebhookError { + #[error(transparent)] + Parse(#[from] WebhookParseError), + #[error(transparent)] + Request(#[from] WebhookRequestError), +} + +/// Enumeration of parsing errors that can occur as `WebhookWorker` sets up a webhook. +#[derive(Error, Debug)] +pub enum WebhookParseError { #[error("{0} is not a valid HttpMethod")] ParseHttpMethodError(String), #[error("error parsing webhook headers")] ParseHeadersError(http::Error), #[error("error parsing webhook url")] ParseUrlError(url::ParseError), - #[error("a webhook could not be delivered but it could be retried later: {error}")] +} + +/// Enumeration of request errors that can occur as `WebhookWorker` sends a request. +#[derive(Error, Debug)] +pub enum WebhookRequestError { RetryableRequestError { error: reqwest::Error, + response: Option, retry_after: Option, }, - #[error("a webhook could not be delivered and it cannot be retried further: {0}")] - NonRetryableRetryableRequestError(reqwest::Error), + NonRetryableRetryableRequestError { + error: reqwest::Error, + response: Option, + }, +} + +/// Enumeration of errors that can occur while handling a `reqwest::Response`. +/// Currently, not consumed anywhere. Grouped here to support a common error type for +/// `utils::first_n_bytes_of_response`. +#[derive(Error, Debug)] +pub enum WebhookResponseError { + #[error("failed to parse a response as UTF8")] + ParseUTF8StringError(#[from] std::str::Utf8Error), + #[error("error while iterating over response body chunks")] + StreamIterationError(#[from] reqwest::Error), + #[error("attempted to slice a chunk of length {0} with an out of bounds index of {1}")] + ChunkOutOfBoundsError(usize, usize), +} + +/// Implement display of `WebhookRequestError` by appending to the underlying `reqwest::Error` +/// any response message if available. +impl fmt::Display for WebhookRequestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + WebhookRequestError::RetryableRequestError { + error, response, .. + } + | WebhookRequestError::NonRetryableRetryableRequestError { error, response } => { + let response_message = match response { + Some(m) => m.to_string(), + None => "No response from the server".to_string(), + }; + writeln!(f, "{}", error)?; + write!(f, "{}", response_message)?; + + Ok(()) + } + } + } +} + +/// Implementation of `WebhookRequestError` designed to further describe the error. +/// In particular, we pass some calls to underyling `reqwest::Error` to provide more details. +impl WebhookRequestError { + pub fn is_timeout(&self) -> bool { + match self { + WebhookRequestError::RetryableRequestError { error, .. } + | WebhookRequestError::NonRetryableRetryableRequestError { error, .. } => { + error.is_timeout() + } + } + } + + pub fn is_status(&self) -> bool { + match self { + WebhookRequestError::RetryableRequestError { error, .. } + | WebhookRequestError::NonRetryableRetryableRequestError { error, .. } => { + error.is_status() + } + } + } + + pub fn status(&self) -> Option { + match self { + WebhookRequestError::RetryableRequestError { error, .. } + | WebhookRequestError::NonRetryableRetryableRequestError { error, .. } => { + error.status() + } + } + } +} + +impl From<&WebhookRequestError> for WebhookJobError { + fn from(error: &WebhookRequestError) -> Self { + if error.is_timeout() { + WebhookJobError::new_timeout(&error.to_string()) + } else if error.is_status() { + WebhookJobError::new_http_status( + error.status().expect("status code is defined").into(), + &error.to_string(), + ) + } else { + // Catch all other errors as `app_metrics::ErrorType::Connection` errors. + // Not all of `reqwest::Error` may strictly be connection errors, so our supported error types may need an extension + // depending on how strict error reporting has to be. + WebhookJobError::new_connection(&error.to_string()) + } + } } /// Enumeration of errors related to initialization and consumption of webhook jobs. diff --git a/hook-worker/src/lib.rs b/hook-worker/src/lib.rs index 22823c9a7e5cd..8488d15b20a36 100644 --- a/hook-worker/src/lib.rs +++ b/hook-worker/src/lib.rs @@ -1,3 +1,4 @@ pub mod config; pub mod error; +pub mod util; pub mod worker; diff --git a/hook-worker/src/util.rs b/hook-worker/src/util.rs new file mode 100644 index 0000000000000..00c5432168645 --- /dev/null +++ b/hook-worker/src/util.rs @@ -0,0 +1,35 @@ +use crate::error::WebhookResponseError; +use futures::StreamExt; +use reqwest::Response; + +pub async fn first_n_bytes_of_response( + response: Response, + n: usize, +) -> Result { + let mut body = response.bytes_stream(); + let mut buffer = String::with_capacity(n); + + while let Some(chunk) = body.next().await { + if buffer.len() >= n { + break; + } + + let chunk = chunk?; + let chunk_str = std::str::from_utf8(&chunk)?; + let upper_bound = std::cmp::min(n - buffer.len(), chunk_str.len()); + + if let Some(partial_chunk_str) = chunk_str.get(0..upper_bound) { + buffer.push_str(partial_chunk_str); + } else { + // For whatever reason we are out of bounds. We should never land here + // given the `std::cmp::min` usage, but I am being extra careful by not + // using a slice index that would panic instead. + return Err(WebhookResponseError::ChunkOutOfBoundsError( + chunk_str.len(), + upper_bound, + )); + } + } + + Ok(buffer) +} diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 5965d26a0801d..824f1e2f23a87 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -18,7 +18,8 @@ use reqwest::header; use tokio::sync; use tracing::error; -use crate::error::{WebhookError, WorkerError}; +use crate::error::{WebhookError, WebhookParseError, WebhookRequestError, WorkerError}; +use crate::util::first_n_bytes_of_response; /// A WebhookJob is any `PgQueueJob` with `WebhookJobParameters` and `WebhookJobMetadata`. trait WebhookJob: PgQueueJob + std::marker::Send { @@ -259,7 +260,7 @@ async fn process_webhook_job( Ok(()) } - Err(WebhookError::ParseHeadersError(e)) => { + Err(WebhookError::Parse(WebhookParseError::ParseHeadersError(e))) => { webhook_job .fail(WebhookJobError::new_parse(&e.to_string())) .await @@ -272,7 +273,7 @@ async fn process_webhook_job( Ok(()) } - Err(WebhookError::ParseHttpMethodError(e)) => { + Err(WebhookError::Parse(WebhookParseError::ParseHttpMethodError(e))) => { webhook_job .fail(WebhookJobError::new_parse(&e)) .await @@ -285,7 +286,7 @@ async fn process_webhook_job( Ok(()) } - Err(WebhookError::ParseUrlError(e)) => { + Err(WebhookError::Parse(WebhookParseError::ParseUrlError(e))) => { webhook_job .fail(WebhookJobError::new_parse(&e.to_string())) .await @@ -298,26 +299,53 @@ async fn process_webhook_job( Ok(()) } - Err(WebhookError::RetryableRequestError { error, retry_after }) => { - let retry_interval = - retry_policy.retry_interval(webhook_job.attempt() as u32, retry_after); - let current_queue = webhook_job.queue(); - let retry_queue = retry_policy.retry_queue(¤t_queue); - - match webhook_job - .retry(WebhookJobError::from(&error), retry_interval, retry_queue) - .await - { - Ok(_) => { - metrics::counter!("webhook_jobs_retried", &labels).increment(1); - - Ok(()) + Err(WebhookError::Request(request_error)) => { + let webhook_job_error = WebhookJobError::from(&request_error); + + match request_error { + WebhookRequestError::RetryableRequestError { + error, retry_after, .. + } => { + let retry_interval = + retry_policy.retry_interval(webhook_job.attempt() as u32, retry_after); + let current_queue = webhook_job.queue(); + let retry_queue = retry_policy.retry_queue(¤t_queue); + + match webhook_job + .retry(webhook_job_error, retry_interval, retry_queue) + .await + { + Ok(_) => { + metrics::counter!("webhook_jobs_retried", &labels).increment(1); + + Ok(()) + } + Err(RetryError::RetryInvalidError(RetryInvalidError { + job: webhook_job, + .. + })) => { + webhook_job + .fail(WebhookJobError::from(&error)) + .await + .map_err(|job_error| { + metrics::counter!("webhook_jobs_database_error", &labels) + .increment(1); + job_error + })?; + + metrics::counter!("webhook_jobs_failed", &labels).increment(1); + + Ok(()) + } + Err(RetryError::DatabaseError(job_error)) => { + metrics::counter!("webhook_jobs_database_error", &labels).increment(1); + Err(WorkerError::from(job_error)) + } + } } - Err(RetryError::RetryInvalidError(RetryInvalidError { - job: webhook_job, .. - })) => { + WebhookRequestError::NonRetryableRetryableRequestError { .. } => { webhook_job - .fail(WebhookJobError::from(&error)) + .fail(webhook_job_error) .await .map_err(|job_error| { metrics::counter!("webhook_jobs_database_error", &labels).increment(1); @@ -328,25 +356,8 @@ async fn process_webhook_job( Ok(()) } - Err(RetryError::DatabaseError(job_error)) => { - metrics::counter!("webhook_jobs_database_error", &labels).increment(1); - Err(WorkerError::from(job_error)) - } } } - Err(WebhookError::NonRetryableRetryableRequestError(error)) => { - webhook_job - .fail(WebhookJobError::from(&error)) - .await - .map_err(|job_error| { - metrics::counter!("webhook_jobs_database_error", &labels).increment(1); - job_error - })?; - - metrics::counter!("webhook_jobs_failed", &labels).increment(1); - - Ok(()) - } } } @@ -367,10 +378,10 @@ async fn send_webhook( body: String, ) -> Result { let method: http::Method = method.into(); - let url: reqwest::Url = (url).parse().map_err(WebhookError::ParseUrlError)?; + let url: reqwest::Url = (url).parse().map_err(WebhookParseError::ParseUrlError)?; let headers: reqwest::header::HeaderMap = (headers) .try_into() - .map_err(WebhookError::ParseHeadersError)?; + .map_err(WebhookParseError::ParseHeadersError)?; let body = reqwest::Body::from(body); let response = client @@ -379,26 +390,36 @@ async fn send_webhook( .body(body) .send() .await - .map_err(|e| WebhookError::RetryableRequestError { + .map_err(|e| WebhookRequestError::RetryableRequestError { error: e, + response: None, retry_after: None, })?; let retry_after = parse_retry_after_header(response.headers()); - match response.error_for_status() { - Ok(response) => Ok(response), + match response.error_for_status_ref() { + Ok(_) => Ok(response), Err(err) => { if is_retryable_status( err.status() .expect("status code is set as error is generated from a response"), ) { - Err(WebhookError::RetryableRequestError { - error: err, - retry_after, - }) + Err(WebhookError::Request( + WebhookRequestError::RetryableRequestError { + error: err, + // TODO: Make amount of bytes configurable. + response: first_n_bytes_of_response(response, 10 * 1024).await.ok(), + retry_after, + }, + )) } else { - Err(WebhookError::NonRetryableRetryableRequestError(err)) + Err(WebhookError::Request( + WebhookRequestError::NonRetryableRetryableRequestError { + error: err, + response: first_n_bytes_of_response(response, 10 * 1024).await.ok(), + }, + )) } } } @@ -574,7 +595,7 @@ mod tests { } #[sqlx::test(migrations = "../migrations")] - async fn test_send_webhook(_: PgPool) { + async fn test_send_webhook(_pg: PgPool) { let method = HttpMethod::POST; let url = "http://localhost:18081/echo"; let headers = collections::HashMap::new(); @@ -591,4 +612,58 @@ mod tests { body.to_owned(), ); } + + #[sqlx::test(migrations = "../migrations")] + async fn test_error_message_contains_response_body(_pg: PgPool) { + let method = HttpMethod::POST; + let url = "http://localhost:18081/fail"; + let headers = collections::HashMap::new(); + let body = "this is an error message"; + let client = reqwest::Client::new(); + + let err = send_webhook(client, &method, url, &headers, body.to_owned()) + .await + .err() + .expect("request didn't fail when it should have failed"); + + assert!(matches!(err, WebhookError::Request(..))); + if let WebhookError::Request(request_error) = err { + assert_eq!(request_error.status(), Some(StatusCode::BAD_REQUEST)); + assert!(request_error.to_string().contains(body)); + // This is the display implementation of reqwest. Just checking it is still there. + // See: https://github.com/seanmonstar/reqwest/blob/master/src/error.rs + assert!(request_error.to_string().contains( + "HTTP status client error (400 Bad Request) for url (http://localhost:18081/fail)" + )); + } + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_error_message_contains_up_to_n_bytes_of_response_body(_pg: PgPool) { + let method = HttpMethod::POST; + let url = "http://localhost:18081/fail"; + let headers = collections::HashMap::new(); + // This is double the current hardcoded amount of bytes. + // TODO: Make this configurable and change it here too. + let body = (0..20 * 1024).map(|_| "a").collect::>().concat(); + let client = reqwest::Client::new(); + + let err = send_webhook(client, &method, url, &headers, body.to_owned()) + .await + .err() + .expect("request didn't fail when it should have failed"); + + assert!(matches!(err, WebhookError::Request(..))); + if let WebhookError::Request(request_error) = err { + assert_eq!(request_error.status(), Some(StatusCode::BAD_REQUEST)); + assert!(request_error.to_string().contains(&body[0..10 * 1024])); + // The 81 bytes account for the reqwest erorr message as described below. + assert_eq!(request_error.to_string().len(), 10 * 1024 + 81); + // This is the display implementation of reqwest. Just checking it is still there. + // See: https://github.com/seanmonstar/reqwest/blob/master/src/error.rs + assert!(request_error.to_string().contains( + "HTTP status client error (400 Bad Request) for url (http://localhost:18081/fail)" + )); + } + } } From e2ce4661867c75ba64682d6fc1d32f61c4beee34 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Fri, 3 May 2024 11:03:44 +0200 Subject: [PATCH 232/247] CI: upgrade to Rust 1.77.2 and Debian bookworm (#34) --- .github/workflows/rust.yml | 13 ++++--------- Dockerfile | 4 ++-- hook-worker/src/dns.rs | 0 3 files changed, 6 insertions(+), 11 deletions(-) create mode 100644 hook-worker/src/dns.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a525f3bf942c1..4c3a3d5241f77 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -17,9 +17,7 @@ jobs: - uses: actions/checkout@v3 - name: Install rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable + uses: dtolnay/rust-toolchain@1.77 - uses: actions/cache@v3 with: @@ -52,9 +50,7 @@ jobs: echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts - name: Install rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable + uses: dtolnay/rust-toolchain@1.77 - uses: actions/cache@v3 with: @@ -73,10 +69,9 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install latest rust - uses: dtolnay/rust-toolchain@master + - name: Install rust + uses: dtolnay/rust-toolchain@1.77 with: - toolchain: stable components: clippy,rustfmt - uses: actions/cache@v3 diff --git a/Dockerfile b/Dockerfile index 67aea7f210f44..a6c59b11a0e33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/lukemathwalker/cargo-chef:latest-rust-1.74.0-buster AS chef +FROM docker.io/lukemathwalker/cargo-chef:latest-rust-1.77-bookworm AS chef ARG BIN WORKDIR /app @@ -20,7 +20,7 @@ RUN cargo chef cook --release --recipe-path recipe.json COPY . . RUN cargo build --release --bin $BIN -FROM debian:bullseye-20230320-slim AS runtime +FROM debian:bookworm-slim AS runtime RUN apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/hook-worker/src/dns.rs b/hook-worker/src/dns.rs new file mode 100644 index 0000000000000..e69de29bb2d1d From cf9b82d97d6a453506560f96a625f4758bd6223a Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 3 May 2024 11:12:28 -0600 Subject: [PATCH 233/247] Bump webhook max size to 5MB (#38) --- hook-api/src/handlers/app.rs | 9 +++++++-- hook-api/src/handlers/webhook.rs | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hook-api/src/handlers/app.rs b/hook-api/src/handlers/app.rs index f8d4b24695b99..fa2bcbc30947a 100644 --- a/hook-api/src/handlers/app.rs +++ b/hook-api/src/handlers/app.rs @@ -1,4 +1,4 @@ -use axum::{routing, Router}; +use axum::{extract::DefaultBodyLimit, routing, Router}; use hook_common::pgqueue::PgQueue; @@ -9,7 +9,12 @@ pub fn add_routes(router: Router, pg_pool: PgQueue) -> Router { .route("/", routing::get(index)) .route("/_readiness", routing::get(index)) .route("/_liveness", routing::get(index)) // No async loop for now, just check axum health - .route("/webhook", routing::post(webhook::post).with_state(pg_pool)) + .route( + "/webhook", + routing::post(webhook::post) + .with_state(pg_pool) + .layer(DefaultBodyLimit::disable()), + ) } pub async fn index() -> &'static str { diff --git a/hook-api/src/handlers/webhook.rs b/hook-api/src/handlers/webhook.rs index e50b8b0241a51..47f21a6638b36 100644 --- a/hook-api/src/handlers/webhook.rs +++ b/hook-api/src/handlers/webhook.rs @@ -9,7 +9,7 @@ use hook_common::pgqueue::{NewJob, PgQueue}; use serde::Serialize; use tracing::{debug, error}; -const MAX_BODY_SIZE: usize = 1_000_000; +pub const MAX_BODY_SIZE: usize = 5_000_000; #[derive(Serialize, Deserialize)] pub struct WebhookPostResponse { @@ -252,7 +252,7 @@ mod tests { let app = add_routes(Router::new(), pg_queue); - let bytes: Vec = vec![b'a'; 1_000_000 * 2]; + let bytes: Vec = vec![b'a'; 5_000_000 * 2]; let long_string = String::from_utf8_lossy(&bytes); let response = app From ee769d9b0a7cfc073c0338be149ba01e67080b15 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 3 May 2024 13:53:46 -0600 Subject: [PATCH 234/247] Make max_body_size configurable via env, drop custom check, use 413 (#39) --- Cargo.lock | 1 + Cargo.toml | 4 ++-- hook-api/Cargo.toml | 1 + hook-api/src/config.rs | 3 +++ hook-api/src/handlers/app.rs | 9 +++++---- hook-api/src/handlers/webhook.rs | 27 +++++++++------------------ hook-api/src/main.rs | 2 +- 7 files changed, 22 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7cc582ee6696c..c56738f84c3f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1076,6 +1076,7 @@ dependencies = [ "sqlx", "tokio", "tower", + "tower-http", "tracing", "tracing-subscriber", "url", diff --git a/Cargo.toml b/Cargo.toml index d1f75274aa629..e097eab2f7339 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ [workspace.lints.rust] # See https://doc.rust-lang.org/stable/rustc/lints/listing/allowed-by-default.html -unsafe_code = "forbid" # forbid cannot be ignored with an annotation +unsafe_code = "forbid" # forbid cannot be ignored with an annotation unstable_features = "forbid" macro_use_extern_crate = "forbid" let_underscore_drop = "deny" @@ -69,7 +69,7 @@ time = { version = "0.3.20", features = [ thiserror = { version = "1.0" } tokio = { version = "1.34.0", features = ["full"] } tower = "0.4.13" -tower-http = { version = "0.5.2", features = ["cors", "trace"] } +tower-http = { version = "0.5.2", features = ["cors", "limit", "trace"] } tracing = "0.1.40" tracing-subscriber = "0.3.18" url = { version = "2.5.0 " } diff --git a/hook-api/Cargo.toml b/hook-api/Cargo.toml index c3528d23da5d2..eb82438c47d65 100644 --- a/hook-api/Cargo.toml +++ b/hook-api/Cargo.toml @@ -19,6 +19,7 @@ serde_json = { workspace = true } sqlx = { workspace = true } tokio = { workspace = true } tower = { workspace = true } +tower-http = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } url = { workspace = true } diff --git a/hook-api/src/config.rs b/hook-api/src/config.rs index 55fa404e5149d..e15f0d3fac77a 100644 --- a/hook-api/src/config.rs +++ b/hook-api/src/config.rs @@ -16,6 +16,9 @@ pub struct Config { #[envconfig(default = "100")] pub max_pg_connections: u32, + + #[envconfig(default = "5000000")] + pub max_body_size: usize, } impl Config { diff --git a/hook-api/src/handlers/app.rs b/hook-api/src/handlers/app.rs index fa2bcbc30947a..7cbbc449e424d 100644 --- a/hook-api/src/handlers/app.rs +++ b/hook-api/src/handlers/app.rs @@ -1,10 +1,11 @@ -use axum::{extract::DefaultBodyLimit, routing, Router}; +use axum::{routing, Router}; +use tower_http::limit::RequestBodyLimitLayer; use hook_common::pgqueue::PgQueue; use super::webhook; -pub fn add_routes(router: Router, pg_pool: PgQueue) -> Router { +pub fn add_routes(router: Router, pg_pool: PgQueue, max_body_size: usize) -> Router { router .route("/", routing::get(index)) .route("/_readiness", routing::get(index)) @@ -13,7 +14,7 @@ pub fn add_routes(router: Router, pg_pool: PgQueue) -> Router { "/webhook", routing::post(webhook::post) .with_state(pg_pool) - .layer(DefaultBodyLimit::disable()), + .layer(RequestBodyLimitLayer::new(max_body_size)), ) } @@ -37,7 +38,7 @@ mod tests { async fn index(db: PgPool) { let pg_queue = PgQueue::new_from_pool("test_index", db).await; - let app = add_routes(Router::new(), pg_queue); + let app = add_routes(Router::new(), pg_queue, 1_000_000); let response = app .oneshot(Request::builder().uri("/").body(Body::empty()).unwrap()) diff --git a/hook-api/src/handlers/webhook.rs b/hook-api/src/handlers/webhook.rs index 47f21a6638b36..808c94878291b 100644 --- a/hook-api/src/handlers/webhook.rs +++ b/hook-api/src/handlers/webhook.rs @@ -9,8 +9,6 @@ use hook_common::pgqueue::{NewJob, PgQueue}; use serde::Serialize; use tracing::{debug, error}; -pub const MAX_BODY_SIZE: usize = 5_000_000; - #[derive(Serialize, Deserialize)] pub struct WebhookPostResponse { #[serde(skip_serializing_if = "Option::is_none")] @@ -37,15 +35,6 @@ pub async fn post( ) -> Result, (StatusCode, Json)> { debug!("received payload: {:?}", payload); - if payload.parameters.body.len() > MAX_BODY_SIZE { - return Err(( - StatusCode::BAD_REQUEST, - Json(WebhookPostResponse { - error: Some("body too large".to_owned()), - }), - )); - } - let url_hostname = get_hostname(&payload.parameters.url)?; // We could cast to i32, but this ensures we are not wrapping. let max_attempts = i32::try_from(payload.max_attempts).map_err(|_| { @@ -125,11 +114,13 @@ mod tests { use crate::handlers::app::add_routes; + const MAX_BODY_SIZE: usize = 1_000_000; + #[sqlx::test(migrations = "../migrations")] async fn webhook_success(db: PgPool) { let pg_queue = PgQueue::new_from_pool("test_index", db).await; - let app = add_routes(Router::new(), pg_queue); + let app = add_routes(Router::new(), pg_queue, MAX_BODY_SIZE); let mut headers = collections::HashMap::new(); headers.insert("Content-Type".to_owned(), "application/json".to_owned()); @@ -171,7 +162,7 @@ mod tests { async fn webhook_bad_url(db: PgPool) { let pg_queue = PgQueue::new_from_pool("test_index", db).await; - let app = add_routes(Router::new(), pg_queue); + let app = add_routes(Router::new(), pg_queue, MAX_BODY_SIZE); let response = app .oneshot( @@ -208,7 +199,7 @@ mod tests { async fn webhook_payload_missing_fields(db: PgPool) { let pg_queue = PgQueue::new_from_pool("test_index", db).await; - let app = add_routes(Router::new(), pg_queue); + let app = add_routes(Router::new(), pg_queue, MAX_BODY_SIZE); let response = app .oneshot( @@ -229,7 +220,7 @@ mod tests { async fn webhook_payload_not_json(db: PgPool) { let pg_queue = PgQueue::new_from_pool("test_index", db).await; - let app = add_routes(Router::new(), pg_queue); + let app = add_routes(Router::new(), pg_queue, MAX_BODY_SIZE); let response = app .oneshot( @@ -250,9 +241,9 @@ mod tests { async fn webhook_payload_body_too_large(db: PgPool) { let pg_queue = PgQueue::new_from_pool("test_index", db).await; - let app = add_routes(Router::new(), pg_queue); + let app = add_routes(Router::new(), pg_queue, MAX_BODY_SIZE); - let bytes: Vec = vec![b'a'; 5_000_000 * 2]; + let bytes: Vec = vec![b'a'; MAX_BODY_SIZE + 1]; let long_string = String::from_utf8_lossy(&bytes); let response = app @@ -283,6 +274,6 @@ mod tests { .await .unwrap(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); } } diff --git a/hook-api/src/main.rs b/hook-api/src/main.rs index 9a9a9fd41c0c2..ad05edef1ff98 100644 --- a/hook-api/src/main.rs +++ b/hook-api/src/main.rs @@ -34,7 +34,7 @@ async fn main() { .await .expect("failed to initialize queue"); - let app = handlers::add_routes(Router::new(), pg_queue); + let app = handlers::add_routes(Router::new(), pg_queue, config.max_body_size); let app = setup_metrics_routes(app); match listen(app, config.bind()).await { From 7715f46ed15cb992dc6eb4d1032d87b0c29ac90b Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 6 May 2024 09:25:18 +0200 Subject: [PATCH 235/247] chore: merge capture & capture-server crates (#36) --- .github/workflows/docker-capture.yml | 2 +- Cargo.lock | 29 +++++---------------- Cargo.toml | 8 ++++-- capture-server/Cargo.toml | 28 -------------------- capture/Cargo.toml | 12 +++++++++ {capture-server => capture}/src/main.rs | 0 {capture-server => capture}/tests/common.rs | 0 {capture-server => capture}/tests/events.rs | 0 8 files changed, 26 insertions(+), 53 deletions(-) delete mode 100644 capture-server/Cargo.toml rename {capture-server => capture}/src/main.rs (100%) rename {capture-server => capture}/tests/common.rs (100%) rename {capture-server => capture}/tests/events.rs (100%) diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-capture.yml index d0efac36b0852..d31e05dcef5ac 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-capture.yml @@ -69,7 +69,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: BIN=capture-server + build-args: BIN=capture - name: Capture image digest run: echo ${{ steps.docker_build_capture.outputs.digest }} diff --git a/Cargo.lock b/Cargo.lock index c56738f84c3f2..5caeea4c0131f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -370,13 +370,19 @@ dependencies = [ "bytes", "envconfig", "flate2", + "futures", "governor", "health", "metrics", "metrics-exporter-prometheus", + "once_cell", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "rand", "rdkafka", "redis", + "reqwest 0.12.3", "serde", "serde_json", "serde_urlencoded", @@ -385,30 +391,9 @@ dependencies = [ "tokio", "tower-http", "tracing", - "uuid", -] - -[[package]] -name = "capture-server" -version = "0.1.0" -dependencies = [ - "anyhow", - "assert-json-diff", - "capture", - "envconfig", - "futures", - "once_cell", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry_sdk", - "rand", - "rdkafka", - "reqwest 0.12.3", - "serde_json", - "tokio", - "tracing", "tracing-opentelemetry", "tracing-subscriber", + "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e097eab2f7339..180355b5ba49a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,6 @@ resolver = "2" members = [ "capture", - "capture-server", "common/health", "hook-api", "hook-common", @@ -44,6 +43,10 @@ http = { version = "1.1.0" } http-body-util = "0.1.0" metrics = "0.22.0" metrics-exporter-prometheus = "0.14.0" +once_cell = "1.18.0" +opentelemetry = { version = "0.22.0", features = ["trace"]} +opentelemetry-otlp = "0.15.0" +opentelemetry_sdk = { version = "0.22.1", features = ["trace", "rt-tokio"] } rand = "0.8.5" rdkafka = { version = "0.36.0", features = ["cmake-build", "ssl", "tracing"] } reqwest = { version = "0.12.3", features = ["stream"] } @@ -71,6 +74,7 @@ tokio = { version = "1.34.0", features = ["full"] } tower = "0.4.13" tower-http = { version = "0.5.2", features = ["cors", "limit", "trace"] } tracing = "0.1.40" -tracing-subscriber = "0.3.18" +tracing-opentelemetry = "0.23.0" +tracing-subscriber = { version="0.3.18", features = ["env-filter"] } url = { version = "2.5.0 " } uuid = { version = "1.6.1", features = ["v7", "serde"] } diff --git a/capture-server/Cargo.toml b/capture-server/Cargo.toml deleted file mode 100644 index 39ee742d2b048..0000000000000 --- a/capture-server/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "capture-server" -version = "0.1.0" -edition = "2021" - -[lints] -workspace = true - -[dependencies] -capture = { path = "../capture" } -envconfig = { workspace = true } -opentelemetry = { version = "0.22.0", features = ["trace"]} -opentelemetry-otlp = "0.15.0" -opentelemetry_sdk = { version = "0.22.1", features = ["trace", "rt-tokio"] } -tokio = { workspace = true } -tracing = { workspace = true } -tracing-opentelemetry = "0.23.0" -tracing-subscriber = { workspace = true, features = ["env-filter"] } - -[dev-dependencies] -anyhow = { workspace = true, features = [] } -assert-json-diff = { workspace = true } -futures = "0.3.29" -once_cell = "1.18.0" -rand = { workspace = true } -rdkafka = { workspace = true } -reqwest = { workspace = true } -serde_json = { workspace = true } diff --git a/capture/Cargo.toml b/capture/Cargo.toml index ae5ad9a3bd127..97d310f03d662 100644 --- a/capture/Cargo.toml +++ b/capture/Cargo.toml @@ -19,6 +19,9 @@ governor = { workspace = true } health = { path = "../common/health" } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry-otlp = { workspace = true } +opentelemetry_sdk = { workspace = true } rand = { workspace = true } rdkafka = { workspace = true } redis = { version = "0.23.3", features = [ @@ -34,8 +37,17 @@ time = { workspace = true } tokio = { workspace = true } tower-http = { workspace = true } tracing = { workspace = true } +tracing-opentelemetry = { workspace = true } +tracing-subscriber = { workspace = true } uuid = { workspace = true } [dev-dependencies] assert-json-diff = { workspace = true } axum-test-helper = { git = "https://github.com/posthog/axum-test-helper.git" } # TODO: remove, directly use reqwest like capture-server tests +anyhow = { workspace = true } +futures = { workspace = true } +once_cell = { workspace = true } +rand = { workspace = true } +rdkafka = { workspace = true } +reqwest = { workspace = true } +serde_json = { workspace = true } diff --git a/capture-server/src/main.rs b/capture/src/main.rs similarity index 100% rename from capture-server/src/main.rs rename to capture/src/main.rs diff --git a/capture-server/tests/common.rs b/capture/tests/common.rs similarity index 100% rename from capture-server/tests/common.rs rename to capture/tests/common.rs diff --git a/capture-server/tests/events.rs b/capture/tests/events.rs similarity index 100% rename from capture-server/tests/events.rs rename to capture/tests/events.rs From e8343d9ad1642f395a87925f6e9ad415d003e791 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 6 May 2024 10:35:37 +0200 Subject: [PATCH 236/247] ci: refactor docker build workflows for all rust crates (#37) --- .../{docker-capture.yml => docker-build.yml} | 26 ++++--- .github/workflows/docker-hook-api.yml | 75 ------------------- .github/workflows/docker-hook-janitor.yml | 75 ------------------- .github/workflows/docker-hook-worker.yml | 75 ------------------- 4 files changed, 15 insertions(+), 236 deletions(-) rename .github/workflows/{docker-capture.yml => docker-build.yml} (77%) delete mode 100644 .github/workflows/docker-hook-api.yml delete mode 100644 .github/workflows/docker-hook-janitor.yml delete mode 100644 .github/workflows/docker-hook-worker.yml diff --git a/.github/workflows/docker-capture.yml b/.github/workflows/docker-build.yml similarity index 77% rename from .github/workflows/docker-capture.yml rename to .github/workflows/docker-build.yml index d31e05dcef5ac..3132a61b0be03 100644 --- a/.github/workflows/docker-capture.yml +++ b/.github/workflows/docker-build.yml @@ -1,4 +1,4 @@ -name: Build capture docker image +name: Build container images on: workflow_dispatch: @@ -6,12 +6,16 @@ on: branches: - "main" -permissions: - packages: write - jobs: build: - name: build and publish capture image + name: Build and publish container image + strategy: + matrix: + image: + - capture + - hook-api + - hook-janitor + - hook-worker runs-on: depot-ubuntu-22.04-4 permissions: id-token: write # allow issuing OIDC tokens for this workflow run @@ -45,7 +49,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/posthog/hog-rs/capture + images: ghcr.io/posthog/hog-rs/${{ matrix.image }} tags: | type=ref,event=pr type=ref,event=branch @@ -57,8 +61,8 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 - - name: Build and push capture - id: docker_build_capture + - name: Build and push image + id: docker_build uses: depot/build-push-action@v1 with: context: ./ @@ -69,7 +73,7 @@ jobs: platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - build-args: BIN=capture + build-args: BIN=${{ matrix.image }} - - name: Capture image digest - run: echo ${{ steps.docker_build_capture.outputs.digest }} + - name: Container image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/docker-hook-api.yml b/.github/workflows/docker-hook-api.yml deleted file mode 100644 index e6f83f1b6b3fb..0000000000000 --- a/.github/workflows/docker-hook-api.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Build hook-api docker image - -on: - workflow_dispatch: - push: - branches: - - "main" - -permissions: - packages: write - -jobs: - build: - name: build and publish hook-api image - runs-on: depot-ubuntu-22.04-4 - permissions: - id-token: write # allow issuing OIDC tokens for this workflow run - contents: read # allow reading the repo contents - packages: write # allow push to ghcr.io - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Depot CLI - uses: depot/setup-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: posthog - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - images: ghcr.io/posthog/hog-rs/hook-api - tags: | - type=ref,event=pr - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Build and push api - id: docker_build_hook_api - uses: depot/build-push-action@v1 - with: - context: ./ - file: ./Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: BIN=hook-api - - - name: Hook-api image digest - run: echo ${{ steps.docker_build_hook_api.outputs.digest }} diff --git a/.github/workflows/docker-hook-janitor.yml b/.github/workflows/docker-hook-janitor.yml deleted file mode 100644 index 33706d0987637..0000000000000 --- a/.github/workflows/docker-hook-janitor.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Build hook-janitor docker image - -on: - workflow_dispatch: - push: - branches: - - "main" - -permissions: - packages: write - -jobs: - build: - name: build and publish hook-janitor image - runs-on: depot-ubuntu-22.04-4 - permissions: - id-token: write # allow issuing OIDC tokens for this workflow run - contents: read # allow reading the repo contents - packages: write # allow push to ghcr.io - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Depot CLI - uses: depot/setup-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: posthog - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - images: ghcr.io/posthog/hog-rs/hook-janitor - tags: | - type=ref,event=pr - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Build and push janitor - id: docker_build_hook_janitor - uses: depot/build-push-action@v1 - with: - context: ./ - file: ./Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: BIN=hook-janitor - - - name: Hook-janitor image digest - run: echo ${{ steps.docker_build_hook_janitor.outputs.digest }} diff --git a/.github/workflows/docker-hook-worker.yml b/.github/workflows/docker-hook-worker.yml deleted file mode 100644 index dc5ca53abef88..0000000000000 --- a/.github/workflows/docker-hook-worker.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Build hook-worker docker image - -on: - workflow_dispatch: - push: - branches: - - "main" - -permissions: - packages: write - -jobs: - build: - name: build and publish hook-worker image - runs-on: depot-ubuntu-22.04-4 - permissions: - id-token: write # allow issuing OIDC tokens for this workflow run - contents: read # allow reading the repo contents - packages: write # allow push to ghcr.io - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Depot CLI - uses: depot/setup-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: posthog - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - images: ghcr.io/posthog/hog-rs/hook-worker - tags: | - type=ref,event=pr - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Build and push worker - id: docker_build_hook_worker - uses: depot/build-push-action@v1 - with: - context: ./ - file: ./Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: BIN=hook-worker - - - name: Hook-worker image digest - run: echo ${{ steps.docker_build_hook_worker.outputs.digest }} From 281af615b4874da1b89915a6ccd36d74be5a04a0 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 6 May 2024 16:31:25 +0200 Subject: [PATCH 237/247] hook-worker: deny traffic to internal IPs and IPv6 (#35) --- hook-worker/src/config.rs | 3 + hook-worker/src/dns.rs | 140 ++++++++++++++++++++++++++++++++++++++ hook-worker/src/error.rs | 20 +++++- hook-worker/src/lib.rs | 1 + hook-worker/src/main.rs | 1 + hook-worker/src/worker.rs | 113 ++++++++++++++++++++++-------- 6 files changed, 248 insertions(+), 30 deletions(-) diff --git a/hook-worker/src/config.rs b/hook-worker/src/config.rs index ceb690f38846e..51b23b7f273c5 100644 --- a/hook-worker/src/config.rs +++ b/hook-worker/src/config.rs @@ -37,6 +37,9 @@ pub struct Config { #[envconfig(default = "1")] pub dequeue_batch_size: u32, + + #[envconfig(default = "false")] + pub allow_internal_ips: bool, } impl Config { diff --git a/hook-worker/src/dns.rs b/hook-worker/src/dns.rs index e69de29bb2d1d..36fd7a005398e 100644 --- a/hook-worker/src/dns.rs +++ b/hook-worker/src/dns.rs @@ -0,0 +1,140 @@ +use std::error::Error as StdError; +use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; +use std::{fmt, io}; + +use futures::FutureExt; +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; +use tokio::task::spawn_blocking; + +pub struct NoPublicIPv4Error; + +impl std::error::Error for NoPublicIPv4Error {} +impl fmt::Display for NoPublicIPv4Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "No public IPv4 found for specified host") + } +} +impl fmt::Debug for NoPublicIPv4Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "No public IPv4 found for specified host") + } +} + +/// Internal reqwest type, copied here as part of Resolving +pub(crate) type BoxError = Box; + +/// Returns [`true`] if the address appears to be a globally reachable IPv4. +/// +/// Trimmed down version of the unstable IpAddr::is_global, move to it when it's stable. +fn is_global_ipv4(addr: &SocketAddr) -> bool { + match addr.ip() { + IpAddr::V4(ip) => { + !(ip.octets()[0] == 0 // "This network" + || ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_broadcast()) + } + IpAddr::V6(_) => false, // Our network does not currently support ipv6, let's ignore for now + } +} + +/// DNS resolver using the stdlib resolver, but filtering results to only pass public IPv4 results. +/// +/// Private and broadcast addresses are filtered out, so are IPv6 results for now (as our infra +/// does not currently support IPv6 routing anyway). +/// This is adapted from the GaiResolver in hyper and reqwest. +pub struct PublicIPv4Resolver {} + +impl Resolve for PublicIPv4Resolver { + fn resolve(&self, name: Name) -> Resolving { + // Closure to call the system's resolver (blocking call) through the ToSocketAddrs trait. + let resolve_host = move || (name.as_str(), 0).to_socket_addrs(); + + // Execute the blocking call in a separate worker thread then process its result asynchronously. + // spawn_blocking returns a JoinHandle that implements Future>. + let future_result = spawn_blocking(resolve_host).map(|result| match result { + Ok(Ok(all_addrs)) => { + // Resolution succeeded, filter the results + let filtered_addr: Vec = all_addrs.filter(is_global_ipv4).collect(); + if filtered_addr.is_empty() { + // No public IPs found, error out with PermissionDenied + let err: BoxError = Box::new(NoPublicIPv4Error); + Err(err) + } else { + // Pass remaining IPs in a boxed iterator for request to use. + let addrs: Addrs = Box::new(filtered_addr.into_iter()); + Ok(addrs) + } + } + Ok(Err(err)) => { + // Resolution failed, pass error through in a Box + let err: BoxError = Box::new(err); + Err(err) + } + Err(join_err) => { + // The tokio task failed, pass as io::Error in a Box + let err: BoxError = Box::new(io::Error::from(join_err)); + Err(err) + } + }); + + // Box the Future to satisfy the Resolving interface. + Box::pin(future_result) + } +} + +#[cfg(test)] +mod tests { + use crate::dns::{NoPublicIPv4Error, PublicIPv4Resolver}; + use reqwest::dns::{Name, Resolve}; + use std::str::FromStr; + + #[tokio::test] + async fn it_resolves_google_com() { + let resolver: PublicIPv4Resolver = PublicIPv4Resolver {}; + let addrs = resolver + .resolve(Name::from_str("google.com").unwrap()) + .await + .expect("lookup has failed"); + assert!(addrs.count() > 0, "empty address list") + } + + #[tokio::test] + async fn it_denies_ipv6_google_com() { + let resolver: PublicIPv4Resolver = PublicIPv4Resolver {}; + match resolver + .resolve(Name::from_str("ipv6.google.com").unwrap()) + .await + { + Ok(_) => panic!("should have failed"), + Err(err) => assert!(err.is::()), + } + } + + #[tokio::test] + async fn it_denies_localhost() { + let resolver: PublicIPv4Resolver = PublicIPv4Resolver {}; + match resolver.resolve(Name::from_str("localhost").unwrap()).await { + Ok(_) => panic!("should have failed"), + Err(err) => assert!(err.is::()), + } + } + + #[tokio::test] + async fn it_bubbles_up_resolution_error() { + let resolver: PublicIPv4Resolver = PublicIPv4Resolver {}; + match resolver + .resolve(Name::from_str("invalid.domain.unknown").unwrap()) + .await + { + Ok(_) => panic!("should have failed"), + Err(err) => { + assert!(!err.is::()); + assert!(err + .to_string() + .contains("failed to lookup address information")) + } + } + } +} diff --git a/hook-worker/src/error.rs b/hook-worker/src/error.rs index 48468bc65f544..764e8d973499c 100644 --- a/hook-worker/src/error.rs +++ b/hook-worker/src/error.rs @@ -1,6 +1,8 @@ +use std::error::Error; use std::fmt; use std::time; +use crate::dns::NoPublicIPv4Error; use hook_common::{pgqueue, webhook::WebhookJobError}; use thiserror::Error; @@ -64,7 +66,11 @@ impl fmt::Display for WebhookRequestError { Some(m) => m.to_string(), None => "No response from the server".to_string(), }; - writeln!(f, "{}", error)?; + if is_error_source::(error) { + writeln!(f, "{}: {}", error, NoPublicIPv4Error)?; + } else { + writeln!(f, "{}", error)?; + } write!(f, "{}", response_message)?; Ok(()) @@ -132,3 +138,15 @@ pub enum WorkerError { #[error("timed out while waiting for jobs to be available")] TimeoutError, } + +/// Check the error and it's sources (recursively) to return true if an error of the given type is found. +/// TODO: use Error::sources() when stable +pub fn is_error_source(err: &(dyn std::error::Error + 'static)) -> bool { + if err.is::() { + return true; + } + match err.source() { + None => false, + Some(source) => is_error_source::(source), + } +} diff --git a/hook-worker/src/lib.rs b/hook-worker/src/lib.rs index 8488d15b20a36..94a07584f1da5 100644 --- a/hook-worker/src/lib.rs +++ b/hook-worker/src/lib.rs @@ -1,4 +1,5 @@ pub mod config; +pub mod dns; pub mod error; pub mod util; pub mod worker; diff --git a/hook-worker/src/main.rs b/hook-worker/src/main.rs index 8a6eeb37435ab..050e2b947c780 100644 --- a/hook-worker/src/main.rs +++ b/hook-worker/src/main.rs @@ -52,6 +52,7 @@ async fn main() -> Result<(), WorkerError> { config.request_timeout.0, config.max_concurrent_jobs, retry_policy_builder.provide(), + config.allow_internal_ips, worker_liveness, ); diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index 824f1e2f23a87..fdb405a2fd18b 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -14,11 +14,14 @@ use hook_common::{ webhook::{HttpMethod, WebhookJobError, WebhookJobMetadata, WebhookJobParameters}, }; use http::StatusCode; -use reqwest::header; +use reqwest::{header, Client}; use tokio::sync; use tracing::error; -use crate::error::{WebhookError, WebhookParseError, WebhookRequestError, WorkerError}; +use crate::dns::{NoPublicIPv4Error, PublicIPv4Resolver}; +use crate::error::{ + is_error_source, WebhookError, WebhookParseError, WebhookRequestError, WorkerError, +}; use crate::util::first_n_bytes_of_response; /// A WebhookJob is any `PgQueueJob` with `WebhookJobParameters` and `WebhookJobMetadata`. @@ -74,6 +77,25 @@ pub struct WebhookWorker<'p> { liveness: HealthHandle, } +pub fn build_http_client( + request_timeout: time::Duration, + allow_internal_ips: bool, +) -> reqwest::Result { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ); + let mut client_builder = reqwest::Client::builder() + .default_headers(headers) + .user_agent("PostHog Webhook Worker") + .timeout(request_timeout); + if !allow_internal_ips { + client_builder = client_builder.dns_resolver(Arc::new(PublicIPv4Resolver {})) + } + client_builder.build() +} + impl<'p> WebhookWorker<'p> { #[allow(clippy::too_many_arguments)] pub fn new( @@ -84,19 +106,10 @@ impl<'p> WebhookWorker<'p> { request_timeout: time::Duration, max_concurrent_jobs: usize, retry_policy: RetryPolicy, + allow_internal_ips: bool, liveness: HealthHandle, ) -> Self { - let mut headers = header::HeaderMap::new(); - headers.insert( - header::CONTENT_TYPE, - header::HeaderValue::from_static("application/json"), - ); - - let client = reqwest::Client::builder() - .default_headers(headers) - .user_agent("PostHog Webhook Worker") - .timeout(request_timeout) - .build() + let client = build_http_client(request_timeout, allow_internal_ips) .expect("failed to construct reqwest client for webhook worker"); Self { @@ -390,10 +403,19 @@ async fn send_webhook( .body(body) .send() .await - .map_err(|e| WebhookRequestError::RetryableRequestError { - error: e, - response: None, - retry_after: None, + .map_err(|e| { + if is_error_source::(&e) { + WebhookRequestError::NonRetryableRetryableRequestError { + error: e, + response: None, + } + } else { + WebhookRequestError::RetryableRequestError { + error: e, + response: None, + retry_after: None, + } + } })?; let retry_after = parse_retry_after_header(response.headers()); @@ -469,6 +491,7 @@ fn parse_retry_after_header(header_map: &reqwest::header::HeaderMap) -> Option Client { + build_http_client(Duration::from_secs(1), true).expect("failed to create client") + } + #[allow(dead_code)] async fn enqueue_job( queue: &PgQueue, @@ -569,6 +598,7 @@ mod tests { time::Duration::from_millis(5000), 10, RetryPolicy::default(), + false, liveness, ); @@ -594,15 +624,14 @@ mod tests { assert!(registry.get_status().healthy) } - #[sqlx::test(migrations = "../migrations")] - async fn test_send_webhook(_pg: PgPool) { + #[tokio::test] + async fn test_send_webhook() { let method = HttpMethod::POST; let url = "http://localhost:18081/echo"; let headers = collections::HashMap::new(); let body = "a very relevant request body"; - let client = reqwest::Client::new(); - let response = send_webhook(client, &method, url, &headers, body.to_owned()) + let response = send_webhook(localhost_client(), &method, url, &headers, body.to_owned()) .await .expect("send_webhook failed"); @@ -613,15 +642,14 @@ mod tests { ); } - #[sqlx::test(migrations = "../migrations")] - async fn test_error_message_contains_response_body(_pg: PgPool) { + #[tokio::test] + async fn test_error_message_contains_response_body() { let method = HttpMethod::POST; let url = "http://localhost:18081/fail"; let headers = collections::HashMap::new(); let body = "this is an error message"; - let client = reqwest::Client::new(); - let err = send_webhook(client, &method, url, &headers, body.to_owned()) + let err = send_webhook(localhost_client(), &method, url, &headers, body.to_owned()) .await .err() .expect("request didn't fail when it should have failed"); @@ -638,17 +666,16 @@ mod tests { } } - #[sqlx::test(migrations = "../migrations")] - async fn test_error_message_contains_up_to_n_bytes_of_response_body(_pg: PgPool) { + #[tokio::test] + async fn test_error_message_contains_up_to_n_bytes_of_response_body() { let method = HttpMethod::POST; let url = "http://localhost:18081/fail"; let headers = collections::HashMap::new(); // This is double the current hardcoded amount of bytes. // TODO: Make this configurable and change it here too. let body = (0..20 * 1024).map(|_| "a").collect::>().concat(); - let client = reqwest::Client::new(); - let err = send_webhook(client, &method, url, &headers, body.to_owned()) + let err = send_webhook(localhost_client(), &method, url, &headers, body.to_owned()) .await .err() .expect("request didn't fail when it should have failed"); @@ -666,4 +693,32 @@ mod tests { )); } } + + #[tokio::test] + async fn test_private_ips_denied() { + let method = HttpMethod::POST; + let url = "http://localhost:18081/echo"; + let headers = collections::HashMap::new(); + let body = "a very relevant request body"; + let filtering_client = + build_http_client(Duration::from_secs(1), false).expect("failed to create client"); + + let err = send_webhook(filtering_client, &method, url, &headers, body.to_owned()) + .await + .err() + .expect("request didn't fail when it should have failed"); + + assert!(matches!(err, WebhookError::Request(..))); + if let WebhookError::Request(request_error) = err { + assert_eq!(request_error.status(), None); + assert!(request_error + .to_string() + .contains("No public IPv4 found for specified host")); + if let WebhookRequestError::RetryableRequestError { .. } = request_error { + panic!("error should not be retryable") + } + } else { + panic!("unexpected error type {:?}", err) + } + } } From ae707cb2ed4a3d4c97df9f399189579b521f3973 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 6 May 2024 17:25:27 +0200 Subject: [PATCH 238/247] chore: cleanup unnecessary clippy allows (#40) --- hook-common/src/kafka_messages/plugin_logs.rs | 2 -- hook-worker/src/worker.rs | 13 +++---------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/hook-common/src/kafka_messages/plugin_logs.rs b/hook-common/src/kafka_messages/plugin_logs.rs index 5a852e6aa3221..039788afe2dc5 100644 --- a/hook-common/src/kafka_messages/plugin_logs.rs +++ b/hook-common/src/kafka_messages/plugin_logs.rs @@ -4,7 +4,6 @@ use uuid::Uuid; use super::serialize_datetime; -#[allow(dead_code)] #[derive(Serialize)] pub enum PluginLogEntrySource { System, @@ -12,7 +11,6 @@ pub enum PluginLogEntrySource { Console, } -#[allow(dead_code)] #[derive(Serialize)] pub enum PluginLogEntryType { Debug, diff --git a/hook-worker/src/worker.rs b/hook-worker/src/worker.rs index fdb405a2fd18b..9dcc4a2f4b7b0 100644 --- a/hook-worker/src/worker.rs +++ b/hook-worker/src/worker.rs @@ -7,9 +7,7 @@ use futures::future::join_all; use health::HealthHandle; use hook_common::pgqueue::PgTransactionBatch; use hook_common::{ - pgqueue::{ - DatabaseError, Job, PgQueue, PgQueueJob, PgTransactionJob, RetryError, RetryInvalidError, - }, + pgqueue::{Job, PgQueue, PgQueueJob, PgTransactionJob, RetryError, RetryInvalidError}, retry::RetryPolicy, webhook::{HttpMethod, WebhookJobError, WebhookJobMetadata, WebhookJobParameters}, }; @@ -489,32 +487,27 @@ fn parse_retry_after_header(header_map: &reqwest::header::HeaderMap) -> Option String { std::process::id().to_string() } /// Get a request client or panic - #[allow(dead_code)] fn localhost_client() -> Client { build_http_client(Duration::from_secs(1), true).expect("failed to create client") } - #[allow(dead_code)] async fn enqueue_job( queue: &PgQueue, max_attempts: i32, From 871441b400d3af766a5a012507dde6c1aaf45c7f Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Tue, 7 May 2024 21:13:01 +0100 Subject: [PATCH 239/247] feat(flags): Basic flags service (#31) --- Cargo.lock | 48 ++++++++++++----- Cargo.toml | 3 +- feature-flags/Cargo.toml | 35 ++++++++++++ feature-flags/src/api.rs | 58 ++++++++++++++++++++ feature-flags/src/config.rs | 24 +++++++++ feature-flags/src/lib.rs | 7 +++ feature-flags/src/main.rs | 39 ++++++++++++++ feature-flags/src/redis.rs | 77 ++++++++++++++++++++++++++ feature-flags/src/router.rs | 19 +++++++ feature-flags/src/server.rs | 31 +++++++++++ feature-flags/src/v0_endpoint.rs | 89 +++++++++++++++++++++++++++++++ feature-flags/src/v0_request.rs | 68 +++++++++++++++++++++++ feature-flags/tests/common.rs | 66 +++++++++++++++++++++++ feature-flags/tests/test_flags.rs | 43 +++++++++++++++ 14 files changed, 594 insertions(+), 13 deletions(-) create mode 100644 feature-flags/Cargo.toml create mode 100644 feature-flags/src/api.rs create mode 100644 feature-flags/src/config.rs create mode 100644 feature-flags/src/lib.rs create mode 100644 feature-flags/src/main.rs create mode 100644 feature-flags/src/redis.rs create mode 100644 feature-flags/src/router.rs create mode 100644 feature-flags/src/server.rs create mode 100644 feature-flags/src/v0_endpoint.rs create mode 100644 feature-flags/src/v0_request.rs create mode 100644 feature-flags/tests/common.rs create mode 100644 feature-flags/tests/test_flags.rs diff --git a/Cargo.lock b/Cargo.lock index 5caeea4c0131f..0f475fa488c04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,7 +182,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.1.0", + "hyper 1.3.1", "hyper-util", "itoa", "matchit", @@ -273,7 +273,7 @@ dependencies = [ "bytes", "http 1.1.0", "http-body 1.0.0", - "hyper 1.1.0", + "hyper 1.3.1", "reqwest 0.11.24", "serde", "tokio", @@ -352,9 +352,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "capture" @@ -691,6 +691,29 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "feature-flags" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-trait", + "axum 0.7.5", + "axum-client-ip", + "bytes", + "envconfig", + "once_cell", + "rand", + "redis", + "reqwest 0.12.3", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "finl_unicode" version = "1.2.0" @@ -1226,9 +1249,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.1.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", @@ -1240,6 +1263,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", + "smallvec", "tokio", "want", ] @@ -1278,7 +1302,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.1.0", + "hyper 1.3.1", "hyper-util", "native-tls", "tokio", @@ -1297,7 +1321,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.1.0", + "hyper 1.3.1", "pin-project-lite", "socket2 0.5.5", "tokio", @@ -1519,7 +1543,7 @@ checksum = "5d58e362dc7206e9456ddbcdbd53c71ba441020e62104703075a69151e38d85f" dependencies = [ "base64 0.22.0", "http-body-util", - "hyper 1.1.0", + "hyper 1.3.1", "hyper-tls", "hyper-util", "indexmap 2.2.2", @@ -2310,7 +2334,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.1.0", + "hyper 1.3.1", "hyper-tls", "hyper-util", "ipnet", @@ -3054,9 +3078,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.36.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 180355b5ba49a..ea5d041027ad8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ resolver = "2" members = [ "capture", "common/health", + "feature-flags", "hook-api", "hook-common", "hook-janitor", @@ -49,7 +50,7 @@ opentelemetry-otlp = "0.15.0" opentelemetry_sdk = { version = "0.22.1", features = ["trace", "rt-tokio"] } rand = "0.8.5" rdkafka = { version = "0.36.0", features = ["cmake-build", "ssl", "tracing"] } -reqwest = { version = "0.12.3", features = ["stream"] } +reqwest = { version = "0.12.3", features = ["json", "stream"] } serde = { version = "1.0", features = ["derive"] } serde_derive = { version = "1.0" } serde_json = { version = "1.0" } diff --git a/feature-flags/Cargo.toml b/feature-flags/Cargo.toml new file mode 100644 index 0000000000000..ddfe0705a157e --- /dev/null +++ b/feature-flags/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "feature-flags" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } +axum = { workspace = true } +axum-client-ip = { workspace = true } +envconfig = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +bytes = { workspace = true } +rand = { workspace = true } +redis = { version = "0.23.3", features = [ + "tokio-comp", + "cluster", + "cluster-async", +] } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } + +[lints] +workspace = true + +[dev-dependencies] +assert-json-diff = { workspace = true } +once_cell = "1.18.0" +reqwest = { workspace = true } + diff --git a/feature-flags/src/api.rs b/feature-flags/src/api.rs new file mode 100644 index 0000000000000..c94eed698357f --- /dev/null +++ b/feature-flags/src/api.rs @@ -0,0 +1,58 @@ +use std::collections::HashMap; + +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Debug, PartialEq, Eq, Deserialize, Serialize)] +pub enum FlagsResponseCode { + Ok = 1, +} + +#[derive(Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FlagsResponse { + pub error_while_computing_flags: bool, + // TODO: better typing here, support bool responses + pub feature_flags: HashMap, +} + +#[derive(Error, Debug)] +pub enum FlagError { + #[error("failed to decode request: {0}")] + RequestDecodingError(String), + #[error("failed to parse request: {0}")] + RequestParsingError(#[from] serde_json::Error), + + #[error("Empty distinct_id in request")] + EmptyDistinctId, + #[error("No distinct_id in request")] + MissingDistinctId, + + #[error("No api_key in request")] + NoTokenError, + #[error("API key is not valid")] + TokenValidationError, + + #[error("rate limited")] + RateLimited, +} + +impl IntoResponse for FlagError { + fn into_response(self) -> Response { + match self { + FlagError::RequestDecodingError(_) + | FlagError::RequestParsingError(_) + | FlagError::EmptyDistinctId + | FlagError::MissingDistinctId => (StatusCode::BAD_REQUEST, self.to_string()), + + FlagError::NoTokenError | FlagError::TokenValidationError => { + (StatusCode::UNAUTHORIZED, self.to_string()) + } + + FlagError::RateLimited => (StatusCode::TOO_MANY_REQUESTS, self.to_string()), + } + .into_response() + } +} diff --git a/feature-flags/src/config.rs b/feature-flags/src/config.rs new file mode 100644 index 0000000000000..3fa6f50e878e1 --- /dev/null +++ b/feature-flags/src/config.rs @@ -0,0 +1,24 @@ +use std::net::SocketAddr; + +use envconfig::Envconfig; + +#[derive(Envconfig, Clone)] +pub struct Config { + #[envconfig(default = "127.0.0.1:0")] + pub address: SocketAddr, + + #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] + pub write_database_url: String, + + #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] + pub read_database_url: String, + + #[envconfig(default = "1024")] + pub max_concurrent_jobs: usize, + + #[envconfig(default = "100")] + pub max_pg_connections: u32, + + #[envconfig(default = "redis://localhost:6379/")] + pub redis_url: String, +} diff --git a/feature-flags/src/lib.rs b/feature-flags/src/lib.rs new file mode 100644 index 0000000000000..9175b5c3974af --- /dev/null +++ b/feature-flags/src/lib.rs @@ -0,0 +1,7 @@ +pub mod api; +pub mod config; +pub mod redis; +pub mod router; +pub mod server; +pub mod v0_endpoint; +pub mod v0_request; diff --git a/feature-flags/src/main.rs b/feature-flags/src/main.rs new file mode 100644 index 0000000000000..980db6973893f --- /dev/null +++ b/feature-flags/src/main.rs @@ -0,0 +1,39 @@ +use envconfig::Envconfig; +use tokio::signal; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{EnvFilter, Layer}; + +use feature_flags::config::Config; +use feature_flags::server::serve; + +async fn shutdown() { + let mut term = signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to register SIGTERM handler"); + + let mut interrupt = signal::unix::signal(signal::unix::SignalKind::interrupt()) + .expect("failed to register SIGINT handler"); + + tokio::select! { + _ = term.recv() => {}, + _ = interrupt.recv() => {}, + }; + + tracing::info!("Shutting down gracefully..."); +} + +#[tokio::main] +async fn main() { + let config = Config::init_from_env().expect("Invalid configuration:"); + + // Basic logging for now: + // - stdout with a level configured by the RUST_LOG envvar (default=ERROR) + let log_layer = tracing_subscriber::fmt::layer().with_filter(EnvFilter::from_default_env()); + tracing_subscriber::registry().with(log_layer).init(); + + // Open the TCP port and start the server + let listener = tokio::net::TcpListener::bind(config.address) + .await + .expect("could not bind port"); + serve(config, listener, shutdown()).await +} diff --git a/feature-flags/src/redis.rs b/feature-flags/src/redis.rs new file mode 100644 index 0000000000000..8c038201698e5 --- /dev/null +++ b/feature-flags/src/redis.rs @@ -0,0 +1,77 @@ +use std::time::Duration; + +use anyhow::Result; +use async_trait::async_trait; +use redis::AsyncCommands; +use tokio::time::timeout; + +// average for all commands is <10ms, check grafana +const REDIS_TIMEOUT_MILLISECS: u64 = 10; + +/// A simple redis wrapper +/// Copied from capture/src/redis.rs. +/// TODO: Modify this to support hincrby, get, and set commands. + +#[async_trait] +pub trait Client { + // A very simplified wrapper, but works for our usage + async fn zrangebyscore(&self, k: String, min: String, max: String) -> Result>; +} + +pub struct RedisClient { + client: redis::Client, +} + +impl RedisClient { + pub fn new(addr: String) -> Result { + let client = redis::Client::open(addr)?; + + Ok(RedisClient { client }) + } +} + +#[async_trait] +impl Client for RedisClient { + async fn zrangebyscore(&self, k: String, min: String, max: String) -> Result> { + let mut conn = self.client.get_async_connection().await?; + + let results = conn.zrangebyscore(k, min, max); + let fut = timeout(Duration::from_secs(REDIS_TIMEOUT_MILLISECS), results).await?; + + Ok(fut?) + } +} + +// TODO: Find if there's a better way around this. +#[derive(Clone)] +pub struct MockRedisClient { + zrangebyscore_ret: Vec, +} + +impl MockRedisClient { + pub fn new() -> MockRedisClient { + MockRedisClient { + zrangebyscore_ret: Vec::new(), + } + } + + pub fn zrangebyscore_ret(&mut self, ret: Vec) -> Self { + self.zrangebyscore_ret = ret; + + self.clone() + } +} + +impl Default for MockRedisClient { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Client for MockRedisClient { + // A very simplified wrapper, but works for our usage + async fn zrangebyscore(&self, _k: String, _min: String, _max: String) -> Result> { + Ok(self.zrangebyscore_ret.clone()) + } +} diff --git a/feature-flags/src/router.rs b/feature-flags/src/router.rs new file mode 100644 index 0000000000000..8824d44efdbde --- /dev/null +++ b/feature-flags/src/router.rs @@ -0,0 +1,19 @@ +use std::sync::Arc; + +use axum::{routing::post, Router}; + +use crate::{redis::Client, v0_endpoint}; + +#[derive(Clone)] +pub struct State { + pub redis: Arc, + // TODO: Add pgClient when ready +} + +pub fn router(redis: Arc) -> Router { + let state = State { redis }; + + Router::new() + .route("/flags", post(v0_endpoint::flags).get(v0_endpoint::flags)) + .with_state(state) +} diff --git a/feature-flags/src/server.rs b/feature-flags/src/server.rs new file mode 100644 index 0000000000000..ffe6b0efb7068 --- /dev/null +++ b/feature-flags/src/server.rs @@ -0,0 +1,31 @@ +use std::future::Future; +use std::net::SocketAddr; +use std::sync::Arc; + +use tokio::net::TcpListener; + +use crate::config::Config; + +use crate::redis::RedisClient; +use crate::router; + +pub async fn serve(config: Config, listener: TcpListener, shutdown: F) +where + F: Future + Send + 'static, +{ + let redis_client = + Arc::new(RedisClient::new(config.redis_url).expect("failed to create redis client")); + + let app = router::router(redis_client); + + // run our app with hyper + // `axum::Server` is a re-export of `hyper::Server` + tracing::info!("listening on {:?}", listener.local_addr().unwrap()); + axum::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(shutdown) + .await + .unwrap() +} diff --git a/feature-flags/src/v0_endpoint.rs b/feature-flags/src/v0_endpoint.rs new file mode 100644 index 0000000000000..8f7761181e050 --- /dev/null +++ b/feature-flags/src/v0_endpoint.rs @@ -0,0 +1,89 @@ +use std::collections::HashMap; + +use axum::{debug_handler, Json}; +use bytes::Bytes; +// TODO: stream this instead +use axum::extract::{MatchedPath, Query, State}; +use axum::http::{HeaderMap, Method}; +use axum_client_ip::InsecureClientIp; +use tracing::instrument; + +use crate::{ + api::{FlagError, FlagsResponse}, + router, + v0_request::{FlagRequest, FlagsQueryParams}, +}; + +/// Feature flag evaluation endpoint. +/// Only supports a specific shape of data, and rejects any malformed data. + +#[instrument( + skip_all, + fields( + path, + token, + batch_size, + user_agent, + content_encoding, + content_type, + version, + compression, + historical_migration + ) +)] +#[debug_handler] +pub async fn flags( + _state: State, + InsecureClientIp(ip): InsecureClientIp, + meta: Query, + headers: HeaderMap, + method: Method, + path: MatchedPath, + body: Bytes, +) -> Result, FlagError> { + let user_agent = headers + .get("user-agent") + .map_or("unknown", |v| v.to_str().unwrap_or("unknown")); + let content_encoding = headers + .get("content-encoding") + .map_or("unknown", |v| v.to_str().unwrap_or("unknown")); + + tracing::Span::current().record("user_agent", user_agent); + tracing::Span::current().record("content_encoding", content_encoding); + tracing::Span::current().record("version", meta.version.clone()); + tracing::Span::current().record("method", method.as_str()); + tracing::Span::current().record("path", path.as_str().trim_end_matches('/')); + tracing::Span::current().record("ip", ip.to_string()); + + let request = match headers + .get("content-type") + .map_or("", |v| v.to_str().unwrap_or("")) + { + "application/x-www-form-urlencoded" => { + return Err(FlagError::RequestDecodingError(String::from( + "invalid form data", + ))); + } + ct => { + tracing::Span::current().record("content_type", ct); + + FlagRequest::from_bytes(body) + } + }?; + + let token = request.extract_and_verify_token()?; + + tracing::Span::current().record("token", &token); + + tracing::debug!("request: {:?}", request); + + // TODO: Some actual processing for evaluating the feature flag + + Ok(Json(FlagsResponse { + error_while_computing_flags: false, + feature_flags: HashMap::from([ + ("beta-feature".to_string(), "variant-1".to_string()), + ("rollout-flag".to_string(), true.to_string()), + ]), + })) +} diff --git a/feature-flags/src/v0_request.rs b/feature-flags/src/v0_request.rs new file mode 100644 index 0000000000000..f2269df1b5f74 --- /dev/null +++ b/feature-flags/src/v0_request.rs @@ -0,0 +1,68 @@ +use std::collections::HashMap; + +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tracing::instrument; + +use crate::api::FlagError; + +#[derive(Deserialize, Default)] +pub struct FlagsQueryParams { + #[serde(alias = "v")] + pub version: Option, +} + +#[derive(Default, Debug, Deserialize, Serialize)] +pub struct FlagRequest { + #[serde( + alias = "$token", + alias = "api_key", + skip_serializing_if = "Option::is_none" + )] + pub token: Option, + #[serde(alias = "$distinct_id", skip_serializing_if = "Option::is_none")] + pub distinct_id: Option, + pub geoip_disable: Option, + #[serde(default)] + pub person_properties: Option>, + #[serde(default)] + pub groups: Option>, + // TODO: better type this since we know its going to be a nested json + #[serde(default)] + pub group_properties: Option>, + #[serde(alias = "$anon_distinct_id", skip_serializing_if = "Option::is_none")] + pub anon_distinct_id: Option, +} + +impl FlagRequest { + /// Takes a request payload and tries to decompress and unmarshall it. + /// While posthog-js sends a compression query param, a sizable portion of requests + /// fail due to it being missing when the body is compressed. + /// Instead of trusting the parameter, we peek at the payload's first three bytes to + /// detect gzip, fallback to uncompressed utf8 otherwise. + #[instrument(skip_all)] + pub fn from_bytes(bytes: Bytes) -> Result { + tracing::debug!(len = bytes.len(), "decoding new request"); + // TODO: Add base64 decoding + let payload = String::from_utf8(bytes.into()).map_err(|e| { + tracing::error!("failed to decode body: {}", e); + FlagError::RequestDecodingError(String::from("invalid body encoding")) + })?; + + tracing::debug!(json = payload, "decoded event data"); + Ok(serde_json::from_str::(&payload)?) + } + + pub fn extract_and_verify_token(&self) -> Result { + let token = match self { + FlagRequest { + token: Some(token), .. + } => token.to_string(), + _ => return Err(FlagError::NoTokenError), + }; + // TODO: Get tokens from redis, confirm this one is valid + // validate_token(&token)?; + Ok(token) + } +} diff --git a/feature-flags/tests/common.rs b/feature-flags/tests/common.rs new file mode 100644 index 0000000000000..f66a11ff37c25 --- /dev/null +++ b/feature-flags/tests/common.rs @@ -0,0 +1,66 @@ +use std::net::SocketAddr; +use std::str::FromStr; +use std::string::ToString; +use std::sync::Arc; + +use once_cell::sync::Lazy; +use rand::distributions::Alphanumeric; +use rand::Rng; +use tokio::net::TcpListener; +use tokio::sync::Notify; + +use feature_flags::config::Config; +use feature_flags::server::serve; + +pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { + address: SocketAddr::from_str("127.0.0.1:0").unwrap(), + redis_url: "redis://localhost:6379/".to_string(), + write_database_url: "postgres://posthog:posthog@localhost:15432/test_database".to_string(), + read_database_url: "postgres://posthog:posthog@localhost:15432/test_database".to_string(), + max_concurrent_jobs: 1024, + max_pg_connections: 100, +}); + +pub struct ServerHandle { + pub addr: SocketAddr, + shutdown: Arc, +} + +impl ServerHandle { + pub async fn for_config(config: Config) -> ServerHandle { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let notify = Arc::new(Notify::new()); + let shutdown = notify.clone(); + + tokio::spawn(async move { + serve(config, listener, async move { notify.notified().await }).await + }); + ServerHandle { addr, shutdown } + } + + pub async fn send_flags_request>(&self, body: T) -> reqwest::Response { + let client = reqwest::Client::new(); + client + .post(format!("http://{:?}/flags", self.addr)) + .body(body) + .send() + .await + .expect("failed to send request") + } +} + +impl Drop for ServerHandle { + fn drop(&mut self) { + self.shutdown.notify_one() + } +} + +pub fn random_string(prefix: &str, length: usize) -> String { + let suffix: String = rand::thread_rng() + .sample_iter(Alphanumeric) + .take(length) + .map(char::from) + .collect(); + format!("{}_{}", prefix, suffix) +} diff --git a/feature-flags/tests/test_flags.rs b/feature-flags/tests/test_flags.rs new file mode 100644 index 0000000000000..82f41f05d81c6 --- /dev/null +++ b/feature-flags/tests/test_flags.rs @@ -0,0 +1,43 @@ +use anyhow::Result; +use assert_json_diff::assert_json_include; + +use reqwest::StatusCode; +use serde_json::{json, Value}; + +use crate::common::*; +mod common; + +#[tokio::test] +async fn it_sends_flag_request() -> Result<()> { + let token = random_string("token", 16); + let distinct_id = "user_distinct_id".to_string(); + + let config = DEFAULT_CONFIG.clone(); + + let server = ServerHandle::for_config(config).await; + + let payload = json!({ + "token": token, + "distinct_id": distinct_id, + "groups": {"group1": "group1"} + }); + let res = server.send_flags_request(payload.to_string()).await; + assert_eq!(StatusCode::OK, res.status()); + + // We don't want to deserialize the data into a flagResponse struct here, + // because we want to assert the shape of the raw json data. + let json_data = res.json::().await?; + + assert_json_include!( + actual: json_data, + expected: json!({ + "errorWhileComputingFlags": false, + "featureFlags": { + "beta-feature": "variant-1", + "rollout-flag": "true", + } + }) + ); + + Ok(()) +} From ebaf596ec571f0c2c2cc8bafbdd4ee932ac28b2a Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 27 May 2024 14:23:07 +0200 Subject: [PATCH 240/247] capture: add overflow_enabled option (#43) --- capture/src/config.rs | 3 +++ capture/src/server.rs | 42 ++++++++++++++++------------- capture/src/sinks/kafka.rs | 14 ++++++---- capture/tests/common.rs | 1 + capture/tests/events.rs | 54 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 23 deletions(-) diff --git a/capture/src/config.rs b/capture/src/config.rs index 07b7f89496d44..d91e7b7241337 100644 --- a/capture/src/config.rs +++ b/capture/src/config.rs @@ -13,6 +13,9 @@ pub struct Config { pub redis_url: String, pub otel_url: Option, + #[envconfig(default = "false")] + pub overflow_enabled: bool, + #[envconfig(default = "100")] pub overflow_per_second_limit: NonZeroU32, diff --git a/capture/src/server.rs b/capture/src/server.rs index 07049874edef9..85850363e762c 100644 --- a/capture/src/server.rs +++ b/capture/src/server.rs @@ -48,24 +48,30 @@ where .register("rdkafka".to_string(), Duration::seconds(30)) .await; - let partition = OverflowLimiter::new( - config.overflow_per_second_limit, - config.overflow_burst_limit, - config.overflow_forced_keys, - ); - if config.export_prometheus { - let partition = partition.clone(); - tokio::spawn(async move { - partition.report_metrics().await; - }); - } - { - // Ensure that the rate limiter state does not grow unbounded - let partition = partition.clone(); - tokio::spawn(async move { - partition.clean_state().await; - }); - } + let partition = match config.overflow_enabled { + false => None, + true => { + let partition = OverflowLimiter::new( + config.overflow_per_second_limit, + config.overflow_burst_limit, + config.overflow_forced_keys, + ); + if config.export_prometheus { + let partition = partition.clone(); + tokio::spawn(async move { + partition.report_metrics().await; + }); + } + { + // Ensure that the rate limiter state does not grow unbounded + let partition = partition.clone(); + tokio::spawn(async move { + partition.clean_state().await; + }); + } + Some(partition) + } + }; let sink = KafkaSink::new(config.kafka, sink_liveness, partition) .expect("failed to start Kafka sink"); diff --git a/capture/src/sinks/kafka.rs b/capture/src/sinks/kafka.rs index 945e581183d37..bc45fa1077f7b 100644 --- a/capture/src/sinks/kafka.rs +++ b/capture/src/sinks/kafka.rs @@ -80,7 +80,7 @@ impl rdkafka::ClientContext for KafkaContext { #[derive(Clone)] pub struct KafkaSink { producer: FutureProducer, - partition: OverflowLimiter, + partition: Option, main_topic: String, historical_topic: String, } @@ -89,7 +89,7 @@ impl KafkaSink { pub fn new( config: KafkaConfig, liveness: HealthHandle, - partition: OverflowLimiter, + partition: Option, ) -> anyhow::Result { info!("connecting to Kafka brokers at {}...", config.kafka_hosts); @@ -150,7 +150,11 @@ impl KafkaSink { DataType::AnalyticsHistorical => (&self.historical_topic, Some(event_key.as_str())), // We never trigger overflow on historical events DataType::AnalyticsMain => { // TODO: deprecate capture-led overflow or move logic in handler - if self.partition.is_limited(&event_key) { + let is_limited = match &self.partition { + None => false, + Some(partition) => partition.is_limited(&event_key), + }; + if is_limited { (&self.main_topic, None) // Analytics overflow goes to the main topic without locality } else { (&self.main_topic, Some(event_key.as_str())) @@ -280,11 +284,11 @@ mod tests { let handle = registry .register("one".to_string(), Duration::seconds(30)) .await; - let limiter = OverflowLimiter::new( + let limiter = Some(OverflowLimiter::new( NonZeroU32::new(10).unwrap(), NonZeroU32::new(10).unwrap(), None, - ); + )); let cluster = MockCluster::new(1).expect("failed to create mock brokers"); let config = config::KafkaConfig { kafka_producer_linger_ms: 0, diff --git a/capture/tests/common.rs b/capture/tests/common.rs index 788e6e28240c7..868b27c120a7f 100644 --- a/capture/tests/common.rs +++ b/capture/tests/common.rs @@ -29,6 +29,7 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { print_sink: false, address: SocketAddr::from_str("127.0.0.1:0").unwrap(), redis_url: "redis://localhost:6379/".to_string(), + overflow_enabled: false, overflow_burst_limit: NonZeroU32::new(5).unwrap(), overflow_per_second_limit: NonZeroU32::new(10).unwrap(), overflow_forced_keys: None, diff --git a/capture/tests/events.rs b/capture/tests/events.rs index 111b02c7f2cb1..7d2defcebd5ff 100644 --- a/capture/tests/events.rs +++ b/capture/tests/events.rs @@ -174,6 +174,7 @@ async fn it_overflows_events_on_burst() -> Result<()> { let mut config = DEFAULT_CONFIG.clone(); config.kafka.kafka_topic = topic.topic_name().to_string(); + config.overflow_enabled = true; config.overflow_burst_limit = NonZeroU32::new(2).unwrap(); config.overflow_per_second_limit = NonZeroU32::new(1).unwrap(); @@ -223,6 +224,7 @@ async fn it_does_not_overflow_team_with_different_ids() -> Result<()> { let mut config = DEFAULT_CONFIG.clone(); config.kafka.kafka_topic = topic.topic_name().to_string(); + config.overflow_enabled = true; config.overflow_burst_limit = NonZeroU32::new(1).unwrap(); config.overflow_per_second_limit = NonZeroU32::new(1).unwrap(); @@ -254,6 +256,58 @@ async fn it_does_not_overflow_team_with_different_ids() -> Result<()> { Ok(()) } +#[tokio::test] +async fn it_skips_overflows_when_disabled() -> Result<()> { + setup_tracing(); + + let token = random_string("token", 16); + let distinct_id = random_string("id", 16); + + let topic = EphemeralTopic::new().await; + + let mut config = DEFAULT_CONFIG.clone(); + config.kafka.kafka_topic = topic.topic_name().to_string(); + config.overflow_enabled = false; + config.overflow_burst_limit = NonZeroU32::new(2).unwrap(); + config.overflow_per_second_limit = NonZeroU32::new(1).unwrap(); + + let server = ServerHandle::for_config(config).await; + + let event = json!([{ + "token": token, + "event": "event1", + "distinct_id": distinct_id + },{ + "token": token, + "event": "event2", + "distinct_id": distinct_id + },{ + "token": token, + "event": "event3", + "distinct_id": distinct_id + }]); + + let res = server.capture_events(event.to_string()).await; + assert_eq!(StatusCode::OK, res.status()); + + assert_eq!( + topic.next_message_key()?.unwrap(), + format!("{}:{}", token, distinct_id) + ); + + assert_eq!( + topic.next_message_key()?.unwrap(), + format!("{}:{}", token, distinct_id) + ); + + // Should have triggered overflow, but has not + assert_eq!( + topic.next_message_key()?.unwrap(), + format!("{}:{}", token, distinct_id) + ); + Ok(()) +} + #[tokio::test] async fn it_trims_distinct_id() -> Result<()> { setup_tracing(); From f71dc0867bc0ce80c91953de69f5ca9f65521d84 Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Wed, 29 May 2024 14:32:41 +0100 Subject: [PATCH 241/247] feat(flags): Do token validation and extract distinct id (#41) --- Cargo.lock | 39 ++++- feature-flags/Cargo.toml | 1 + feature-flags/src/api.rs | 9 ++ feature-flags/src/config.rs | 2 +- feature-flags/src/lib.rs | 9 ++ feature-flags/src/redis.rs | 73 +++++---- feature-flags/src/team.rs | 139 ++++++++++++++++++ feature-flags/src/test_utils.rs | 50 +++++++ feature-flags/src/v0_endpoint.rs | 23 +-- feature-flags/src/v0_request.rs | 90 ++++++++++-- .../tests/{common.rs => common/mod.rs} | 27 ++-- feature-flags/tests/test_flags.rs | 46 +++++- 12 files changed, 442 insertions(+), 66 deletions(-) create mode 100644 feature-flags/src/team.rs create mode 100644 feature-flags/src/test_utils.rs rename feature-flags/tests/{common.rs => common/mod.rs} (76%) diff --git a/Cargo.lock b/Cargo.lock index 0f475fa488c04..8642adee54b3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -707,6 +707,7 @@ dependencies = [ "redis", "reqwest 0.12.3", "serde", + "serde-pickle", "serde_json", "thiserror", "tokio", @@ -1395,6 +1396,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "iter-read" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c397ca3ea05ad509c4ec451fea28b4771236a376ca1c69fd5143aae0cf8f93c4" + [[package]] name = "itertools" version = "0.12.1" @@ -1680,6 +1687,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -1705,11 +1722,10 @@ checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] @@ -1726,9 +1742,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", @@ -2533,6 +2549,19 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-pickle" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c762ad136a26407c6a80825813600ceeab5e613660d93d79a41f0ec877171e71" +dependencies = [ + "byteorder", + "iter-read", + "num-bigint", + "num-traits", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.196" diff --git a/feature-flags/Cargo.toml b/feature-flags/Cargo.toml index ddfe0705a157e..1e0c111d71263 100644 --- a/feature-flags/Cargo.toml +++ b/feature-flags/Cargo.toml @@ -24,6 +24,7 @@ redis = { version = "0.23.3", features = [ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +serde-pickle = { version = "1.1.1"} [lints] workspace = true diff --git a/feature-flags/src/api.rs b/feature-flags/src/api.rs index c94eed698357f..ccf4735e5b04a 100644 --- a/feature-flags/src/api.rs +++ b/feature-flags/src/api.rs @@ -37,6 +37,11 @@ pub enum FlagError { #[error("rate limited")] RateLimited, + + #[error("failed to parse redis cache data")] + DataParsingError, + #[error("redis unavailable")] + RedisUnavailable, } impl IntoResponse for FlagError { @@ -52,6 +57,10 @@ impl IntoResponse for FlagError { } FlagError::RateLimited => (StatusCode::TOO_MANY_REQUESTS, self.to_string()), + + FlagError::DataParsingError | FlagError::RedisUnavailable => { + (StatusCode::SERVICE_UNAVAILABLE, self.to_string()) + } } .into_response() } diff --git a/feature-flags/src/config.rs b/feature-flags/src/config.rs index 3fa6f50e878e1..cc7ad37bf72c1 100644 --- a/feature-flags/src/config.rs +++ b/feature-flags/src/config.rs @@ -4,7 +4,7 @@ use envconfig::Envconfig; #[derive(Envconfig, Clone)] pub struct Config { - #[envconfig(default = "127.0.0.1:0")] + #[envconfig(default = "127.0.0.1:3001")] pub address: SocketAddr, #[envconfig(default = "postgres://posthog:posthog@localhost:15432/test_database")] diff --git a/feature-flags/src/lib.rs b/feature-flags/src/lib.rs index 9175b5c3974af..195a55c88095d 100644 --- a/feature-flags/src/lib.rs +++ b/feature-flags/src/lib.rs @@ -3,5 +3,14 @@ pub mod config; pub mod redis; pub mod router; pub mod server; +pub mod team; pub mod v0_endpoint; pub mod v0_request; + +// Test modules don't need to be compiled with main binary +// #[cfg(test)] +// TODO: To use in integration tests, we need to compile with binary +// or make it a separate feature using cfg(feature = "integration-tests") +// and then use this feature only in tests. +// For now, ok to just include in binary +pub mod test_utils; diff --git a/feature-flags/src/redis.rs b/feature-flags/src/redis.rs index 8c038201698e5..89dde421d0abc 100644 --- a/feature-flags/src/redis.rs +++ b/feature-flags/src/redis.rs @@ -2,20 +2,38 @@ use std::time::Duration; use anyhow::Result; use async_trait::async_trait; -use redis::AsyncCommands; +use redis::{AsyncCommands, RedisError}; +use thiserror::Error; use tokio::time::timeout; // average for all commands is <10ms, check grafana const REDIS_TIMEOUT_MILLISECS: u64 = 10; +#[derive(Error, Debug)] +pub enum CustomRedisError { + #[error("Not found in redis")] + NotFound, + + #[error("Pickle error: {0}")] + PickleError(#[from] serde_pickle::Error), + + #[error("Redis error: {0}")] + Other(#[from] RedisError), + + #[error("Timeout error")] + Timeout(#[from] tokio::time::error::Elapsed), +} /// A simple redis wrapper /// Copied from capture/src/redis.rs. -/// TODO: Modify this to support hincrby, get, and set commands. +/// TODO: Modify this to support hincrby #[async_trait] pub trait Client { // A very simplified wrapper, but works for our usage async fn zrangebyscore(&self, k: String, min: String, max: String) -> Result>; + + async fn get(&self, k: String) -> Result; + async fn set(&self, k: String, v: String) -> Result<()>; } pub struct RedisClient { @@ -40,38 +58,39 @@ impl Client for RedisClient { Ok(fut?) } -} -// TODO: Find if there's a better way around this. -#[derive(Clone)] -pub struct MockRedisClient { - zrangebyscore_ret: Vec, -} + async fn get(&self, k: String) -> Result { + let mut conn = self.client.get_async_connection().await?; -impl MockRedisClient { - pub fn new() -> MockRedisClient { - MockRedisClient { - zrangebyscore_ret: Vec::new(), + let results = conn.get(k); + let fut: Result, RedisError> = + timeout(Duration::from_secs(REDIS_TIMEOUT_MILLISECS), results).await?; + + // return NotFound error when empty or not found + if match &fut { + Ok(v) => v.is_empty(), + Err(_) => false, + } { + return Err(CustomRedisError::NotFound); } - } - pub fn zrangebyscore_ret(&mut self, ret: Vec) -> Self { - self.zrangebyscore_ret = ret; + // TRICKY: We serialise data to json, then django pickles it. + // Here we deserialize the bytes using serde_pickle, to get the json string. + let string_response: String = serde_pickle::from_slice(&fut?, Default::default())?; - self.clone() + Ok(string_response) } -} -impl Default for MockRedisClient { - fn default() -> Self { - Self::new() - } -} + async fn set(&self, k: String, v: String) -> Result<()> { + // TRICKY: We serialise data to json, then django pickles it. + // Here we serialize the json string to bytes using serde_pickle. + let bytes = serde_pickle::to_vec(&v, Default::default())?; -#[async_trait] -impl Client for MockRedisClient { - // A very simplified wrapper, but works for our usage - async fn zrangebyscore(&self, _k: String, _min: String, _max: String) -> Result> { - Ok(self.zrangebyscore_ret.clone()) + let mut conn = self.client.get_async_connection().await?; + + let results = conn.set(k, bytes); + let fut = timeout(Duration::from_secs(REDIS_TIMEOUT_MILLISECS), results).await?; + + Ok(fut?) } } diff --git a/feature-flags/src/team.rs b/feature-flags/src/team.rs new file mode 100644 index 0000000000000..ac62ea9ba55cb --- /dev/null +++ b/feature-flags/src/team.rs @@ -0,0 +1,139 @@ +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tracing::instrument; + +use crate::{ + api::FlagError, + redis::{Client, CustomRedisError}, +}; + +// TRICKY: This cache data is coming from django-redis. If it ever goes out of sync, we'll bork. +// TODO: Add integration tests across repos to ensure this doesn't happen. +pub const TEAM_TOKEN_CACHE_PREFIX: &str = "posthog:1:team_token:"; + +#[derive(Debug, Deserialize, Serialize)] +pub struct Team { + pub id: i64, + pub name: String, + pub api_token: String, +} + +impl Team { + /// Validates a token, and returns a team if it exists. + + #[instrument(skip_all)] + pub async fn from_redis( + client: Arc, + token: String, + ) -> Result { + // TODO: Instead of failing here, i.e. if not in redis, fallback to pg + let serialized_team = client + .get(format!("{TEAM_TOKEN_CACHE_PREFIX}{}", token)) + .await + .map_err(|e| match e { + CustomRedisError::NotFound => FlagError::TokenValidationError, + CustomRedisError::PickleError(_) => { + tracing::error!("failed to fetch data: {}", e); + FlagError::DataParsingError + } + _ => { + tracing::error!("Unknown redis error: {}", e); + FlagError::RedisUnavailable + } + })?; + + let team: Team = serde_json::from_str(&serialized_team).map_err(|e| { + tracing::error!("failed to parse data to team: {}", e); + FlagError::DataParsingError + })?; + + Ok(team) + } +} + +#[cfg(test)] +mod tests { + use rand::Rng; + use redis::AsyncCommands; + + use super::*; + use crate::{ + team, + test_utils::{insert_new_team_in_redis, random_string, setup_redis_client}, + }; + + #[tokio::test] + async fn test_fetch_team_from_redis() { + let client = setup_redis_client(None); + + let team = insert_new_team_in_redis(client.clone()).await.unwrap(); + + let target_token = team.api_token; + + let team_from_redis = Team::from_redis(client.clone(), target_token.clone()) + .await + .unwrap(); + assert_eq!(team_from_redis.api_token, target_token); + assert_eq!(team_from_redis.id, team.id); + } + + #[tokio::test] + async fn test_fetch_invalid_team_from_redis() { + let client = setup_redis_client(None); + + match Team::from_redis(client.clone(), "banana".to_string()).await { + Err(FlagError::TokenValidationError) => (), + _ => panic!("Expected TokenValidationError"), + }; + } + + #[tokio::test] + async fn test_cant_connect_to_redis_error_is_not_token_validation_error() { + let client = setup_redis_client(Some("redis://localhost:1111/".to_string())); + + match Team::from_redis(client.clone(), "banana".to_string()).await { + Err(FlagError::RedisUnavailable) => (), + _ => panic!("Expected RedisUnavailable"), + }; + } + + #[tokio::test] + async fn test_corrupted_data_in_redis_is_handled() { + // TODO: Extend this test with fallback to pg + let id = rand::thread_rng().gen_range(0..10_000_000); + let token = random_string("phc_", 12); + let team = Team { + id, + name: "team".to_string(), + api_token: token, + }; + let serialized_team = serde_json::to_string(&team).expect("Failed to serialise team"); + + // manually insert non-pickled data in redis + let client = + redis::Client::open("redis://localhost:6379/").expect("Failed to create redis client"); + let mut conn = client + .get_async_connection() + .await + .expect("Failed to get redis connection"); + conn.set::( + format!( + "{}{}", + team::TEAM_TOKEN_CACHE_PREFIX, + team.api_token.clone() + ), + serialized_team, + ) + .await + .expect("Failed to write data to redis"); + + // now get client connection for data + let client = setup_redis_client(None); + + match Team::from_redis(client.clone(), team.api_token.clone()).await { + Err(FlagError::DataParsingError) => (), + Err(other) => panic!("Expected DataParsingError, got {:?}", other), + Ok(_) => panic!("Expected DataParsingError"), + }; + } +} diff --git a/feature-flags/src/test_utils.rs b/feature-flags/src/test_utils.rs new file mode 100644 index 0000000000000..75db86d3878ee --- /dev/null +++ b/feature-flags/src/test_utils.rs @@ -0,0 +1,50 @@ +use anyhow::Error; +use std::sync::Arc; + +use crate::{ + redis::{Client, RedisClient}, + team::{self, Team}, +}; +use rand::{distributions::Alphanumeric, Rng}; + +pub fn random_string(prefix: &str, length: usize) -> String { + let suffix: String = rand::thread_rng() + .sample_iter(Alphanumeric) + .take(length) + .map(char::from) + .collect(); + format!("{}{}", prefix, suffix) +} + +pub async fn insert_new_team_in_redis(client: Arc) -> Result { + let id = rand::thread_rng().gen_range(0..10_000_000); + let token = random_string("phc_", 12); + let team = Team { + id, + name: "team".to_string(), + api_token: token, + }; + + let serialized_team = serde_json::to_string(&team)?; + client + .set( + format!( + "{}{}", + team::TEAM_TOKEN_CACHE_PREFIX, + team.api_token.clone() + ), + serialized_team, + ) + .await?; + + Ok(team) +} + +pub fn setup_redis_client(url: Option) -> Arc { + let redis_url = match url { + Some(value) => value, + None => "redis://localhost:6379/".to_string(), + }; + let client = RedisClient::new(redis_url).expect("Failed to create redis client"); + Arc::new(client) +} diff --git a/feature-flags/src/v0_endpoint.rs b/feature-flags/src/v0_endpoint.rs index 8f7761181e050..ba4bcef8fec47 100644 --- a/feature-flags/src/v0_endpoint.rs +++ b/feature-flags/src/v0_endpoint.rs @@ -33,7 +33,7 @@ use crate::{ )] #[debug_handler] pub async fn flags( - _state: State, + state: State, InsecureClientIp(ip): InsecureClientIp, meta: Query, headers: HeaderMap, @@ -59,21 +59,26 @@ pub async fn flags( .get("content-type") .map_or("", |v| v.to_str().unwrap_or("")) { - "application/x-www-form-urlencoded" => { - return Err(FlagError::RequestDecodingError(String::from( - "invalid form data", - ))); + "application/json" => { + tracing::Span::current().record("content_type", "application/json"); + FlagRequest::from_bytes(body) } ct => { - tracing::Span::current().record("content_type", ct); - - FlagRequest::from_bytes(body) + return Err(FlagError::RequestDecodingError(format!( + "unsupported content type: {}", + ct + ))); } }?; - let token = request.extract_and_verify_token()?; + let token = request + .extract_and_verify_token(state.redis.clone()) + .await?; + + let distinct_id = request.extract_distinct_id()?; tracing::Span::current().record("token", &token); + tracing::Span::current().record("distinct_id", &distinct_id); tracing::debug!("request: {:?}", request); diff --git a/feature-flags/src/v0_request.rs b/feature-flags/src/v0_request.rs index f2269df1b5f74..63b26b455f6f4 100644 --- a/feature-flags/src/v0_request.rs +++ b/feature-flags/src/v0_request.rs @@ -1,11 +1,11 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; use bytes::Bytes; use serde::{Deserialize, Serialize}; use serde_json::Value; use tracing::instrument; -use crate::api::FlagError; +use crate::{api::FlagError, redis::Client, team::Team}; #[derive(Deserialize, Default)] pub struct FlagsQueryParams { @@ -36,11 +36,8 @@ pub struct FlagRequest { } impl FlagRequest { - /// Takes a request payload and tries to decompress and unmarshall it. - /// While posthog-js sends a compression query param, a sizable portion of requests - /// fail due to it being missing when the body is compressed. - /// Instead of trusting the parameter, we peek at the payload's first three bytes to - /// detect gzip, fallback to uncompressed utf8 otherwise. + /// Takes a request payload and tries to read it. + /// Only supports base64 encoded payloads or uncompressed utf-8 as json. #[instrument(skip_all)] pub fn from_bytes(bytes: Bytes) -> Result { tracing::debug!(len = bytes.len(), "decoding new request"); @@ -54,15 +51,88 @@ impl FlagRequest { Ok(serde_json::from_str::(&payload)?) } - pub fn extract_and_verify_token(&self) -> Result { + pub async fn extract_and_verify_token( + &self, + redis_client: Arc, + ) -> Result { let token = match self { FlagRequest { token: Some(token), .. } => token.to_string(), _ => return Err(FlagError::NoTokenError), }; - // TODO: Get tokens from redis, confirm this one is valid - // validate_token(&token)?; + + // validate token + Team::from_redis(redis_client, token.clone()).await?; + + // TODO: fallback when token not found in redis + Ok(token) } + + pub fn extract_distinct_id(&self) -> Result { + let distinct_id = match &self.distinct_id { + None => return Err(FlagError::MissingDistinctId), + Some(id) => id, + }; + + match distinct_id.len() { + 0 => Err(FlagError::EmptyDistinctId), + 1..=200 => Ok(distinct_id.to_owned()), + _ => Ok(distinct_id.chars().take(200).collect()), + } + } +} + +#[cfg(test)] +mod tests { + use crate::api::FlagError; + use crate::v0_request::FlagRequest; + use bytes::Bytes; + use serde_json::json; + + #[test] + fn empty_distinct_id_not_accepted() { + let json = json!({ + "distinct_id": "", + "token": "my_token1", + }); + let bytes = Bytes::from(json.to_string()); + + let flag_payload = FlagRequest::from_bytes(bytes).expect("failed to parse request"); + + match flag_payload.extract_distinct_id() { + Err(FlagError::EmptyDistinctId) => (), + _ => panic!("expected empty distinct id error"), + }; + } + + #[test] + fn too_large_distinct_id_is_truncated() { + let json = json!({ + "distinct_id": std::iter::repeat("a").take(210).collect::(), + "token": "my_token1", + }); + let bytes = Bytes::from(json.to_string()); + + let flag_payload = FlagRequest::from_bytes(bytes).expect("failed to parse request"); + + assert_eq!(flag_payload.extract_distinct_id().unwrap().len(), 200); + } + + #[test] + fn distinct_id_is_returned_correctly() { + let json = json!({ + "$distinct_id": "alakazam", + "token": "my_token1", + }); + let bytes = Bytes::from(json.to_string()); + + let flag_payload = FlagRequest::from_bytes(bytes).expect("failed to parse request"); + + match flag_payload.extract_distinct_id() { + Ok(id) => assert_eq!(id, "alakazam"), + _ => panic!("expected distinct id"), + }; + } } diff --git a/feature-flags/tests/common.rs b/feature-flags/tests/common/mod.rs similarity index 76% rename from feature-flags/tests/common.rs rename to feature-flags/tests/common/mod.rs index f66a11ff37c25..c8644fe1f4542 100644 --- a/feature-flags/tests/common.rs +++ b/feature-flags/tests/common/mod.rs @@ -4,8 +4,7 @@ use std::string::ToString; use std::sync::Arc; use once_cell::sync::Lazy; -use rand::distributions::Alphanumeric; -use rand::Rng; +use reqwest::header::CONTENT_TYPE; use tokio::net::TcpListener; use tokio::sync::Notify; @@ -44,6 +43,21 @@ impl ServerHandle { client .post(format!("http://{:?}/flags", self.addr)) .body(body) + .header(CONTENT_TYPE, "application/json") + .send() + .await + .expect("failed to send request") + } + + pub async fn send_invalid_header_for_flags_request>( + &self, + body: T, + ) -> reqwest::Response { + let client = reqwest::Client::new(); + client + .post(format!("http://{:?}/flags", self.addr)) + .body(body) + .header(CONTENT_TYPE, "xyz") .send() .await .expect("failed to send request") @@ -55,12 +69,3 @@ impl Drop for ServerHandle { self.shutdown.notify_one() } } - -pub fn random_string(prefix: &str, length: usize) -> String { - let suffix: String = rand::thread_rng() - .sample_iter(Alphanumeric) - .take(length) - .map(char::from) - .collect(); - format!("{}_{}", prefix, suffix) -} diff --git a/feature-flags/tests/test_flags.rs b/feature-flags/tests/test_flags.rs index 82f41f05d81c6..2ceba24efd712 100644 --- a/feature-flags/tests/test_flags.rs +++ b/feature-flags/tests/test_flags.rs @@ -5,14 +5,20 @@ use reqwest::StatusCode; use serde_json::{json, Value}; use crate::common::*; -mod common; + +use feature_flags::test_utils::{insert_new_team_in_redis, setup_redis_client}; + +pub mod common; #[tokio::test] async fn it_sends_flag_request() -> Result<()> { - let token = random_string("token", 16); + let config = DEFAULT_CONFIG.clone(); + let distinct_id = "user_distinct_id".to_string(); - let config = DEFAULT_CONFIG.clone(); + let client = setup_redis_client(Some(config.redis_url.clone())); + let team = insert_new_team_in_redis(client.clone()).await.unwrap(); + let token = team.api_token; let server = ServerHandle::for_config(config).await; @@ -41,3 +47,37 @@ async fn it_sends_flag_request() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn it_rejects_invalid_headers_flag_request() -> Result<()> { + let config = DEFAULT_CONFIG.clone(); + + let distinct_id = "user_distinct_id".to_string(); + + let client = setup_redis_client(Some(config.redis_url.clone())); + let team = insert_new_team_in_redis(client.clone()).await.unwrap(); + let token = team.api_token; + + let server = ServerHandle::for_config(config).await; + + let payload = json!({ + "token": token, + "distinct_id": distinct_id, + "groups": {"group1": "group1"} + }); + let res = server + .send_invalid_header_for_flags_request(payload.to_string()) + .await; + assert_eq!(StatusCode::BAD_REQUEST, res.status()); + + // We don't want to deserialize the data into a flagResponse struct here, + // because we want to assert the shape of the raw json data. + let response_text = res.text().await?; + + assert_eq!( + response_text, + "failed to decode request: unsupported content type: xyz" + ); + + Ok(()) +} From ff0780cf5badd14c5fdf9a84c27ee222120639cd Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 29 May 2024 16:26:11 +0200 Subject: [PATCH 242/247] capture: add broker rtt latency and timeout metrics (#44) --- capture/src/sinks/kafka.rs | 41 +++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/capture/src/sinks/kafka.rs b/capture/src/sinks/kafka.rs index bc45fa1077f7b..bff61b56419d4 100644 --- a/capture/src/sinks/kafka.rs +++ b/capture/src/sinks/kafka.rs @@ -36,12 +36,11 @@ impl rdkafka::ClientContext for KafkaContext { for (topic, stats) in stats.topics { gauge!( "capture_kafka_produce_avg_batch_size_bytes", - "topic" => topic.clone() + "topic" => topic.clone() ) .set(stats.batchsize.avg as f64); gauge!( "capture_kafka_produce_avg_batch_size_events", - "topic" => topic ) .set(stats.batchcnt.avg as f64); @@ -49,30 +48,58 @@ impl rdkafka::ClientContext for KafkaContext { for (_, stats) in stats.brokers { let id_string = format!("{}", stats.nodeid); + if let Some(rtt) = stats.rtt { + gauge!( + "capture_kafka_produce_rtt_latency_ms", + "quantile" => "p50", + "broker" => id_string.clone() + ) + .set(rtt.p50 as f64); + gauge!( + "capture_kafka_produce_rtt_latency_ms", + "quantile" => "p90", + "broker" => id_string.clone() + ) + .set(rtt.p90 as f64); + gauge!( + "capture_kafka_produce_rtt_latency_ms", + "quantile" => "p95", + "broker" => id_string.clone() + ) + .set(rtt.p95 as f64); + gauge!( + "capture_kafka_produce_rtt_latency_ms", + "quantile" => "p99", + "broker" => id_string.clone() + ) + .set(rtt.p99 as f64); + } + gauge!( "capture_kafka_broker_requests_pending", - "broker" => id_string.clone() ) .set(stats.outbuf_cnt as f64); gauge!( "capture_kafka_broker_responses_awaiting", - "broker" => id_string.clone() ) .set(stats.waitresp_cnt as f64); counter!( "capture_kafka_broker_tx_errors_total", - "broker" => id_string.clone() ) .absolute(stats.txerrs); counter!( "capture_kafka_broker_rx_errors_total", - - "broker" => id_string + "broker" => id_string.clone() ) .absolute(stats.rxerrs); + counter!( + "capture_kafka_broker_request_timeouts", + "broker" => id_string + ) + .absolute(stats.req_timeouts); } } } From 8d69910b091cb2b9ff2ada1a2946b92804983c89 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Thu, 30 May 2024 15:59:20 +0200 Subject: [PATCH 243/247] capture: fix produce_rtt_latency metric unit (#46) --- capture/src/sinks/kafka.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/capture/src/sinks/kafka.rs b/capture/src/sinks/kafka.rs index bff61b56419d4..b82d3c342a115 100644 --- a/capture/src/sinks/kafka.rs +++ b/capture/src/sinks/kafka.rs @@ -50,25 +50,25 @@ impl rdkafka::ClientContext for KafkaContext { let id_string = format!("{}", stats.nodeid); if let Some(rtt) = stats.rtt { gauge!( - "capture_kafka_produce_rtt_latency_ms", + "capture_kafka_produce_rtt_latency_us", "quantile" => "p50", "broker" => id_string.clone() ) .set(rtt.p50 as f64); gauge!( - "capture_kafka_produce_rtt_latency_ms", + "capture_kafka_produce_rtt_latency_us", "quantile" => "p90", "broker" => id_string.clone() ) .set(rtt.p90 as f64); gauge!( - "capture_kafka_produce_rtt_latency_ms", + "capture_kafka_produce_rtt_latency_us", "quantile" => "p95", "broker" => id_string.clone() ) .set(rtt.p95 as f64); gauge!( - "capture_kafka_produce_rtt_latency_ms", + "capture_kafka_produce_rtt_latency_us", "quantile" => "p99", "broker" => id_string.clone() ) From 63db2a6c20182445d5d3cc71aff55fb099e6c60a Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Thu, 30 May 2024 17:18:28 +0100 Subject: [PATCH 244/247] feat(flags): Extract flag definitions from redis (#42) --- feature-flags/src/flag_definitions.rs | 200 ++++++++++++++++++++++++++ feature-flags/src/lib.rs | 1 + feature-flags/src/test_utils.rs | 43 ++++++ 3 files changed, 244 insertions(+) create mode 100644 feature-flags/src/flag_definitions.rs diff --git a/feature-flags/src/flag_definitions.rs b/feature-flags/src/flag_definitions.rs new file mode 100644 index 0000000000000..29ec8d8c38c49 --- /dev/null +++ b/feature-flags/src/flag_definitions.rs @@ -0,0 +1,200 @@ +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tracing::instrument; + +use crate::{ + api::FlagError, + redis::{Client, CustomRedisError}, +}; + +// TRICKY: This cache data is coming from django-redis. If it ever goes out of sync, we'll bork. +// TODO: Add integration tests across repos to ensure this doesn't happen. +pub const TEAM_FLAGS_CACHE_PREFIX: &str = "posthog:1:team_feature_flags_"; + +// TODO: Hmm, revisit when dealing with groups, but seems like +// ideal to just treat it as a u8 and do our own validation on top +#[derive(Debug, Deserialize, Serialize)] +pub enum GroupTypeIndex {} + +#[derive(Debug, Deserialize, Serialize)] +pub enum OperatorType { + #[serde(rename = "exact")] + Exact, + #[serde(rename = "is_not")] + IsNot, + #[serde(rename = "icontains")] + Icontains, + #[serde(rename = "not_icontains")] + NotIcontains, + #[serde(rename = "regex")] + Regex, + #[serde(rename = "not_regex")] + NotRegex, + #[serde(rename = "gt")] + Gt, + #[serde(rename = "lt")] + Lt, + #[serde(rename = "gte")] + Gte, + #[serde(rename = "lte")] + Lte, + #[serde(rename = "is_set")] + IsSet, + #[serde(rename = "is_not_set")] + IsNotSet, + #[serde(rename = "is_date_exact")] + IsDateExact, + #[serde(rename = "is_date_after")] + IsDateAfter, + #[serde(rename = "is_date_before")] + IsDateBefore, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct PropertyFilter { + pub key: String, + pub value: serde_json::Value, + pub operator: Option, + #[serde(rename = "type")] + pub prop_type: String, + pub group_type_index: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct FlagGroupType { + pub properties: Option>, + pub rollout_percentage: Option, + pub variant: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct MultivariateFlagVariant { + pub key: String, + pub name: Option, + pub rollout_percentage: f32, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct MultivariateFlagOptions { + pub variants: Vec, +} + +// TODO: test name with https://www.fileformat.info/info/charset/UTF-16/list.htm values, like '𝖕𝖗𝖔𝖕𝖊𝖗𝖙𝖞': `𝓿𝓪𝓵𝓾𝓮` + +#[derive(Debug, Deserialize, Serialize)] +pub struct FlagFilters { + pub groups: Vec, + pub multivariate: Option, + pub aggregation_group_type_index: Option, + pub payloads: Option, + pub super_groups: Option>, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct FeatureFlag { + pub id: i64, + pub team_id: i64, + pub name: Option, + pub key: String, + pub filters: FlagFilters, + #[serde(default)] + pub deleted: bool, + #[serde(default)] + pub active: bool, + #[serde(default)] + pub ensure_experience_continuity: bool, +} + +#[derive(Debug, Deserialize, Serialize)] + +pub struct FeatureFlagList { + pub flags: Vec, +} + +impl FeatureFlagList { + /// Returns feature flags given a team_id + + #[instrument(skip_all)] + pub async fn from_redis( + client: Arc, + team_id: i64, + ) -> Result { + // TODO: Instead of failing here, i.e. if not in redis, fallback to pg + let serialized_flags = client + .get(format!("{TEAM_FLAGS_CACHE_PREFIX}{}", team_id)) + .await + .map_err(|e| match e { + CustomRedisError::NotFound => FlagError::TokenValidationError, + CustomRedisError::PickleError(_) => { + tracing::error!("failed to fetch data: {}", e); + println!("failed to fetch data: {}", e); + FlagError::DataParsingError + } + _ => { + tracing::error!("Unknown redis error: {}", e); + FlagError::RedisUnavailable + } + })?; + + let flags_list: Vec = + serde_json::from_str(&serialized_flags).map_err(|e| { + tracing::error!("failed to parse data to flags list: {}", e); + println!("failed to parse data: {}", e); + + FlagError::DataParsingError + })?; + + Ok(FeatureFlagList { flags: flags_list }) + } +} + +#[cfg(test)] +mod tests { + use rand::Rng; + + use super::*; + use crate::test_utils::{ + insert_flags_for_team_in_redis, insert_new_team_in_redis, setup_redis_client, + }; + + #[tokio::test] + async fn test_fetch_flags_from_redis() { + let client = setup_redis_client(None); + + let team = insert_new_team_in_redis(client.clone()).await.unwrap(); + + insert_flags_for_team_in_redis(client.clone(), team.id, None) + .await + .expect("Failed to insert flags"); + + let flags_from_redis = FeatureFlagList::from_redis(client.clone(), team.id) + .await + .unwrap(); + assert_eq!(flags_from_redis.flags.len(), 1); + let flag = flags_from_redis.flags.get(0).unwrap(); + assert_eq!(flag.key, "flag1"); + assert_eq!(flag.team_id, team.id); + assert_eq!(flag.filters.groups.len(), 1); + assert_eq!(flag.filters.groups[0].properties.as_ref().unwrap().len(), 1); + } + + #[tokio::test] + async fn test_fetch_invalid_team_from_redis() { + let client = setup_redis_client(None); + + match FeatureFlagList::from_redis(client.clone(), 1234).await { + Err(FlagError::TokenValidationError) => (), + _ => panic!("Expected TokenValidationError"), + }; + } + + #[tokio::test] + async fn test_cant_connect_to_redis_error_is_not_token_validation_error() { + let client = setup_redis_client(Some("redis://localhost:1111/".to_string())); + + match FeatureFlagList::from_redis(client.clone(), 1234).await { + Err(FlagError::RedisUnavailable) => (), + _ => panic!("Expected RedisUnavailable"), + }; + } +} diff --git a/feature-flags/src/lib.rs b/feature-flags/src/lib.rs index 195a55c88095d..0352c21c3382a 100644 --- a/feature-flags/src/lib.rs +++ b/feature-flags/src/lib.rs @@ -1,5 +1,6 @@ pub mod api; pub mod config; +pub mod flag_definitions; pub mod redis; pub mod router; pub mod server; diff --git a/feature-flags/src/test_utils.rs b/feature-flags/src/test_utils.rs index 75db86d3878ee..0cefb7eda8e7c 100644 --- a/feature-flags/src/test_utils.rs +++ b/feature-flags/src/test_utils.rs @@ -1,7 +1,9 @@ use anyhow::Error; +use serde_json::json; use std::sync::Arc; use crate::{ + flag_definitions, redis::{Client, RedisClient}, team::{self, Team}, }; @@ -40,6 +42,47 @@ pub async fn insert_new_team_in_redis(client: Arc) -> Result, + team_id: i64, + json_value: Option, +) -> Result<(), Error> { + let payload = match json_value { + Some(value) => value, + None => json!([{ + "id": 1, + "key": "flag1", + "name": "flag1 description", + "active": true, + "deleted": false, + "team_id": team_id, + "filters": { + "groups": [ + { + "properties": [ + { + "key": "email", + "value": "a@b.com", + "type": "person", + }, + ] + }, + ], + }, + }]) + .to_string(), + }; + + client + .set( + format!("{}{}", flag_definitions::TEAM_FLAGS_CACHE_PREFIX, team_id), + payload, + ) + .await?; + + Ok(()) +} + pub fn setup_redis_client(url: Option) -> Arc { let redis_url = match url { Some(value) => value, From cf302723ef9e0e2abc8e32307c8e9e7c3072a407 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Thu, 6 Jun 2024 16:31:06 +0200 Subject: [PATCH 245/247] capture: subdivide process_events_error cause (#48) --- capture/src/v0_endpoint.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/capture/src/v0_endpoint.rs b/capture/src/v0_endpoint.rs index 3849e29328efa..ff4b90f2662e2 100644 --- a/capture/src/v0_endpoint.rs +++ b/capture/src/v0_endpoint.rs @@ -150,7 +150,14 @@ pub async fn event( tracing::debug!(context=?context, events=?events, "decoded request"); if let Err(err) = process_events(state.sink.clone(), &events, &context).await { - report_dropped_events("process_events_error", events.len() as u64); + let cause = match err { + // TODO: automate this with a macro + CaptureError::EmptyDistinctId => "empty_distinct_id", + CaptureError::MissingDistinctId => "missing_distinct_id", + CaptureError::MissingEventName => "missing_event_name", + _ => "process_events_error", + }; + report_dropped_events(cause, events.len() as u64); tracing::log::warn!("rejected invalid payload: {}", err); return Err(err); } From f28466a0f9699441109060ba1ba4e7baf3705a8f Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Mon, 10 Jun 2024 10:12:32 +0100 Subject: [PATCH 246/247] feat(flags): Match flags on rollout percentage (#45) --- Cargo.lock | 1 + feature-flags/Cargo.toml | 1 + feature-flags/src/flag_definitions.rs | 81 +- feature-flags/src/flag_matching.rs | 161 +++ feature-flags/src/lib.rs | 1 + feature-flags/src/team.rs | 1 + feature-flags/src/test_utils.rs | 35 +- .../tests/test_flag_matching_consistency.rs | 1209 +++++++++++++++++ 8 files changed, 1454 insertions(+), 36 deletions(-) create mode 100644 feature-flags/src/flag_matching.rs create mode 100644 feature-flags/tests/test_flag_matching_consistency.rs diff --git a/Cargo.lock b/Cargo.lock index 8642adee54b3a..b9f226bb08bb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -709,6 +709,7 @@ dependencies = [ "serde", "serde-pickle", "serde_json", + "sha1", "thiserror", "tokio", "tracing", diff --git a/feature-flags/Cargo.toml b/feature-flags/Cargo.toml index 1e0c111d71263..4993930362857 100644 --- a/feature-flags/Cargo.toml +++ b/feature-flags/Cargo.toml @@ -25,6 +25,7 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } serde-pickle = { version = "1.1.1"} +sha1 = "0.10.6" [lints] workspace = true diff --git a/feature-flags/src/flag_definitions.rs b/feature-flags/src/flag_definitions.rs index 29ec8d8c38c49..1f4582c606bd7 100644 --- a/feature-flags/src/flag_definitions.rs +++ b/feature-flags/src/flag_definitions.rs @@ -1,4 +1,4 @@ -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use std::sync::Arc; use tracing::instrument; @@ -13,44 +13,30 @@ pub const TEAM_FLAGS_CACHE_PREFIX: &str = "posthog:1:team_feature_flags_"; // TODO: Hmm, revisit when dealing with groups, but seems like // ideal to just treat it as a u8 and do our own validation on top -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize)] pub enum GroupTypeIndex {} -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum OperatorType { - #[serde(rename = "exact")] Exact, - #[serde(rename = "is_not")] IsNot, - #[serde(rename = "icontains")] Icontains, - #[serde(rename = "not_icontains")] NotIcontains, - #[serde(rename = "regex")] Regex, - #[serde(rename = "not_regex")] NotRegex, - #[serde(rename = "gt")] Gt, - #[serde(rename = "lt")] Lt, - #[serde(rename = "gte")] Gte, - #[serde(rename = "lte")] Lte, - #[serde(rename = "is_set")] IsSet, - #[serde(rename = "is_not_set")] IsNotSet, - #[serde(rename = "is_date_exact")] IsDateExact, - #[serde(rename = "is_date_after")] IsDateAfter, - #[serde(rename = "is_date_before")] IsDateBefore, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize)] pub struct PropertyFilter { pub key: String, pub value: serde_json::Value, @@ -60,28 +46,28 @@ pub struct PropertyFilter { pub group_type_index: Option, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize)] pub struct FlagGroupType { pub properties: Option>, - pub rollout_percentage: Option, + pub rollout_percentage: Option, pub variant: Option, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize)] pub struct MultivariateFlagVariant { pub key: String, pub name: Option, - pub rollout_percentage: f32, + pub rollout_percentage: f64, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize)] pub struct MultivariateFlagOptions { pub variants: Vec, } // TODO: test name with https://www.fileformat.info/info/charset/UTF-16/list.htm values, like '𝖕𝖗𝖔𝖕𝖊𝖗𝖙𝖞': `𝓿𝓪𝓵𝓾𝓮` -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize)] pub struct FlagFilters { pub groups: Vec, pub multivariate: Option, @@ -90,7 +76,7 @@ pub struct FlagFilters { pub super_groups: Option>, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize)] pub struct FeatureFlag { pub id: i64, pub team_id: i64, @@ -105,15 +91,31 @@ pub struct FeatureFlag { pub ensure_experience_continuity: bool, } -#[derive(Debug, Deserialize, Serialize)] +impl FeatureFlag { + pub fn get_group_type_index(&self) -> Option { + self.filters.aggregation_group_type_index + } + + pub fn get_conditions(&self) -> &Vec { + &self.filters.groups + } + + pub fn get_variants(&self) -> Vec { + self.filters + .multivariate + .clone() + .map_or(vec![], |m| m.variants) + } +} + +#[derive(Debug, Deserialize)] pub struct FeatureFlagList { pub flags: Vec, } impl FeatureFlagList { - /// Returns feature flags given a team_id - + /// Returns feature flags from redis given a team_id #[instrument(skip_all)] pub async fn from_redis( client: Arc, @@ -126,6 +128,8 @@ impl FeatureFlagList { .map_err(|e| match e { CustomRedisError::NotFound => FlagError::TokenValidationError, CustomRedisError::PickleError(_) => { + // TODO: Implement From trait for FlagError so we don't need to map + // CustomRedisError ourselves tracing::error!("failed to fetch data: {}", e); println!("failed to fetch data: {}", e); FlagError::DataParsingError @@ -150,8 +154,6 @@ impl FeatureFlagList { #[cfg(test)] mod tests { - use rand::Rng; - use super::*; use crate::test_utils::{ insert_flags_for_team_in_redis, insert_new_team_in_redis, setup_redis_client, @@ -161,7 +163,9 @@ mod tests { async fn test_fetch_flags_from_redis() { let client = setup_redis_client(None); - let team = insert_new_team_in_redis(client.clone()).await.unwrap(); + let team = insert_new_team_in_redis(client.clone()) + .await + .expect("Failed to insert team"); insert_flags_for_team_in_redis(client.clone(), team.id, None) .await @@ -169,13 +173,20 @@ mod tests { let flags_from_redis = FeatureFlagList::from_redis(client.clone(), team.id) .await - .unwrap(); + .expect("Failed to fetch flags from redis"); assert_eq!(flags_from_redis.flags.len(), 1); - let flag = flags_from_redis.flags.get(0).unwrap(); + let flag = flags_from_redis.flags.get(0).expect("Empty flags in redis"); assert_eq!(flag.key, "flag1"); assert_eq!(flag.team_id, team.id); assert_eq!(flag.filters.groups.len(), 1); - assert_eq!(flag.filters.groups[0].properties.as_ref().unwrap().len(), 1); + assert_eq!( + flag.filters.groups[0] + .properties + .as_ref() + .expect("Properties don't exist on flag") + .len(), + 1 + ); } #[tokio::test] diff --git a/feature-flags/src/flag_matching.rs b/feature-flags/src/flag_matching.rs new file mode 100644 index 0000000000000..c59b594f32e98 --- /dev/null +++ b/feature-flags/src/flag_matching.rs @@ -0,0 +1,161 @@ +use crate::flag_definitions::{FeatureFlag, FlagGroupType}; +use sha1::{Digest, Sha1}; +use std::fmt::Write; + +#[derive(Debug, PartialEq, Eq)] +pub struct FeatureFlagMatch { + pub matches: bool, + pub variant: Option, + //reason + //condition_index + //payload +} + +// TODO: Rework FeatureFlagMatcher - python has a pretty awkward interface, where we pass in all flags, and then again +// the flag to match. I don't think there's any reason anymore to store the flags in the matcher, since we can just +// pass the flag to match directly to the get_match method. This will also make the matcher more stateless. +// Potentially, we could also make the matcher a long-lived object, with caching for group keys and such. +// It just takes in the flag and distinct_id and returns the match... +// Or, make this fully stateless +// and have a separate cache struct for caching group keys, cohort definitions, etc. - and check size, if we can keep it in memory +// for all teams. If not, we can have a LRU cache, or a cache that stores only the most recent N keys. +// But, this can be a future refactor, for now just focusing on getting the basic matcher working, write lots and lots of tests +// and then we can easily refactor stuff around. +#[derive(Debug)] +pub struct FeatureFlagMatcher { + // pub flags: Vec, + pub distinct_id: String, +} + +const LONG_SCALE: u64 = 0xfffffffffffffff; + +impl FeatureFlagMatcher { + pub fn new(distinct_id: String) -> Self { + FeatureFlagMatcher { + // flags, + distinct_id, + } + } + + pub fn get_match(&self, feature_flag: &FeatureFlag) -> FeatureFlagMatch { + if self.hashed_identifier(feature_flag).is_none() { + return FeatureFlagMatch { + matches: false, + variant: None, + }; + } + + // TODO: super groups for early access + // TODO: Variant overrides condition sort + + for (index, condition) in feature_flag.get_conditions().iter().enumerate() { + let (is_match, _evaluation_reason) = + self.is_condition_match(feature_flag, condition, index); + + if is_match { + // TODO: This is a bit awkward, we should handle overrides only when variants exist. + let variant = match condition.variant.clone() { + Some(variant_override) => { + if feature_flag + .get_variants() + .iter() + .any(|v| v.key == variant_override) + { + Some(variant_override) + } else { + self.get_matching_variant(feature_flag) + } + } + None => self.get_matching_variant(feature_flag), + }; + + // let payload = self.get_matching_payload(is_match, variant, feature_flag); + return FeatureFlagMatch { + matches: true, + variant, + }; + } + } + FeatureFlagMatch { + matches: false, + variant: None, + } + } + + pub fn is_condition_match( + &self, + feature_flag: &FeatureFlag, + condition: &FlagGroupType, + _index: usize, + ) -> (bool, String) { + let rollout_percentage = condition.rollout_percentage.unwrap_or(100.0); + let mut condition_match = true; + if condition.properties.is_some() { + // TODO: Handle matching conditions + if !condition.properties.as_ref().unwrap().is_empty() { + condition_match = false; + } + } + + if !condition_match { + return (false, "NO_CONDITION_MATCH".to_string()); + } else if rollout_percentage == 100.0 { + // TODO: Check floating point schenanigans if any + return (true, "CONDITION_MATCH".to_string()); + } + + if self.get_hash(feature_flag, "") > (rollout_percentage / 100.0) { + return (false, "OUT_OF_ROLLOUT_BOUND".to_string()); + } + + (true, "CONDITION_MATCH".to_string()) + } + + pub fn hashed_identifier(&self, feature_flag: &FeatureFlag) -> Option { + if feature_flag.get_group_type_index().is_none() { + // TODO: Use hash key overrides for experience continuity + Some(self.distinct_id.clone()) + } else { + // TODO: Handle getting group key + Some("".to_string()) + } + } + + /// This function takes a identifier and a feature flag key and returns a float between 0 and 1. + /// Given the same identifier and key, it'll always return the same float. These floats are + /// uniformly distributed between 0 and 1, so if we want to show this feature to 20% of traffic + /// we can do _hash(key, identifier) < 0.2 + pub fn get_hash(&self, feature_flag: &FeatureFlag, salt: &str) -> f64 { + // check if hashed_identifier is None + let hashed_identifier = self + .hashed_identifier(feature_flag) + .expect("hashed_identifier is None when computing hash"); + let hash_key = format!("{}.{}{}", feature_flag.key, hashed_identifier, salt); + let mut hasher = Sha1::new(); + hasher.update(hash_key.as_bytes()); + let result = hasher.finalize(); + // :TRICKY: Convert the first 15 characters of the digest to a hexadecimal string + // not sure if this is correct, padding each byte as 2 characters + let hex_str: String = result.iter().fold(String::new(), |mut acc, byte| { + let _ = write!(acc, "{:02x}", byte); + acc + })[..15] + .to_string(); + let hash_val = u64::from_str_radix(&hex_str, 16).unwrap(); + + hash_val as f64 / LONG_SCALE as f64 + } + + pub fn get_matching_variant(&self, feature_flag: &FeatureFlag) -> Option { + let hash = self.get_hash(feature_flag, "variant"); + let mut total_percentage = 0.0; + + for variant in feature_flag.get_variants() { + total_percentage += variant.rollout_percentage / 100.0; + if hash < total_percentage { + return Some(variant.key.clone()); + } + } + None + } +} diff --git a/feature-flags/src/lib.rs b/feature-flags/src/lib.rs index 0352c21c3382a..edc2a2963ff3b 100644 --- a/feature-flags/src/lib.rs +++ b/feature-flags/src/lib.rs @@ -1,6 +1,7 @@ pub mod api; pub mod config; pub mod flag_definitions; +pub mod flag_matching; pub mod redis; pub mod router; pub mod server; diff --git a/feature-flags/src/team.rs b/feature-flags/src/team.rs index ac62ea9ba55cb..e872aa477968f 100644 --- a/feature-flags/src/team.rs +++ b/feature-flags/src/team.rs @@ -42,6 +42,7 @@ impl Team { } })?; + // TODO: Consider an LRU cache for teams as well, with small TTL to skip redis/pg lookups let team: Team = serde_json::from_str(&serialized_team).map_err(|e| { tracing::error!("failed to parse data to team: {}", e); FlagError::DataParsingError diff --git a/feature-flags/src/test_utils.rs b/feature-flags/src/test_utils.rs index 0cefb7eda8e7c..92bc8a4ff4494 100644 --- a/feature-flags/src/test_utils.rs +++ b/feature-flags/src/test_utils.rs @@ -3,7 +3,7 @@ use serde_json::json; use std::sync::Arc; use crate::{ - flag_definitions, + flag_definitions::{self, FeatureFlag}, redis::{Client, RedisClient}, team::{self, Team}, }; @@ -91,3 +91,36 @@ pub fn setup_redis_client(url: Option) -> Arc { let client = RedisClient::new(redis_url).expect("Failed to create redis client"); Arc::new(client) } + +pub fn create_flag_from_json(json_value: Option) -> Vec { + let payload = match json_value { + Some(value) => value, + None => json!([{ + "id": 1, + "key": "flag1", + "name": "flag1 description", + "active": true, + "deleted": false, + "team_id": 1, + "filters": { + "groups": [ + { + "properties": [ + { + "key": "email", + "value": "a@b.com", + "type": "person", + }, + ], + "rollout_percentage": 50, + }, + ], + }, + }]) + .to_string(), + }; + + let flags: Vec = + serde_json::from_str(&payload).expect("Failed to parse data to flags list"); + flags +} diff --git a/feature-flags/tests/test_flag_matching_consistency.rs b/feature-flags/tests/test_flag_matching_consistency.rs new file mode 100644 index 0000000000000..4a24b0e16d50e --- /dev/null +++ b/feature-flags/tests/test_flag_matching_consistency.rs @@ -0,0 +1,1209 @@ +/// These tests are common between all libraries doing local evaluation of feature flags. +/// This ensures there are no mismatches between implementations. +use feature_flags::flag_matching::{FeatureFlagMatch, FeatureFlagMatcher}; + +use feature_flags::test_utils::create_flag_from_json; +use serde_json::json; + +#[test] +fn it_is_consistent_with_rollout_calculation_for_simple_flags() { + let flags = create_flag_from_json(Some( + json!([{ + "id": 1, + "key": "simple-flag", + "name": "Simple flag", + "active": true, + "deleted": false, + "team_id": 1, + "filters": { + "groups": [ + { + "properties": [], + "rollout_percentage": 45, + }, + ], + }, + }]) + .to_string(), + )); + + let results = vec![ + false, true, true, false, true, false, false, true, false, true, false, true, true, false, + true, false, false, false, true, true, false, true, false, false, true, false, true, true, + false, false, false, true, true, true, true, false, false, false, false, false, false, + true, true, false, true, true, false, false, false, true, true, false, false, false, false, + true, false, true, false, true, false, true, true, false, true, false, true, false, true, + true, false, false, true, false, false, true, false, true, false, false, true, false, + false, false, true, true, false, true, true, false, true, true, true, true, true, false, + true, true, false, false, true, true, true, true, false, false, true, false, true, true, + true, false, false, false, false, false, true, false, false, true, true, true, false, + false, true, false, true, false, false, true, false, false, false, false, false, false, + false, false, true, true, false, false, true, false, false, true, true, false, false, true, + false, true, false, true, true, true, false, false, false, true, false, false, false, + false, true, true, false, true, true, false, true, false, true, true, false, true, false, + true, true, true, false, true, false, false, true, true, false, true, false, true, true, + false, false, true, true, true, true, false, true, true, false, false, true, false, true, + false, false, true, true, false, true, false, true, false, false, false, false, false, + false, false, true, false, true, true, false, false, true, false, true, false, false, + false, true, false, true, false, false, false, true, false, false, true, false, true, true, + false, false, false, false, true, false, false, false, false, false, false, false, false, + false, false, false, false, false, true, true, false, true, false, true, true, false, true, + false, true, false, false, false, true, true, true, true, false, false, false, false, + false, true, true, true, false, false, true, true, false, false, false, false, false, true, + false, true, true, true, true, false, true, true, true, false, false, true, false, true, + false, false, true, true, true, false, true, false, false, false, true, true, false, true, + false, true, false, true, true, true, true, true, false, false, true, false, true, false, + true, true, true, false, true, false, true, true, false, true, true, true, true, true, + false, false, false, false, false, true, false, true, false, false, true, true, false, + false, false, true, false, true, true, true, true, false, false, false, false, true, true, + false, false, true, true, false, true, true, true, true, false, true, true, true, false, + false, true, true, false, false, true, false, false, true, false, false, false, false, + false, false, false, false, false, false, true, true, false, false, true, false, false, + true, false, true, false, false, true, false, false, false, false, false, false, true, + false, false, false, false, false, false, false, false, false, true, true, true, false, + false, false, true, false, true, false, false, false, true, false, false, false, false, + false, false, false, true, false, false, false, false, false, false, false, false, true, + false, true, false, true, true, true, false, false, false, true, true, true, false, true, + false, true, true, false, false, false, true, false, false, false, false, true, false, + true, false, true, true, false, true, false, false, false, true, false, false, true, true, + false, true, false, false, false, false, false, false, true, true, false, false, true, + false, false, true, true, true, false, false, false, true, false, false, false, false, + true, false, true, false, false, false, true, false, true, true, false, true, false, true, + false, true, false, false, true, false, false, true, false, true, false, true, false, true, + false, false, true, true, true, true, false, true, false, false, false, false, false, true, + false, false, true, false, false, true, true, false, false, false, false, true, true, true, + false, false, true, false, false, true, true, true, true, false, false, false, true, false, + false, false, true, false, false, true, true, true, true, false, false, true, true, false, + true, false, true, false, false, true, true, false, true, true, true, true, false, false, + true, false, false, true, true, false, true, false, true, false, false, true, false, false, + false, false, true, true, true, false, true, false, false, true, false, false, true, false, + false, false, false, true, false, true, false, true, true, false, false, true, false, true, + true, true, false, false, false, false, true, true, false, true, false, false, false, true, + false, false, false, false, true, true, true, false, false, false, true, true, true, true, + false, true, true, false, true, true, true, false, true, false, false, true, false, true, + true, true, true, false, true, false, true, false, true, false, false, true, true, false, + false, true, false, true, false, false, false, false, true, false, true, false, false, + false, true, true, true, false, false, false, true, false, true, true, false, false, false, + false, false, true, false, true, false, false, true, true, false, true, true, true, true, + false, false, true, false, false, true, false, true, false, true, true, false, false, + false, true, false, true, true, false, false, false, true, false, true, false, true, true, + false, true, false, false, true, false, false, false, true, true, true, false, false, + false, false, false, true, false, false, true, true, true, true, true, false, false, false, + false, false, false, false, false, true, true, true, false, false, true, true, false, true, + true, false, true, false, true, false, false, false, true, false, false, true, false, + false, true, true, true, true, false, false, true, false, true, true, false, false, true, + false, false, true, true, false, true, false, false, true, true, true, false, false, false, + false, false, true, false, true, false, false, false, false, false, true, true, false, + true, true, true, false, false, false, false, true, true, true, true, false, true, true, + false, true, false, true, false, true, false, false, false, false, true, true, true, true, + false, false, true, false, true, true, false, false, false, false, false, false, true, + false, true, false, true, true, false, false, true, true, true, true, false, false, true, + false, true, true, false, false, true, true, true, false, true, false, false, true, true, + false, false, false, true, false, false, true, false, false, false, true, true, true, true, + false, true, false, true, false, true, false, true, false, false, true, false, false, true, + false, true, true, + ]; + + for i in 0..1000 { + let distinct_id = format!("distinct_id_{}", i); + + let feature_flag_match = FeatureFlagMatcher::new(distinct_id).get_match(&flags[0]); + + if results[i] { + assert_eq!( + feature_flag_match, + FeatureFlagMatch { + matches: true, + variant: None, + } + ); + } else { + assert_eq!( + feature_flag_match, + FeatureFlagMatch { + matches: false, + variant: None, + } + ); + } + } +} + +#[test] +fn it_is_consistent_with_rollout_calculation_for_multivariate_flags() { + let flags = create_flag_from_json(Some( + json!([{ + "id": 1, + "key": "multivariate-flag", + "name": "Multivariate flag", + "active": true, + "deleted": false, + "team_id": 1, + "filters": { + "groups": [ + { + "properties": [], + "rollout_percentage": 55, + }, + ], + "multivariate": { + "variants": [ + { + "key": "first-variant", + "name": "First Variant", + "rollout_percentage": 50, + }, + { + "key": "second-variant", + "name": "Second Variant", + "rollout_percentage": 20, + }, + { + "key": "third-variant", + "name": "Third Variant", + "rollout_percentage": 20, + }, + { + "key": "fourth-variant", + "name": "Fourth Variant", + "rollout_percentage": 5, + }, + { + "key": "fifth-variant", + "name": "Fifth Variant", + "rollout_percentage": 5, + }, + ], + }, + }, + }]) + .to_string(), + )); + + let results = vec![ + Some("second-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("fourth-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + None, + None, + None, + None, + None, + Some("first-variant".to_string()), + Some("fifth-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + None, + None, + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + None, + Some("third-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + None, + None, + None, + Some("first-variant".to_string()), + None, + None, + None, + None, + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + None, + Some("fifth-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("fifth-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("fourth-variant".to_string()), + None, + None, + None, + None, + Some("third-variant".to_string()), + None, + None, + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("second-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + None, + None, + Some("second-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("fifth-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("second-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("fifth-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + None, + None, + Some("third-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + None, + Some("third-variant".to_string()), + None, + None, + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + None, + Some("fourth-variant".to_string()), + Some("fourth-variant".to_string()), + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("fifth-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("fifth-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("second-variant".to_string()), + Some("fifth-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + None, + Some("third-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("fifth-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("fourth-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("fifth-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("fourth-variant".to_string()), + Some("fifth-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + None, + Some("third-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + None, + None, + Some("third-variant".to_string()), + Some("fourth-variant".to_string()), + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + None, + Some("fourth-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("fourth-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("fourth-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + None, + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("fifth-variant".to_string()), + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("fourth-variant".to_string()), + None, + None, + None, + Some("fourth-variant".to_string()), + None, + None, + Some("third-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("second-variant".to_string()), + None, + None, + Some("fifth-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + None, + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("third-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + None, + None, + None, + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + None, + None, + None, + None, + Some("first-variant".to_string()), + None, + None, + None, + None, + None, + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + None, + None, + None, + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + None, + None, + Some("second-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + None, + None, + None, + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("fifth-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + None, + None, + None, + None, + None, + Some("fourth-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("fifth-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("fifth-variant".to_string()), + None, + None, + Some("third-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + None, + None, + None, + None, + Some("first-variant".to_string()), + None, + None, + None, + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("fifth-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("third-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("third-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + None, + Some("second-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("fifth-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("fifth-variant".to_string()), + None, + None, + None, + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("second-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + None, + None, + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + Some("second-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("second-variant".to_string()), + None, + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("fifth-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("fourth-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("fifth-variant".to_string()), + None, + None, + None, + Some("second-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("second-variant".to_string()), + None, + None, + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("third-variant".to_string()), + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + None, + None, + Some("first-variant".to_string()), + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("third-variant".to_string()), + Some("second-variant".to_string()), + Some("third-variant".to_string()), + None, + None, + Some("third-variant".to_string()), + Some("first-variant".to_string()), + None, + Some("first-variant".to_string()), + ]; + + for i in 0..1000 { + let distinct_id = format!("distinct_id_{}", i); + + let feature_flag_match = FeatureFlagMatcher::new(distinct_id).get_match(&flags[0]); + + if results[i].is_some() { + assert_eq!( + feature_flag_match, + FeatureFlagMatch { + matches: true, + variant: results[i].clone(), + } + ); + } else { + assert_eq!( + feature_flag_match, + FeatureFlagMatch { + matches: false, + variant: None, + } + ); + } + } +} From f6569a710a8a505aa4d170abc36c0bb49dfa3bdc Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Mon, 10 Jun 2024 10:45:23 +0100 Subject: [PATCH 247/247] feat(flags): Add basic property matching (#47) --- Cargo.lock | 9 +- feature-flags/Cargo.toml | 1 + feature-flags/README.md | 36 + feature-flags/src/flag_definitions.rs | 5 +- feature-flags/src/flag_matching.rs | 1 - feature-flags/src/lib.rs | 1 + feature-flags/src/property_matching.rs | 1647 ++++++++++++++++++++++++ 7 files changed, 1694 insertions(+), 6 deletions(-) create mode 100644 feature-flags/README.md create mode 100644 feature-flags/src/property_matching.rs diff --git a/Cargo.lock b/Cargo.lock index b9f226bb08bb9..804ab47416080 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -705,6 +705,7 @@ dependencies = [ "once_cell", "rand", "redis", + "regex", "reqwest 0.12.3", "serde", "serde-pickle", @@ -1033,9 +1034,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.5" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -2250,9 +2251,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", diff --git a/feature-flags/Cargo.toml b/feature-flags/Cargo.toml index 4993930362857..08ff21eaed0d8 100644 --- a/feature-flags/Cargo.toml +++ b/feature-flags/Cargo.toml @@ -26,6 +26,7 @@ serde_json = { workspace = true } thiserror = { workspace = true } serde-pickle = { version = "1.1.1"} sha1 = "0.10.6" +regex = "1.10.4" [lints] workspace = true diff --git a/feature-flags/README.md b/feature-flags/README.md new file mode 100644 index 0000000000000..1c9500900aade --- /dev/null +++ b/feature-flags/README.md @@ -0,0 +1,36 @@ + +# Testing + +``` +cargo test --package feature-flags +``` + +### To watch changes + +``` +brew install cargo-watch +``` + +and then run: + +``` +cargo watch -x test --package feature-flags +``` + +To run a specific test: + +``` +cargo watch -x "test --package feature-flags --lib -- property_matching::tests::test_match_properties_math_operators --exact --show-output" +``` + +# Running + +``` +RUST_LOG=debug cargo run --bin feature-flags +``` + +# Format code + +``` +cargo fmt --package feature-flags +``` \ No newline at end of file diff --git a/feature-flags/src/flag_definitions.rs b/feature-flags/src/flag_definitions.rs index 1f4582c606bd7..fbbd0445b5998 100644 --- a/feature-flags/src/flag_definitions.rs +++ b/feature-flags/src/flag_definitions.rs @@ -16,7 +16,7 @@ pub const TEAM_FLAGS_CACHE_PREFIX: &str = "posthog:1:team_feature_flags_"; #[derive(Debug, Deserialize)] pub enum GroupTypeIndex {} -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] #[serde(rename_all = "snake_case")] pub enum OperatorType { Exact, @@ -39,6 +39,9 @@ pub enum OperatorType { #[derive(Debug, Clone, Deserialize)] pub struct PropertyFilter { pub key: String, + // TODO: Probably need a default for value? + // incase operators like is_set, is_not_set are used + // not guaranteed to have a value, if say created via api pub value: serde_json::Value, pub operator: Option, #[serde(rename = "type")] diff --git a/feature-flags/src/flag_matching.rs b/feature-flags/src/flag_matching.rs index c59b594f32e98..510fc153dc87a 100644 --- a/feature-flags/src/flag_matching.rs +++ b/feature-flags/src/flag_matching.rs @@ -135,7 +135,6 @@ impl FeatureFlagMatcher { hasher.update(hash_key.as_bytes()); let result = hasher.finalize(); // :TRICKY: Convert the first 15 characters of the digest to a hexadecimal string - // not sure if this is correct, padding each byte as 2 characters let hex_str: String = result.iter().fold(String::new(), |mut acc, byte| { let _ = write!(acc, "{:02x}", byte); acc diff --git a/feature-flags/src/lib.rs b/feature-flags/src/lib.rs index edc2a2963ff3b..7f03747b9ee6d 100644 --- a/feature-flags/src/lib.rs +++ b/feature-flags/src/lib.rs @@ -2,6 +2,7 @@ pub mod api; pub mod config; pub mod flag_definitions; pub mod flag_matching; +pub mod property_matching; pub mod redis; pub mod router; pub mod server; diff --git a/feature-flags/src/property_matching.rs b/feature-flags/src/property_matching.rs new file mode 100644 index 0000000000000..9f7d9ea173963 --- /dev/null +++ b/feature-flags/src/property_matching.rs @@ -0,0 +1,1647 @@ +use std::collections::HashMap; + +use crate::flag_definitions::{OperatorType, PropertyFilter}; +use regex::Regex; +use serde_json::Value; + +#[derive(Debug, PartialEq, Eq)] +pub enum FlagMatchingError { + ValidationError(String), + MissingProperty(String), + InconclusiveOperatorMatch, + InvalidRegexPattern, +} + +pub fn to_string_representation(value: &Value) -> String { + if value.is_string() { + return value + .as_str() + .expect("string slice should always exist for string value") + .to_string(); + } + value.to_string() +} + +pub fn to_f64_representation(value: &Value) -> Option { + if value.is_number() { + return value.as_f64(); + } + to_string_representation(value).parse::().ok() +} + +pub fn match_property( + property: &PropertyFilter, + matching_property_values: &HashMap, + partial_props: bool, +) -> Result { + // only looks for matches where key exists in override_property_values + // doesn't support operator is_not_set with partial_props + + if partial_props && !matching_property_values.contains_key(&property.key) { + return Err(FlagMatchingError::MissingProperty(format!( + "can't match properties without a value. Missing property: {}", + property.key + ))); + } + + let key = &property.key; + let operator = property.operator.clone().unwrap_or(OperatorType::Exact); + let value = &property.value; + let match_value = matching_property_values.get(key); + + match operator { + OperatorType::Exact | OperatorType::IsNot => { + let compute_exact_match = |value: &Value, override_value: &Value| -> bool { + if is_truthy_or_falsy_property_value(value) { + // Do boolean handling, such that passing in "true" or "True" or "false" or "False" as matching value is equivalent + let truthy = is_truthy_property_value(value); + return override_value.to_string().to_lowercase() + == truthy.to_string().to_lowercase(); + } + + if value.is_array() { + return value + .as_array() + .expect("expected array value") + .iter() + .map(|v| to_string_representation(v).to_lowercase()) + .collect::>() + .contains(&to_string_representation(override_value).to_lowercase()); + } + to_string_representation(value).to_lowercase() + == to_string_representation(override_value).to_lowercase() + }; + + if let Some(match_value) = match_value { + if operator == OperatorType::Exact { + Ok(compute_exact_match(value, match_value)) + } else { + Ok(!compute_exact_match(value, match_value)) + } + } else { + Ok(false) + } + } + OperatorType::IsSet => Ok(matching_property_values.contains_key(key)), + OperatorType::IsNotSet => { + if partial_props { + if matching_property_values.contains_key(key) { + Ok(false) + } else { + Err(FlagMatchingError::InconclusiveOperatorMatch) + } + } else { + Ok(!matching_property_values.contains_key(key)) + } + } + OperatorType::Icontains | OperatorType::NotIcontains => { + if let Some(match_value) = match_value { + // TODO: Check eq_ignore_ascii_case and to_ascii_lowercase + // see https://doc.rust-lang.org/std/string/struct.String.html#method.to_lowercase + // do we want to lowercase non-ascii stuff? + let is_contained = to_string_representation(match_value) + .to_lowercase() + .contains(&to_string_representation(value).to_lowercase()); + + if operator == OperatorType::Icontains { + Ok(is_contained) + } else { + Ok(!is_contained) + } + } else { + // When value doesn't exist, it's not a match + Ok(false) + } + } + OperatorType::Regex | OperatorType::NotRegex => { + if match_value.is_none() { + return Ok(false); + } + + let pattern = match Regex::new(&to_string_representation(value)) { + Ok(pattern) => pattern, + Err(_) => return Ok(false), + //TODO: Should we return Err here and handle elsewhere? + //Err(FlagMatchingError::InvalidRegexPattern) + // python just returns false here + }; + let haystack = to_string_representation(match_value.unwrap_or(&Value::Null)); + let match_ = pattern.find(&haystack); + + if operator == OperatorType::Regex { + Ok(match_.is_some()) + } else { + Ok(match_.is_none()) + } + } + OperatorType::Gt | OperatorType::Gte | OperatorType::Lt | OperatorType::Lte => { + if match_value.is_none() { + return Ok(false); + } + // TODO: Move towards only numeric matching of these operators??? + + let compare = |lhs: f64, rhs: f64, operator: OperatorType| -> bool { + match operator { + OperatorType::Gt => lhs > rhs, + OperatorType::Gte => lhs >= rhs, + OperatorType::Lt => lhs < rhs, + OperatorType::Lte => lhs <= rhs, + _ => false, + } + }; + + let parsed_value = match to_f64_representation(match_value.unwrap_or(&Value::Null)) { + Some(parsed_value) => parsed_value, + None => { + return Err(FlagMatchingError::ValidationError( + "value is not a number".to_string(), + )) + } + }; + + if let Some(override_value) = to_f64_representation(value) { + Ok(compare(parsed_value, override_value, operator)) + } else { + Err(FlagMatchingError::ValidationError( + "override value is not a number".to_string(), + )) + } + } + OperatorType::IsDateExact | OperatorType::IsDateAfter | OperatorType::IsDateBefore => { + // TODO: Handle date operators + Ok(false) + // let parsed_date = determine_parsed_date_for_property_matching(match_value); + + // if parsed_date.is_none() { + // return Ok(false); + // } + + // if let Some(override_value) = value.as_str() { + // let override_date = match parser::parse(override_value) { + // Ok(override_date) => override_date, + // Err(_) => return Ok(false), + // }; + + // match operator { + // OperatorType::IsDateBefore => Ok(override_date < parsed_date.unwrap()), + // OperatorType::IsDateAfter => Ok(override_date > parsed_date.unwrap()), + // _ => Ok(false), + // } + // } else { + // Ok(false) + // } + } + } +} + +fn is_truthy_or_falsy_property_value(value: &Value) -> bool { + if value.is_boolean() { + return true; + } + + if value.is_string() { + let parsed_value = value + .as_str() + .expect("expected string value") + .to_lowercase(); + return parsed_value == "true" || parsed_value == "false"; + } + + if value.is_array() { + return value + .as_array() + .expect("expected array value") + .iter() + .all(is_truthy_or_falsy_property_value); + } + + false +} + +fn is_truthy_property_value(value: &Value) -> bool { + if value.is_boolean() { + return value.as_bool().expect("expected boolean value"); + } + + if value.is_string() { + let parsed_value = value + .as_str() + .expect("expected string value") + .to_lowercase(); + return parsed_value == "true"; + } + + if value.is_array() { + return value + .as_array() + .expect("expected array value") + .iter() + .all(is_truthy_property_value); + } + + false +} + +/// Copy of https://github.com/PostHog/posthog/blob/master/posthog/queries/test/test_base.py#L35 +/// with some modifications to match Rust's behavior +/// and to test the match_property function +#[cfg(test)] +mod test_match_properties { + use super::*; + use serde_json::json; + + #[test] + fn test_match_properties_exact_with_partial_props() { + let property_a = PropertyFilter { + key: "key".to_string(), + value: json!("value"), + operator: None, + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("value"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("value2"))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(""))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(null))]), + true + ) + .expect("expected match to exist"), + false + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key2".to_string(), json!("value"))]), + true + ) + .is_err(), + true + ); + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key2".to_string(), json!("value"))]), + true + ) + .err() + .expect("expected match to exist"), + FlagMatchingError::MissingProperty( + "can't match properties without a value. Missing property: key".to_string() + ) + ); + assert_eq!( + match_property(&property_a, &HashMap::from([]), true).is_err(), + true + ); + + let property_b = PropertyFilter { + key: "key".to_string(), + value: json!("value"), + operator: Some(OperatorType::Exact), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!("value"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!("value2"))]), + true + ) + .expect("expected match to exist"), + false + ); + + let property_c = PropertyFilter { + key: "key".to_string(), + value: json!(["value1", "value2", "value3"]), + operator: Some(OperatorType::Exact), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value1"))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value2"))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value3"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value4"))]), + true + ) + .expect("expected match to exist"), + false + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key2".to_string(), json!("value"))]), + true + ) + .is_err(), + true + ); + } + + #[test] + fn test_match_properties_is_not() { + let property_a = PropertyFilter { + key: "key".to_string(), + value: json!("value"), + operator: Some(OperatorType::IsNot), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("value2"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(""))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(null))]), + true + ) + .expect("expected match to exist"), + true + ); + + // partial mode returns error when key doesn't exist + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key2".to_string(), json!("value1"))]), + true + ) + .is_err(), + true + ); + + let property_c = PropertyFilter { + key: "key".to_string(), + value: json!(["value1", "value2", "value3"]), + operator: Some(OperatorType::IsNot), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value4"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value5"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value6"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!(""))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!(null))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value2"))]), + true + ) + .expect("expected match to exist"), + false + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value3"))]), + true + ) + .expect("expected match to exist"), + false + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value1"))]), + true + ) + .expect("expected match to exist"), + false + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key2".to_string(), json!("value1"))]), + true + ) + .is_err(), + true + ); + } + + #[test] + fn test_match_properties_is_set() { + let property_a = PropertyFilter { + key: "key".to_string(), + value: json!("value"), + operator: Some(OperatorType::IsSet), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("value"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("value2"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(""))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(null))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key2".to_string(), json!("value1"))]), + true + ) + .is_err(), + true + ); + + assert_eq!( + match_property(&property_a, &HashMap::from([]), true).is_err(), + true + ); + } + + #[test] + fn test_match_properties_icontains() { + let property_a = PropertyFilter { + key: "key".to_string(), + value: json!("valUe"), + operator: Some(OperatorType::Icontains), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("value"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("value2"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("value3"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("vaLue4"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("343tfvalue5"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("Alakazam"))]), + true + ) + .expect("expected match to exist"), + false + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(123))]), + true + ) + .expect("expected match to exist"), + false + ); + + let property_b = PropertyFilter { + key: "key".to_string(), + value: json!("3"), + operator: Some(OperatorType::Icontains), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!("3"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!(323))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!("val3"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!("three"))]), + true + ) + .expect("expected match to exist"), + false + ); + } + + #[test] + fn test_match_properties_regex() { + let property_a = PropertyFilter { + key: "key".to_string(), + value: json!(r"\.com$"), + operator: Some(OperatorType::Regex), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("value.com"))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("value2.com"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(".com343tfvalue5"))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("Alakazam"))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(123))]), + true + ) + .expect("expected match to exist"), + false + ); + + let property_b = PropertyFilter { + key: "key".to_string(), + value: json!("3"), + operator: Some(OperatorType::Regex), + prop_type: "person".to_string(), + group_type_index: None, + }; + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!("3"))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!(323))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!("val3"))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!("three"))]), + true + ) + .expect("expected match to exist"), + false + ); + + // invalid regex + let property_c = PropertyFilter { + key: "key".to_string(), + value: json!(r"?*"), + operator: Some(OperatorType::Regex), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value"))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("value2"))]), + true + ) + .expect("expected match to exist"), + false + ); + + // non string value + let property_d = PropertyFilter { + key: "key".to_string(), + value: json!(4), + operator: Some(OperatorType::Regex), + prop_type: "person".to_string(), + group_type_index: None, + }; + assert_eq!( + match_property( + &property_d, + &HashMap::from([("key".to_string(), json!("4"))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_d, + &HashMap::from([("key".to_string(), json!(4))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_d, + &HashMap::from([("key".to_string(), json!("value"))]), + true + ) + .expect("expected match to exist"), + false + ); + } + + #[test] + fn test_match_properties_math_operators() { + let property_a = PropertyFilter { + key: "key".to_string(), + value: json!(1), + operator: Some(OperatorType::Gt), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(2))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(3))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(0))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(-1))]), + true + ) + .expect("expected match to exist"), + false + ); + + // # we handle type mismatches so this should be true + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("23"))]), + true + ) + .expect("expected match to exist"), + true + ); + + let property_b = PropertyFilter { + key: "key".to_string(), + value: json!(1), + operator: Some(OperatorType::Lt), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!(0))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!(-1))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!(-3))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!(1))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!("1"))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!("3"))]), + true + ) + .expect("expected match to exist"), + false + ); + + let property_c = PropertyFilter { + key: "key".to_string(), + value: json!(1), + operator: Some(OperatorType::Gte), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!(1))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!(2))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!(0))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!(-1))]), + true + ) + .expect("expected match to exist"), + false + ); + // # now we handle type mismatches so this should be true + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("3"))]), + true + ) + .expect("expected match to exist"), + true + ); + + let property_d = PropertyFilter { + key: "key".to_string(), + value: json!("43"), + operator: Some(OperatorType::Lt), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_d, + &HashMap::from([("key".to_string(), json!("41"))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_d, + &HashMap::from([("key".to_string(), json!("42"))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_d, + &HashMap::from([("key".to_string(), json!(42))]), + true + ) + .expect("expected match to exist"), + true + ); + + assert_eq!( + match_property( + &property_d, + &HashMap::from([("key".to_string(), json!("43"))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_d, + &HashMap::from([("key".to_string(), json!("44"))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_d, + &HashMap::from([("key".to_string(), json!(44))]), + true + ) + .expect("expected match to exist"), + false + ); + + let property_e = PropertyFilter { + key: "key".to_string(), + value: json!("30"), + operator: Some(OperatorType::Lt), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_e, + &HashMap::from([("key".to_string(), json!("29"))]), + true + ) + .expect("expected match to exist"), + true + ); + + // # depending on the type of override, we adjust type comparison + // This is wonky, do we want to continue this behavior? :/ + // TODO: Come back to this + // assert_eq!( + // match_property( + // &property_e, + // &HashMap::from([("key".to_string(), json!("100"))]), + // true + // ) + // .expect("expected match to exist"), + // true + // ); + assert_eq!( + match_property( + &property_e, + &HashMap::from([("key".to_string(), json!(100))]), + true + ) + .expect("expected match to exist"), + false + ); + + // let property_f = PropertyFilter { + // key: "key".to_string(), + // value: json!("123aloha"), + // operator: Some(OperatorType::Gt), + // prop_type: "person".to_string(), + // group_type_index: None, + // }; + + // TODO: This test fails because 123aloha is not a number + // and currently we don't support string comparison.. + // assert_eq!( + // match_property( + // &property_f, + // &HashMap::from([("key".to_string(), json!("123"))]), + // true + // ) + // .expect("expected match to exist"), + // false + // ); + // assert_eq!( + // match_property( + // &property_f, + // &HashMap::from([("key".to_string(), json!(122))]), + // true + // ) + // .expect("expected match to exist"), + // false + // ); + + // # this turns into a string comparison + // TODO: Fix + // assert_eq!( + // match_property( + // &property_f, + // &HashMap::from([("key".to_string(), json!(129))]), + // true + // ) + // .expect("expected match to exist"), + // true + // ); + } + + #[test] + fn test_none_property_value_with_all_operators() { + let property_a = PropertyFilter { + key: "key".to_string(), + value: json!("null"), + operator: Some(OperatorType::IsNot), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!(null))]), + true + ) + .expect("expected match to exist"), + false + ); + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key".to_string(), json!("non"))]), + true + ) + .expect("expected match to exist"), + true + ); + + let property_b = PropertyFilter { + key: "key".to_string(), + value: json!(null), + operator: Some(OperatorType::IsSet), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_b, + &HashMap::from([("key".to_string(), json!(null))]), + true + ) + .expect("expected match to exist"), + true + ); + + let property_c = PropertyFilter { + key: "key".to_string(), + value: json!("nu"), + operator: Some(OperatorType::Icontains), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!(null))]), + true + ) + .expect("expected match to exist"), + true + ); + assert_eq!( + match_property( + &property_c, + &HashMap::from([("key".to_string(), json!("smh"))]), + true + ) + .expect("expected match to exist"), + false + ); + + let property_d = PropertyFilter { + key: "key".to_string(), + value: json!("Nu"), + operator: Some(OperatorType::Regex), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_d, + &HashMap::from([("key".to_string(), json!(null))]), + true + ) + .expect("expected match to exist"), + false + ); + + let property_d_upper_case = PropertyFilter { + key: "key".to_string(), + value: json!("Nu"), + operator: Some(OperatorType::Regex), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_d_upper_case, + &HashMap::from([("key".to_string(), json!(null))]), + true + ) + .expect("expected match to exist"), + false + ); + + // TODO: Fails because not a number + // let property_e = PropertyFilter { + // key: "key".to_string(), + // value: json!(1), + // operator: Some(OperatorType::Gt), + // prop_type: "person".to_string(), + // group_type_index: None, + // }; + + // assert_eq!( + // match_property(&property_e, &HashMap::from([("key".to_string(), json!(null))]), true) + // .expect("expected match to exist"), + // true + // ); + } + + #[test] + fn test_match_properties_all_operators_with_full_props() { + let property_a = PropertyFilter { + key: "key".to_string(), + value: json!("value"), + operator: None, + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_a, + &HashMap::from([("key2".to_string(), json!("value"))]), + false + ) + .expect("Expected no errors with full props mode for non-existent keys"), + false + ); + assert_eq!( + match_property(&property_a, &HashMap::from([]), false), + Ok(false) + ); + + let property_exact = PropertyFilter { + key: "key".to_string(), + value: json!(["value1", "value2", "value3"]), + operator: Some(OperatorType::Exact), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_exact, + &HashMap::from([("key2".to_string(), json!("value"))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + let property_is_set = PropertyFilter { + key: "key".to_string(), + value: json!("value"), + operator: Some(OperatorType::IsSet), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_is_set, + &HashMap::from([("key2".to_string(), json!("value"))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + let property_is_not_set = PropertyFilter { + key: "key".to_string(), + value: json!(null), + operator: Some(OperatorType::IsNotSet), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_is_not_set, + &HashMap::from([("key2".to_string(), json!("value"))]), + false + ) + .expect("Expected no errors with full props mode"), + true + ); + assert_eq!( + match_property( + &property_is_not_set, + &HashMap::from([("key".to_string(), json!("value"))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + // is not set with partial props returns false when key exists + assert_eq!( + match_property( + &property_is_not_set, + &HashMap::from([("key".to_string(), json!("value"))]), + true + ) + .expect("Expected no errors with full props mode"), + false + ); + // is not set returns error when key doesn't exist + assert_eq!( + match_property( + &property_is_not_set, + &HashMap::from([("key2".to_string(), json!("value"))]), + true + ) + .is_err(), + true + ); + + let property_icontains = PropertyFilter { + key: "key".to_string(), + value: json!("valUe"), + operator: Some(OperatorType::Icontains), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_icontains, + &HashMap::from([("key2".to_string(), json!("value"))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + let property_not_icontains = PropertyFilter { + key: "key".to_string(), + value: json!("valUe"), + operator: Some(OperatorType::NotIcontains), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_not_icontains, + &HashMap::from([("key2".to_string(), json!("value"))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + let property_regex = PropertyFilter { + key: "key".to_string(), + value: json!(r"\.com$"), + operator: Some(OperatorType::Regex), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_regex, + &HashMap::from([("key2".to_string(), json!("value.com"))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + let property_not_regex = PropertyFilter { + key: "key".to_string(), + value: json!(r"\.com$"), + operator: Some(OperatorType::NotRegex), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_not_regex, + &HashMap::from([("key2".to_string(), json!("value.com"))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + let property_gt = PropertyFilter { + key: "key".to_string(), + value: json!(1), + operator: Some(OperatorType::Gt), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_gt, + &HashMap::from([("key2".to_string(), json!(2))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + let property_gte = PropertyFilter { + key: "key".to_string(), + value: json!(1), + operator: Some(OperatorType::Gte), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_gte, + &HashMap::from([("key2".to_string(), json!(2))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + let property_lt = PropertyFilter { + key: "key".to_string(), + value: json!(1), + operator: Some(OperatorType::Lt), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_lt, + &HashMap::from([("key2".to_string(), json!(0))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + let property_lte = PropertyFilter { + key: "key".to_string(), + value: json!(1), + operator: Some(OperatorType::Lte), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_lte, + &HashMap::from([("key2".to_string(), json!(0))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + + // TODO: Handle date operators + let property_is_date_before = PropertyFilter { + key: "key".to_string(), + value: json!("2021-01-01"), + operator: Some(OperatorType::IsDateBefore), + prop_type: "person".to_string(), + group_type_index: None, + }; + + assert_eq!( + match_property( + &property_is_date_before, + &HashMap::from([("key2".to_string(), json!("2021-01-02"))]), + false + ) + .expect("Expected no errors with full props mode"), + false + ); + } +}