chore: move livestream to posthog monorepo (#23044)

* 🔥 initial commit * update readme * Update README.md * Update README.md * deploy scripts * very basic consumer setup * add some configs and docker-compose * formatting for testing * add tailscale * flip from dev to prod flag * set default to be not prod * default for group_id * tailscale up * update gitignore * basic geolocation * remove unused localServer * document mmdb * just make configs an example * drop raw print * add a start script (downloads the mmdb) * add readme and update configs.example * ts working * if in start * update start script * fix start * fix start * fix more * add sql endpoints for tokenId and Person lookups * work towards filter * sub channel * fix subChan * hardcode team2 token * add cors * only allow get and head * add atomicbool * add channel to kafka * add logs * verbose logs * make array * drop sub ptrs * more logs * helps to loop * drop some logigng * move sub branch * logging * drop log * hog * Deal with numeric distinct ids later * logs * api_key * send 1/1000 * remove log * remove more logs * change response payload * set timestamp if needed * fill in person_id if team_id is set * require teamid, convert to token * clean up subs on disconnect * log * check for token in another place * clean up subs on disconnect * drop modulo and log * fix no assign * don't reuse db conn for now * drop a log * add back commented out log * Don't block on send to client channel * add geo bool * only geo events * use wrapper ip * don't require team in geo mode * add an endpoint and stats keeper for teams * remove stats keeper * start stats keeper * wire it up * change the shape of the response * omit empty error * omit empty on the stats as well * enable logging on back pressure * add jwt endpoint for testing * support multiple event types * Get Auth Setup * jwt team is float so turn that into int * logs * add auth for stats endpoint * remove tailscale and use autoTLS on public endpoints * default to :443 for auto tls * remove un-needed endpoints and handlers * Use compression because... a lot of data (#9) * add dockerfile and CI/CD (#10) * add dockerfile and CI/CD * Use ubuntu not alpine couldn't build in alpine :'( * Add MMDB download to Dockerfile (#11) * Use clearer name for MMDB * Don't connect to Kafka over SSL in dev * Fix JWT token in example config * Add postgres.url to example config * Add expected scope * Fix const syntax * Put scope validation where claims are known * Fix audience validation * moves * ignore livestream for ci * main -> master * move GA to root * docker lint fix * fix typo * fixes for docker builds * test docker build * livestream build docker * dang * Update .github/workflows/livestream-docker-image.yml Co-authored-by: Neil Kakkar <[email protected]> * Update .github/workflows/livestream-docker-image.yml Co-authored-by: Neil Kakkar <[email protected]> * don't build posthog container when PR is pushed for rust or livestream * Update .github/workflows/livestream-docker-image.yml Co-authored-by: Neil Kakkar <[email protected]> * add a lot of paths-ignore * Update .github/workflows/livestream-docker-image.yml Co-authored-by: Neil Kakkar <[email protected]> * Dorny filters are handling most of what I was trying to do * remove tailscale to speed up builds * maybe? * push container to github.com/posthog/postog * don't build container on PR * remove more filters because dorny --------- Co-authored-by: Brett Hoerner <[email protected]> Co-authored-by: Zach Waterfield <[email protected]> Co-authored-by: Frank Hamand <[email protected]> Co-authored-by: Michael Matloka <[email protected]> Co-authored-by: Neil Kakkar <[email protected]>
PostHog · Jun 18, 2024 · 59eaa99 · 59eaa99
1 parent 3f90216
commit 59eaa99
Show file tree

Hide file tree

Showing 28 changed files with 1,627 additions and 1 deletion.
diff --git a/.github/workflows/build-and-deploy-prod.yml b/.github/workflows/build-and-deploy-prod.yml
@@ -9,6 +9,7 @@ on:
             - master
         paths-ignore:
             - 'rust/**'
+            - 'livestream/**'
 
 jobs:
     slack:

diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml
@@ -8,12 +8,12 @@ on:
     push:
         branches:
             - master
-    pull_request:
     workflow_dispatch:
         inputs:
             clickhouseServerVersion:
                 description: ClickHouse server version. Leave blank for default
                 type: string
+    pull_request:
 
 concurrency:
     group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

diff --git a/.github/workflows/ci-hog.yml b/.github/workflows/ci-hog.yml
@@ -8,7 +8,13 @@ on:
     push:
         branches:
             - master
+        paths-ignore:
+            - rust/**
+            - livestream/**
     pull_request:
+        paths-ignore:
+            - rust/**
+            - livestream/**
 
 jobs:
     # Job to decide if we should run backend ci

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -56,6 +56,8 @@ jobs:
                       build-mode: none
                     - language: python
                       build-mode: none
+                    - language: go
+                      build-mode: autobuild
                 # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
                 # Use `c-cpp` to analyze code written in C, C++ or both
                 # Use 'java-kotlin' to analyze code written in Java, Kotlin or both

diff --git a/.github/workflows/container-images-cd.yml b/.github/workflows/container-images-cd.yml
@@ -15,6 +15,7 @@ on:
             - master
         paths-ignore:
             - 'rust/**'
+            - 'livestream/**'
     workflow_dispatch:
 
 jobs:

diff --git a/.github/workflows/container-images-ci.yml b/.github/workflows/container-images-ci.yml
@@ -2,6 +2,9 @@ name: Container Images CI
 
 on:
     pull_request:
+        paths-ignore:
+            - 'rust/**'
+            - 'livestream/**'
 
 concurrency:
     group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

diff --git a/.github/workflows/livestream-docker-image.yml b/.github/workflows/livestream-docker-image.yml
@@ -0,0 +1,88 @@
+name: Livestream Docker CI/CD
+
+on:
+    push:
+        branches:
+            - master
+        paths:
+            - 'livestream/**'
+            - '.github/workflows/livestream-docker-image.yml'
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+
+        permissions:
+            contents: read
+            packages: write
+
+        outputs:
+            sha: ${{ steps.push.outputs.digest }}
+
+        steps:
+            - name: Check out livestream code
+              uses: actions/checkout@v4
+              with:
+                  sparse-checkout: 'livestream/'
+                  sparse-checkout-cone-mode: false
+
+            - name: Log in to the Container registry
+              uses: docker/login-action@v3
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+                  logout: false
+
+            - name: Extract metadata (tags, labels) for Docker
+              id: meta
+              uses: docker/metadata-action@v5
+              with:
+                  images: ghcr.io/posthog/posthog/livestream
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v2
+
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v2
+
+            - name: Build and push Docker image
+              id: push
+              if: github.ref == 'refs/heads/master'
+              uses: docker/build-push-action@v5
+              with:
+                  context: livestream/
+                  file: livestream/Dockerfile
+                  push: true
+                  platforms: linux/amd64,linux/arm64
+                  tags: ${{ steps.meta.outputs.tags }}
+                  labels: ${{ steps.meta.outputs.labels }}
+
+    # deploy:
+    #   runs-on: ubuntu-latest
+    #   needs: build
+    #   steps:
+    #   - name: get deployer token
+    #     id: deployer
+    #     uses: getsentry/action-github-app-token@v3
+    #     with:
+    #         app_id: ${{ secrets.DEPLOYER_APP_ID }}
+    #         private_key: ${{ secrets.DEPLOYER_APP_PRIVATE_KEY }}
+
+    #   - name: Trigger livestream deployment
+    #     uses: peter-evans/repository-dispatch@v3
+    #     with:
+    #         token: ${{ steps.deployer.outputs.token }}
+    #         repository: PostHog/charts
+    #         event-type: commit_state_update
+    #         client-payload: |
+    #             {
+    #               "values": {
+    #                 "image": {
+    #                   "sha": "${{ needs.build.outputs.sha }}"
+    #                 }
+    #               },
+    #               "release": "livestream",
+    #               "commit": ${{ toJson(github.event.head_commit) }},
+    #               "repository": ${{ toJson(github.repository) }}
+    #             }
diff --git a/livestream/.gitignore b/livestream/.gitignore
@@ -0,0 +1,32 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# binary / build
+livestream
+
+# tsnet 
+.tsnet-state/
+
+# configs
+configs/configs.yml
+mmdb.db
diff --git a/livestream/Dockerfile b/livestream/Dockerfile
@@ -0,0 +1,24 @@
+FROM golang:1.22 as builder
+WORKDIR /code
+COPY go.sum go.mod ./
+RUN go mod download -x
+
+COPY . ./
+RUN go get ./...
+RUN go build -v -o /livestream ./...
+
+# Fetch the GeoLite2-City database that will be used for IP geolocation within Django.
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    "ca-certificates" \
+    "curl" \
+    "brotli" \
+    && \
+    rm -rf /var/lib/apt/lists/* && \
+    mkdir share && \
+    ( curl -s -L "https://mmdbcdn.posthog.net/" --http1.1 | brotli --decompress --output=/GeoLite2-City.mmdb ) && \
+    chmod -R 755 /GeoLite2-City.mmdb
+
+FROM ubuntu
+COPY --from=builder /livestream /GeoLite2-City.mmdb /
+CMD ["/livestream"]
diff --git a/livestream/README.md b/livestream/README.md
@@ -0,0 +1,19 @@
+<img src="https://github.com/PostHog/livestream/assets/391319/d4a4964d-4b19-4605-b268-157366817863" width="280" height="280" />
+
+# RealTime Hog 3000
+
+The start of something fresh.
+
+## Installing
+
+```bash
+curl https://mmdbcdn.posthog.net/ | brotli -d > mmdb.db
+```
+
+Config the configs in `configs/config.yml`. You can take a peak at the examples in `configs/configs.example.yml`
+
+Run it!
+
+```bash
+go run .
+```
diff --git a/livestream/configs.go b/livestream/configs.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/spf13/viper"
+)
+
+func loadConfigs() {
+	viper.SetConfigName("configs")
+	viper.AddConfigPath("configs/")
+
+	viper.SetDefault("kafka.group_id", "livestream")
+	viper.SetDefault("prod", false)
+
+	err := viper.ReadInConfig()
+	if err != nil {
+		panic(fmt.Errorf("fatal error config file: %w", err))
+	}
+
+	viper.OnConfigChange(func(e fsnotify.Event) {
+		fmt.Println("Config file changed:", e.Name)
+	})
+	viper.WatchConfig()
+}
diff --git a/livestream/configs/configs.example.yml b/livestream/configs/configs.example.yml
@@ -0,0 +1,11 @@
+prod: true
+kafka:
+    brokers: 'localhost:9092'
+    topic: ''
+    group_id: 'livestream-dev'
+mmdb:
+    path: 'mmdb.db'
+jwt:
+    token: '<randomly generated secret key>'
+postgres:
+    url: 'postgres://postgres:postgres@localhost:5432/postgres'
diff --git a/livestream/db.go b/livestream/db.go
@@ -0,0 +1,18 @@
+package main
+
+import (
+	"context"
+	"log"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/spf13/viper"
+)
+
+func getPGConn() *pgx.Conn {
+	url := viper.GetString("postgres.url")
+	conn, err := pgx.Connect(context.Background(), url)
+	if err != nil {
+		log.Panicf("Unable to connect to database: %v\n", err)
+	}
+	return conn
+}
diff --git a/livestream/deploy.sh b/livestream/deploy.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+env GOOS=linux GOARCH=arm64 go build -o dist/livestream
+scp dist/livestream [email protected]:
+
diff --git a/livestream/docker-compose.yml b/livestream/docker-compose.yml
@@ -0,0 +1,62 @@
+version: "3.8"
+
+services:
+  postgres:
+    image: postgres:16-alpine
+    restart: always
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: liveevents
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+
+  redis:
+    image: redis:alpine
+    restart: always
+    ports:
+      - "6379:6379"
+
+  redpanda:
+    image: vectorized/redpanda:v23.2.17
+    command:
+      - redpanda start
+      - --smp 1
+      - --overprovisioned
+      - --node-id 0
+      - --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
+      - --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
+      - --pandaproxy-addr 0.0.0.0:8082
+      - --advertise-pandaproxy-addr localhost:8082
+    ports:
+      - 8081:8081
+      - 8082:8082
+      - 9092:9092
+      - 29092:29092
+
+  console:
+    image: docker.redpanda.com/redpandadata/console:v2.3.8
+    restart: on-failure
+    entrypoint: /bin/sh
+    command: -c "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console"
+    environment:
+      CONFIG_FILEPATH: /tmp/config.yml
+      CONSOLE_CONFIG_FILE: |
+        kafka:
+          brokers: ["redpanda:29092"]
+          schemaRegistry:
+            enabled: true
+            urls: ["http://redpanda:8081"]
+        connect:
+          enabled: true
+          clusters:
+            - name: datagen
+              url: http://connect:8083
+    ports:
+      - "8088:8088"
+    depends_on:
+      - redpanda
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,6 +9,7 @@ on: @@
                 - master
             paths-ignore:
                 - 'rust/**'
+                - 'livestream/**'
     jobs:
         slack:
@@ Expand Down @@