chore: move livestream to posthog monorepo

.github/workflows/build-and-deploy-prod.yml

@@ -9,6 +9,7 @@ on:
             - master
         paths-ignore:
             - 'rust/**'
+            - 'livestream/**'
 
 jobs:
     slack:

.github/workflows/ci-backend.yml

@@ -8,12 +8,12 @@ on:
     push:
         branches:
             - master
-    pull_request:
     workflow_dispatch:
         inputs:
             clickhouseServerVersion:
                 description: ClickHouse server version. Leave blank for default
                 type: string
+    pull_request:
 
 concurrency:
     group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

.github/workflows/ci-hog.yml

@@ -8,7 +8,13 @@ on:
     push:
         branches:
             - master
+        paths-ignore:
+            - rust/**
+            - livestream/**
     pull_request:
+        paths-ignore:
+            - rust/**
+            - livestream/**
 
 jobs:
     # Job to decide if we should run backend ci

.github/workflows/container-images-cd.yml

@@ -15,6 +15,7 @@ on:
             - master
         paths-ignore:
             - 'rust/**'
+            - 'livestream/**'
     workflow_dispatch:
 
 jobs:

.github/workflows/container-images-ci.yml

@@ -2,6 +2,9 @@ name: Container Images CI
 
 on:
     pull_request:
+        paths-ignore:
+            - 'rust/**'
+            - 'livestream/**'
 
 concurrency:
     group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

.github/workflows/livestream-docker-image.yml

@@ -0,0 +1,88 @@
+name: Livestream Docker CI/CD
+
+on:
+    push:
+        branches:
+            - master
+        paths:
+            - 'livestream/**'
+            - '.github/workflows/livestream-docker-image.yml'
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+
+        permissions:
+            contents: read
+            packages: write
+
+        outputs:
+            sha: ${{ steps.push.outputs.digest }}
+
+        steps:
+            - name: Check out livestream code
+              uses: actions/checkout@v4
+              with:
+                  sparse-checkout: 'livestream/'
+                  sparse-checkout-cone-mode: false
+
+            - name: Log in to the Container registry
+              uses: docker/login-action@v3
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+                  logout: false
+
+            - name: Extract metadata (tags, labels) for Docker
+              id: meta
+              uses: docker/metadata-action@v5
+              with:
+                  images: ghcr.io/posthog/posthog/livestream
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v2
+
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v2
+
+            - name: Build and push Docker image
+              id: push
+              if: github.ref != 'refs/heads/master'
+              uses: docker/build-push-action@v5
+              with:
+                  context: livestream/
+                  file: livestream/Dockerfile
+                  push: true
+                  platforms: linux/amd64,linux/arm64
+                  tags: ${{ steps.meta.outputs.tags }}
+                  labels: ${{ steps.meta.outputs.labels }}
+
+    # deploy:
+    #   runs-on: ubuntu-latest
+    #   needs: build
+    #   steps:
+    #   - name: get deployer token
+    #     id: deployer
+    #     uses: getsentry/action-github-app-token@v3
+    #     with:
+    #         app_id: ${{ secrets.DEPLOYER_APP_ID }}
+    #         private_key: ${{ secrets.DEPLOYER_APP_PRIVATE_KEY }}
+
+    #   - name: Trigger livestream deployment
+    #     uses: peter-evans/repository-dispatch@v3
+    #     with:
+    #         token: ${{ steps.deployer.outputs.token }}
+    #         repository: PostHog/charts
+    #         event-type: commit_state_update
+    #         client-payload: |
+    #             {
+    #               "values": {
+    #                 "image": {
+    #                   "sha": "${{ needs.build.outputs.sha }}"
+    #                 }
+    #               },
+    #               "release": "livestream",
+    #               "commit": ${{ toJson(github.event.head_commit) }},
+    #               "repository": ${{ toJson(github.repository) }}
+    #             }

.github/workflows/rust.yml

@@ -3,8 +3,14 @@ name: Rust test CI
 on:
     workflow_dispatch:
     push:
+        paths:
+            - 'rust/**'
+            - '.github/workflows/rust.yml'
         branches: [master, main]
     pull_request:
+        paths:
+            - 'rust/**'
+            - '.github/workflows/rust.yml'
 
 env:
     CARGO_TERM_COLOR: always

livestream/.gitignore

@@ -0,0 +1,32 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# binary / build
+livestream
+
+# tsnet 
+.tsnet-state/
+
+# configs
+configs/configs.yml
+mmdb.db

livestream/Dockerfile

@@ -0,0 +1,24 @@
+FROM golang:1.22 as builder
+WORKDIR /code
+COPY go.sum go.mod ./
+RUN go mod download -x
+
+COPY . ./
+RUN go get ./...
+RUN go build -v -o /livestream ./...
+
+# Fetch the GeoLite2-City database that will be used for IP geolocation within Django.
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    "ca-certificates" \
+    "curl" \
+    "brotli" \
+    && \
+    rm -rf /var/lib/apt/lists/* && \
+    mkdir share && \
+    ( curl -s -L "https://mmdbcdn.posthog.net/" --http1.1 | brotli --decompress --output=/GeoLite2-City.mmdb ) && \
+    chmod -R 755 /GeoLite2-City.mmdb
+
+FROM ubuntu
+COPY --from=builder /livestream /GeoLite2-City.mmdb /
+CMD ["/livestream"]

livestream/README.md

@@ -0,0 +1,19 @@
+<img src="https://github.com/PostHog/livestream/assets/391319/d4a4964d-4b19-4605-b268-157366817863" width="280" height="280" />
+
+# RealTime Hog 3000
+
+The start of something fresh.
+
+## Installing
+
+```bash
+curl https://mmdbcdn.posthog.net/ | brotli -d > mmdb.db
+```
+
+Config the configs in `configs/config.yml`. You can take a peak at the examples in `configs/configs.example.yml`
+
+Run it!
+
+```bash
+go run .
+```

livestream/configs.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/spf13/viper"
+)
+
+func loadConfigs() {
+	viper.SetConfigName("configs")
+	viper.AddConfigPath("configs/")
+
+	viper.SetDefault("kafka.group_id", "livestream")
+	viper.SetDefault("prod", false)
+
+	err := viper.ReadInConfig()
+	if err != nil {
+		panic(fmt.Errorf("fatal error config file: %w", err))
+	}
+
+	viper.OnConfigChange(func(e fsnotify.Event) {
+		fmt.Println("Config file changed:", e.Name)
+	})
+	viper.WatchConfig()
+}

livestream/configs/configs.example.yml

@@ -0,0 +1,11 @@
+prod: true
+kafka:
+    brokers: 'localhost:9092'
+    topic: ''
+    group_id: 'livestream-dev'
+mmdb:
+    path: 'mmdb.db'
+jwt:
+    token: '<randomly generated secret key>'
+postgres:
+    url: 'postgres://postgres:postgres@localhost:5432/postgres'

livestream/db.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"context"
+	"log"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/spf13/viper"
+)
+
+func getPGConn() *pgx.Conn {
+	url := viper.GetString("postgres.url")
+	conn, err := pgx.Connect(context.Background(), url)
+	if err != nil {
+		log.Panicf("Unable to connect to database: %v\n", err)
+	}
+	return conn
+}

livestream/deploy.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+env GOOS=linux GOARCH=arm64 go build -o dist/livestream
+scp dist/livestream [email protected]:
+

livestream/docker-compose.yml

@@ -0,0 +1,62 @@
+version: "3.8"
+
+services:
+  postgres:
+    image: postgres:16-alpine
+    restart: always
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: liveevents
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+
+  redis:
+    image: redis:alpine
+    restart: always
+    ports:
+      - "6379:6379"
+
+  redpanda:
+    image: vectorized/redpanda:v23.2.17
+    command:
+      - redpanda start
+      - --smp 1
+      - --overprovisioned
+      - --node-id 0
+      - --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
+      - --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
+      - --pandaproxy-addr 0.0.0.0:8082
+      - --advertise-pandaproxy-addr localhost:8082
+    ports:
+      - 8081:8081
+      - 8082:8082
+      - 9092:9092
+      - 29092:29092
+
+  console:
+    image: docker.redpanda.com/redpandadata/console:v2.3.8
+    restart: on-failure
+    entrypoint: /bin/sh
+    command: -c "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console"
+    environment:
+      CONFIG_FILEPATH: /tmp/config.yml
+      CONSOLE_CONFIG_FILE: |
+        kafka:
+          brokers: ["redpanda:29092"]
+          schemaRegistry:
+            enabled: true
+            urls: ["http://redpanda:8081"]
+        connect:
+          enabled: true
+          clusters:
+            - name: datagen
+              url: http://connect:8083
+    ports:
+      - "8088:8088"
+    depends_on:
+      - redpanda