chore: hobby deployment on PR (#18733)

* Check docker before installing it
PostHog · Nov 29, 2023 · b0a9ce2 · b0a9ce2
1 parent ba96af9
commit b0a9ce2
Show file tree

Hide file tree

Showing 6 changed files with 150 additions and 74 deletions.
diff --git a/.github/pr-deploy/hobby.yaml.tmpl b/.github/pr-deploy/hobby.yaml.tmpl
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hobby-$HOSTNAME
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hobby-$HOSTNAME
+  template:
+    metadata:
+      labels:
+        app: hobby-$HOSTNAME
+    spec:
+      containers:
+      - name: hobby-$HOSTNAME
+        image: 169684386827.dkr.ecr.us-east-1.amazonaws.com/hobby-base:latest
+        imagePullPolicy: Always
+        securityContext:
+          privileged: true
+        ports:
+        - containerPort: 2375
+        - containerPort: 80
+        env:
+        - name: BRANCH_NAME
+          value: $BRANCH_NAME
+        - name: REGISTRY_URL
+          value: $REGISTRY_URL
+        - name: DOCKER_TAG
+          value: $DOCKER_TAG
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hobby-service-$HOSTNAME
+  annotations:
+    tailscale.com/expose: "true"
+    tailscale.com/hostname: hobby-$HOSTNAME
+    tailscale.com/tags: tag:service
+  finalizers:
+    - tailscale.com/finalizer
+spec:
+  selector:
+    app: hobby-$HOSTNAME
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80
+  type: ClusterIP
diff --git a/.github/workflows/container-images-ci.yml b/.github/workflows/container-images-ci.yml
@@ -14,7 +14,7 @@ jobs:
         permissions:
             id-token: write # allow issuing OIDC tokens for this workflow run
             contents: read # allow at least reading the repo contents, add other permissions if necessary
-
+    
         steps:
             # If this run wasn't initiated by PostHog Bot (meaning: snapshot update),
             # cancel previous runs of snapshot update-inducing workflows

diff --git a/.github/workflows/pr-cleanup.yml b/.github/workflows/pr-cleanup.yml
@@ -10,30 +10,27 @@ jobs:
     deploy_preview_cleanup:
         name: Deploy Preview Cleanup
         runs-on: ubuntu-latest
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'deploy') }}
 
         permissions:
             id-token: write
             contents: read
             deployments: write
 
         steps:
-            - name: Checkout chart
-              uses: actions/checkout@v3
-              with:
-                  repository: posthog/charts-clickhouse
-                  path: charts-clickhouse
-
             - name: Checkout
               uses: actions/checkout@v3
-              with:
-                  path: posthog
 
             - uses: aws-actions/configure-aws-credentials@v1
               with:
                   aws-region: us-east-1
                   role-to-assume: arn:aws:iam::169684386827:role/github-terraform-infra-role
                   role-duration-seconds: 3600
 
+            - name: Login to Amazon ECR
+              id: aws-ecr
+              uses: aws-actions/amazon-ecr-login@v1
+
             - name: connect to tailscale
               uses: tailscale/github-action@8b804aa882ac3429b804a2a22f9803a2101a0db9
               env:
@@ -43,18 +40,26 @@ jobs:
                   authkey: ${{ secrets.TAILSCALE_OAUTH_SECRET }}
                   args: --advertise-tags tag:github-runner
 
-            - name: Delete preview deployment
+            - name: Delete hobby deployment
               id: cleanup
               run: |
-                  aws eks update-kubeconfig --name "posthog-dev"
-                  kubectl config set-cluster arn:aws:eks:us-east-1:169684386827:cluster/posthog-dev --server=https://eks-posthog-dev --tls-server-name 8BD6E1D2FEBDE47C8177E29CAC9E6C61.gr7.us-east-1.eks.amazonaws.com
+                aws eks update-kubeconfig --name "posthog-dev"
+                kubectl config set-cluster arn:aws:eks:us-east-1:169684386827:cluster/posthog-dev --server=https://eks-posthog-dev --tls-server-name 8BD6E1D2FEBDE47C8177E29CAC9E6C61.gr7.us-east-1.eks.amazonaws.com
+
+                export PR_NUM=${{ github.event.number }}
+                export BRANCH_NAME=${{ github.head_ref }}
+                export BRANCH_NAME_HASH=$(echo -n ${{ github.head_ref }} | md5sum | cut -c 1-8)
+                export HOSTNAME=pr-$PR_NUM-${BRANCH_NAME_HASH}
+                export HOSTNAME=${HOSTNAME:0:38}
+                export HOSTNAME=${HOSTNAME%%-}
+                export NAMESPACE="hobby"
+                export REGISTRY_URL=${{ steps.aws-ecr.outputs.registry }}/pr-test
+                export DOCKER_TAG=${{ github.event.pull_request.head.sha }}
 
-                  export PR_NUM=${{ github.event.number }}
-                  export BRANCH_NAME=${{ github.head_ref }}
-                  export NAMESPACE=pr-$PR_NUM-${BRANCH_NAME//\//-}
+                envsubst < .github/pr-deploy/hobby.yaml.tmpl > .github/pr-deploy/hobby.yaml
 
-                  kubectl -n $NAMESPACE delete clickhouseinstallations.clickhouse.altinity.com/posthog || true
-                  kubectl delete namespace $NAMESPACE || true
+                #Clean and deploy
+                kubectl -n $NAMESPACE delete -f .github/pr-deploy/hobby.yaml || true
 
             - name: delete deployment
               uses: bobheadxi/[email protected]

diff --git a/.github/workflows/pr-deploy.yml b/.github/workflows/pr-deploy.yml
@@ -18,17 +18,24 @@ jobs:
             deployments: write
 
         steps:
-            - name: Checkout chart
-              uses: actions/checkout@v3
-              with:
-                  repository: posthog/charts-clickhouse
-                  path: charts-clickhouse
-
             - name: Checkout
               uses: actions/checkout@v3
-              with:
-                  path: posthog
 
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v2
+
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v2
+
+            - name: Set up Depot CLI
+              uses: depot/setup-action@v1
+
+            - name: Login to DockerHub
+              uses: docker/login-action@v2
+              with:
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+
             - uses: aws-actions/configure-aws-credentials@v1
               with:
                   aws-region: us-east-1
@@ -44,6 +51,20 @@ jobs:
                   authkey: ${{ secrets.TAILSCALE_OAUTH_SECRET }}
                   args: --advertise-tags tag:github-runner
 
+            - name: Login to Amazon ECR
+              id: aws-ecr
+              uses: aws-actions/amazon-ecr-login@v1
+
+            - name: Build and push PR test image
+              id: build-unit
+              uses: depot/build-push-action@v1
+              with:
+                  buildx-fallback: false # the fallback is so slow it's better to just fail
+                  push: true
+                  tags: ${{ steps.aws-ecr.outputs.registry }}/pr-test:${{ github.event.pull_request.head.sha }}
+                  platforms: linux/arm64,linux/amd64
+                  build-args: COMMIT_HASH=${{ github.event.pull_request.head.sha }}
+
             - name: start deployment
               uses: bobheadxi/[email protected]
               id: deployment
@@ -52,45 +73,29 @@ jobs:
                   env: pr-${{ github.head_ref }}
                   ref: ${{ github.head_ref }}
 
-            - name: Install PostHog using the Helm chart
-              id: helm_install
+            - name: Deploy hobby
+              id: deploy_hobby
               run: |
                   aws eks update-kubeconfig --name "posthog-dev"
                   kubectl config set-cluster arn:aws:eks:us-east-1:169684386827:cluster/posthog-dev --server=https://eks-posthog-dev --tls-server-name 8BD6E1D2FEBDE47C8177E29CAC9E6C61.gr7.us-east-1.eks.amazonaws.com
 
-                  export COMMIT_SHA=${{ github.sha }}
                   export PR_NUM=${{ github.event.number }}
                   export BRANCH_NAME=${{ github.head_ref }}
-                  export RELEASE_NAME=posthog
-                  export NAMESPACE=pr-$PR_NUM-${BRANCH_NAME//\//-}
-                  export NAMESPACE=${NAMESPACE:0:38}
-                  export NAMESPACE=${NAMESPACE%%-}
-                  export HOSTNAME=$NAMESPACE
-                  export TAILNET_NAME=hedgehog-kitefin
-                  export TS_AUTHKEY=${{ secrets.TAILSCALE_SERVICE_AUTHKEY }}
-
-                  export POSTHOG_WEB_SERVICE_NAME=POSTHOG_WEB
-                  export POSTHOG_WEB_SERVICE_PORT=8000
-                  envsubst < posthog/.github/pr-deploy/tailscale.yaml.tmpl > posthog/.github/pr-deploy/tailscale.yaml
-                  envsubst < posthog/.github/pr-deploy/values.yaml.tmpl > posthog/.github/pr-deploy/values.yaml
-
-                  helm upgrade --install \
-                  -f posthog/.github/pr-deploy/values.yaml \
-                  --set image.tag=pr-$PR_NUM \
-                  --timeout 30m \
-                  --create-namespace \
-                  --namespace $NAMESPACE \
-                  $RELEASE_NAME charts-clickhouse/charts/posthog \
-                  --wait \
-                  --wait-for-jobs
-
-                  POD_NAME=$(kubectl -n $NAMESPACE get pod | grep web | cut -d' ' -f1)
-                  kubectl -n $NAMESPACE exec $POD_NAME -- python manage.py shell -c "import sys;from posthog.models import Team; sys.exit(not Team.objects.exists())" \
-                  || kubectl -n $NAMESPACE exec $POD_NAME -- python manage.py generate_demo_data \
-                  && kubectl -n $NAMESPACE exec $POD_NAME -- python manage.py shell -c "from posthog.models import Team; t = Team.objects.first(); t.is_demo = False; t.save()"
-
-                  kubectl -n $NAMESPACE apply -f posthog/.github/pr-deploy/tailscale.yaml
-                  echo "url=$NAMESPACE.hedgehog-kitefin.ts.net" >> $GITHUB_OUTPUT
+                  export BRANCH_NAME_HASH=$(echo -n ${{ github.head_ref }} | md5sum | cut -c 1-8)
+                  export HOSTNAME=pr-$PR_NUM-${BRANCH_NAME_HASH}
+                  export HOSTNAME=${HOSTNAME:0:38}
+                  export HOSTNAME=${HOSTNAME%%-}
+                  export NAMESPACE="hobby"
+                  export REGISTRY_URL=${{ steps.aws-ecr.outputs.registry }}/pr-test
+                  export DOCKER_TAG=${{ github.event.pull_request.head.sha }}
+
+                  envsubst < .github/pr-deploy/hobby.yaml.tmpl > .github/pr-deploy/hobby.yaml
+
+                  #Clean and deploy
+                  kubectl -n $NAMESPACE delete -f .github/pr-deploy/hobby.yaml || true
+                  kubectl -n $NAMESPACE apply -f .github/pr-deploy/hobby.yaml
+
+                  echo "url=$NAMESPACE-${HOSTNAME}.hedgehog-kitefin.ts.net" >> $GITHUB_OUTPUT
 
             - name: update deployment status
               uses: bobheadxi/deployments@v1
@@ -99,5 +104,5 @@ jobs:
                   status: ${{ job.status }}
                   env: ${{ steps.deployment.outputs.env }}
                   ref: ${{ github.head_ref }}
-                  env_url: https://${{ steps.helm_install.outputs.url }}
+                  env_url: http://${{ steps.deploy_hobby.outputs.url }}
                   deployment_id: ${{ steps.deployment.outputs.deployment_id }}
diff --git a/bin/deploy-hobby b/bin/deploy-hobby
@@ -90,6 +90,10 @@ elif [[ "$POSTHOG_APP_TAG" = "latest" ]]
 then
     echo "Pulling latest from current branch: $(git branch --show-current)"
     git pull
+elif [[ "$POSTHOG_APP_TAG" =~ ^[0-9a-f]{40}$ ]]
+then
+    echo "Checking out specific commit hash: $POSTHOG_APP_TAG"
+    git checkout $POSTHOG_APP_TAG
 else
     releaseTag="${POSTHOG_APP_TAG/release-/""}"
     git fetch --tags
@@ -104,13 +108,18 @@ then
 export TLS_BLOCK="acme_ca https://acme-staging-v02.api.letsencrypt.org/directory"
 fi
 
+if [ "$REGISTRY_URL" == "" ]
+then
+export REGISTRY_URL="posthog/posthog"
+fi
+
 # rewrite caddyfile
 rm -f Caddyfile
 envsubst > Caddyfile <<EOF
 {
 $TLS_BLOCK
 }
-$DOMAIN, :80, :443 {
+$DOMAIN, http://, https:// {
 reverse_proxy http://web:8000
 }
 EOF
@@ -166,14 +175,21 @@ EOF
 chmod +x compose/wait
 
 # setup docker
-echo "Setting up Docker"
-sudo apt install -y apt-transport-https ca-certificates curl software-properties-common
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -E apt-key add -
-sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
-sudo apt update
-sudo apt-cache policy docker-ce
-sudo apt install -y docker-ce
-
+# setup docker
+# Check if Docker is already installed
+if ! command -v docker &> /dev/null; then
+    echo "Docker is not installed. Setting up Docker."
+
+    # Setup Docker
+    sudo apt install -y apt-transport-https ca-certificates curl software-properties-common
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -E apt-key add -
+    sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
+    sudo apt update
+    sudo apt-cache policy docker-ce
+    sudo apt install -y docker-ce
+else
+    echo "Docker is already installed. Skipping installation."
+fi
 
 # setup docker-compose
 echo "Setting up Docker Compose"
@@ -190,7 +206,7 @@ cp posthog/docker-compose.base.yml docker-compose.base.yml
 cp posthog/docker-compose.hobby.yml docker-compose.yml.tmpl
 envsubst < docker-compose.yml.tmpl > docker-compose.yml
 rm docker-compose.yml.tmpl
-echo "Starting the stack!"
+echo "Starting the stack!" 
 sudo -E docker-compose -f docker-compose.yml up -d
 
 echo "We will need to wait ~5-10 minutes for things to settle down, migrations to finish, and TLS certs to be issued"

diff --git a/docker-compose.hobby.yml b/docker-compose.hobby.yml
@@ -61,15 +61,15 @@ services:
             SENTRY_DSN: $SENTRY_DSN
             SITE_URL: https://$DOMAIN
             SECRET_KEY: $POSTHOG_SECRET
-        image: posthog/posthog:$POSTHOG_APP_TAG
+        image: $REGISTRY_URL:$POSTHOG_APP_TAG
     web:
         extends:
             file: docker-compose.base.yml
             service: web
         command: /compose/start
         volumes:
             - ./compose:/compose
-        image: posthog/posthog:$POSTHOG_APP_TAG
+        image: $REGISTRY_URL:$POSTHOG_APP_TAG
         environment:
             SENTRY_DSN: $SENTRY_DSN
             SITE_URL: https://$DOMAIN
@@ -78,7 +78,7 @@ services:
         extends:
             file: docker-compose.base.yml
             service: plugins
-        image: posthog/posthog:$POSTHOG_APP_TAG
+        image: $REGISTRY_URL:$POSTHOG_APP_TAG
         environment:
             SENTRY_DSN: $SENTRY_DSN
             SITE_URL: https://$DOMAIN
@@ -106,7 +106,7 @@ services:
         extends:
             file: docker-compose.base.yml
             service: asyncmigrationscheck
-        image: posthog/posthog:$POSTHOG_APP_TAG
+        image: $REGISTRY_URL:$POSTHOG_APP_TAG
         environment:
             SENTRY_DSN: $SENTRY_DSN
             SITE_URL: https://$DOMAIN
@@ -141,7 +141,7 @@ services:
             service: temporal-django-worker
         volumes:
             - ./compose:/compose
-        image: posthog/posthog:$POSTHOG_APP_TAG
+        image: $REGISTRY_URL:$POSTHOG_APP_TAG
         environment:
             SENTRY_DSN: $SENTRY_DSN
             SITE_URL: https://$DOMAIN