sst config with lambda/neon and standalone servers

electric-sql · Dec 3, 2024 · b53d87b · b53d87b
1 parent d8bbb95
commit b53d87b
Show file tree

Hide file tree

Showing 11 changed files with 656 additions and 51 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,3 @@
+
+# sst
+.sst
diff --git a/examples/yjs-provider/Dockerfile b/examples/yjs-provider/Dockerfile
@@ -0,0 +1,31 @@
+FROM node:lts-alpine AS base
+
+# Stage 1: Install dependencies
+FROM base AS deps
+WORKDIR /app
+
+RUN npm install -g pnpm
+
+COPY pnpm-*.yaml ./
+COPY package.json ./
+COPY tsconfig.build.json ./
+COPY packages/typescript-client packages/typescript-client/
+COPY packages/react-hooks packages/react-hooks/
+COPY examples/yjs-provider/ examples/yjs-provider/
+
+# Install dependencies
+RUN pnpm install --frozen-lockfile
+RUN pnpm run -r build 
+
+
+# Need to make production image more clean
+FROM node:lts-alpine AS prod
+WORKDIR /app
+
+ENV NODE_ENV=production
+COPY --from=deps /app/ ./
+
+WORKDIR /app/examples/yjs-provider/
+
+EXPOSE 3000
+CMD ["npm", "run", "start"]
diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts
@@ -1,5 +1,10 @@
 import { pool } from "../../db"
 import { NextResponse } from "next/server"
+import { neon } from "@neondatabase/serverless"
+
+const sql = process.env.POOLED_DATABASE_URL
+  ? neon(process.env.POOLED_DATABASE_URL)
+  : undefined
 
 export async function POST(request: Request) {
   try {
@@ -19,23 +24,37 @@ export async function POST(request: Request) {
 }
 
 async function saveOperation(room: string, op: string) {
-  pool.query(
-    `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`,
-    [room, op]
-  )
+  if (sql) {
+    await sql`
+      INSERT INTO ydoc_operations (room, op) VALUES (${room}, decode(${op}, 'base64'))
+    `
+  } else {
+    await pool!.query(
+      `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`,
+      [room, op]
+    )
+  }
 }
 
 async function saveAwarenessOperation(
   room: string,
   op: string,
   clientId: string
 ) {
-  await pool.query(
-    `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) 
+  if (sql) {
+    await sql`
+      INSERT INTO ydoc_awareness (room, clientId, op) VALUES (${room}, ${clientId}, decode(${op}, 'base64'))
+      ON CONFLICT (clientId, room)
+      DO UPDATE SET op = decode(${op}, 'base64')
+    `
+  } else {
+    await pool!.query(
+      `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64'))
        ON CONFLICT (clientId, room)
        DO UPDATE SET op = decode($3, 'base64')`,
-    [room, clientId, op]
-  )
+      [room, clientId, op]
+    )
+  }
 }
 
 async function getRequestParams(

diff --git a/examples/yjs-provider/app/db.ts b/examples/yjs-provider/app/db.ts
@@ -2,7 +2,8 @@ import pgPkg from "pg"
 const { Pool } = pgPkg
 
 const connectionString =
-  process.env.POOLED_DATABASE_URL ||
+  // process.env.POOLED_DATABASE_URL || disabled so that we use neon client for pooled connection
+  process.env.DATABASE_URL ||
   `postgresql://postgres:password@localhost:54321/electric`
 
 const pool = new Pool({

diff --git a/examples/yjs-provider/app/shape-proxy/[...table]/route.ts b/examples/yjs-provider/app/shape-proxy/[...table]/route.ts
@@ -2,8 +2,8 @@ export async function GET(request: Request) {
   const url = new URL(request.url)
   const originUrl = new URL(
     process.env.ELECTRIC_URL
-      ? `${process.env.ELECTRIC_URL}/v1/shape`
-      : `http://localhost:3000/v1/shape`
+      ? `${process.env.ELECTRIC_URL}/v1/shape/`
+      : `http://localhost:3000/v1/shape/`
   )
 
   url.searchParams.forEach((value, key) => {
@@ -19,6 +19,8 @@ export async function GET(request: Request) {
     originUrl.searchParams.set(`token`, process.env.ELECTRIC_TOKEN)
   }
 
+  console.log("database_id", process.env.DATABASE_ID)
+  console.log("electric_token", process.env.ELECTRIC_TOKEN)
   console.log(originUrl.toString())
 
   const newRequest = new Request(originUrl.toString(), {

diff --git a/examples/yjs-provider/app/y-electric.ts b/examples/yjs-provider/app/y-electric.ts
@@ -120,7 +120,7 @@ export class ElectricProvider extends ObservableV2<ObservableProvider> {
   }
 
   private get awarenessUrl() {
-    return this.serverUrl + `/v1/shape/`
+    return this.serverUrl + `/v1/shape`
   }
 
   get synced() {

diff --git a/examples/yjs-provider/next.config.mjs b/examples/yjs-provider/next.config.mjs
@@ -2,11 +2,6 @@
  * @type {import('next').NextConfig}
  */
 const nextConfig = {
-    // logging: {
-    //   fetches: {
-    //     fullUrl: true,
-    //     hmrRefreshes: true,
-    //   },
-    // },
-  }
-  export default nextConfig
+}
+
+export default nextConfig
diff --git a/examples/yjs-provider/package.json b/examples/yjs-provider/package.json
@@ -22,10 +22,11 @@
     "@codemirror/view": "^6.32.0",
     "@electric-sql/client": "workspace:*",
     "@electric-sql/react": "workspace:*",
+    "@neondatabase/serverless": "^0.10.4",
     "codemirror": "^6.0.1",
     "lib0": "^0.2.96",
-    "next": "^14.2.5",
-    "pg": "^8.12.0",
+    "next": "^14.2.9",
+    "pg": "^8.13.1",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "sst": "^3.3.35",

diff --git a/examples/yjs-provider/sst-env.d.ts b/examples/yjs-provider/sst-env.d.ts
@@ -6,5 +6,17 @@ import "sst"
 export {}
 declare module "sst" {
   export interface Resource {
+    "yjs": {
+      "type": "sst.aws.Nextjs"
+      "url": string
+    }
+    "yjs-service-vbalegas": {
+      "service": string
+      "type": "sst.aws.Service"
+      "url": string
+    }
+    "yjs-vpc-vbalegas": {
+      "type": "sst.aws.Vpc"
+    }
   }
 }
diff --git a/examples/yjs-provider/sst.config.ts b/examples/yjs-provider/sst.config.ts
@@ -33,21 +33,26 @@ export default $config({
     })
 
     const databaseUri = getNeonDbUri(project, db)
-    const pooledDatabaseUri = getNeonDbUri(project, db, true)
     try {
       databaseUri.apply(applyMigrations)
 
       const electricInfo = databaseUri.apply((uri) =>
         addDatabaseToElectric(uri)
       )
 
-      const website = deployApp(electricInfo, databaseUri, pooledDatabaseUri)
+      const website = deployServerlessApp(
+        electricInfo,
+        databaseUri,
+        databaseUri
+      )
+
+      const server = deployApp(electricInfo, databaseUri)
       return {
         databaseUri,
-        pooledUri: pooledDatabaseUri,
         database_id: electricInfo.id,
         electric_token: electricInfo.token,
         website: website.url,
+        server: server.url,
       }
     } catch (e) {
       console.error(`Failed to deploy yjs example stack`, e)
@@ -65,6 +70,39 @@ function applyMigrations(uri: string) {
 }
 
 function deployApp(
+  { id, token }: $util.Output<{ id: string; token: string }>,
+  uri: $util.Output<string>
+) {
+  const vpc = new sst.aws.Vpc(`yjs-vpc-${$app.stage}`)
+  const cluster = new sst.aws.Cluster(`yjs-cluster-${$app.stage}`, { vpc })
+
+  const service = cluster.addService(`yjs-service-${$app.stage}`, {
+    loadBalancer: {
+      ports: [{ listen: "443/https", forward: "3000/http" }],
+      domain: {
+        name: `yjs-server-${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`,
+        dns: sst.cloudflare.dns(),
+      },
+    },
+    environment: {
+      ELECTRIC_URL: process.env.ELECTRIC_API!,
+      DATABASE_URL: uri,
+      DATABASE_ID: id,
+      ELECTRIC_TOKEN: token,
+    },
+    image: {
+      context: "../..",
+      dockerfile: "Dockerfile",
+    },
+    dev: {
+      command: "npm run dev",
+    },
+  })
+
+  return service
+}
+
+function deployServerlessApp(
   electricInfo: $util.Output<{ id: string; token: string }>,
   uri: $util.Output<string>,
   pooledUri: $util.Output<string>
@@ -75,7 +113,7 @@ function deployApp(
       ELECTRIC_TOKEN: electricInfo.token,
       DATABASE_ID: electricInfo.id,
       DATABASE_URL: uri,
-      POOLED_DATABASE_URL: pooledUri,
+      //  POOLED_DATABASE_URL: TODO
     },
     domain: {
       name: `yjs${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`,
@@ -86,26 +124,15 @@ function deployApp(
 
 function getNeonDbUri(
   project: $util.Output<neon.GetProjectResult>,
-  db: neon.Database,
-  pool: boolean = false
+  db: neon.Database
 ) {
   const passwordOutput = neon.getBranchRolePasswordOutput({
     projectId: project.id,
     branchId: project.defaultBranchId,
     roleName: db.ownerName,
   })
 
-  return $interpolate`postgresql://${passwordOutput.roleName}:${passwordOutput.password}@${project.databaseHost}/${db.name}?sslmode=require`.apply(
-    (v) => {
-      if (pool) {
-        return v.replace(
-          process.env.NEON_PROJECT_ID!,
-          process.env.NEON_PROJECT_ID + `-pooler`
-        )
-      }
-      return v
-    }
-  )
+  return $interpolate`postgresql://${passwordOutput.roleName}:${passwordOutput.password}@${project.databaseHost}/${db.name}?sslmode=require`
 }
 
 async function addDatabaseToElectric(