Infra for setting S3 bucket for storing datasets

apps/infra/Pulumi.core.yaml

@@ -2,6 +2,10 @@ encryptionsalt: v1:K+dTqOgU40c=:v1:xzsAOOiJEEAsdCQ4:Oe7NKHjXdYdZrcrrBa1/0yolY5M3
 config:
   infra:DATABASE_PASSWORD:
     secure: v1:WABt/tJjfsAKMplU:RRLtj5mTu301x4sn2Tr91u4O0Msd3mWVJutcDg==
+  infra:LATITUDE_LLM_AWS_ACCESS_KEY:
+    secure: v1:4SUt0/rEtwZMk0ts:BarCGxQjAzapKjZlsyNw9Vb6lyMGiMAdPlYCkrpfd4SU+fEb
+  infra:LATITUDE_LLM_AWS_ACCESS_SECRET:
+    secure: v1:/+DMPM/zWsNsNHDg:JIlxC1v0ocfdfX1TsDXXTZClL8Uza74CMq5YSst7Ymu+CPulRNgUejrSQEFVW3F6BkXMA1QU3s4=
   infra:MAILER_API_KEY:
     secure: v1:Br02wGwX0UOAZiNp:wIzQFfWCzWygBRCVJNDs/ZP65EXEAIFhduPOOniN6EOeXPyJ4UHjOefPyO2bhOBkR5E3CWLmQBsVjIGrGwAHZYZCQg==
   infra:albSecurityGroup:

apps/infra/README.md

@@ -0,0 +1,28 @@
+# Infra
+
+Infra about how we do the setup of the infra for the project.
+
+## Pulumi setup
+
+Go to pulumi CLI [installation guide](https://www.pulumi.com/docs/install/) if
+you didn't install it yet.
+
+### Set pulumi config values (can be encrypted with --secret)
+
+Pulumi store screts encrypted in the state files like `Pulumi.core.yaml` or
+`Pulumi.web-production.yaml1`.
+
+```bash
+pulumi config set AWS_ACCESS_KEY [your-access-key] --secret
+```
+
+### Do changes
+
+Make your changes and run. It will show the changes and ask for permision
+
+```bash
+pulumi up
+```
+
+It will ask you for Pulumi Passphrase. Is in 1Password. ask for permissions if
+you can't find wit "Pulumi Passphrase".

apps/infra/src/core/index.ts

@@ -4,4 +4,5 @@ export * from './elasticCache'
 export * from './ec2'
 export * from './ecs'
 export * from './iam'
+export * from './s3'
 export * from './secrets'

apps/infra/src/core/s3.ts

@@ -0,0 +1,20 @@
+import * as aws from '@pulumi/aws'
+
+const regionProvider = new aws.Provider('euCentral1RegionProvider', {
+  region: 'eu-central-1',
+})
+
+export const bucket = new aws.s3.BucketV2(
+  'mainLatitudeBucketResouce',
+  {
+    acl: 'private', // Canned ACL
+    bucket: 'latitude-llm-bucket-production',
+    tags: {
+      Name: 'Latitude LLM bucket',
+      Environment: 'Production',
+    },
+  },
+  { provider: regionProvider },
+)
+
+export const bucketName = bucket.bucket

apps/infra/src/core/secrets.ts

@@ -1,16 +1,41 @@
 import * as aws from '@pulumi/aws'
 import * as pulumi from '@pulumi/pulumi'
 
-const cfg = new pulumi.Config()
+const config = new pulumi.Config()
 
 const mailerApiKey = new aws.secretsmanager.Secret('MAILER_API_KEY', {
   description: 'API key for the mailer service',
   name: 'MAILER_API_KEY',
 })
 
+const awsAccessKey = new aws.secretsmanager.Secret(
+  'LATITUDE_LLM_AWS_ACCESS_KEY',
+  {
+    description: 'AWS access key',
+    name: 'LATITUDE_LLM_AWS_ACCESS_KEY',
+  },
+)
+const awsAccessSecret = new aws.secretsmanager.Secret(
+  'LATITUDE_LLM_AWS_ACCESS_SECRET',
+  {
+    description: 'AWS access secret',
+    name: 'LATITUDE_LLM_AWS_ACCESS_SECRET',
+  },
+)
+
 new aws.secretsmanager.SecretVersion('MAILER_API_KEY_VERSION', {
   secretId: mailerApiKey.id,
-  secretString: cfg.requireSecret('MAILER_API_KEY'),
+  secretString: config.requireSecret('MAILER_API_KEY'),
+})
+new aws.secretsmanager.SecretVersion('LATITUDE_LLM_AWS_ACCESS_KEY_VERSION', {
+  secretId: awsAccessKey.id,
+  secretString: config.requireSecret('LATITUDE_LLM_AWS_ACCESS_KEY'),
+})
+new aws.secretsmanager.SecretVersion('LATITUDE_LLM_AWS_ACCESS_SECRET_VERSION', {
+  secretId: awsAccessSecret.id,
+  secretString: config.requireSecret('LATITUDE_LLM_AWS_ACCESS_SECRET'),
 })
 
 export const mailerApiKeyArn = mailerApiKey.arn
+export const awsAccessKeyArn = awsAccessKey.arn
+export const awsAccessSecretArn = awsAccessSecret.arn

apps/infra/src/deployments/shared.ts

@@ -25,6 +25,25 @@ const mailerApiKey = mailerApiKeyArn.apply((arn) => {
   return secret.secretString
 })
 
+const awsAccessKeyArn = coreStack.requireOutput('latitudeLlmAwsAccessKeyArn')
+const awsAccessKey = awsAccessKeyArn.apply((arn) => {
+  const secret = aws.secretsmanager.getSecretVersionOutput({
+    secretId: arn,
+  })
+
+  return secret.secretString
+})
+const awsAccessSecretArn = coreStack.requireOutput(
+  'latitudeLlmAwsAccessSecretArn',
+)
+const awsAccessSecret = awsAccessSecretArn.apply((arn) => {
+  const secret = aws.secretsmanager.getSecretVersionOutput({
+    secretId: arn,
+  })
+
+  return secret.secretString
+})
+
 export const dbUrl = pulumi.interpolate`postgresql://${dbUsername}:${dbPassword}@${dbEndpoint}/${dbName}?sslmode=verify-full&sslrootcert=/app/packages/core/src/assets/eu-central-1-bundle.pem`
 export const environment = pulumi
   .all([cacheEndpoint, dbUrl, mailerApiKey])
@@ -40,5 +59,10 @@ export const environment = pulumi
       { name: 'LATITUDE_URL', value: 'https://app.latitude.so' },
       { name: 'FROM_MAILER_EMAIL', value: '[email protected]' },
       { name: 'MAILER_API_KEY', value: mailerApiKey },
+      { name: 'DRIVE_DISK', value: 's3' },
+      { name: 'ASW_REGION', value: 'eu-central-1' },
+      { name: 'S3_BUCKET', value: 'latitude-llm-bucket-production' },
+      { name: 'AWS_ACCESS_KEY', value: awsAccessKey },
+      { name: 'AWS_ACCESS_SECRET', value: awsAccessSecret },
     ]
   })

apps/web/package.json

@@ -5,7 +5,7 @@
   "scripts": {
     "build": "next build",
     "dev": "next dev",
-    "dev:debug": "NODE_OPTIONS='--inspect=0.0.0.0:9229' ./node_modules/.bin/next dev",
+    "dev:debug": "NODE_DEBUG=flydrive:s3 NODE_OPTIONS='--inspect=0.0.0.0:9229' ./node_modules/.bin/next dev",
     "dev:local": "USE_LOCALHOST=true pnpm run dev",
     "lint": "next lint",
     "prettier": "prettier --write \"**/*.{ts,tsx,md}\"",

apps/web/src/actions/datasets/create.ts

@@ -21,8 +21,8 @@ const DELIMITER_VALUES = {
   space: ' ',
 }
 
-const MAX_SIZE = 3
-const MAX_UPLOAD_SIZE_IN_MB = 3 * 1024 * 1024
+const MAX_SIZE = 15
+const MAX_UPLOAD_SIZE_IN_MB = MAX_SIZE * 1024 * 1024
 export const createDatasetAction = authProcedure
   .createServerAction()
   .input(
@@ -36,7 +36,7 @@ export const createDatasetAction = authProcedure
               async (name) => {
                 const scope = new DatasetsRepository(ctx.workspace.id)
                 const existing = await scope.findByName(name)
-                return !existing
+                return !existing.length
               },
               {
                 message:

package.json

@@ -39,6 +39,8 @@
     "@ai-sdk/google": "^0.0.46",
     "@ai-sdk/mistral": "^0.0.38",
     "@ai-sdk/openai": "^0.0.54",
+    "@aws-sdk/client-s3": "^3.645.0",
+    "@aws-sdk/s3-request-presigner": "^3.645.0",
     "@monaco-editor/react": "^4.6.0",
     "@plunk/node": "^3.0.2",
     "@radix-ui/react-avatar": "^1.1.0",

packages/core/package.json

@@ -38,6 +38,8 @@
     "@ai-sdk/google": "^0.0.46",
     "@ai-sdk/mistral": "^0.0.38",
     "@ai-sdk/openai": "^0.0.54",
+    "@aws-sdk/client-s3": "^3.645.0",
+    "@aws-sdk/s3-request-presigner": "^3.645.0",
     "@faker-js/faker": "^8.4.1",
     "@latitude-data/compiler": "workspace:^",
     "@latitude-data/env": "workspace:*",
@@ -75,6 +77,8 @@
     "@ai-sdk/google": "^0.0.46",
     "@ai-sdk/mistral": "^0.0.38",
     "@ai-sdk/openai": "^0.0.54",
+    "@aws-sdk/client-s3": "^3.645.0",
+    "@aws-sdk/s3-request-presigner": "^3.645.0",
     "@latitude-data/compiler": "workspace:^",
     "@latitude-data/env": "workspace:^",
     "@latitude-data/jobs": "workspace:^",

packages/core/src/lib/disk.ts

@@ -1,24 +1,42 @@
+import { debuglog } from 'node:util'
 import { Readable } from 'stream'
 
+import { HeadBucketCommand, S3Client } from '@aws-sdk/client-s3'
 import { Result } from '@latitude-data/core/lib/Result'
 import { env } from '@latitude-data/env'
 import { Disk, errors } from 'flydrive'
 import { FSDriver } from 'flydrive/drivers/fs'
 import { S3Driver } from 'flydrive/drivers/s3'
 import { WriteOptions } from 'flydrive/types'
 
+var debug_default = debuglog('flydrive:s3')
+
 const generateUrl = (publicPath: string) => async (key: string) =>
   `/${publicPath}/${key}`
 
-function getAwsCredentials() {
+/**
+ * These env variables are set in production.
+ * If you want to test this locally, you need to set them in your machine.
+ * Create a .env.development file in packages/env/.env.development and add the following:
+ *
+ * S3_BUCKET=[your-bucket-name]
+ * AWS_REGION=[your-region]
+ * AWS_ACCESS_KEY=[your-key]
+ * AWS_ACCESS_SECRET=[your-secret]
+ */
+function getAwsConfig() {
   const accessKeyId = env.AWS_ACCESS_KEY
+  const bucket = env.S3_BUCKET
+  const region = env.AWS_REGION
   const secretAccessKey = env.AWS_ACCESS_SECRET
 
-  if (!accessKeyId || !secretAccessKey) {
-    throw new Error('AWS credentials not configured')
+  if (!accessKeyId || !secretAccessKey || !bucket || !region) {
+    throw new Error(
+      'AWS credentials not configured. Check you setup AWS_ACCESS_KEY, AWS_ACCESS_SECRET, S3_BUCKET and AWS_REGION in your .env file.',
+    )
   }
 
-  return { accessKeyId, secretAccessKey }
+  return { region, bucket, credentials: { accessKeyId, secretAccessKey } }
 }
 
 async function getReadableStreamFromFile(file: File) {
@@ -39,6 +57,27 @@ export class DiskWrapper {
     this.disk = new Disk(this.buildDisk(args))
   }
 
+  async pingBucket() {
+    const bucket = env.S3_BUCKET
+    debug_default('pinging bucket %s', bucket)
+    const awsConfig = getAwsConfig()
+    const client = new S3Client({
+      credentials: awsConfig.credentials,
+      region: awsConfig.region,
+    })
+    const input = {
+      Bucket: env.S3_BUCKET,
+      ExpectedBucketOwner: 'TODO_FIND_ID',
+    }
+    const command = new HeadBucketCommand(input)
+    try {
+      await client.send(command)
+    } catch (error) {
+      debug_default('error pinging bucket %s', bucket)
+      debug_default('error pinging bucket %s', error)
+    }
+  }
+
   file(key: string) {
     return this.disk.file(key)
   }
@@ -99,10 +138,11 @@ export class DiskWrapper {
       })
     }
 
+    const awsConfig = getAwsConfig()
     return new S3Driver({
-      credentials: getAwsCredentials(),
-      region: env.AWS_REGION,
-      bucket: env.S3_BUCKET,
+      credentials: awsConfig.credentials,
+      region: awsConfig.region,
+      bucket: awsConfig.bucket,
       visibility: 'private',
     })
   }

packages/env/src/index.ts

@@ -25,7 +25,8 @@ if (environment !== 'production') {
       LATITUDE_DOMAIN: 'latitude.so',
       LATITUDE_URL: 'http://localhost:3000',
       FROM_MAILER_EMAIL: '[email protected]',
-      DRIVE_DISK: 'local',
+      /* DRIVE_DISK: 'local', */
+      DRIVE_DISK: 's3',
     },
     { path: pathToEnv },
   )
@@ -57,7 +58,6 @@ export const env = createEnv({
   },
   runtimeEnv: {
     ...process.env,
-    // TODO: Remove once s3 is implemented
-    DRIVE_DISK: 'local',
+    DRIVE_DISK: process.env.DRIVE_DISK || 'local',
   },
 })