Skip to content

Commit

Permalink
Infra for setting S3 bucket for storing datasets
Browse files Browse the repository at this point in the history
  • Loading branch information
andresgutgon committed Sep 9, 2024
1 parent 79aa65c commit b6e3ee9
Show file tree
Hide file tree
Showing 13 changed files with 9,474 additions and 11,136 deletions.
4 changes: 4 additions & 0 deletions apps/infra/Pulumi.core.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ encryptionsalt: v1:K+dTqOgU40c=:v1:xzsAOOiJEEAsdCQ4:Oe7NKHjXdYdZrcrrBa1/0yolY5M3
config:
infra:DATABASE_PASSWORD:
secure: v1:WABt/tJjfsAKMplU:RRLtj5mTu301x4sn2Tr91u4O0Msd3mWVJutcDg==
infra:LATITUDE_LLM_AWS_ACCESS_KEY:
secure: v1:4SUt0/rEtwZMk0ts:BarCGxQjAzapKjZlsyNw9Vb6lyMGiMAdPlYCkrpfd4SU+fEb
infra:LATITUDE_LLM_AWS_ACCESS_SECRET:
secure: v1:/+DMPM/zWsNsNHDg:JIlxC1v0ocfdfX1TsDXXTZClL8Uza74CMq5YSst7Ymu+CPulRNgUejrSQEFVW3F6BkXMA1QU3s4=
infra:MAILER_API_KEY:
secure: v1:Br02wGwX0UOAZiNp:wIzQFfWCzWygBRCVJNDs/ZP65EXEAIFhduPOOniN6EOeXPyJ4UHjOefPyO2bhOBkR5E3CWLmQBsVjIGrGwAHZYZCQg==
infra:albSecurityGroup:
Expand Down
28 changes: 28 additions & 0 deletions apps/infra/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Infra

Infra about how we do the setup of the infra for the project.

## Pulumi setup

Go to pulumi CLI [installation guide](https://www.pulumi.com/docs/install/) if
you didn't install it yet.

### Set pulumi config values (can be encrypted with --secret)

Pulumi store screts encrypted in the state files like `Pulumi.core.yaml` or
`Pulumi.web-production.yaml1`.

```bash
pulumi config set AWS_ACCESS_KEY [your-access-key] --secret
```

### Do changes

Make your changes and run. It will show the changes and ask for permision

```bash
pulumi up
```

It will ask you for Pulumi Passphrase. Is in 1Password. ask for permissions if
you can't find wit "Pulumi Passphrase".
1 change: 1 addition & 0 deletions apps/infra/src/core/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@ export * from './elasticCache'
export * from './ec2'
export * from './ecs'
export * from './iam'
export * from './s3'
export * from './secrets'
20 changes: 20 additions & 0 deletions apps/infra/src/core/s3.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import * as aws from '@pulumi/aws'

const regionProvider = new aws.Provider('euCentral1RegionProvider', {
region: 'eu-central-1',
})

export const bucket = new aws.s3.BucketV2(
'mainLatitudeBucketResouce',
{
acl: 'private', // Canned ACL
bucket: 'latitude-llm-bucket-production',
tags: {
Name: 'Latitude LLM bucket',
Environment: 'Production',
},
},
{ provider: regionProvider },
)

export const bucketName = bucket.bucket
29 changes: 27 additions & 2 deletions apps/infra/src/core/secrets.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,41 @@
import * as aws from '@pulumi/aws'
import * as pulumi from '@pulumi/pulumi'

const cfg = new pulumi.Config()
const config = new pulumi.Config()

const mailerApiKey = new aws.secretsmanager.Secret('MAILER_API_KEY', {
description: 'API key for the mailer service',
name: 'MAILER_API_KEY',
})

const awsAccessKey = new aws.secretsmanager.Secret(
'LATITUDE_LLM_AWS_ACCESS_KEY',
{
description: 'AWS access key',
name: 'LATITUDE_LLM_AWS_ACCESS_KEY',
},
)
const awsAccessSecret = new aws.secretsmanager.Secret(
'LATITUDE_LLM_AWS_ACCESS_SECRET',
{
description: 'AWS access secret',
name: 'LATITUDE_LLM_AWS_ACCESS_SECRET',
},
)

new aws.secretsmanager.SecretVersion('MAILER_API_KEY_VERSION', {
secretId: mailerApiKey.id,
secretString: cfg.requireSecret('MAILER_API_KEY'),
secretString: config.requireSecret('MAILER_API_KEY'),
})
new aws.secretsmanager.SecretVersion('LATITUDE_LLM_AWS_ACCESS_KEY_VERSION', {
secretId: awsAccessKey.id,
secretString: config.requireSecret('LATITUDE_LLM_AWS_ACCESS_KEY'),
})
new aws.secretsmanager.SecretVersion('LATITUDE_LLM_AWS_ACCESS_SECRET_VERSION', {
secretId: awsAccessSecret.id,
secretString: config.requireSecret('LATITUDE_LLM_AWS_ACCESS_SECRET'),
})

export const mailerApiKeyArn = mailerApiKey.arn
export const awsAccessKeyArn = awsAccessKey.arn
export const awsAccessSecretArn = awsAccessSecret.arn
24 changes: 24 additions & 0 deletions apps/infra/src/deployments/shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,25 @@ const mailerApiKey = mailerApiKeyArn.apply((arn) => {
return secret.secretString
})

const awsAccessKeyArn = coreStack.requireOutput('latitudeLlmAwsAccessKeyArn')
const awsAccessKey = awsAccessKeyArn.apply((arn) => {
const secret = aws.secretsmanager.getSecretVersionOutput({
secretId: arn,
})

return secret.secretString
})
const awsAccessSecretArn = coreStack.requireOutput(
'latitudeLlmAwsAccessSecretArn',
)
const awsAccessSecret = awsAccessSecretArn.apply((arn) => {
const secret = aws.secretsmanager.getSecretVersionOutput({
secretId: arn,
})

return secret.secretString
})

export const dbUrl = pulumi.interpolate`postgresql://${dbUsername}:${dbPassword}@${dbEndpoint}/${dbName}?sslmode=verify-full&sslrootcert=/app/packages/core/src/assets/eu-central-1-bundle.pem`
export const environment = pulumi
.all([cacheEndpoint, dbUrl, mailerApiKey])
Expand All @@ -40,5 +59,10 @@ export const environment = pulumi
{ name: 'LATITUDE_URL', value: 'https://app.latitude.so' },
{ name: 'FROM_MAILER_EMAIL', value: '[email protected]' },
{ name: 'MAILER_API_KEY', value: mailerApiKey },
{ name: 'DRIVE_DISK', value: 's3' },
{ name: 'ASW_REGION', value: 'eu-central-1' },
{ name: 'S3_BUCKET', value: 'latitude-llm-bucket-production' },
{ name: 'AWS_ACCESS_KEY', value: awsAccessKey },
{ name: 'AWS_ACCESS_SECRET', value: awsAccessSecret },
]
})
2 changes: 1 addition & 1 deletion apps/web/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"scripts": {
"build": "next build",
"dev": "next dev",
"dev:debug": "NODE_OPTIONS='--inspect=0.0.0.0:9229' ./node_modules/.bin/next dev",
"dev:debug": "NODE_DEBUG=flydrive:s3 NODE_OPTIONS='--inspect=0.0.0.0:9229' ./node_modules/.bin/next dev",
"dev:local": "USE_LOCALHOST=true pnpm run dev",
"lint": "next lint",
"prettier": "prettier --write \"**/*.{ts,tsx,md}\"",
Expand Down
6 changes: 3 additions & 3 deletions apps/web/src/actions/datasets/create.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ const DELIMITER_VALUES = {
space: ' ',
}

const MAX_SIZE = 3
const MAX_UPLOAD_SIZE_IN_MB = 3 * 1024 * 1024
const MAX_SIZE = 15
const MAX_UPLOAD_SIZE_IN_MB = MAX_SIZE * 1024 * 1024
export const createDatasetAction = authProcedure
.createServerAction()
.input(
Expand All @@ -36,7 +36,7 @@ export const createDatasetAction = authProcedure
async (name) => {
const scope = new DatasetsRepository(ctx.workspace.id)
const existing = await scope.findByName(name)
return !existing
return !existing.length
},
{
message:
Expand Down
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
"@ai-sdk/google": "^0.0.46",
"@ai-sdk/mistral": "^0.0.38",
"@ai-sdk/openai": "^0.0.54",
"@aws-sdk/client-s3": "^3.645.0",
"@aws-sdk/s3-request-presigner": "^3.645.0",
"@monaco-editor/react": "^4.6.0",
"@plunk/node": "^3.0.2",
"@radix-ui/react-avatar": "^1.1.0",
Expand Down
4 changes: 4 additions & 0 deletions packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
"@ai-sdk/google": "^0.0.46",
"@ai-sdk/mistral": "^0.0.38",
"@ai-sdk/openai": "^0.0.54",
"@aws-sdk/client-s3": "^3.645.0",
"@aws-sdk/s3-request-presigner": "^3.645.0",
"@faker-js/faker": "^8.4.1",
"@latitude-data/compiler": "workspace:^",
"@latitude-data/env": "workspace:*",
Expand Down Expand Up @@ -75,6 +77,8 @@
"@ai-sdk/google": "^0.0.46",
"@ai-sdk/mistral": "^0.0.38",
"@ai-sdk/openai": "^0.0.54",
"@aws-sdk/client-s3": "^3.645.0",
"@aws-sdk/s3-request-presigner": "^3.645.0",
"@latitude-data/compiler": "workspace:^",
"@latitude-data/env": "workspace:^",
"@latitude-data/jobs": "workspace:^",
Expand Down
54 changes: 47 additions & 7 deletions packages/core/src/lib/disk.ts
Original file line number Diff line number Diff line change
@@ -1,24 +1,42 @@
import { debuglog } from 'node:util'
import { Readable } from 'stream'

import { HeadBucketCommand, S3Client } from '@aws-sdk/client-s3'
import { Result } from '@latitude-data/core/lib/Result'
import { env } from '@latitude-data/env'
import { Disk, errors } from 'flydrive'
import { FSDriver } from 'flydrive/drivers/fs'
import { S3Driver } from 'flydrive/drivers/s3'
import { WriteOptions } from 'flydrive/types'

var debug_default = debuglog('flydrive:s3')

const generateUrl = (publicPath: string) => async (key: string) =>
`/${publicPath}/${key}`

function getAwsCredentials() {
/**
* These env variables are set in production.
* If you want to test this locally, you need to set them in your machine.
* Create a .env.development file in packages/env/.env.development and add the following:
*
* S3_BUCKET=[your-bucket-name]
* AWS_REGION=[your-region]
* AWS_ACCESS_KEY=[your-key]
* AWS_ACCESS_SECRET=[your-secret]
*/
function getAwsConfig() {
const accessKeyId = env.AWS_ACCESS_KEY
const bucket = env.S3_BUCKET
const region = env.AWS_REGION
const secretAccessKey = env.AWS_ACCESS_SECRET

if (!accessKeyId || !secretAccessKey) {
throw new Error('AWS credentials not configured')
if (!accessKeyId || !secretAccessKey || !bucket || !region) {
throw new Error(
'AWS credentials not configured. Check you setup AWS_ACCESS_KEY, AWS_ACCESS_SECRET, S3_BUCKET and AWS_REGION in your .env file.',
)
}

return { accessKeyId, secretAccessKey }
return { region, bucket, credentials: { accessKeyId, secretAccessKey } }
}

async function getReadableStreamFromFile(file: File) {
Expand All @@ -39,6 +57,27 @@ export class DiskWrapper {
this.disk = new Disk(this.buildDisk(args))
}

async pingBucket() {
const bucket = env.S3_BUCKET
debug_default('pinging bucket %s', bucket)
const awsConfig = getAwsConfig()
const client = new S3Client({
credentials: awsConfig.credentials,
region: awsConfig.region,
})
const input = {
Bucket: env.S3_BUCKET,
ExpectedBucketOwner: 'TODO_FIND_ID',
}
const command = new HeadBucketCommand(input)
try {
await client.send(command)
} catch (error) {
debug_default('error pinging bucket %s', bucket)
debug_default('error pinging bucket %s', error)
}
}

file(key: string) {
return this.disk.file(key)
}
Expand Down Expand Up @@ -99,10 +138,11 @@ export class DiskWrapper {
})
}

const awsConfig = getAwsConfig()
return new S3Driver({
credentials: getAwsCredentials(),
region: env.AWS_REGION,
bucket: env.S3_BUCKET,
credentials: awsConfig.credentials,
region: awsConfig.region,
bucket: awsConfig.bucket,
visibility: 'private',
})
}
Expand Down
2 changes: 0 additions & 2 deletions packages/env/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,5 @@ export const env = createEnv({
},
runtimeEnv: {
...process.env,
// TODO: Remove once s3 is implemented
DRIVE_DISK: 'local',
},
})
Loading

0 comments on commit b6e3ee9

Please sign in to comment.