-
Notifications
You must be signed in to change notification settings - Fork 60
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Infra for setting S3 bucket for storing datasets
- Loading branch information
1 parent
79aa65c
commit b6e3ee9
Showing
13 changed files
with
9,474 additions
and
11,136 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
# Infra | ||
|
||
Infra about how we do the setup of the infra for the project. | ||
|
||
## Pulumi setup | ||
|
||
Go to pulumi CLI [installation guide](https://www.pulumi.com/docs/install/) if | ||
you didn't install it yet. | ||
|
||
### Set pulumi config values (can be encrypted with --secret) | ||
|
||
Pulumi store screts encrypted in the state files like `Pulumi.core.yaml` or | ||
`Pulumi.web-production.yaml1`. | ||
|
||
```bash | ||
pulumi config set AWS_ACCESS_KEY [your-access-key] --secret | ||
``` | ||
|
||
### Do changes | ||
|
||
Make your changes and run. It will show the changes and ask for permision | ||
|
||
```bash | ||
pulumi up | ||
``` | ||
|
||
It will ask you for Pulumi Passphrase. Is in 1Password. ask for permissions if | ||
you can't find wit "Pulumi Passphrase". |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
import * as aws from '@pulumi/aws' | ||
|
||
const regionProvider = new aws.Provider('euCentral1RegionProvider', { | ||
region: 'eu-central-1', | ||
}) | ||
|
||
export const bucket = new aws.s3.BucketV2( | ||
'mainLatitudeBucketResouce', | ||
{ | ||
acl: 'private', // Canned ACL | ||
bucket: 'latitude-llm-bucket-production', | ||
tags: { | ||
Name: 'Latitude LLM bucket', | ||
Environment: 'Production', | ||
}, | ||
}, | ||
{ provider: regionProvider }, | ||
) | ||
|
||
export const bucketName = bucket.bucket |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,16 +1,41 @@ | ||
import * as aws from '@pulumi/aws' | ||
import * as pulumi from '@pulumi/pulumi' | ||
|
||
const cfg = new pulumi.Config() | ||
const config = new pulumi.Config() | ||
|
||
const mailerApiKey = new aws.secretsmanager.Secret('MAILER_API_KEY', { | ||
description: 'API key for the mailer service', | ||
name: 'MAILER_API_KEY', | ||
}) | ||
|
||
const awsAccessKey = new aws.secretsmanager.Secret( | ||
'LATITUDE_LLM_AWS_ACCESS_KEY', | ||
{ | ||
description: 'AWS access key', | ||
name: 'LATITUDE_LLM_AWS_ACCESS_KEY', | ||
}, | ||
) | ||
const awsAccessSecret = new aws.secretsmanager.Secret( | ||
'LATITUDE_LLM_AWS_ACCESS_SECRET', | ||
{ | ||
description: 'AWS access secret', | ||
name: 'LATITUDE_LLM_AWS_ACCESS_SECRET', | ||
}, | ||
) | ||
|
||
new aws.secretsmanager.SecretVersion('MAILER_API_KEY_VERSION', { | ||
secretId: mailerApiKey.id, | ||
secretString: cfg.requireSecret('MAILER_API_KEY'), | ||
secretString: config.requireSecret('MAILER_API_KEY'), | ||
}) | ||
new aws.secretsmanager.SecretVersion('LATITUDE_LLM_AWS_ACCESS_KEY_VERSION', { | ||
secretId: awsAccessKey.id, | ||
secretString: config.requireSecret('LATITUDE_LLM_AWS_ACCESS_KEY'), | ||
}) | ||
new aws.secretsmanager.SecretVersion('LATITUDE_LLM_AWS_ACCESS_SECRET_VERSION', { | ||
secretId: awsAccessSecret.id, | ||
secretString: config.requireSecret('LATITUDE_LLM_AWS_ACCESS_SECRET'), | ||
}) | ||
|
||
export const mailerApiKeyArn = mailerApiKey.arn | ||
export const awsAccessKeyArn = awsAccessKey.arn | ||
export const awsAccessSecretArn = awsAccessSecret.arn |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -25,6 +25,25 @@ const mailerApiKey = mailerApiKeyArn.apply((arn) => { | |
return secret.secretString | ||
}) | ||
|
||
const awsAccessKeyArn = coreStack.requireOutput('latitudeLlmAwsAccessKeyArn') | ||
const awsAccessKey = awsAccessKeyArn.apply((arn) => { | ||
const secret = aws.secretsmanager.getSecretVersionOutput({ | ||
secretId: arn, | ||
}) | ||
|
||
return secret.secretString | ||
}) | ||
const awsAccessSecretArn = coreStack.requireOutput( | ||
'latitudeLlmAwsAccessSecretArn', | ||
) | ||
const awsAccessSecret = awsAccessSecretArn.apply((arn) => { | ||
const secret = aws.secretsmanager.getSecretVersionOutput({ | ||
secretId: arn, | ||
}) | ||
|
||
return secret.secretString | ||
}) | ||
|
||
export const dbUrl = pulumi.interpolate`postgresql://${dbUsername}:${dbPassword}@${dbEndpoint}/${dbName}?sslmode=verify-full&sslrootcert=/app/packages/core/src/assets/eu-central-1-bundle.pem` | ||
export const environment = pulumi | ||
.all([cacheEndpoint, dbUrl, mailerApiKey]) | ||
|
@@ -40,5 +59,10 @@ export const environment = pulumi | |
{ name: 'LATITUDE_URL', value: 'https://app.latitude.so' }, | ||
{ name: 'FROM_MAILER_EMAIL', value: '[email protected]' }, | ||
{ name: 'MAILER_API_KEY', value: mailerApiKey }, | ||
{ name: 'DRIVE_DISK', value: 's3' }, | ||
{ name: 'ASW_REGION', value: 'eu-central-1' }, | ||
{ name: 'S3_BUCKET', value: 'latitude-llm-bucket-production' }, | ||
{ name: 'AWS_ACCESS_KEY', value: awsAccessKey }, | ||
{ name: 'AWS_ACCESS_SECRET', value: awsAccessSecret }, | ||
] | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.