diff --git a/apps/web/package.json b/apps/web/package.json index cd91c57ca..6f784dd3b 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -5,7 +5,7 @@ "scripts": { "build": "next build", "dev": "next dev", - "dev:debug": "NODE_DEBUG=flydrive:s3 NODE_OPTIONS='--inspect=0.0.0.0:9229' ./node_modules/.bin/next dev", + "dev:debug": "NODE_DEBUG=flydrive:s3,latitude:debug NODE_OPTIONS='--inspect=0.0.0.0:9229' ./node_modules/.bin/next dev", "dev:local": "USE_LOCALHOST=true pnpm run dev", "lint": "next lint", "prettier": "prettier --write \"**/*.{ts,tsx,md}\"", diff --git a/packages/core/src/lib/disk.ts b/packages/core/src/lib/disk.ts index 24cd00c19..8652541b2 100644 --- a/packages/core/src/lib/disk.ts +++ b/packages/core/src/lib/disk.ts @@ -1,7 +1,11 @@ import { debuglog } from 'node:util' import { Readable } from 'stream' -import { HeadBucketCommand, S3Client } from '@aws-sdk/client-s3' +import { + HeadBucketCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3' import { Result } from '@latitude-data/core/lib/Result' import { env } from '@latitude-data/env' import { Disk, errors } from 'flydrive' @@ -9,11 +13,30 @@ import { FSDriver } from 'flydrive/drivers/fs' import { S3Driver } from 'flydrive/drivers/s3' import { WriteOptions } from 'flydrive/types' -var debug_default = debuglog('flydrive:s3') +var debug_default = debuglog('latitude:debug') const generateUrl = (publicPath: string) => async (key: string) => `/${publicPath}/${key}` +function logRequestMiddleware() { + // @ts-ignore + const logRequestMiddleware = (next, _context) => async (args) => { + debug_default("Request: %O", args.request.headers); + + return next(args); + }; + + // @ts-ignore + logRequestMiddleware.applyToStack = (clientStack) => { + clientStack.add(logRequestMiddleware, { + step: "finalizeRequest", // use 'serialize' step for v3 of aws-sdk + name: "logRequestMiddleware", + priority: "high", // Ensure this middleware is run first + }); + }; + + return logRequestMiddleware; +} /** * These env variables are set in production. * If you want to test this locally, you need to set them in your machine. @@ -57,30 +80,6 @@ export class DiskWrapper { this.disk = new Disk(this.buildDisk(args)) } - async pingBucket() { - debugger - const bucket = env.S3_BUCKET - debug_default('pinging bucket %s', bucket) - const awsConfig = getAwsConfig() - const client = new S3Client({ - credentials: awsConfig.credentials, - region: awsConfig.region, - }) - const input = { - Bucket: env.S3_BUCKET, - ExpectedBucketOwner: - 'FIND_THE_OWNER_ID_OF_THE_BUCKET_AND_REPLACE_THIS_STRING', - } - const command = new HeadBucketCommand(input) - try { - debugger - await client.send(command) - } catch (error) { - debug_default('error pinging bucket %s', bucket) - debug_default('error pinging bucket %s', error) - } - } - file(key: string) { return this.disk.file(key) } @@ -96,10 +95,48 @@ export class DiskWrapper { return this.putStream(key, contents) } + async rawS3ClientPutStream( + key: string, + contents: Readable, + options?: WriteOptions, + ) { + debugger + const awsConfig = getAwsConfig() + const client = new S3Client({ + credentials: awsConfig.credentials, + region: awsConfig.region, + }) + client.middlewareStack.use(logRequestMiddleware()) + return new Promise((resolve, reject) => { + debugger + contents.once('error', reject) + const commandOptions = { + Bucket: awsConfig.bucket, + Key: key, + ContentType: 'text/csv', + Body: contents, + } + const command = new PutObjectCommand(commandOptions) + return client + .send(command) + .then((data) => { + debugger + debug_default('file %s:%s written', awsConfig.bucket, key) + resolve(data) + }) + .catch((e) => { + debugger + debug_default('error writing file %s:%s', awsConfig.bucket, key) + debug_default('Error: ', e) + reject(e) + }) + }) + } + async putStream(key: string, contents: Readable, options?: WriteOptions) { try { - await this.pingBucket() - await this.disk.putStream(key, contents, options) + await this.rawS3ClientPutStream(key, contents, options) + /* await this.disk.putStream(key, contents, options) */ return Result.nil() } catch (e) { if (e instanceof errors.E_CANNOT_WRITE_FILE) { @@ -147,6 +184,7 @@ export class DiskWrapper { credentials: awsConfig.credentials, region: awsConfig.region, bucket: awsConfig.bucket, + supportsACL: false, visibility: 'private', }) } diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts index dace068c5..d99672b81 100644 --- a/packages/env/src/index.ts +++ b/packages/env/src/index.ts @@ -24,7 +24,8 @@ if (environment !== 'production') { LATITUDE_DOMAIN: 'latitude.so', LATITUDE_URL: 'http://localhost:3000', FROM_MAILER_EMAIL: 'hello@latitude.so', - DRIVE_DISK: 'local', + /* DRIVE_DISK: 'local', */ + DRIVE_DISK: 's3', }, { path: pathToEnv }, )