diff --git a/package-lock.json b/package-lock.json index 72a218fda..d9b02f752 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25753,7 +25753,6 @@ "version": "0.0.0", "dependencies": { "@aws-sdk/client-s3": "3.292.0", - "@aws-sdk/client-sqs": "3.292.0", "@elastic/elasticsearch": "7.13.0", "@octokit/graphql": "4.8.0", "@octokit/rest": "19.0.7", @@ -30971,7 +30970,6 @@ "version": "file:packages/backend", "requires": { "@aws-sdk/client-s3": "3.292.0", - "@aws-sdk/client-sqs": "3.292.0", "@elastic/elasticsearch": "7.13.0", "@octokit/graphql": "4.8.0", "@octokit/rest": "19.0.7", diff --git a/packages/backend/package.json b/packages/backend/package.json index 07f9567a6..3b8f7069f 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -25,7 +25,6 @@ }, "dependencies": { "@aws-sdk/client-s3": "3.292.0", - "@aws-sdk/client-sqs": "3.292.0", "@elastic/elasticsearch": "7.13.0", "@octokit/graphql": "4.8.0", "@octokit/rest": "19.0.7", diff --git a/packages/backend/src/controllers/avatars.v1.ts b/packages/backend/src/controllers/avatars.v1.ts index a8cf9c439..c483211ce 100644 --- a/packages/backend/src/controllers/avatars.v1.ts +++ b/packages/backend/src/controllers/avatars.v1.ts @@ -16,9 +16,10 @@ import { NoSuchKey } from '@aws-sdk/client-s3'; import { getLogger } from '@iex/shared/logger'; +import { Storage } from '@iex/shared/storage'; import { Response, Request } from 'express'; +import Container from 'typedi'; -import { streamFromS3 } from '../lib/storage'; import { getType } from '../shared/mime'; const logger = getLogger('avatars.v1'); @@ -32,7 +33,8 @@ export const getAvatar = async (req: Request, res: Response): Promise => { logger.debug(`Attempting to load avatar: ${path}`); try { - const readable = await streamFromS3(path); + const storage = Container.get(Storage); + const readable = await storage.streamFile({ path }); if (req.query['content-type']) { res.contentType(req.query['content-type'] as string); diff --git a/packages/backend/src/controllers/drafts.v1.ts b/packages/backend/src/controllers/drafts.v1.ts index ba826b444..a02112848 100644 --- a/packages/backend/src/controllers/drafts.v1.ts +++ b/packages/backend/src/controllers/drafts.v1.ts @@ -16,9 +16,9 @@ import { NoSuchKey } from '@aws-sdk/client-s3'; import { getLogger } from '@iex/shared/logger'; +import { Storage } from '@iex/shared/storage'; import { Response, Request } from 'express'; - -import { streamFromS3 } from '../lib/storage'; +import Container from 'typedi'; const logger = getLogger('drafts.v1'); @@ -30,7 +30,8 @@ export const getDraftAttachment = async (req: Request, res: Response): Promise => { const filePath = req.params.filepath + req.params[0]; - const key = `insights/${req.params.namespace}/${req.params.name}/files/${filePath}`; + const path = `insights/${req.params.namespace}/${req.params.name}/files/${filePath}`; logger.debug(`Attempting to HEAD insight attachment: ${filePath}`); - const headObject = await headFromS3(key); + const storage = Container.get(Storage); + const headObject = await storage.rawHead({ path }); if (headObject === undefined) { res.status(404); @@ -55,14 +57,15 @@ export const headInsightFile = async (req: Request, res: Response): Promise => { const filePath = req.params.filepath + req.params[0]; - const key = `insights/${req.params.namespace}/${req.params.name}/files/${filePath}`; + const path = `insights/${req.params.namespace}/${req.params.name}/files/${filePath}`; logger.debug(`Attempting to load insight attachment: ${filePath}`); const range = Array.isArray(req.headers['range']) ? req.headers['range'][0] : req.headers['range']; try { - const response = await getFromS3(key, range); + const storage = Container.get(Storage); + const response = await storage.rawGet({ path, range }); if (response.AcceptRanges) { res.set('Accept-Ranges', response.AcceptRanges); diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index de70f7336..3c018d103 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -37,6 +37,8 @@ import { deployMappings } from './lib/elasticsearch'; import { createServer } from './server'; import { syncExampleInsights } from './lib/init'; import { getLogger } from '@iex/shared/logger'; +import { Storage } from '@iex/shared/storage'; +import Container from 'typedi'; const logger = getLogger('index'); @@ -53,6 +55,9 @@ process.on('uncaughtException', (uncaughtException) => { }); const startup = async (): Promise => { + // Add storage class to the Container since it doesn't use an annotation + Container.set(Storage, new Storage()); + // Deploy Elasticsearch Index mappings logger.debug('Deploying elasticsearch indices'); await pRetry(() => deployMappings(), { diff --git a/packages/backend/src/lib/backends/file-system.sync.ts b/packages/backend/src/lib/backends/file-system.sync.ts index 9994fe5a5..f86707fa0 100644 --- a/packages/backend/src/lib/backends/file-system.sync.ts +++ b/packages/backend/src/lib/backends/file-system.sync.ts @@ -21,9 +21,11 @@ import { PersonType } from '@iex/models/person-type'; import { RepositoryType } from '@iex/models/repository-type'; import { MessageQueue } from '@iex/mq/message-queue'; import { getLogger } from '@iex/shared/logger'; +import { Storage } from '@iex/shared/storage'; import { nanoid } from 'nanoid'; import pMap from 'p-map'; import readingTime from 'reading-time'; +import Container from 'typedi'; import { InsightYaml } from '@iex/backend/models/insight-yaml'; @@ -33,7 +35,6 @@ import { ActivityService } from '../../services/activity.service'; import { UserService } from '../../services/user.service'; import { getTypeAsync } from '../../shared/mime'; import { GitInstance, INSIGHT_YAML_FILE } from '../git-instance'; -import { writeToS3 } from '../storage'; import { BaseSync, INDEXABLE_MIME_TYPES, READONLY_FILES, THUMBNAIL_LOCATIONS } from './base.sync'; @@ -325,6 +326,8 @@ const syncFiles = async ( insight: IndexedInsight, previousInsight: IndexedInsight | null ): Promise => { + const storage = Container.get(Storage); + // Message Queue for converting files const conversionMq = new MessageQueue({ region: process.env.S3_REGION, queueUrl: process.env.CONVERSION_SQS_URL }); @@ -362,7 +365,7 @@ const syncFiles = async ( const targetS3Path = `insights/${insight.fullName}/files/${wf.path}`; - await writeToS3(contents!, targetS3Path); + await storage.writeFile({ body: contents!, path: targetS3Path }); if (file.conversions !== undefined) { // Submit a conversion request for this file diff --git a/packages/backend/src/lib/backends/github.sync.ts b/packages/backend/src/lib/backends/github.sync.ts index 2a6b4675c..58de697ad 100644 --- a/packages/backend/src/lib/backends/github.sync.ts +++ b/packages/backend/src/lib/backends/github.sync.ts @@ -22,13 +22,13 @@ import { PersonType } from '@iex/models/person-type'; import { RepositoryType } from '@iex/models/repository-type'; import { MessageQueue } from '@iex/mq/message-queue'; import { getLogger } from '@iex/shared/logger'; +import { Storage } from '@iex/shared/storage'; import { nanoid } from 'nanoid'; import pMap from 'p-map'; import readingTime from 'reading-time'; import Container from 'typedi'; import { GitInstance, INSIGHT_YAML_FILE } from '../../lib/git-instance'; -import { writeToS3 } from '../../lib/storage'; import { Insight } from '../../models/insight'; import { InsightFile, InsightFileConversion } from '../../models/insight-file'; import { InsightYaml } from '../../models/insight-yaml'; @@ -452,6 +452,8 @@ const syncFiles = async ( insight: IndexedInsight, previousInsight: IndexedInsight | null ): Promise => { + const storage = Container.get(Storage); + // Message Queue for converting files const conversionMq = new MessageQueue({ region: process.env.S3_REGION, queueUrl: process.env.CONVERSION_SQS_URL }); @@ -489,7 +491,7 @@ const syncFiles = async ( const targetS3Path = `insights/${insight.fullName}/files/${wf.path}`; - await writeToS3(contents!, targetS3Path); + await storage.writeFile({ body: contents!, path: targetS3Path }); if (file.conversions !== undefined) { // Submit a conversion request for this file diff --git a/packages/backend/src/lib/import.ts b/packages/backend/src/lib/import.ts index da82e7e1a..27af0ee49 100644 --- a/packages/backend/src/lib/import.ts +++ b/packages/backend/src/lib/import.ts @@ -23,11 +23,10 @@ import { nanoid } from 'nanoid'; import { parse as htmlParse, HTMLElement } from 'node-html-parser'; import TurndownService from 'turndown'; import * as turndownPluginGfm from 'turndown-plugin-gfm'; +import Container from 'typedi'; import { DraftDataInput, DraftKey } from '../models/draft'; -import { AttachmentService } from '../services/attachment.service'; import { DraftService } from '../services/draft.service'; -import { TemplateService } from '../services/template.service'; const logger = getLogger('import'); @@ -194,7 +193,7 @@ export function convertToDraft(request: ImportRequest): DraftDataInput { export async function importToNewDraft(request: ImportRequest): Promise { logger.info(`Importing web page ${request.url}`); - const draftService = new DraftService(new AttachmentService(), new TemplateService()); + const draftService = Container.get(DraftService); const draftKey = draftService.newDraftKey(); const draftData = convertToDraft(request); diff --git a/packages/backend/src/lib/storage.ts b/packages/backend/src/lib/storage.ts deleted file mode 100644 index a0530d496..000000000 --- a/packages/backend/src/lib/storage.ts +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright 2021 Expedia, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - GetObjectCommand, - GetObjectCommandOutput, - HeadObjectCommand, - HeadObjectCommandOutput, - NotFound, - PutObjectCommand, - S3Client, - S3ClientConfig -} from '@aws-sdk/client-s3'; -import { getLogger } from '@iex/shared/logger'; -import type { ReadStream } from 'fs-extra'; - -const logger = getLogger('storage'); - -const defaultOptions: S3ClientConfig = { - region: process.env.S3_REGION, - maxAttempts: 4, - - endpoint: process.env.S3_ENDPOINT !== '' ? process.env.S3_ENDPOINT : undefined, - - // S3 Path-style requests are deprecated - // But some S3-compatible APIs may use them (e.g. Minio) - forcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true' ? true : undefined -}; - -export function createS3Client(options: S3ClientConfig = defaultOptions): S3Client { - return new S3Client({ ...defaultOptions, ...options }); -} - -export const defaultS3Client = createS3Client(); - -/** - * Writes data to the Insights Explorer bucket. - * - * @note The function executes a putobject() request - * @param {Buffer | String} body file content to write to S3 - * @param {string} key S3 bucket key to write to - * @returns {string} S3 bucket URI to the uploaded file - * @throws {Error} If putobject request fails - */ -export async function writeToS3(body: Buffer | string, key: string): Promise { - const bucket = process.env.S3_BUCKET!; - - const response = await defaultS3Client.send(new PutObjectCommand({ Body: body, Bucket: bucket, Key: key })); - const uri = `s3://${bucket}/${key}`; - - logger.info(`S3 file successfully uploaded with Etag: ${response.ETag} and URI: ${uri}`); - return uri; -} - -/** - * Writes a stream to the Insights Explorer bucket. - * - * @note The function executes a putobject() request - * @param {ReadStream} stream stream content to write to S3 - * @param {number} fileSize Size of the file - * @param {string} key S3 bucket key to write to - * @returns {string} S3 bucket URI to the uploaded file - * @throws {Error} If putobject request fails - */ -export async function streamToS3(stream: ReadStream, fileSize: number, key: string): Promise { - const bucket = process.env.S3_BUCKET!; - - const response = await defaultS3Client.send( - new PutObjectCommand({ Body: stream, Bucket: bucket, ContentLength: fileSize, Key: key }) - ); - const uri = `s3://${bucket}/${key}`; - - logger.info(`S3 file successfully uploaded with Etag: ${response.ETag} and URI: ${uri}`); - return uri; -} - -/** - * Streams data from the Insights Explorer S3 bucket. - * - * @note The function executes a getObject() request - * @param {string} key Key to get file from bucket - * @range {string} range Optional range to retrieve - * @returns {Promise} Returns requested S3 file stream - */ -export async function streamFromS3(key: string, range?: string): Promise { - const bucket = process.env.S3_BUCKET!; - - logger.info(`Streaming from s3://${bucket}/${key}`); - - const response = await defaultS3Client.send(new GetObjectCommand({ Bucket: bucket, Key: key, Range: range })); - - return response.Body as unknown as ReadStream; -} - -/** - * Gets an Object from the Insights Explorer S3 bucket. - * - * @note The function executes a getObject() request - * @param {string} key Key to get file from bucket - * @range {string} range Optional range to retrieve - * @returns {GetObjectCommandOutput} Returns requested S3 GetObject response - */ -export function getFromS3(key: string, range?: string): Promise { - const bucket = process.env.S3_BUCKET!; - - logger.info(`Streaming from s3://${bucket}/${key}`); - - return defaultS3Client.send(new GetObjectCommand({ Bucket: bucket, Key: key, Range: range })); -} - -/** - * Returns a file head request. - * - * @note The function executes a headObject() request - * @param {string} key Key of file to check in bucket - * @returns {Request} Returns requested S3 HeadObject response - */ -export async function headFromS3(key: string): Promise { - const bucket = process.env.S3_BUCKET!; - - logger.info(`Checking existance of s3://${bucket}/${key}`); - try { - return await defaultS3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); - } catch (error: any) { - if (error instanceof NotFound) return undefined; - - logger.error(JSON.stringify(error, null, 2)); - throw error; - } -} - -/** - * Checks if a file exists in S3 and returns true/false - * - * @note The function executes a headObject() request - * @param {string} key Key of file to check in bucket - * @returns {Promise} Returns true if the file exists, else false - */ -export async function existsInS3(key: string): Promise { - const bucket = process.env.S3_BUCKET!; - - logger.info(`Checking existance of s3://${bucket}/${key}`); - try { - await defaultS3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); - return true; - } catch (error: any) { - if (error instanceof NotFound) return false; - throw error; - } -} diff --git a/packages/backend/src/services/attachment.service.ts b/packages/backend/src/services/attachment.service.ts index 17955ced1..89ccd43cd 100644 --- a/packages/backend/src/services/attachment.service.ts +++ b/packages/backend/src/services/attachment.service.ts @@ -14,15 +14,17 @@ * limitations under the License. */ -import { ReadStream } from 'fs'; +import { Readable } from 'stream'; +import { Storage } from '@iex/shared/storage'; import { Service } from 'typedi'; -import { streamFromS3, streamToS3 } from '../lib/storage'; import { InsightFile } from '../models/insight-file'; @Service() export class AttachmentService { + constructor(private storage: Storage) {} + getAvatarPath(avatarKey: string): string { return `avatars/${avatarKey}`; } @@ -31,15 +33,19 @@ export class AttachmentService { return `drafts/${draftKey}/files/${attachment.id}`; } - uploadAvatar(avatarKey: string, size: number, readStream: ReadStream): Promise { - return streamToS3(readStream, size, this.getAvatarPath(avatarKey)); + uploadAvatar(avatarKey: string, size: number, stream: Readable): Promise { + return this.storage.writeFile({ stream, fileSize: size, path: this.getAvatarPath(avatarKey) }); } - uploadToDraft(draftKey: string, attachment: Partial, readStream: ReadStream): Promise { - return streamToS3(readStream, attachment.size!, this.getDraftAttachmentPath(draftKey, attachment)); + uploadToDraft(draftKey: string, attachment: Partial, stream: Readable): Promise { + return this.storage.writeFile({ + stream, + fileSize: attachment.size!, + path: this.getDraftAttachmentPath(draftKey, attachment) + }); } - streamFromDraft(draftKey: string, attachment: Partial): Promise { - return streamFromS3(this.getDraftAttachmentPath(draftKey, attachment)); + streamFromDraft(draftKey: string, attachment: Partial): Promise { + return this.storage.streamFile({ path: this.getDraftAttachmentPath(draftKey, attachment) }); } } diff --git a/packages/backend/src/services/draft.service.ts b/packages/backend/src/services/draft.service.ts index ba19ab991..7d3f2e332 100644 --- a/packages/backend/src/services/draft.service.ts +++ b/packages/backend/src/services/draft.service.ts @@ -18,11 +18,11 @@ import { IndexedInsight } from '@iex/models/indexed/indexed-insight'; import { InsightFileAction } from '@iex/models/insight-file-action'; import { ItemType } from '@iex/models/item-type'; import { getLogger } from '@iex/shared/logger'; +import { Storage } from '@iex/shared/storage'; import { nanoid } from 'nanoid'; import pMap from 'p-map'; import { Service } from 'typedi'; -import { streamFromS3 } from '../lib/storage'; import { DraftKey } from '../models/draft'; import { Draft, DraftInput } from '../models/draft'; import { User } from '../models/user'; @@ -36,6 +36,7 @@ const logger = getLogger('draft.service'); export class DraftService { constructor( private readonly attachmentService: AttachmentService, + private readonly storage: Storage, private readonly templateService: TemplateService ) { logger.trace('[DRAFT.SERVICE] Constructing New Draft Service'); @@ -220,8 +221,8 @@ export class DraftService { async (file) => { // Copy each file from the template // and upload to the draft - const key = `insights/${template.fullName}/files/${file.path}`; - const readable = await streamFromS3(key); + const path = `insights/${template.fullName}/files/${file.path}`; + const readable = await this.storage.streamFile({ path }); // Duplicate file with new ID and 'add' action const newFile = { @@ -280,8 +281,8 @@ export class DraftService { async (file) => { // Copy each file from the source insight // and upload to the draft - const key = `insights/${sourceInsight.fullName}/files/${file.path}`; - const readable = await streamFromS3(key); + const path = `insights/${sourceInsight.fullName}/files/${file.path}`; + const readable = await this.storage.streamFile({ path }); // Duplicate file with new ID and 'add' action const newFile = { diff --git a/packages/backend/src/services/insight.service.ts b/packages/backend/src/services/insight.service.ts index 174350042..e78b87d16 100644 --- a/packages/backend/src/services/insight.service.ts +++ b/packages/backend/src/services/insight.service.ts @@ -25,6 +25,7 @@ import { RepositoryPermission } from '@iex/models/repository-permission'; import { RepositoryType } from '@iex/models/repository-type'; import { sort } from '@iex/shared/dataloader-util'; import { getLogger } from '@iex/shared/logger'; +import { Storage } from '@iex/shared/storage'; import { ApolloError } from 'apollo-server-express'; import DataLoader from 'dataloader'; import { raw } from 'objection'; @@ -52,7 +53,6 @@ import { ElasticIndex } from '../lib/elasticsearch'; import { GitInstance } from '../lib/git-instance'; -import { streamFromS3 } from '../lib/storage'; import { Activity, ActivityType, IndexedInsightActivityDetails } from '../models/activity'; import { GitHubRepository, RepositoryVisibility } from '../models/backends/github'; import { Comment } from '../models/comment'; @@ -133,7 +133,11 @@ export class InsightService { return sort(insightIds, result, 'insightId').map((row) => row?.userIds || []); }); - constructor(private readonly activityService: ActivityService, private readonly userService: UserService) { + constructor( + private readonly activityService: ActivityService, + private readonly storage: Storage, + private readonly userService: UserService + ) { logger.trace('Constructing New Insight Service'); } @@ -466,8 +470,9 @@ export class InsightService { // Add newly uploaded files case InsightFileAction.ADD: { logger.debug(`Adding new file: ${file.path}`); - const fileObjectKey = `drafts/${draftKey}/files/${file.id}`; - const readable = await streamFromS3(fileObjectKey); + const path = `drafts/${draftKey}/files/${file.id}`; + const readable = await this.storage.streamFile({ path }); + return gitInstance.putFileFromStream(file.path!, readable); } diff --git a/packages/backend/src/shared/mime.ts b/packages/backend/src/shared/mime.ts index 5d1029835..c082aa7fb 100644 --- a/packages/backend/src/shared/mime.ts +++ b/packages/backend/src/shared/mime.ts @@ -14,8 +14,9 @@ * limitations under the License. */ +import { Readable } from 'node:stream'; + import * as fileType from 'file-type'; -import { ReadStream } from 'fs-extra'; import * as mime from 'mime'; const fileNameOverrides: Record = { @@ -68,7 +69,7 @@ export async function getTypeAsync({ }: { fileName?: string; buffer?: Buffer | null; - stream?: ReadStream; + stream?: Readable; }): Promise { if (fileName !== undefined) { const type = mime.getType(fileName); diff --git a/packages/shared/src/storage.ts b/packages/shared/src/storage.ts index c893329ee..8e1a7ff90 100644 --- a/packages/shared/src/storage.ts +++ b/packages/shared/src/storage.ts @@ -16,32 +16,49 @@ import type { Readable } from 'stream'; -import { S3 } from 'aws-sdk'; -import type { ReadStream } from 'fs-extra'; +import type { + S3ClientConfig, + GetObjectCommandOutput, + PutObjectCommandInput, + HeadObjectCommandOutput +} from '@aws-sdk/client-s3'; +import { GetObjectCommand, HeadObjectCommand, NotFound, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { getLogger } from '@iex/shared/logger'; -const logger = getLogger('storage'); +export type StorageOptions = S3ClientConfig; -export type StorageOptions = S3.Types.ClientConfiguration; +export interface RequiredStorageUriOptions { + uri: string; + bucket: string; + path: string; +} -export interface StorageUriOptions { - uri?: string; - bucket?: string; - path?: string; +export interface OptionalStorageUriOptions { + range?: string; } +export type StorageUriOptions = Partial & OptionalStorageUriOptions; + export type StorageWriteOptions = StorageUriOptions & { body?: Buffer | string; fileSize?: number; - stream?: ReadStream; + stream?: Readable; }; export type StorageReadOptions = StorageUriOptions; -const defaultOptions: S3.Types.ClientConfiguration = { +export type NormalizedStorageUriOptions = StorageUriOptions & RequiredStorageUriOptions; + +const defaultOptions: S3ClientConfig = { region: process.env.S3_REGION, - maxRetries: 3 + maxAttempts: 4, + + endpoint: process.env.S3_ENDPOINT !== '' ? process.env.S3_ENDPOINT : undefined, + + // S3 Path-style requests are deprecated + // But some S3-compatible APIs may use them (e.g. Minio) + forcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true' ? true : undefined }; /** @@ -50,10 +67,12 @@ const defaultOptions: S3.Types.ClientConfiguration = { * abstraction layer across multiple storage engines, if needed. */ export class Storage { - private s3: S3; + private logger = getLogger('storage'); + + private s3Client: S3Client; - constructor(readonly options: StorageOptions = defaultOptions) { - this.s3 = new S3({ ...defaultOptions, ...options }); + constructor(readonly options: StorageOptions = {}) { + this.s3Client = new S3Client({ ...defaultOptions, ...options }); } static toUri(bucket: string, path: string): string { @@ -71,7 +90,7 @@ export class Storage { throw new Error('Unable to parse URI: ' + uri); } - static normalizeUriOptions(options: StorageUriOptions): Required { + static normalizeUriOptions(options: StorageUriOptions): NormalizedStorageUriOptions { let { bucket, path, uri } = options; if (uri === undefined) { if (bucket === undefined) { @@ -89,7 +108,7 @@ export class Storage { path = parsed.path; } - return { bucket, path, uri }; + return { ...options, bucket, path, uri }; } /** @@ -100,17 +119,18 @@ export class Storage { const { body, fileSize, stream } = options; if (body !== undefined) { - const response = await this.s3.putObject({ Body: body, Bucket: bucket, Key: path }).promise(); - logger.debug(`S3 file successfully uploaded with Etag: ${response.ETag} and URI: ${uri}`); + const response = await this.s3Client.send(new PutObjectCommand({ Body: body, Bucket: bucket, Key: path })); + + this.logger.debug(`S3 file successfully uploaded with Etag: ${response.ETag} and URI: ${uri}`); } else if (stream !== undefined) { - const uploadOptions: S3.PutObjectRequest = { Body: stream, Bucket: bucket, Key: path }; + const uploadOptions: PutObjectCommandInput = { Body: stream, Bucket: bucket, Key: path }; if (fileSize !== undefined) { uploadOptions.ContentLength = fileSize; } - const response = await this.s3.upload(uploadOptions).promise(); - logger.debug(`S3 file successfully streamed with Etag: ${response.ETag} and URI: ${uri}`); + const response = await this.s3Client.send(new PutObjectCommand(uploadOptions)); + this.logger.debug(`S3 file successfully streamed with Etag: ${response.ETag} and URI: ${uri}`); } else { throw new Error('Either body or stream options must be set.'); } @@ -119,15 +139,15 @@ export class Storage { } /** - * Returns a data stream of a file + * Returns a data stream of a file. */ async streamFile(options: StorageReadOptions): Promise { - const { bucket, path, uri } = Storage.normalizeUriOptions(options); + const { bucket, path, uri, range } = Storage.normalizeUriOptions(options); - logger.debug(`Streaming from ${uri}`); - const response = this.s3.getObject({ Bucket: bucket, Key: path }).createReadStream(); + this.logger.debug(`Streaming from ${uri}`); - return response; + const response = await this.s3Client.send(new GetObjectCommand({ Bucket: bucket, Key: path, Range: range })); + return response.Body as unknown as Readable; } /** @@ -136,12 +156,43 @@ export class Storage { async exists(options: StorageReadOptions): Promise { const { bucket, path, uri } = Storage.normalizeUriOptions(options); - logger.debug(`Checking existance of ${uri}`); + this.logger.debug(`Checking existance of ${uri}`); try { - await this.s3.headObject({ Bucket: bucket, Key: path }).promise(); + await this.s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: path })); return true; } catch (error: any) { - if (error.code == 'NotFound') return false; + if (error instanceof NotFound) return false; + throw error; + } + } + + /** + * Returns the raw response from the storage backend. + * + * @note The function executes a getObject() request + * @param {string} key Key to get file from bucket + * @range {string} range Optional range to retrieve + * @returns {GetObjectCommandOutput} Returns requested S3 GetObject response + */ + async rawGet(options: StorageReadOptions): Promise { + const { bucket, path, uri, range } = Storage.normalizeUriOptions(options); + + this.logger.debug(`Streaming from ${uri}`); + + return this.s3Client.send(new GetObjectCommand({ Bucket: bucket, Key: path, Range: range })); + } + + /** + * Checks if a file exists and returns the raw HEAD response + */ + async rawHead(options: StorageReadOptions): Promise { + const { bucket, path, uri } = Storage.normalizeUriOptions(options); + + this.logger.debug(`Checking existance of ${uri}`); + try { + return await this.s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: path })); + } catch (error: any) { + if (error instanceof NotFound) return undefined; throw error; } }