fix: Adopt shared storage class and consolidate usage

Removes storage.ts from backend in favor of the version in the shared package.  Migrated AWS SDK v3 changes to the shared package.  Cleaned up some DI mess.

packages/backend/package.json

@@ -25,7 +25,6 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "3.292.0",
-    "@aws-sdk/client-sqs": "3.292.0",
     "@elastic/elasticsearch": "7.13.0",
     "@octokit/graphql": "4.8.0",
     "@octokit/rest": "19.0.7",

packages/backend/src/controllers/avatars.v1.ts

@@ -16,9 +16,10 @@
 
 import { NoSuchKey } from '@aws-sdk/client-s3';
 import { getLogger } from '@iex/shared/logger';
+import { Storage } from '@iex/shared/storage';
 import { Response, Request } from 'express';
+import Container from 'typedi';
 
-import { streamFromS3 } from '../lib/storage';
 import { getType } from '../shared/mime';
 
 const logger = getLogger('avatars.v1');
@@ -32,7 +33,8 @@ export const getAvatar = async (req: Request, res: Response): Promise<void> => {
   logger.debug(`Attempting to load avatar: ${path}`);
 
   try {
-    const readable = await streamFromS3(path);
+    const storage = Container.get(Storage);
+    const readable = await storage.streamFile({ path });
 
     if (req.query['content-type']) {
       res.contentType(req.query['content-type'] as string);

packages/backend/src/controllers/drafts.v1.ts

@@ -16,9 +16,9 @@
 
 import { NoSuchKey } from '@aws-sdk/client-s3';
 import { getLogger } from '@iex/shared/logger';
+import { Storage } from '@iex/shared/storage';
 import { Response, Request } from 'express';
-
-import { streamFromS3 } from '../lib/storage';
+import Container from 'typedi';
 
 const logger = getLogger('drafts.v1');
 
@@ -30,7 +30,8 @@ export const getDraftAttachment = async (req: Request, res: Response): Promise<v
   logger.debug(`Attempting to load draft attachment: ${draftKey}, ${req.params.attachmentKey}`);
 
   try {
-    const readable = await streamFromS3(draftKey);
+    const storage = Container.get(Storage);
+    const readable = await storage.streamFile({ path: draftKey });
 
     if (req.query['content-type']) {
       res.contentType(req.query['content-type'] as string);

packages/backend/src/controllers/insights.v1.ts

@@ -18,9 +18,10 @@ import { Readable } from 'node:stream';
 
 import { NoSuchKey } from '@aws-sdk/client-s3';
 import { getLogger } from '@iex/shared/logger';
+import { Storage } from '@iex/shared/storage';
 import { Response, Request } from 'express';
+import Container from 'typedi';
 
-import { getFromS3, headFromS3 } from '../lib/storage';
 import { getType } from '../shared/mime';
 
 const logger = getLogger('insights.v1');
@@ -30,11 +31,12 @@ const logger = getLogger('insights.v1');
  */
 export const headInsightFile = async (req: Request, res: Response): Promise<void> => {
   const filePath = req.params.filepath + req.params[0];
-  const key = `insights/${req.params.namespace}/${req.params.name}/files/${filePath}`;
+  const path = `insights/${req.params.namespace}/${req.params.name}/files/${filePath}`;
 
   logger.debug(`Attempting to HEAD insight attachment: ${filePath}`);
 
-  const headObject = await headFromS3(key);
+  const storage = Container.get(Storage);
+  const headObject = await storage.rawHead({ path });
 
   if (headObject === undefined) {
     res.status(404);
@@ -55,14 +57,15 @@ export const headInsightFile = async (req: Request, res: Response): Promise<void
  */
 export const getInsightFile = async (req: Request, res: Response): Promise<void> => {
   const filePath = req.params.filepath + req.params[0];
-  const key = `insights/${req.params.namespace}/${req.params.name}/files/${filePath}`;
+  const path = `insights/${req.params.namespace}/${req.params.name}/files/${filePath}`;
 
   logger.debug(`Attempting to load insight attachment: ${filePath}`);
 
   const range = Array.isArray(req.headers['range']) ? req.headers['range'][0] : req.headers['range'];
 
   try {
-    const response = await getFromS3(key, range);
+    const storage = Container.get(Storage);
+    const response = await storage.rawGet({ path, range });
 
     if (response.AcceptRanges) {
       res.set('Accept-Ranges', response.AcceptRanges);

packages/backend/src/index.ts

@@ -37,6 +37,8 @@ import { deployMappings } from './lib/elasticsearch';
 import { createServer } from './server';
 import { syncExampleInsights } from './lib/init';
 import { getLogger } from '@iex/shared/logger';
+import { Storage } from '@iex/shared/storage';
+import Container from 'typedi';
 
 const logger = getLogger('index');
 
@@ -53,6 +55,9 @@ process.on('uncaughtException', (uncaughtException) => {
 });
 
 const startup = async (): Promise<Server> => {
+  // Add storage class to the Container since it doesn't use an annotation
+  Container.set(Storage, new Storage());
+
   // Deploy Elasticsearch Index mappings
   logger.debug('Deploying elasticsearch indices');
   await pRetry(() => deployMappings(), {

packages/backend/src/lib/backends/file-system.sync.ts

@@ -21,9 +21,11 @@ import { PersonType } from '@iex/models/person-type';
 import { RepositoryType } from '@iex/models/repository-type';
 import { MessageQueue } from '@iex/mq/message-queue';
 import { getLogger } from '@iex/shared/logger';
+import { Storage } from '@iex/shared/storage';
 import { nanoid } from 'nanoid';
 import pMap from 'p-map';
 import readingTime from 'reading-time';
+import Container from 'typedi';
 
 import { InsightYaml } from '@iex/backend/models/insight-yaml';
 
@@ -33,7 +35,6 @@ import { ActivityService } from '../../services/activity.service';
 import { UserService } from '../../services/user.service';
 import { getTypeAsync } from '../../shared/mime';
 import { GitInstance, INSIGHT_YAML_FILE } from '../git-instance';
-import { writeToS3 } from '../storage';
 
 import { BaseSync, INDEXABLE_MIME_TYPES, READONLY_FILES, THUMBNAIL_LOCATIONS } from './base.sync';
 
@@ -325,6 +326,8 @@ const syncFiles = async (
   insight: IndexedInsight,
   previousInsight: IndexedInsight | null
 ): Promise<void> => {
+  const storage = Container.get(Storage);
+
   // Message Queue for converting files
   const conversionMq = new MessageQueue({ region: process.env.S3_REGION, queueUrl: process.env.CONVERSION_SQS_URL });
 
@@ -362,7 +365,7 @@ const syncFiles = async (
 
         const targetS3Path = `insights/${insight.fullName}/files/${wf.path}`;
 
-        await writeToS3(contents!, targetS3Path);
+        await storage.writeFile({ body: contents!, path: targetS3Path });
 
         if (file.conversions !== undefined) {
           // Submit a conversion request for this file

packages/backend/src/lib/backends/github.sync.ts

@@ -22,13 +22,13 @@ import { PersonType } from '@iex/models/person-type';
 import { RepositoryType } from '@iex/models/repository-type';
 import { MessageQueue } from '@iex/mq/message-queue';
 import { getLogger } from '@iex/shared/logger';
+import { Storage } from '@iex/shared/storage';
 import { nanoid } from 'nanoid';
 import pMap from 'p-map';
 import readingTime from 'reading-time';
 import Container from 'typedi';
 
 import { GitInstance, INSIGHT_YAML_FILE } from '../../lib/git-instance';
-import { writeToS3 } from '../../lib/storage';
 import { Insight } from '../../models/insight';
 import { InsightFile, InsightFileConversion } from '../../models/insight-file';
 import { InsightYaml } from '../../models/insight-yaml';
@@ -452,6 +452,8 @@ const syncFiles = async (
   insight: IndexedInsight,
   previousInsight: IndexedInsight | null
 ): Promise<void> => {
+  const storage = Container.get(Storage);
+
   // Message Queue for converting files
   const conversionMq = new MessageQueue({ region: process.env.S3_REGION, queueUrl: process.env.CONVERSION_SQS_URL });
 
@@ -489,7 +491,7 @@ const syncFiles = async (
 
         const targetS3Path = `insights/${insight.fullName}/files/${wf.path}`;
 
-        await writeToS3(contents!, targetS3Path);
+        await storage.writeFile({ body: contents!, path: targetS3Path });
 
         if (file.conversions !== undefined) {
           // Submit a conversion request for this file

packages/backend/src/lib/import.ts

@@ -23,11 +23,10 @@ import { nanoid } from 'nanoid';
 import { parse as htmlParse, HTMLElement } from 'node-html-parser';
 import TurndownService from 'turndown';
 import * as turndownPluginGfm from 'turndown-plugin-gfm';
+import Container from 'typedi';
 
 import { DraftDataInput, DraftKey } from '../models/draft';
-import { AttachmentService } from '../services/attachment.service';
 import { DraftService } from '../services/draft.service';
-import { TemplateService } from '../services/template.service';
 
 const logger = getLogger('import');
 
@@ -194,7 +193,7 @@ export function convertToDraft(request: ImportRequest): DraftDataInput {
 export async function importToNewDraft(request: ImportRequest): Promise<DraftKey> {
   logger.info(`Importing web page ${request.url}`);
 
-  const draftService = new DraftService(new AttachmentService(), new TemplateService());
+  const draftService = Container.get(DraftService);
   const draftKey = draftService.newDraftKey();
 
   const draftData = convertToDraft(request);