Skip to content

Commit

Permalink
Merge pull request #179 from konecty/feat/server-file-storage
Browse files Browse the repository at this point in the history
Feature: server file storage
  • Loading branch information
marcusdemoura authored Oct 31, 2024
2 parents 93f0db1 + 9cfad3a commit ab71f7b
Show file tree
Hide file tree
Showing 14 changed files with 710 additions and 481 deletions.
1 change: 0 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@
"luxon": "^3.4.3",
"mime-types": "^2.1.35",
"mkdirp": "^3.0.1",
"mmmagic": "^0.5.3",
"moment": "^2.29.4",
"moment-timezone": "^0.5.43",
"mongodb": "^6.1.0",
Expand Down
15 changes: 13 additions & 2 deletions src/imports/file/file.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,24 @@
import findIndex from 'lodash/findIndex';
import first from 'lodash/first';
import get from 'lodash/get';
import size from 'lodash/size';
import first from 'lodash/first';

import { MetaObject } from '@imports/model/MetaObject';
import { getUserSafe } from '../auth/getUser';
import { update } from '../data/data';
import { logger } from '../utils/logger';
import { getUserSafe } from '../auth/getUser';

/**
*
* @param {object} payload
* @param {string} payload.document
* @param {string} payload.fieldName
* @param {string} payload.recordCode
* @param {object} payload.body
* @param {string} [payload.authTokenId]
* @param {import('@imports/model/User').User} [payload.contextUser]
* @returns {Promise<Record<string, unknown>>}
*/
export async function fileUpload({ authTokenId = null, document, fieldName, recordCode, body, contextUser = null }) {
const { success, data: user, errors } = await getUserSafe(authTokenId, contextUser);

Expand Down
15 changes: 12 additions & 3 deletions src/imports/model/Namespace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,27 @@ const CommonStorageProps = z.object({
.optional(),
});

const S3Storage = CommonStorageProps.extend({
export const S3StorageCfg = CommonStorageProps.extend({
type: z.literal('s3'),
config: z.record(z.any()).optional(),
bucket: z.string().optional(),
publicUrl: z.string().optional(),
});

const FSStorage = CommonStorageProps.extend({
export const FSStorageCfg = CommonStorageProps.extend({
type: z.literal('fs'),
directory: z.string().optional(),
});

export const ServerStorageCfg = CommonStorageProps.extend({
type: z.literal('server'),
config: z.object({
upload: z.string(),
preview: z.string(),
headers: z.record(z.string()).optional(),
}),
});

export const NamespaceSchema = z
.object({
type: z.literal('Namespace'),
Expand All @@ -35,7 +44,7 @@ export const NamespaceSchema = z
dateFormat: z.string().optional(),
logoURL: z.string().optional(),

storage: z.discriminatedUnion('type', [S3Storage, FSStorage]).optional(),
storage: z.discriminatedUnion('type', [S3StorageCfg, FSStorageCfg, ServerStorageCfg]).optional(),
RocketChat: z
.object({
accessToken: z.string(),
Expand Down
108 changes: 108 additions & 0 deletions src/imports/storage/FSStorage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import FileStorage, { FileContext, FileData } from './FileStorage';

import { ALLOWED_CORS_FILE_TYPES, DEFAULT_EXPIRATION } from '@imports/consts';
import { fileUpload } from '@imports/file/file';
import { logger } from '@imports/utils/logger';

import crypto from 'crypto';
import { readFile, unlink, writeFile } from 'fs/promises';
import mime from 'mime-types';
import { mkdirp } from 'mkdirp';
import path from 'path';

import { FSStorageCfg } from '@imports/model/Namespace';
import BluebirdPromise from 'bluebird';
import { z } from 'zod';

export default class FSStorage implements FileStorage {
storageCfg: FileStorage['storageCfg'];

constructor(storageCfg: FileStorage['storageCfg']) {
this.storageCfg = storageCfg;
}

async sendFile(fullUrl: string, filePath: string, reply: any) {
logger.trace(`Proxying file ${filePath} from FS`);
const storageCfg = this.storageCfg as z.infer<typeof FSStorageCfg>;
const ext = filePath.split('.').pop()?.toLowerCase() ?? '';
const directory = storageCfg.directory ?? '/tmp';
const fullPath = path.join(directory, filePath);

try {
const fileContent = await readFile(fullPath);
const etag = crypto.createHash('md5').update(fileContent).digest('hex');
const contentType = mime.lookup(filePath) || 'application/octet-stream';
return reply
.headers({
'content-type': contentType,
'content-length': fileContent.length,
'keep-alive': 'timeout=5',
etag,
'cache-control': `public, max-age=${DEFAULT_EXPIRATION}`,
'access-control-allow-origin': ALLOWED_CORS_FILE_TYPES.includes(ext) ? '*' : 'same-origin',
})
.send(fileContent);
} catch (error) {
logger.error(error, `Error proxying file ${filePath} from FS`);
return reply.status(404).send('Not found');
}
}

async upload(fileData: FileData, filesToSave: { name: string; content: Buffer }[], context: FileContext) {
fileData.etag = crypto.createHash('md5').update(filesToSave[0].content).digest('hex');
const storageCfg = this.storageCfg as z.infer<typeof FSStorageCfg>;
const storageDirectory = storageCfg.directory ?? '/tmp';
const fileDirectory = path.join(storageDirectory, fileData.key.replace(fileData.name, ''));

await mkdirp(fileDirectory);

await BluebirdPromise.each(filesToSave, async ({ name, content }) => {
const filePath = path.join(fileDirectory, name);
await writeFile(filePath, content);
});

const coreResponse = await fileUpload({
contextUser: context.user,
document: context.document,
fieldName: context.fieldName,
recordCode: context.recordId,
body: fileData,
});

if (coreResponse.success === false) {
await BluebirdPromise.each(filesToSave, async ({ name }) => {
await unlink(path.join(fileDirectory, name));
});
}

return coreResponse;
}

async delete(directory: string, fileName: string) {
const storageCfg = this.storageCfg as z.infer<typeof FSStorageCfg>;
directory = `${storageCfg.directory ?? '/tmp'}/${directory}`;

const fullPath = path.join(directory, decodeURIComponent(fileName));
const thumbnailFullPath = path.join(directory, 'thumbnail', decodeURIComponent(fileName));
const watermarkFullPath = path.join(directory, 'watermark', decodeURIComponent(fileName));
try {
unlink(fullPath);
} catch (error) {
logger.error(error, `Error deleting file ${fileName} from FS`);
}

try {
unlink(thumbnailFullPath);
} catch (error) {
logger.error(error, `Error deleting thumbnail file ${fileName} from FS`);
}

if (storageCfg?.wm != null) {
try {
unlink(watermarkFullPath);
} catch (error) {
logger.error(error, `Error deleting watermark file ${fileName} from FS`);
}
}
}
}
51 changes: 51 additions & 0 deletions src/imports/storage/FileStorage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import { Namespace } from '@imports/model/Namespace';
import { User } from '@imports/model/User';
import { IncomingHttpHeaders } from 'undici/types/header';
import FSStorage from './FSStorage';
import S3Storage from './S3Storage';
import ServerStorage from './ServerStorage';

export default abstract class FileStorage {
storageCfg: Required<Namespace>['storage'];

static fromNamespaceStorage(storageCfg?: FileStorage['storageCfg']): FileStorage {
switch (storageCfg?.type) {
case 's3':
return new S3Storage(storageCfg);
case 'server':
return new ServerStorage(storageCfg);
case 'fs':
default:
return new FSStorage(storageCfg ?? { type: 'fs' });
}
}

constructor(storageCfg: FileStorage['storageCfg']) {
this.storageCfg = storageCfg;
}

abstract sendFile(fullUrl: string, filePath: string, reply: any): Promise<void>;
abstract upload(fileData: FileData, filesToSave: { name: string; content: Buffer }[], context: FileContext): Promise<Record<string, unknown>>;
abstract delete(directory: string, fileName: string, context: FileContext): Promise<void>;
}

export type FileData = {
key: string;
kind: string;
size: number;
name: string;
etag?: string;
version?: string;
};

export type FileContext = {
namespace?: string;
accessId?: string;
document: string;
recordId: string;
fieldName: string;
fileName: string;
user: User;
authTokenId?: string;
headers: IncomingHttpHeaders;
};
Loading

0 comments on commit ab71f7b

Please sign in to comment.