-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #179 from konecty/feat/server-file-storage
Feature: server file storage
- Loading branch information
Showing
14 changed files
with
710 additions
and
481 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,108 @@ | ||
import FileStorage, { FileContext, FileData } from './FileStorage'; | ||
|
||
import { ALLOWED_CORS_FILE_TYPES, DEFAULT_EXPIRATION } from '@imports/consts'; | ||
import { fileUpload } from '@imports/file/file'; | ||
import { logger } from '@imports/utils/logger'; | ||
|
||
import crypto from 'crypto'; | ||
import { readFile, unlink, writeFile } from 'fs/promises'; | ||
import mime from 'mime-types'; | ||
import { mkdirp } from 'mkdirp'; | ||
import path from 'path'; | ||
|
||
import { FSStorageCfg } from '@imports/model/Namespace'; | ||
import BluebirdPromise from 'bluebird'; | ||
import { z } from 'zod'; | ||
|
||
export default class FSStorage implements FileStorage { | ||
storageCfg: FileStorage['storageCfg']; | ||
|
||
constructor(storageCfg: FileStorage['storageCfg']) { | ||
this.storageCfg = storageCfg; | ||
} | ||
|
||
async sendFile(fullUrl: string, filePath: string, reply: any) { | ||
logger.trace(`Proxying file ${filePath} from FS`); | ||
const storageCfg = this.storageCfg as z.infer<typeof FSStorageCfg>; | ||
const ext = filePath.split('.').pop()?.toLowerCase() ?? ''; | ||
const directory = storageCfg.directory ?? '/tmp'; | ||
const fullPath = path.join(directory, filePath); | ||
|
||
try { | ||
const fileContent = await readFile(fullPath); | ||
const etag = crypto.createHash('md5').update(fileContent).digest('hex'); | ||
const contentType = mime.lookup(filePath) || 'application/octet-stream'; | ||
return reply | ||
.headers({ | ||
'content-type': contentType, | ||
'content-length': fileContent.length, | ||
'keep-alive': 'timeout=5', | ||
etag, | ||
'cache-control': `public, max-age=${DEFAULT_EXPIRATION}`, | ||
'access-control-allow-origin': ALLOWED_CORS_FILE_TYPES.includes(ext) ? '*' : 'same-origin', | ||
}) | ||
.send(fileContent); | ||
} catch (error) { | ||
logger.error(error, `Error proxying file ${filePath} from FS`); | ||
return reply.status(404).send('Not found'); | ||
} | ||
} | ||
|
||
async upload(fileData: FileData, filesToSave: { name: string; content: Buffer }[], context: FileContext) { | ||
fileData.etag = crypto.createHash('md5').update(filesToSave[0].content).digest('hex'); | ||
const storageCfg = this.storageCfg as z.infer<typeof FSStorageCfg>; | ||
const storageDirectory = storageCfg.directory ?? '/tmp'; | ||
const fileDirectory = path.join(storageDirectory, fileData.key.replace(fileData.name, '')); | ||
|
||
await mkdirp(fileDirectory); | ||
|
||
await BluebirdPromise.each(filesToSave, async ({ name, content }) => { | ||
const filePath = path.join(fileDirectory, name); | ||
await writeFile(filePath, content); | ||
}); | ||
|
||
const coreResponse = await fileUpload({ | ||
contextUser: context.user, | ||
document: context.document, | ||
fieldName: context.fieldName, | ||
recordCode: context.recordId, | ||
body: fileData, | ||
}); | ||
|
||
if (coreResponse.success === false) { | ||
await BluebirdPromise.each(filesToSave, async ({ name }) => { | ||
await unlink(path.join(fileDirectory, name)); | ||
}); | ||
} | ||
|
||
return coreResponse; | ||
} | ||
|
||
async delete(directory: string, fileName: string) { | ||
const storageCfg = this.storageCfg as z.infer<typeof FSStorageCfg>; | ||
directory = `${storageCfg.directory ?? '/tmp'}/${directory}`; | ||
|
||
const fullPath = path.join(directory, decodeURIComponent(fileName)); | ||
const thumbnailFullPath = path.join(directory, 'thumbnail', decodeURIComponent(fileName)); | ||
const watermarkFullPath = path.join(directory, 'watermark', decodeURIComponent(fileName)); | ||
try { | ||
unlink(fullPath); | ||
} catch (error) { | ||
logger.error(error, `Error deleting file ${fileName} from FS`); | ||
} | ||
|
||
try { | ||
unlink(thumbnailFullPath); | ||
} catch (error) { | ||
logger.error(error, `Error deleting thumbnail file ${fileName} from FS`); | ||
} | ||
|
||
if (storageCfg?.wm != null) { | ||
try { | ||
unlink(watermarkFullPath); | ||
} catch (error) { | ||
logger.error(error, `Error deleting watermark file ${fileName} from FS`); | ||
} | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
import { Namespace } from '@imports/model/Namespace'; | ||
import { User } from '@imports/model/User'; | ||
import { IncomingHttpHeaders } from 'undici/types/header'; | ||
import FSStorage from './FSStorage'; | ||
import S3Storage from './S3Storage'; | ||
import ServerStorage from './ServerStorage'; | ||
|
||
export default abstract class FileStorage { | ||
storageCfg: Required<Namespace>['storage']; | ||
|
||
static fromNamespaceStorage(storageCfg?: FileStorage['storageCfg']): FileStorage { | ||
switch (storageCfg?.type) { | ||
case 's3': | ||
return new S3Storage(storageCfg); | ||
case 'server': | ||
return new ServerStorage(storageCfg); | ||
case 'fs': | ||
default: | ||
return new FSStorage(storageCfg ?? { type: 'fs' }); | ||
} | ||
} | ||
|
||
constructor(storageCfg: FileStorage['storageCfg']) { | ||
this.storageCfg = storageCfg; | ||
} | ||
|
||
abstract sendFile(fullUrl: string, filePath: string, reply: any): Promise<void>; | ||
abstract upload(fileData: FileData, filesToSave: { name: string; content: Buffer }[], context: FileContext): Promise<Record<string, unknown>>; | ||
abstract delete(directory: string, fileName: string, context: FileContext): Promise<void>; | ||
} | ||
|
||
export type FileData = { | ||
key: string; | ||
kind: string; | ||
size: number; | ||
name: string; | ||
etag?: string; | ||
version?: string; | ||
}; | ||
|
||
export type FileContext = { | ||
namespace?: string; | ||
accessId?: string; | ||
document: string; | ||
recordId: string; | ||
fieldName: string; | ||
fileName: string; | ||
user: User; | ||
authTokenId?: string; | ||
headers: IncomingHttpHeaders; | ||
}; |
Oops, something went wrong.