From ef6c2692c1151e63be2284f6b0a746e4e84e643a Mon Sep 17 00:00:00 2001 From: Murderlon Date: Mon, 29 Jan 2024 11:06:30 +0100 Subject: [PATCH] Introduce @tus/utils to simplify building/publishing --- .../configstores/FileConfigstore.ts | 48 ------- .../configstores/MemoryConfigstore.ts | 36 ------ .../configstores/RedisConfigstore.ts | 40 ------ packages/file-store/configstores/Types.ts | 9 -- packages/file-store/configstores/index.ts | 10 +- packages/file-store/index.ts | 2 +- packages/file-store/package.json | 5 +- packages/file-store/test.ts | 2 +- packages/gcs-store/index.ts | 3 +- packages/gcs-store/package.json | 4 +- packages/s3-store/index.ts | 2 +- packages/s3-store/package.json | 5 +- packages/s3-store/test.ts | 2 +- packages/server/package.json | 1 + packages/server/src/handlers/BaseHandler.ts | 6 +- packages/server/src/handlers/DeleteHandler.ts | 3 +- packages/server/src/handlers/GetHandler.ts | 2 +- packages/server/src/handlers/HeadHandler.ts | 3 +- .../server/src/handlers/OptionsHandler.ts | 2 +- packages/server/src/handlers/PatchHandler.ts | 3 +- packages/server/src/handlers/PostHandler.ts | 12 +- packages/server/src/index.ts | 4 +- packages/server/src/lockers/MemoryLocker.ts | 3 +- packages/server/src/server.ts | 10 +- packages/server/src/types.ts | 2 +- .../server/src/validators/HeaderValidator.ts | 3 +- packages/server/test/BaseHandler.test.ts | 4 +- packages/server/test/DataStore.test.ts | 2 +- packages/server/test/DeleteHandler.test.ts | 4 +- packages/server/test/GetHandler.test.ts | 3 +- packages/server/test/HeadHandler.test.ts | 3 +- packages/server/test/HeaderValidator.test.ts | 2 +- packages/server/test/Metadata.test.ts | 2 +- packages/server/test/OptionsHandler.test.ts | 3 +- packages/server/test/PatchHandler.test.ts | 3 +- packages/server/test/PostHandler.test.ts | 3 +- packages/server/test/Server.test.ts | 3 +- packages/server/test/StreamSplitter.test.ts | 2 +- packages/server/test/Uid.test.ts | 2 +- packages/server/test/Upload.test.ts | 3 +- packages/utils/package.json | 37 ++++++ packages/{server => utils}/src/constants.ts | 0 packages/utils/src/index.ts | 3 + .../src/kvstores/FileKvStore.ts | 0 .../src/kvstores/MemoryKvStore.ts | 0 .../src/kvstores/RedisKvStore.ts | 0 .../{server => utils}/src/kvstores/Types.ts | 0 .../{server => utils}/src/kvstores/index.ts | 0 .../{server => utils}/src/models/Context.ts | 0 .../{server => utils}/src/models/DataStore.ts | 0 .../{server => utils}/src/models/Locker.ts | 0 .../{server => utils}/src/models/Metadata.ts | 0 .../src/models/StreamLimiter.ts | 0 .../src/models/StreamSplitter.ts | 0 packages/{server => utils}/src/models/Uid.ts | 0 .../{server => utils}/src/models/Upload.ts | 0 .../{server => utils}/src/models/index.ts | 1 + packages/utils/test/Metadata.test.ts | 120 ++++++++++++++++++ packages/utils/test/StreamSplitter.test.ts | 54 ++++++++ packages/utils/test/Uid.test.ts | 23 ++++ packages/utils/test/Upload.test.ts | 29 +++++ packages/utils/tsconfig.json | 8 ++ yarn.lock | 27 +++- 63 files changed, 348 insertions(+), 215 deletions(-) delete mode 100644 packages/file-store/configstores/FileConfigstore.ts delete mode 100644 packages/file-store/configstores/MemoryConfigstore.ts delete mode 100644 packages/file-store/configstores/RedisConfigstore.ts delete mode 100644 packages/file-store/configstores/Types.ts create mode 100644 packages/utils/package.json rename packages/{server => utils}/src/constants.ts (100%) create mode 100644 packages/utils/src/index.ts rename packages/{server => utils}/src/kvstores/FileKvStore.ts (100%) rename packages/{server => utils}/src/kvstores/MemoryKvStore.ts (100%) rename packages/{server => utils}/src/kvstores/RedisKvStore.ts (100%) rename packages/{server => utils}/src/kvstores/Types.ts (100%) rename packages/{server => utils}/src/kvstores/index.ts (100%) rename packages/{server => utils}/src/models/Context.ts (100%) rename packages/{server => utils}/src/models/DataStore.ts (100%) rename packages/{server => utils}/src/models/Locker.ts (100%) rename packages/{server => utils}/src/models/Metadata.ts (100%) rename packages/{server => utils}/src/models/StreamLimiter.ts (100%) rename packages/{server => utils}/src/models/StreamSplitter.ts (100%) rename packages/{server => utils}/src/models/Uid.ts (100%) rename packages/{server => utils}/src/models/Upload.ts (100%) rename packages/{server => utils}/src/models/index.ts (86%) create mode 100644 packages/utils/test/Metadata.test.ts create mode 100644 packages/utils/test/StreamSplitter.test.ts create mode 100644 packages/utils/test/Uid.test.ts create mode 100644 packages/utils/test/Upload.test.ts create mode 100644 packages/utils/tsconfig.json diff --git a/packages/file-store/configstores/FileConfigstore.ts b/packages/file-store/configstores/FileConfigstore.ts deleted file mode 100644 index e118e349..00000000 --- a/packages/file-store/configstores/FileConfigstore.ts +++ /dev/null @@ -1,48 +0,0 @@ -import fs from 'node:fs/promises' -import path from 'node:path' -import {Upload} from '@tus/server' - -import {Configstore} from './Types' - -/** - * FileConfigstore writes the `Upload` JSON metadata to disk next the uploaded file itself. - * It uses a queue which only processes one operation at a time to prevent unsafe concurrent access. - */ -export class FileConfigstore implements Configstore { - directory: string - - constructor(path: string) { - this.directory = path - } - - async get(key: string): Promise { - try { - const buffer = await fs.readFile(this.resolve(key), 'utf8') - return JSON.parse(buffer as string) - } catch { - return undefined - } - } - - async set(key: string, value: Upload): Promise { - await fs.writeFile(this.resolve(key), JSON.stringify(value)) - } - - async delete(key: string): Promise { - await fs.rm(this.resolve(key)) - } - - async list(): Promise> { - const files = await fs.readdir(this.directory) - const sorted = files.sort((a, b) => a.localeCompare(b)) - const name = (file: string) => path.basename(file, '.json') - // To only return tus file IDs we check if the file has a corresponding JSON info file - return sorted.filter( - (file, idx) => idx < sorted.length - 1 && name(file) === name(sorted[idx + 1]) - ) - } - - private resolve(key: string): string { - return path.resolve(this.directory, `${key}.json`) - } -} diff --git a/packages/file-store/configstores/MemoryConfigstore.ts b/packages/file-store/configstores/MemoryConfigstore.ts deleted file mode 100644 index 2c310dd6..00000000 --- a/packages/file-store/configstores/MemoryConfigstore.ts +++ /dev/null @@ -1,36 +0,0 @@ -import {Upload} from '@tus/server' -import {Configstore} from './Types' - -/** - * Memory based configstore. - * Used mostly for unit tests. - * - * @author Mitja Puzigaća - */ -export class MemoryConfigstore implements Configstore { - data: Map = new Map() - - async get(key: string): Promise { - return this.deserializeValue(this.data.get(key)) - } - - async set(key: string, value: Upload): Promise { - this.data.set(key, this.serializeValue(value)) - } - - async delete(key: string): Promise { - this.data.delete(key) - } - - async list(): Promise> { - return [...this.data.keys()] - } - - private serializeValue(value: Upload): string { - return JSON.stringify(value) - } - - private deserializeValue(buffer: string | undefined): Upload | undefined { - return buffer ? new Upload(JSON.parse(buffer)) : undefined - } -} diff --git a/packages/file-store/configstores/RedisConfigstore.ts b/packages/file-store/configstores/RedisConfigstore.ts deleted file mode 100644 index b380abf1..00000000 --- a/packages/file-store/configstores/RedisConfigstore.ts +++ /dev/null @@ -1,40 +0,0 @@ -import {RedisClientType} from '@redis/client' - -import {Upload} from '@tus/server' -import {Configstore} from './Types' - -/** - * Redis based configstore. - * - * @author Mitja Puzigaća - */ -export class RedisConfigstore implements Configstore { - constructor(private redis: RedisClientType, private prefix: string = '') { - this.redis = redis - this.prefix = prefix - } - - async get(key: string): Promise { - return this.deserializeValue(await this.redis.get(this.prefix + key)) - } - - async set(key: string, value: Upload): Promise { - await this.redis.set(this.prefix + key, this.serializeValue(value)) - } - - async delete(key: string): Promise { - await this.redis.del(this.prefix + key) - } - - async list(): Promise> { - return this.redis.keys(this.prefix + '*') - } - - private serializeValue(value: Upload): string { - return JSON.stringify(value) - } - - private deserializeValue(buffer: string | null): Upload | undefined { - return buffer ? JSON.parse(buffer) : undefined - } -} diff --git a/packages/file-store/configstores/Types.ts b/packages/file-store/configstores/Types.ts deleted file mode 100644 index fcf8f739..00000000 --- a/packages/file-store/configstores/Types.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {Upload} from '@tus/server' - -export interface Configstore { - get(key: string): Promise - set(key: string, value: Upload): Promise - delete(key: string): Promise - - list?(): Promise> -} diff --git a/packages/file-store/configstores/index.ts b/packages/file-store/configstores/index.ts index 906b0000..9352a76a 100644 --- a/packages/file-store/configstores/index.ts +++ b/packages/file-store/configstores/index.ts @@ -1,4 +1,6 @@ -export {FileConfigstore} from './FileConfigstore' -export {MemoryConfigstore} from './MemoryConfigstore' -export {RedisConfigstore} from './RedisConfigstore' -export {Configstore} from './Types' +export { + FileKvStore as FileConfigstore, + MemoryKvStore as MemoryConfigstore, + RedisKvStore as RedisConfigstore, + KvStore as Configstore, +} from '@tus/utils' diff --git a/packages/file-store/index.ts b/packages/file-store/index.ts index 89c53d9e..06a8d926 100644 --- a/packages/file-store/index.ts +++ b/packages/file-store/index.ts @@ -8,7 +8,7 @@ import http from 'node:http' import debug from 'debug' import {Configstore, FileConfigstore} from './configstores' -import {DataStore, Upload, ERRORS} from '@tus/server' +import {DataStore, Upload, ERRORS} from '@tus/utils' export * from './configstores' diff --git a/packages/file-store/package.json b/packages/file-store/package.json index f8745967..62cc99ab 100644 --- a/packages/file-store/package.json +++ b/packages/file-store/package.json @@ -21,10 +21,10 @@ "test": "mocha test.ts --exit --extension ts --require ts-node/register" }, "dependencies": { + "@tus/utils": "workspace:*", "debug": "^4.3.4" }, "devDependencies": { - "@tus/server": "workspace:^", "@types/debug": "^4.1.12", "@types/mocha": "^10.0.6", "@types/node": "^20.11.5", @@ -34,9 +34,6 @@ "should": "^13.2.3", "typescript": "^5.3.3" }, - "peerDependencies": { - "@tus/server": "workspace:^" - }, "optionalDependencies": { "@redis/client": "^1.5.13" }, diff --git a/packages/file-store/test.ts b/packages/file-store/test.ts index 6ba8c5ce..0433f84c 100644 --- a/packages/file-store/test.ts +++ b/packages/file-store/test.ts @@ -8,7 +8,7 @@ import path from 'node:path' import sinon from 'sinon' import {FileStore, FileConfigstore} from './' -import {Upload} from '@tus/server' +import {Upload} from '@tus/utils' import * as shared from '../../test/stores.test' diff --git a/packages/gcs-store/index.ts b/packages/gcs-store/index.ts index 205e2839..c92e1786 100644 --- a/packages/gcs-store/index.ts +++ b/packages/gcs-store/index.ts @@ -3,8 +3,7 @@ import stream from 'node:stream' import http from 'node:http' import debug from 'debug' -import {ERRORS, TUS_RESUMABLE} from '@tus/server' -import {Upload, DataStore} from '@tus/server' +import {ERRORS, TUS_RESUMABLE, Upload, DataStore} from '@tus/utils' const log = debug('tus-node-server:stores:gcsstore') diff --git a/packages/gcs-store/package.json b/packages/gcs-store/package.json index 10ffb824..46818025 100644 --- a/packages/gcs-store/package.json +++ b/packages/gcs-store/package.json @@ -21,6 +21,7 @@ "test": "mocha test.ts --timeout 30000 --exit --extension ts --require ts-node/register" }, "dependencies": { + "@tus/utils": "workspace:*", "debug": "^4.3.4" }, "devDependencies": { @@ -36,8 +37,7 @@ "typescript": "^5.3.3" }, "peerDependencies": { - "@google-cloud/storage": "*", - "@tus/server": "workspace:^" + "@google-cloud/storage": "*" }, "engines": { "node": ">=16" diff --git a/packages/s3-store/index.ts b/packages/s3-store/index.ts index 5282c0fc..44af250c 100644 --- a/packages/s3-store/index.ts +++ b/packages/s3-store/index.ts @@ -15,7 +15,7 @@ import { TUS_RESUMABLE, KvStore, MemoryKvStore, -} from '@tus/server' +} from '@tus/utils' const log = debug('tus-node-server:stores:s3store') diff --git a/packages/s3-store/package.json b/packages/s3-store/package.json index a1e71e17..59d8dea4 100644 --- a/packages/s3-store/package.json +++ b/packages/s3-store/package.json @@ -22,10 +22,10 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.490.0", + "@tus/utils": "workspace:*", "debug": "^4.3.4" }, "devDependencies": { - "@tus/server": "workspace:^", "@types/debug": "^4.1.12", "@types/mocha": "^10.0.6", "@types/node": "^20.11.5", @@ -35,9 +35,6 @@ "should": "^13.2.3", "typescript": "^5.3.3" }, - "peerDependencies": { - "@tus/server": "workspace:^" - }, "engines": { "node": ">=16" } diff --git a/packages/s3-store/test.ts b/packages/s3-store/test.ts index 30c39e8a..e4604176 100644 --- a/packages/s3-store/test.ts +++ b/packages/s3-store/test.ts @@ -7,7 +7,7 @@ import sinon from 'sinon' import {S3Store} from './' import * as shared from '../../test/stores.test' -import {Upload, Uid} from '@tus/server' +import {Upload, Uid} from '@tus/utils' const fixturesPath = path.resolve('../', '../', 'test', 'fixtures') const storePath = path.resolve('../', '../', 'test', 'output') diff --git a/packages/server/package.json b/packages/server/package.json index f9955e6e..b06c6d10 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -21,6 +21,7 @@ "test": "mocha --timeout 40000 --exit --extension ts --require ts-node/register" }, "dependencies": { + "@tus/utils": "workspace:*", "debug": "^4.3.4" }, "devDependencies": { diff --git a/packages/server/src/handlers/BaseHandler.ts b/packages/server/src/handlers/BaseHandler.ts index 10a8d88e..c6f8a1d9 100644 --- a/packages/server/src/handlers/BaseHandler.ts +++ b/packages/server/src/handlers/BaseHandler.ts @@ -1,13 +1,11 @@ import EventEmitter from 'node:events' import type {ServerOptions} from '../types' -import type {DataStore, CancellationContext} from '../models' +import type {DataStore, CancellationContext} from '@tus/utils' import type http from 'node:http' -import {Upload} from '../models' -import {ERRORS} from '../constants' +import {ERRORS, Upload, StreamLimiter} from '@tus/utils' import stream from 'node:stream/promises' import {addAbortSignal, PassThrough} from 'stream' -import {StreamLimiter} from '../models/StreamLimiter' const reExtractFileID = /([^/]+)\/?$/ const reForwardedHost = /host="?([^";]+)/ diff --git a/packages/server/src/handlers/DeleteHandler.ts b/packages/server/src/handlers/DeleteHandler.ts index e36cdf18..cd92afba 100644 --- a/packages/server/src/handlers/DeleteHandler.ts +++ b/packages/server/src/handlers/DeleteHandler.ts @@ -1,6 +1,5 @@ import {BaseHandler} from './BaseHandler' -import {ERRORS, EVENTS} from '../constants' -import {CancellationContext} from '../models' +import {ERRORS, EVENTS, CancellationContext} from '@tus/utils' import type http from 'node:http' diff --git a/packages/server/src/handlers/GetHandler.ts b/packages/server/src/handlers/GetHandler.ts index d8c34990..717294e2 100644 --- a/packages/server/src/handlers/GetHandler.ts +++ b/packages/server/src/handlers/GetHandler.ts @@ -1,7 +1,7 @@ import stream from 'node:stream' import {BaseHandler} from './BaseHandler' -import {ERRORS} from '../constants' +import {ERRORS} from '@tus/utils' import type http from 'node:http' import type {RouteHandler} from '../types' diff --git a/packages/server/src/handlers/HeadHandler.ts b/packages/server/src/handlers/HeadHandler.ts index 86846aab..2e7ae7cc 100644 --- a/packages/server/src/handlers/HeadHandler.ts +++ b/packages/server/src/handlers/HeadHandler.ts @@ -1,7 +1,6 @@ import {BaseHandler} from './BaseHandler' -import {ERRORS} from '../constants' -import {Metadata, Upload, CancellationContext} from '../models' +import {ERRORS, Metadata, Upload, CancellationContext} from '@tus/utils' import type http from 'node:http' diff --git a/packages/server/src/handlers/OptionsHandler.ts b/packages/server/src/handlers/OptionsHandler.ts index 944318cd..9bb19664 100644 --- a/packages/server/src/handlers/OptionsHandler.ts +++ b/packages/server/src/handlers/OptionsHandler.ts @@ -1,5 +1,5 @@ import {BaseHandler} from './BaseHandler' -import {ALLOWED_METHODS, MAX_AGE, HEADERS} from '../constants' +import {ALLOWED_METHODS, MAX_AGE, HEADERS} from '@tus/utils' import type http from 'node:http' diff --git a/packages/server/src/handlers/PatchHandler.ts b/packages/server/src/handlers/PatchHandler.ts index 787b844c..b07d8365 100644 --- a/packages/server/src/handlers/PatchHandler.ts +++ b/packages/server/src/handlers/PatchHandler.ts @@ -1,10 +1,9 @@ import debug from 'debug' import {BaseHandler} from './BaseHandler' -import {ERRORS, EVENTS} from '../constants' import type http from 'node:http' -import {CancellationContext, Upload} from '../models' +import {ERRORS, EVENTS, CancellationContext, Upload} from '@tus/utils' const log = debug('tus-node-server:handlers:patch') diff --git a/packages/server/src/handlers/PostHandler.ts b/packages/server/src/handlers/PostHandler.ts index 413e1023..43abaff4 100644 --- a/packages/server/src/handlers/PostHandler.ts +++ b/packages/server/src/handlers/PostHandler.ts @@ -1,13 +1,19 @@ import debug from 'debug' import {BaseHandler} from './BaseHandler' -import {Upload, Uid, Metadata} from '../models' +import { + Upload, + Uid, + Metadata, + EVENTS, + ERRORS, + DataStore, + CancellationContext, +} from '@tus/utils' import {validateHeader} from '../validators/HeaderValidator' -import {EVENTS, ERRORS} from '../constants' import type http from 'node:http' import type {ServerOptions, WithRequired} from '../types' -import {DataStore, CancellationContext} from '../models' const log = debug('tus-node-server:handlers:post') diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index b7942282..c353f788 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1,6 +1,4 @@ export {Server} from './server' export * from './types' -export * from './models' export * from './lockers' -export * from './constants' -export * from './kvstores' +export * from '@tus/utils' diff --git a/packages/server/src/lockers/MemoryLocker.ts b/packages/server/src/lockers/MemoryLocker.ts index 8e68fdbe..da70777c 100644 --- a/packages/server/src/lockers/MemoryLocker.ts +++ b/packages/server/src/lockers/MemoryLocker.ts @@ -1,5 +1,4 @@ -import {ERRORS} from '../constants' -import {Lock, Locker, RequestRelease} from '../models' +import {ERRORS, Lock, Locker, RequestRelease} from '@tus/utils' /** * MemoryLocker is an implementation of the Locker interface that manages locks in memory. diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index e9cee2bb..04694571 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -11,17 +11,11 @@ import {PostHandler} from './handlers/PostHandler' import {DeleteHandler} from './handlers/DeleteHandler' import {validateHeader} from './validators/HeaderValidator' -import { - EVENTS, - ERRORS, - EXPOSED_HEADERS, - REQUEST_METHODS, - TUS_RESUMABLE, -} from './constants' +import {EVENTS, ERRORS, EXPOSED_HEADERS, REQUEST_METHODS, TUS_RESUMABLE} from '@tus/utils' import type stream from 'node:stream' import type {ServerOptions, RouteHandler, WithOptional} from './types' -import type {DataStore, Upload, CancellationContext} from './models' +import type {DataStore, Upload, CancellationContext} from '@tus/utils' import {MemoryLocker} from './lockers' type Handlers = { diff --git a/packages/server/src/types.ts b/packages/server/src/types.ts index 7477bc54..12f2df81 100644 --- a/packages/server/src/types.ts +++ b/packages/server/src/types.ts @@ -1,6 +1,6 @@ import type http from 'node:http' -import type {Locker, Upload} from './models' +import type {Locker, Upload} from '@tus/utils' /** * Represents the configuration options for a server. diff --git a/packages/server/src/validators/HeaderValidator.ts b/packages/server/src/validators/HeaderValidator.ts index ce8e48a5..ed0f5af5 100644 --- a/packages/server/src/validators/HeaderValidator.ts +++ b/packages/server/src/validators/HeaderValidator.ts @@ -1,5 +1,4 @@ -import {TUS_VERSION, TUS_RESUMABLE} from '../constants' -import {Metadata} from '../models' +import {TUS_VERSION, TUS_RESUMABLE, Metadata} from '@tus/utils' type validator = (value?: string) => boolean diff --git a/packages/server/test/BaseHandler.test.ts b/packages/server/test/BaseHandler.test.ts index c534d68f..291650a7 100644 --- a/packages/server/test/BaseHandler.test.ts +++ b/packages/server/test/BaseHandler.test.ts @@ -3,8 +3,8 @@ import http from 'node:http' import httpMocks from 'node-mocks-http' -import {BaseHandler} from '../src/handlers//BaseHandler' -import {DataStore} from '../src/models' +import {BaseHandler} from '../src/handlers/BaseHandler' +import {DataStore} from '@tus/utils' import {MemoryLocker} from '../src' describe('BaseHandler', () => { diff --git a/packages/server/test/DataStore.test.ts b/packages/server/test/DataStore.test.ts index cf93baf4..8e26a66d 100644 --- a/packages/server/test/DataStore.test.ts +++ b/packages/server/test/DataStore.test.ts @@ -1,7 +1,7 @@ import 'should' import {strict as assert} from 'node:assert' -import {DataStore} from '../src/models/DataStore' +import {DataStore} from '@tus/utils' describe('DataStore', () => { const datastore = new DataStore() diff --git a/packages/server/test/DeleteHandler.test.ts b/packages/server/test/DeleteHandler.test.ts index d255073d..4c0fe2d6 100644 --- a/packages/server/test/DeleteHandler.test.ts +++ b/packages/server/test/DeleteHandler.test.ts @@ -6,10 +6,8 @@ import type http from 'node:http' import sinon from 'sinon' import httpMocks from 'node-mocks-http' -import {DataStore} from '../src/models/DataStore' +import {ERRORS, EVENTS, DataStore, CancellationContext} from '@tus/utils' import {DeleteHandler} from '../src/handlers/DeleteHandler' -import {ERRORS, EVENTS} from '../src/constants' -import {CancellationContext} from '../src/models' import {MemoryLocker} from '../src' describe('DeleteHandler', () => { diff --git a/packages/server/test/GetHandler.test.ts b/packages/server/test/GetHandler.test.ts index db9e3253..cf9811c9 100644 --- a/packages/server/test/GetHandler.test.ts +++ b/packages/server/test/GetHandler.test.ts @@ -9,9 +9,8 @@ import sinon from 'sinon' import httpMocks from 'node-mocks-http' import {GetHandler} from '../src/handlers/GetHandler' -import {DataStore} from '../src/models/DataStore' +import {DataStore, Upload} from '@tus/utils' import {FileStore} from '@tus/file-store' -import {Upload} from '../src/models/Upload' import {MemoryLocker} from '../src' describe('GetHandler', () => { diff --git a/packages/server/test/HeadHandler.test.ts b/packages/server/test/HeadHandler.test.ts index 243e9f60..5a33340f 100644 --- a/packages/server/test/HeadHandler.test.ts +++ b/packages/server/test/HeadHandler.test.ts @@ -4,9 +4,8 @@ import http from 'node:http' import sinon from 'sinon' import httpMocks from 'node-mocks-http' -import {DataStore, Upload, CancellationContext} from '../src/models' +import {ERRORS, DataStore, Upload, CancellationContext} from '@tus/utils' import {HeadHandler} from '../src/handlers/HeadHandler' -import {ERRORS} from '../src/constants' import {MemoryLocker} from '../src' describe('HeadHandler', () => { diff --git a/packages/server/test/HeaderValidator.test.ts b/packages/server/test/HeaderValidator.test.ts index 06ef0cdf..4c8d6355 100644 --- a/packages/server/test/HeaderValidator.test.ts +++ b/packages/server/test/HeaderValidator.test.ts @@ -1,7 +1,7 @@ import {strict as assert} from 'node:assert' import {validateHeader} from '../src/validators/HeaderValidator' -import {TUS_RESUMABLE} from '../src/constants' +import {TUS_RESUMABLE} from '@tus/utils' describe('HeaderValidator', () => { describe('upload-offset', () => { diff --git a/packages/server/test/Metadata.test.ts b/packages/server/test/Metadata.test.ts index b6ce0eab..dc6fafb7 100644 --- a/packages/server/test/Metadata.test.ts +++ b/packages/server/test/Metadata.test.ts @@ -1,5 +1,5 @@ import {strict as assert} from 'node:assert' -import {parse, stringify} from '../src/models/Metadata' +import {parse, stringify} from '@tus/utils/Metadata' describe('Metadata', () => { it('parse valid metadata string', () => { diff --git a/packages/server/test/OptionsHandler.test.ts b/packages/server/test/OptionsHandler.test.ts index 5a3aedb2..5a161057 100644 --- a/packages/server/test/OptionsHandler.test.ts +++ b/packages/server/test/OptionsHandler.test.ts @@ -6,8 +6,7 @@ import http from 'node:http' import httpMocks from 'node-mocks-http' import {OptionsHandler} from '../src/handlers/OptionsHandler' -import {DataStore} from '../src/models/DataStore' -import {ALLOWED_METHODS, ALLOWED_HEADERS, MAX_AGE} from '../src/constants' +import {DataStore, ALLOWED_METHODS, ALLOWED_HEADERS, MAX_AGE} from '@tus/utils' import {MemoryLocker} from '../src' describe('OptionsHandler', () => { diff --git a/packages/server/test/PatchHandler.test.ts b/packages/server/test/PatchHandler.test.ts index 456d6282..e3a41f34 100644 --- a/packages/server/test/PatchHandler.test.ts +++ b/packages/server/test/PatchHandler.test.ts @@ -7,8 +7,7 @@ import sinon from 'sinon' import httpMocks from 'node-mocks-http' import {PatchHandler} from '../src/handlers/PatchHandler' -import {Upload, DataStore, CancellationContext} from '../src/models' -import {EVENTS} from '../src/constants' +import {EVENTS, Upload, DataStore, CancellationContext} from '@tus/utils' import {EventEmitter} from 'node:events' import {addPipableStreamBody} from './utils' import {MemoryLocker} from '../src' diff --git a/packages/server/test/PostHandler.test.ts b/packages/server/test/PostHandler.test.ts index 56b72c8d..bbca9cbd 100644 --- a/packages/server/test/PostHandler.test.ts +++ b/packages/server/test/PostHandler.test.ts @@ -7,9 +7,8 @@ import http from 'node:http' import httpMocks from 'node-mocks-http' import sinon from 'sinon' -import {Upload, DataStore, CancellationContext} from '../src/models' +import {EVENTS, Upload, DataStore, CancellationContext} from '@tus/utils' import {PostHandler} from '../src/handlers/PostHandler' -import {EVENTS} from '../src/constants' import {addPipableStreamBody} from './utils' import {MemoryLocker} from '../src' diff --git a/packages/server/test/Server.test.ts b/packages/server/test/Server.test.ts index 8cea26c1..9ebed4dd 100644 --- a/packages/server/test/Server.test.ts +++ b/packages/server/test/Server.test.ts @@ -10,8 +10,7 @@ import request from 'supertest' import {Server} from '../src' import {FileStore} from '@tus/file-store' -import {DataStore} from '../src/models' -import {TUS_RESUMABLE, EVENTS} from '../src/constants' +import {TUS_RESUMABLE, EVENTS, DataStore} from '@tus/utils' import httpMocks from 'node-mocks-http' import sinon from 'sinon' diff --git a/packages/server/test/StreamSplitter.test.ts b/packages/server/test/StreamSplitter.test.ts index c9cc5907..902209a4 100644 --- a/packages/server/test/StreamSplitter.test.ts +++ b/packages/server/test/StreamSplitter.test.ts @@ -3,7 +3,7 @@ import fs from 'node:fs' import stream from 'node:stream/promises' import {strict as assert} from 'node:assert' -import {StreamSplitter} from '../src/models' +import {StreamSplitter} from '@tus/utils' import {Readable} from 'node:stream' const fileSize = 20_971_520 diff --git a/packages/server/test/Uid.test.ts b/packages/server/test/Uid.test.ts index b7134df2..fbe67a61 100644 --- a/packages/server/test/Uid.test.ts +++ b/packages/server/test/Uid.test.ts @@ -1,6 +1,6 @@ import {strict as assert} from 'node:assert' -import {Uid} from '../src/models' +import {Uid} from '@tus/utils' describe('Uid', () => { it('returns a 32 char string', (done) => { diff --git a/packages/server/test/Upload.test.ts b/packages/server/test/Upload.test.ts index a3cbb542..81a6fa46 100644 --- a/packages/server/test/Upload.test.ts +++ b/packages/server/test/Upload.test.ts @@ -1,8 +1,7 @@ import 'should' import {strict as assert} from 'node:assert' -import {Upload} from '../src/models/Upload' -import {Uid} from '../src/models/Uid' +import {Upload, Uid} from '@tus/utils' describe('Upload', () => { describe('constructor', () => { diff --git a/packages/utils/package.json b/packages/utils/package.json new file mode 100644 index 00000000..1773c27e --- /dev/null +++ b/packages/utils/package.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json.schemastore.org/package.json", + "name": "@tus/utils", + "version": "1.0.0", + "description": "Internal utils for tus Node.js server and stores", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "homepage": "https://github.com/tus/tus-node-server#readme", + "bugs": "https://github.com/tus/tus-node-server/issues", + "repository": "tus/tus-node-server", + "license": "MIT", + "files": [ + "README.md", + "LICENSE", + "dist" + ], + "scripts": { + "build": "tsc", + "lint": "eslint .", + "format": "eslint --fix .", + "test": "mocha --timeout 40000 --exit --extension ts --require ts-node/register" + }, + "devDependencies": { + "@types/debug": "^4.1.12", + "@types/mocha": "^10.0.6", + "@types/node": "^20.11.5", + "eslint": "^8.56.0", + "eslint-config-custom": "workspace:*", + "mocha": "^10.2.0", + "should": "^13.2.3", + "ts-node": "^10.9.2", + "typescript": "^5.3.3" + }, + "engines": { + "node": ">=16" + } +} diff --git a/packages/server/src/constants.ts b/packages/utils/src/constants.ts similarity index 100% rename from packages/server/src/constants.ts rename to packages/utils/src/constants.ts diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts new file mode 100644 index 00000000..78b880f9 --- /dev/null +++ b/packages/utils/src/index.ts @@ -0,0 +1,3 @@ +export * from './models' +export * from './constants' +export * from './kvstores' diff --git a/packages/server/src/kvstores/FileKvStore.ts b/packages/utils/src/kvstores/FileKvStore.ts similarity index 100% rename from packages/server/src/kvstores/FileKvStore.ts rename to packages/utils/src/kvstores/FileKvStore.ts diff --git a/packages/server/src/kvstores/MemoryKvStore.ts b/packages/utils/src/kvstores/MemoryKvStore.ts similarity index 100% rename from packages/server/src/kvstores/MemoryKvStore.ts rename to packages/utils/src/kvstores/MemoryKvStore.ts diff --git a/packages/server/src/kvstores/RedisKvStore.ts b/packages/utils/src/kvstores/RedisKvStore.ts similarity index 100% rename from packages/server/src/kvstores/RedisKvStore.ts rename to packages/utils/src/kvstores/RedisKvStore.ts diff --git a/packages/server/src/kvstores/Types.ts b/packages/utils/src/kvstores/Types.ts similarity index 100% rename from packages/server/src/kvstores/Types.ts rename to packages/utils/src/kvstores/Types.ts diff --git a/packages/server/src/kvstores/index.ts b/packages/utils/src/kvstores/index.ts similarity index 100% rename from packages/server/src/kvstores/index.ts rename to packages/utils/src/kvstores/index.ts diff --git a/packages/server/src/models/Context.ts b/packages/utils/src/models/Context.ts similarity index 100% rename from packages/server/src/models/Context.ts rename to packages/utils/src/models/Context.ts diff --git a/packages/server/src/models/DataStore.ts b/packages/utils/src/models/DataStore.ts similarity index 100% rename from packages/server/src/models/DataStore.ts rename to packages/utils/src/models/DataStore.ts diff --git a/packages/server/src/models/Locker.ts b/packages/utils/src/models/Locker.ts similarity index 100% rename from packages/server/src/models/Locker.ts rename to packages/utils/src/models/Locker.ts diff --git a/packages/server/src/models/Metadata.ts b/packages/utils/src/models/Metadata.ts similarity index 100% rename from packages/server/src/models/Metadata.ts rename to packages/utils/src/models/Metadata.ts diff --git a/packages/server/src/models/StreamLimiter.ts b/packages/utils/src/models/StreamLimiter.ts similarity index 100% rename from packages/server/src/models/StreamLimiter.ts rename to packages/utils/src/models/StreamLimiter.ts diff --git a/packages/server/src/models/StreamSplitter.ts b/packages/utils/src/models/StreamSplitter.ts similarity index 100% rename from packages/server/src/models/StreamSplitter.ts rename to packages/utils/src/models/StreamSplitter.ts diff --git a/packages/server/src/models/Uid.ts b/packages/utils/src/models/Uid.ts similarity index 100% rename from packages/server/src/models/Uid.ts rename to packages/utils/src/models/Uid.ts diff --git a/packages/server/src/models/Upload.ts b/packages/utils/src/models/Upload.ts similarity index 100% rename from packages/server/src/models/Upload.ts rename to packages/utils/src/models/Upload.ts diff --git a/packages/server/src/models/index.ts b/packages/utils/src/models/index.ts similarity index 86% rename from packages/server/src/models/index.ts rename to packages/utils/src/models/index.ts index 908f53ca..9d85b32e 100644 --- a/packages/server/src/models/index.ts +++ b/packages/utils/src/models/index.ts @@ -1,6 +1,7 @@ export {DataStore} from './DataStore' export * as Metadata from './Metadata' export {StreamSplitter} from './StreamSplitter' +export {StreamLimiter} from './StreamLimiter' export {Uid} from './Uid' export {Upload} from './Upload' export {Locker, Lock, RequestRelease} from './Locker' diff --git a/packages/utils/test/Metadata.test.ts b/packages/utils/test/Metadata.test.ts new file mode 100644 index 00000000..b6ce0eab --- /dev/null +++ b/packages/utils/test/Metadata.test.ts @@ -0,0 +1,120 @@ +import {strict as assert} from 'node:assert' +import {parse, stringify} from '../src/models/Metadata' + +describe('Metadata', () => { + it('parse valid metadata string', () => { + const str = + 'file/name dGVzdC5tcDQ=,size OTYwMjQ0,type! dmlkZW8vbXA0,video,withWhitespace ' + const obj = { + 'file/name': 'test.mp4', + size: '960244', + 'type!': 'video/mp4', + video: null, + withWhitespace: null, + } + const decoded = parse(str) + assert.deepStrictEqual(decoded, obj) + }) + + it('check length of metadata string', () => { + const obj = { + filename: 'test.mp4', + size: '960244', + type: 'video/mp4', + video: null, + withWhitespace: null, + } + const encoded = stringify(obj) + + assert.strictEqual(encoded.split(',').length, Object.entries(obj).length) + }) + + it('verify metadata stringification', () => { + assert.strictEqual(stringify({filename: 'test.mp4'}), 'filename dGVzdC5tcDQ=') + assert.strictEqual(stringify({size: '960244'}), 'size OTYwMjQ0') + assert.strictEqual(stringify({type: 'video/mp4'}), 'type dmlkZW8vbXA0') + // Multiple valid options + assert.notStrictEqual(['video', 'video '].indexOf(stringify({video: null})), -1) + assert.notStrictEqual( + ['withWhitespace', 'withWhitespace '].indexOf(stringify({withWhitespace: null})), + -1 + ) + }) + + it('verify metadata parsing', () => { + assert.deepStrictEqual(parse('filename dGVzdC5tcDQ='), { + filename: 'test.mp4', + }) + assert.deepStrictEqual(parse('size OTYwMjQ0'), {size: '960244'}) + assert.deepStrictEqual(parse('type dmlkZW8vbXA0'), { + type: 'video/mp4', + }) + assert.deepStrictEqual(parse('video'), {video: null}) + assert.deepStrictEqual(parse('video '), {video: null}) + assert.deepStrictEqual(parse('withWhitespace'), { + withWhitespace: null, + }) + assert.deepStrictEqual(parse('withWhitespace '), { + withWhitespace: null, + }) + }) + + it('cyclic test', () => { + const obj = { + filename: 'world_domination_plan.pdf', + is_confidential: null, + } + // Object -> string -> object + assert.deepStrictEqual(parse(stringify(obj)), obj) + }) + + describe('verify invalid metadata string', () => { + it('duplicate keys', () => { + assert.throws(() => { + parse('filename dGVzdC5tcDQ=, filename cGFja2FnZS5qc29u') + }) + assert.throws(() => { + parse('video ,video dHJ1ZQ==') + }) + assert.throws(() => { + parse('size,size ') + }) + assert.throws(() => { + parse('') + }) + assert.throws(() => { + parse('\t\n') + }) + }) + + it('invalid key', () => { + assert.throws(() => { + parse('🦁 ZW1vamk=') + }) + assert.throws(() => { + parse('€¢ß') + }) + assert.throws(() => { + parse('test, te st ') + }) + assert.throws(() => { + parse('test,,test') + }) + }) + + it('invalid base64 value', () => { + assert.throws(() => { + parse('key ZW1vamk') + }) // Value is not a multiple of 4 characters + assert.throws(() => { + parse('key invalid-base64==') + }) + assert.throws(() => { + parse('key =ZW1vamk') + }) // Padding can not be at the beginning + assert.throws(() => { + parse('key ') + }) // Only single whitespace is allowed + }) + }) +}) diff --git a/packages/utils/test/StreamSplitter.test.ts b/packages/utils/test/StreamSplitter.test.ts new file mode 100644 index 00000000..c9cc5907 --- /dev/null +++ b/packages/utils/test/StreamSplitter.test.ts @@ -0,0 +1,54 @@ +import os from 'node:os' +import fs from 'node:fs' +import stream from 'node:stream/promises' +import {strict as assert} from 'node:assert' + +import {StreamSplitter} from '../src/models' +import {Readable} from 'node:stream' + +const fileSize = 20_971_520 + +describe('StreamSplitter', () => { + it('should buffer chunks until optimal part size', async () => { + const readStream = fs.createReadStream('../../test/fixtures/test.pdf') + const optimalChunkSize = 8 * 1024 * 1024 + const parts = [optimalChunkSize, optimalChunkSize, fileSize - optimalChunkSize * 2] + let offset = 0 + let index = 0 + const splitterStream = new StreamSplitter({ + chunkSize: optimalChunkSize, + directory: os.tmpdir(), + }).on('chunkFinished', ({size}) => { + offset += size + assert.equal(parts[index], size) + index++ + }) + await stream.pipeline(readStream, splitterStream) + assert.equal(offset, fileSize) + }) + + it('should split to multiple chunks when single buffer exceeds chunk size', async () => { + const optimalChunkSize = 1024 + const expectedChunks = 7 + + const readStream = Readable.from([Buffer.alloc(expectedChunks * optimalChunkSize)]) + + let chunksStarted = 0 + let chunksFinished = 0 + const splitterStream = new StreamSplitter({ + chunkSize: optimalChunkSize, + directory: os.tmpdir(), + }) + .on('chunkStarted', () => { + chunksStarted++ + }) + .on('chunkFinished', () => { + chunksFinished++ + }) + + await stream.pipeline(readStream, splitterStream) + + assert.equal(chunksStarted, expectedChunks) + assert.equal(chunksFinished, expectedChunks) + }) +}) diff --git a/packages/utils/test/Uid.test.ts b/packages/utils/test/Uid.test.ts new file mode 100644 index 00000000..b7134df2 --- /dev/null +++ b/packages/utils/test/Uid.test.ts @@ -0,0 +1,23 @@ +import {strict as assert} from 'node:assert' + +import {Uid} from '../src/models' + +describe('Uid', () => { + it('returns a 32 char string', (done) => { + const id = Uid.rand() + assert.equal(typeof id, 'string') + assert.equal(id.length, 32) + done() + }) + + it('returns a different string every time', (done) => { + const ids: Record = {} + for (let i = 0; i < 16; i++) { + const id = Uid.rand() + assert(!ids[id], 'id was encountered multiple times') + ids[id] = true + } + + done() + }) +}) diff --git a/packages/utils/test/Upload.test.ts b/packages/utils/test/Upload.test.ts new file mode 100644 index 00000000..a3cbb542 --- /dev/null +++ b/packages/utils/test/Upload.test.ts @@ -0,0 +1,29 @@ +import 'should' +import {strict as assert} from 'node:assert' + +import {Upload} from '../src/models/Upload' +import {Uid} from '../src/models/Uid' + +describe('Upload', () => { + describe('constructor', () => { + it('must require a file_name', () => { + assert.throws(() => { + // @ts-expect-error TS(2554): Expected 4 arguments, but got 0. + new Upload() + }, Error) + }) + + it('should set properties given', () => { + const id = Uid.rand() + const size = 1234 + const offset = 0 + const metadata = {foo: 'bar'} + const upload = new Upload({id, size, offset, metadata}) + assert.equal(upload.id, id) + assert.equal(upload.size, size) + assert.equal(upload.offset, offset) + assert.equal(upload.sizeIsDeferred, false) + assert.equal(upload.metadata, metadata) + }) + }) +}) diff --git a/packages/utils/tsconfig.json b/packages/utils/tsconfig.json new file mode 100644 index 00000000..962ac078 --- /dev/null +++ b/packages/utils/tsconfig.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig.json", + "extends": "../../tsconfig.json", + "include": ["src"], + "compilerOptions": { + "outDir": "./dist" + } +} diff --git a/yarn.lock b/yarn.lock index d337613d..e4d1dc1d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1970,7 +1970,7 @@ __metadata: resolution: "@tus/file-store@workspace:packages/file-store" dependencies: "@redis/client": "npm:^1.5.13" - "@tus/server": "workspace:^" + "@tus/utils": "workspace:*" "@types/debug": "npm:^4.1.12" "@types/mocha": "npm:^10.0.6" "@types/node": "npm:^20.11.5" @@ -1980,8 +1980,6 @@ __metadata: mocha: "npm:^10.2.0" should: "npm:^13.2.3" typescript: "npm:^5.3.3" - peerDependencies: - "@tus/server": "workspace:^" dependenciesMeta: "@redis/client": optional: true @@ -1994,6 +1992,7 @@ __metadata: dependencies: "@google-cloud/storage": "npm:^6.12.0" "@tus/server": "workspace:^" + "@tus/utils": "workspace:*" "@types/debug": "npm:^4.1.12" "@types/mocha": "npm:^10.0.6" "@types/node": "npm:^20.11.5" @@ -2005,7 +2004,6 @@ __metadata: typescript: "npm:^5.3.3" peerDependencies: "@google-cloud/storage": "*" - "@tus/server": "workspace:^" languageName: unknown linkType: soft @@ -2014,7 +2012,7 @@ __metadata: resolution: "@tus/s3-store@workspace:packages/s3-store" dependencies: "@aws-sdk/client-s3": "npm:^3.490.0" - "@tus/server": "workspace:^" + "@tus/utils": "workspace:*" "@types/debug": "npm:^4.1.12" "@types/mocha": "npm:^10.0.6" "@types/node": "npm:^20.11.5" @@ -2024,8 +2022,6 @@ __metadata: mocha: "npm:^10.2.0" should: "npm:^13.2.3" typescript: "npm:^5.3.3" - peerDependencies: - "@tus/server": "workspace:^" languageName: unknown linkType: soft @@ -2034,6 +2030,7 @@ __metadata: resolution: "@tus/server@workspace:packages/server" dependencies: "@redis/client": "npm:^1.5.13" + "@tus/utils": "workspace:*" "@types/debug": "npm:^4.1.12" "@types/mocha": "npm:^10.0.6" "@types/node": "npm:^20.11.5" @@ -2055,6 +2052,22 @@ __metadata: languageName: unknown linkType: soft +"@tus/utils@workspace:*, @tus/utils@workspace:packages/utils": + version: 0.0.0-use.local + resolution: "@tus/utils@workspace:packages/utils" + dependencies: + "@types/debug": "npm:^4.1.12" + "@types/mocha": "npm:^10.0.6" + "@types/node": "npm:^20.11.5" + eslint: "npm:^8.56.0" + eslint-config-custom: "workspace:*" + mocha: "npm:^10.2.0" + should: "npm:^13.2.3" + ts-node: "npm:^10.9.2" + typescript: "npm:^5.3.3" + languageName: unknown + linkType: soft + "@types/body-parser@npm:*": version: 1.19.5 resolution: "@types/body-parser@npm:1.19.5"