From a11f0e70a1dc71814495ffb9131bb9b1a81f28f6 Mon Sep 17 00:00:00 2001 From: Harminder Virk Date: Wed, 1 May 2024 13:23:49 +0530 Subject: [PATCH] feat: add debug calls --- drivers/fs/debug.ts | 12 ++++++++++ drivers/fs/driver.ts | 21 +++++++++++++++++ drivers/gcs/driver.ts | 52 +++++++++++++++++++++++++++++++++++++++++-- package.json | 3 ++- src/types.ts | 4 ---- 5 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 drivers/fs/debug.ts diff --git a/drivers/fs/debug.ts b/drivers/fs/debug.ts new file mode 100644 index 0000000..8d5dde2 --- /dev/null +++ b/drivers/fs/debug.ts @@ -0,0 +1,12 @@ +/* + * @flydrive/core + * + * (c) FlyDrive + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +import { debuglog } from 'node:util' + +export default debuglog('flydrive:fs') diff --git a/drivers/fs/driver.ts b/drivers/fs/driver.ts index 7aba90f..a7ce72e 100644 --- a/drivers/fs/driver.ts +++ b/drivers/fs/driver.ts @@ -18,6 +18,7 @@ import { Retrier } from '@humanwhocodes/retry' import { RuntimeException } from '@poppinss/utils' import { dirname, join, relative } from 'node:path' +import debug from './debug.js' import type { FSDriverOptions } from './types.js' import { DriveFile } from '../../src/driver_file.js' import { DriveDirectory } from '../../src/drive_directory.js' @@ -56,6 +57,8 @@ export class FSDriver implements DriverContract { constructor(public options: FSDriverOptions) { this.#rootUrl = typeof options.location === 'string' ? options.location : fileURLToPath(options.location) + + debug('driver config %O', options) } /** @@ -85,6 +88,7 @@ export class FSDriver implements DriverContract { * Returns a boolean indicating if the file exists or not. */ async exist(key: string): Promise { + debug('checking if file exists %s:%s', this.#rootUrl, key) const location = join(this.#rootUrl, key) try { const object = await fsp.stat(location) @@ -102,6 +106,7 @@ export class FSDriver implements DriverContract { * exception is thrown when the file is missing. */ async get(key: string): Promise { + debug('reading file contents %s:%s', this.#rootUrl, key) return this.#read(key).then((value) => value.toString('utf-8')) } @@ -110,6 +115,7 @@ export class FSDriver implements DriverContract { * exception is thrown when the file is missing. */ async getStream(key: string): Promise { + debug('reading file contents as a stream %s:%s', this.#rootUrl, key) const location = join(this.#rootUrl, key) return createReadStream(location) } @@ -119,6 +125,7 @@ export class FSDriver implements DriverContract { * exception is thrown when the file is missing. */ async getArrayBuffer(key: string): Promise { + debug('reading file contents as array buffer %s:%s', this.#rootUrl, key) return this.#read(key).then((value) => new Uint8Array(value.buffer)) } @@ -126,6 +133,7 @@ export class FSDriver implements DriverContract { * Returns the metadata of a file. */ async getMetaData(key: string): Promise { + debug('fetching file metadata %s:%s', this.#rootUrl, key) const location = join(this.#rootUrl, key) const stats = await fsp.stat(location) @@ -157,6 +165,7 @@ export class FSDriver implements DriverContract { const location = join(this.#rootUrl, key) const generateURL = this.options.urlBuilder?.generateURL if (generateURL) { + debug('generating public URL %s:%s', this.#rootUrl, key) return generateURL(key, location) } @@ -182,6 +191,7 @@ export class FSDriver implements DriverContract { */ const generateSignedURL = this.options.urlBuilder?.generateSignedURL if (generateSignedURL) { + debug('generating signed URL %s:%s', this.#rootUrl, key) return generateSignedURL(key, location, normalizedOptions) } @@ -201,6 +211,7 @@ export class FSDriver implements DriverContract { * - Existing file will be overwritten. */ put(key: string, contents: string | Uint8Array, options?: WriteOptions): Promise { + debug('creating/updating file %s:%s', this.#rootUrl, key) return this.#write(key, contents, { signal: options?.signal }) } @@ -212,6 +223,7 @@ export class FSDriver implements DriverContract { * - Existing file will be overwritten. */ putStream(key: string, contents: Readable, options?: WriteOptions): Promise { + debug('creating/updating file using readable stream %s:%s', this.#rootUrl, key) return new Promise((resolve, reject) => { contents.once('error', (error) => reject(error)) return this.#write(key, contents, { signal: options?.signal }).then(resolve).catch(reject) @@ -223,8 +235,10 @@ export class FSDriver implements DriverContract { * be within the root location. */ copy(source: string, destination: string): Promise { + debug('copying file from %s to %s', source, destination) const sourceLocation = join(this.#rootUrl, source) const destinationLocation = join(this.#rootUrl, destination) + return this.#retrier.retry(async () => { await fsp.mkdir(dirname(destinationLocation), { recursive: true }) await fsp.copyFile(sourceLocation, destinationLocation) @@ -236,8 +250,10 @@ export class FSDriver implements DriverContract { * be within the root location. */ move(source: string, destination: string): Promise { + debug('moving file from %s to %s', source, destination) const sourceLocation = join(this.#rootUrl, source) const destinationLocation = join(this.#rootUrl, destination) + return this.#retrier.retry(async () => { await fsp.mkdir(dirname(destinationLocation), { recursive: true }) await fsp.copyFile(sourceLocation, destinationLocation) @@ -251,7 +267,9 @@ export class FSDriver implements DriverContract { * a noop. */ delete(key: string): Promise { + debug('deleting file %s:%s', this.#rootUrl, key) const location = join(this.#rootUrl, key) + return this.#retrier.retry(async () => { try { await fsp.unlink(location) @@ -269,7 +287,9 @@ export class FSDriver implements DriverContract { * command */ deleteAll(prefix: string): Promise { + debug('deleting all files in folder %s:%s', this.#rootUrl, prefix) const location = join(this.#rootUrl, prefix) + return this.#retrier.retry(async () => { return fsp.rm(location, { recursive: true, force: true }) }) @@ -292,6 +312,7 @@ export class FSDriver implements DriverContract { const self = this const location = join(this.#rootUrl, prefix) const { recursive } = Object.assign({ recursive: false }, options) + debug('listing files from folder %s:%s %O', this.#rootUrl, prefix, options) /** * Reading files with their types. diff --git a/drivers/gcs/driver.ts b/drivers/gcs/driver.ts index c3a8dd2..c3a9abf 100644 --- a/drivers/gcs/driver.ts +++ b/drivers/gcs/driver.ts @@ -42,6 +42,13 @@ export class GCSDriver implements DriverContract { if (options.usingUniformAcl !== undefined) { this.#usingUniformAcl = options.usingUniformAcl } + + if (debug.enabled) { + debug('driver config %O', { + ...options, + credentials: 'REDACTED', + }) + } } /** @@ -85,7 +92,7 @@ export class GCSDriver implements DriverContract { gcsOptions.predefinedAcl = gcsOptions.public ? 'publicRead' : 'private' } - debug('gcs write options %s', gcsOptions) + debug('gcs write options %O', gcsOptions) return gcsOptions } @@ -101,6 +108,7 @@ export class GCSDriver implements DriverContract { lastModified: new Date(apiFile.updated!), } + debug('file metadata %O', this.options.bucket, metaData) return metaData } @@ -118,6 +126,8 @@ export class GCSDriver implements DriverContract { options: GetFilesOptions ): Promise<{ files: FileMetadata[]; prefixes: string[]; paginationToken?: string }> { const bucket = this.#storage.bucket(this.options.bucket) + debug('fetching files list %O', options) + return new Promise((resolve, reject) => { bucket.request( { @@ -126,8 +136,10 @@ export class GCSDriver implements DriverContract { }, (error, response) => { if (error) { + debug('list files API error %O', error) reject(error) } else { + debug('list files API response %O', response) resolve({ files: response.items || [], paginationToken: response.nextPageToken, @@ -144,7 +156,9 @@ export class GCSDriver implements DriverContract { * or not. */ async exist(key: string): Promise { + debug('checking if file exists %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) + const response = await bucket.file(key).exists() return response[0] } @@ -154,7 +168,9 @@ export class GCSDriver implements DriverContract { * exception is thrown when object is missing. */ async get(key: string): Promise { + debug('reading file contents %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) + const response = await bucket.file(key).download() return response[0].toString('utf-8') } @@ -164,7 +180,9 @@ export class GCSDriver implements DriverContract { * exception is thrown when the file is missing. */ async getStream(key: string): Promise { + debug('reading file contents as a stream %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) + return bucket.file(key).createReadStream() } @@ -173,7 +191,9 @@ export class GCSDriver implements DriverContract { * exception is thrown when the file is missing. */ async getArrayBuffer(key: string): Promise { + debug('reading file contents as array buffer %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) + const response = await bucket.file(key).download() return new Uint8Array(response[0]) } @@ -182,9 +202,10 @@ export class GCSDriver implements DriverContract { * Returns the file metadata. */ async getMetaData(key: string): Promise { + debug('fetching file metadata %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) - const response = await bucket.file(key).getMetadata() + const response = await bucket.file(key).getMetadata() return this.#createFileMetaData(response[0]) } @@ -192,7 +213,9 @@ export class GCSDriver implements DriverContract { * Returns the visibility of a file */ async getVisibility(key: string): Promise { + debug('fetching file visibility %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) + const [isFilePublic] = await bucket.file(key).isPublic() return isFilePublic ? 'public' : 'private' } @@ -207,9 +230,11 @@ export class GCSDriver implements DriverContract { */ const generateURL = this.options.urlBuilder?.generateURL if (generateURL) { + debug('using custom implementation for generating public URL %s:%s', this.options.bucket, key) return generateURL(key, this.options.bucket, this.#storage) } + debug('generating public URL %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) const file = bucket.file(key) return file.publicUrl() @@ -242,9 +267,11 @@ export class GCSDriver implements DriverContract { */ const generateSignedURL = this.options.urlBuilder?.generateSignedURL if (generateSignedURL) { + debug('using custom implementation for generating signed URL %s:%s', this.options.bucket, key) return generateSignedURL(key, this.options.bucket, signedURLOptions, this.#storage) } + debug('generating signed URL %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) const file = bucket.file(key) @@ -256,6 +283,7 @@ export class GCSDriver implements DriverContract { * Updates the visibility of a file */ async setVisibility(key: string, visibility: ObjectVisibility): Promise { + debug('updating file visibility %s:%s to %s', this.options.bucket, key, visibility) const bucket = this.#storage.bucket(this.options.bucket) const file = bucket.file(key) @@ -274,6 +302,7 @@ export class GCSDriver implements DriverContract { contents: string | Uint8Array, options?: WriteOptions | undefined ): Promise { + debug('creating/updating file %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) await bucket.file(key).save(Buffer.from(contents), this.#getSaveOptions(options)) } @@ -282,7 +311,9 @@ export class GCSDriver implements DriverContract { * Writes a file to the bucket for the given key and stream */ putStream(key: string, contents: Readable, options?: WriteOptions | undefined): Promise { + debug('creating/updating file using readable stream %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) + return new Promise((resolve, reject) => { /** * GCS internally creates a pipeline of stream and invokes the "_destroy" method @@ -304,6 +335,13 @@ export class GCSDriver implements DriverContract { * be within the root location. */ async copy(source: string, destination: string, options?: WriteOptions): Promise { + debug( + 'copying file from %s:%s to %s:%s', + this.options.bucket, + source, + this.options.bucket, + destination + ) const bucket = this.#storage.bucket(this.options.bucket) options = options || {} @@ -325,6 +363,14 @@ export class GCSDriver implements DriverContract { * be within the root location. */ async move(source: string, destination: string, options?: WriteOptions): Promise { + debug( + 'moving file from %s:%s to %s:%s', + this.options.bucket, + source, + this.options.bucket, + destination + ) + const bucket = this.#storage.bucket(this.options.bucket) options = options || {} @@ -345,6 +391,7 @@ export class GCSDriver implements DriverContract { * Deletes the object from the bucket */ async delete(key: string) { + debug('removing file %s:%s', this.options.bucket, key) const bucket = this.#storage.bucket(this.options.bucket) await bucket.file(key).delete({ ignoreNotFound: true }) } @@ -355,6 +402,7 @@ export class GCSDriver implements DriverContract { */ async deleteAll(prefix: string): Promise { const bucket = this.#storage.bucket(this.options.bucket) + debug('removing all files matching prefix %s:%s', this.options.bucket, prefix) await bucket.deleteFiles({ prefix: `${prefix.replace(/\/$/, '')}/` }) } diff --git a/package.json b/package.json index 05aa8b6..da1bf1a 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "format": "prettier --write .", "quick:test": "node --import=./tsnode.esm.js --enable-source-maps bin/test.ts", "pretest": "npm run lint", - "test": "c8 npm run quick:test", + "test": "cross-env NODE_DEBUG=flydrive:* c8 npm run quick:test", "prebuild": "npm run lint && npm run clean", "build": "tsup-node && tsc --emitDeclarationOnly --declaration", "postbuild": "npm run copy:templates", @@ -60,6 +60,7 @@ "@types/node": "^20.12.7", "c8": "^9.1.0", "copyfiles": "^2.4.1", + "cross-env": "^7.0.3", "del-cli": "^5.1.0", "eslint": "^8.57.0", "get-stream": "^9.0.1", diff --git a/src/types.ts b/src/types.ts index e05e6b8..12f53c6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -48,11 +48,7 @@ export type WriteOptions = { export type SignedURLOptions = { expiresIn?: string | number contentType?: string - // contentLanguage?: string - // contentEncoding?: string contentDisposition?: string - // cacheControl?: string - // contentLength?: number } & { [key: string]: any }