feat: provide minio provider as alternative

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+* @teknologi-umum/frontend-web @teknologi-umum/backend-javascript
+.github @teknologi-umum/infrastructure

README.md

@@ -149,7 +149,7 @@ Thank you again for wanting to contribute! We look forward to working with you.
 ```
 The MIT License (MIT)
 
-Copyright (c) 2023 Teknologi Umum
+Copyright (c) 2024 Teknologi Umum <[email protected]>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package.json

@@ -48,21 +48,22 @@
  },
  "homepage": "https://github.com/teknologi-umum/blob-js#readme",
  "devDependencies": {
- "@rollup/plugin-typescript": "^11.1.1",
+ "@rollup/plugin-typescript": "^11.1.6",
  "@teknologi-umum/eslint-config-typescript": "^0.0.10",
- "@types/node": "^20.3.2",
- "@vitest/coverage-v8": "^0.33.0",
- "eslint": "^8.43.0",
+ "@types/node": "^20.14.12",
+ "@vitest/coverage-v8": "^2.0.4",
+ "eslint": "^8.57.0",
  "lorem-ipsum": "^2.0.8",
- "rollup": "^3.25.3",
- "tslib": "^2.6.0",
- "typescript": "^5.1.5",
- "vitest": "^0.33.0"
+ "rollup": "^4.19.1",
+ "tslib": "^2.6.3",
+ "typescript": "^5.5.4",
+ "vitest": "^2.0.4"
  },
  "dependencies": {
- "@aws-sdk/client-s3": "^3.374.0",
- "@azure/storage-blob": "^12.14.0",
- "@google-cloud/storage": "^6.11.0"
+ "@aws-sdk/client-s3": "^3.620.0",
+ "@azure/storage-blob": "^12.24.0",
+ "@google-cloud/storage": "^6.11.0",
+ "minio": "^8.0.1"
  },
  "directories": {
  "lib": "./src",

src/s3/minio.ts

@@ -0,0 +1,135 @@
+import { join } from "node:path";
+import { buffer } from "node:stream/consumers";
+
+import * as Minio from "minio";
+import { Readable, Writable } from "stream";
+
+import { ConnectionString } from "../connectionString";
+import type { IObjectStorage, PutOptions, StatResponse } from "../interface";
+import { UnimplementedError } from "../errors";
+
+export class MinioStorage implements IObjectStorage {
+ private readonly client: Minio.Client;
+ private readonly bucketName: string;
+
+ constructor(config: ConnectionString) {
+ const clientOptions: Minio.ClientOptions = {
+ endPoint: "",
+ accessKey: "",
+ secretKey: ""
+ };
+ if (config.username !== undefined && config.username !== "" && config.password !== undefined && config.password !== "") {
+ clientOptions.accessKey = config.username;
+ clientOptions.secretKey = config.password;
+ }
+
+ if (config.parameters !== undefined) {
+ if ("region" in config.parameters && typeof config.parameters.region === "string") {
+ clientOptions.region = config.parameters.region;
+ } else {
+ // See https://github.com/aws/aws-sdk-js-v3/issues/1845#issuecomment-754832210
+ clientOptions.region = "us-east-1";
+ clientOptions.pathStyle = true;
+ }
+
+ if ("forcePathStyle" in config.parameters) {
+ clientOptions.pathStyle = Boolean(config.parameters.forcePathStyle);
+ }
+
+ if ("useAccelerateEndpoint" in config.parameters) {
+ clientOptions.s3AccelerateEndpoint = config.parameters.useAccelerateEndpoint;
+ }
+
+ if ("endpoint" in config.parameters && typeof config.parameters.endpoint === "string") {
+ clientOptions.endPoint = config.parameters.endpoint;
+ }
+
+ if ("useSSL" in config.parameters) {
+ clientOptions.useSSL = Boolean(config.parameters.useSSL);
+ }
+
+ if ("port" in config.parameters) {
+ clientOptions.port = Number.parseInt(config.parameters.port);
+ }
+ }
+
+ this.client = new Minio.Client(clientOptions);
+ this.bucketName = config.bucketName;
+ }
+
+ async put(path: string, content: string | Buffer, options?: PutOptions | undefined): Promise<void> {
+ await this.client.putObject(this.bucketName, path, content, undefined, options?.metadata);
+ }
+
+ putStream(path: string, options?: PutOptions | undefined): Promise<Writable> {
+ throw new UnimplementedError();
+ }
+
+ async get(path: string, encoding?: string | undefined): Promise<Buffer> {
+ const response = await this.client.getObject(this.bucketName, path);
+ return buffer(response);
+ }
+
+ getStream(path: string): Promise<Readable> {
+ return this.client.getObject(this.bucketName, path);
+ }
+
+ async stat(path: string): Promise<StatResponse> {
+ const response = await this.client.statObject(this.bucketName, path);
+ return {
+ size: response.size,
+ lastModified: response.lastModified,
+ createdTime: new Date(0),
+ etag: response.etag,
+ metadata: response.metaData
+ };
+ }
+
+ list(path?: string | undefined): Promise<Iterable<string>> {
+ return new Promise((resolve, reject) => {
+ const listStream = this.client.listObjectsV2(this.bucketName, path, false);
+ const objects: string[] = [];
+ listStream.on("end", () => {
+ resolve(objects);
+ });
+
+ listStream.on("data", (item) => {
+ if (item.name !== undefined) {
+ objects.push(item.name);
+ }
+ });
+
+ listStream.on("error", (error) => {
+ reject(error);
+ });
+ });
+ }
+
+ async exists(path: string): Promise<boolean> {
+ try {
+ await this.client.statObject(this.bucketName, path);
+ return true;
+ } catch (error: unknown) {
+ if (error instanceof Minio.S3Error) {
+ if (error.code === "NoSuchKey") {
+ return false;
+ }
+ }
+
+ throw error;
+ }
+ }
+
+ async delete(path: string): Promise<void> {
+ await this.client.removeObject(this.bucketName, path);
+ }
+
+ async copy(sourcePath: string, destinationPath: string): Promise<void> {
+ await this.client.copyObject(this.bucketName, destinationPath, join(this.bucketName, sourcePath));
+ }
+
+ async move(sourcePath: string, destinationPath: string): Promise<void> {
+ await this.copy(sourcePath, destinationPath);
+ await this.delete(sourcePath);
+ }
+}

src/storage.ts

@@ -3,28 +3,29 @@ import { type IObjectStorage, PutOptions, StatResponse } from "./interface";
 import { parseConnectionString } from "./connectionString";
 import { FileStorage } from "./file/file";
 import { S3Storage } from "./s3/s3";
+import { MinioStorage } from "./s3/minio";
 
 /**
- *
- * The Storage class implements the IObjectStorage interface and provides a way to interact
+ * The Storage class implements the `IObjectStorage` interface and provides a way to interact
  * with an object storage service. The class takes a connection string as input and uses it
  * to initialize the appropriate implementation class for the specified provider.
  */
 export class Storage implements IObjectStorage {
  readonly #implementation: IObjectStorage;
 
  /**
- * The Storage constructor takes a connectionString as input and uses it to initialize
+ * The Storage constructor takes a `connectionString` as input and uses it to initialize
  * the appropriate implementation class for the specified provider.
- * The connectionString is a string that specifies the connection information for
- * the object storage service. The format of the connectionString depends on the provider.
+ * The `connectionString` is a string that specifies the connection information for
+ * the object storage service. The format of the `connectionString` depends on the provider.
  *
  * The following are the supported providers and their respective connection string formats:
  *
  * * AWS (or any other S3-compatible storage): s3://access_key:secret_key@bucket_name?forcePathStyle=true&customUserAgent=your_user_agent&more_config=more_value
  * * Google Cloud: gcs://bucket_name
  * * Azure Blob: azblob://my_container
  * * Filesystem: file:///path/to/directory
+ * 
  * Note: Using any other schemes than s3, gcs, azblob or file will throw a TypeError.
  *
  * For example, the following is a valid connectionString for AWS S3:
@@ -40,7 +41,11 @@ export class Storage implements IObjectStorage {
  this.#implementation = new FileStorage(parsedConnectionString.bucketName);
  break;
  case "s3":
- this.#implementation = new S3Storage(parsedConnectionString);
+ if (parsedConnectionString.parameters?.useMinioSdk === "true") {
+ this.#implementation = new MinioStorage(parsedConnectionString);
+ } else {
+ this.#implementation = new S3Storage(parsedConnectionString);
+ }
  break;
  // case "azblob":
  // this.#implementation = new AzureBlobStorage(parsedConnectionString);

tests/s3/integration.test.ts → tests/s3/aws.integration.test.ts

@@ -1,5 +1,5 @@
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { destroyBucket, removeAllObject, setupBucket } from "./util";
+import { destroyBucket, removeAllObject, setupBucket } from "./aws.util";
 import { S3Client } from "@aws-sdk/client-s3";
 import { loremIpsum } from "lorem-ipsum";
 import { createHash } from "node:crypto";

tests/s3/minio.integration.test.ts

@@ -0,0 +1,71 @@
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { destroyBucket, removeAllObject, setupBucket } from "./minio.util";
+import { loremIpsum } from "lorem-ipsum";
+import { createHash } from "node:crypto";
+import { ConnectionString } from "../../src/connectionString";
+import { MinioStorage } from "../../src/s3/minio";
+import { Client } from "minio";
+
+describe("S3 Provider - Integration", () => {
+ const s3Host = process.env.S3_HOST ?? "http://localhost:9000";
+ const s3Access = process.env.S3_ACCESS ?? "teknologi-umum";
+ const s3Secret = process.env.S3_SECRET ?? "very-strong-password";
+ const bucketName = "blob-js";
+ const s3Client = new Client({
+ endPoint: s3Host,
+ accessKey: s3Access,
+ secretKey: s3Secret,
+ useSSL: false,
+ pathStyle: true,
+ region: "us-east-1"
+ });
+
+ const connectionStringConfig: ConnectionString = {
+ provider: "s3",
+ username: s3Access,
+ password: s3Secret,
+ bucketName: bucketName,
+ parameters: {
+ useMinioSdk: "true",
+ endpoint: s3Host,
+ disableHostPrefix: "true",
+ forcePathStyle: "true"
+ }
+ };
+
+ beforeAll(async () => {
+ // Create S3 bucket
+ await setupBucket(s3Client, bucketName);
+ });
+
+ afterAll(async () => {
+ await removeAllObject(s3Client, bucketName);
+ await destroyBucket(s3Client, bucketName);
+ });
+
+ it("should be able to create, read and delete file", async () => {
+ const content = loremIpsum({count: 1024, units: "sentences"});
+ const hashFunc = createHash("md5");
+ hashFunc.update(content);
+ const checksum = hashFunc.digest("base64");
+
+ const s3Client = new MinioStorage(connectionStringConfig);
+
+ await s3Client.put("lorem-ipsum.txt", content, {contentMD5: checksum});
+
+ expect(s3Client.exists("lorem-ipsum.txt"))
+ .resolves
+ .toStrictEqual(true);
+
+ // GetObjectAttributes is not available on MinIO
+ // const fileStat = await s3Client.stat("lorem-ipsum.txt");
+ // expect(fileStat.size).toStrictEqual(content.length);
+
+ const fileContent = await s3Client.get("lorem-ipsum.txt");
+ expect(fileContent.toString()).toStrictEqual(content);
+
+ expect(s3Client.delete("lorem-ipsum.txt"))
+ .resolves
+ .ok;
+ });
+});

tests/s3/minio.util.ts

@@ -0,0 +1,38 @@
+import {
+ BucketAlreadyExists,
+ BucketAlreadyOwnedByYou,
+ CreateBucketCommand,
+ DeleteBucketCommand,
+ DeleteObjectsCommand,
+ ListObjectsV2Command,
+ S3Client
+} from "@aws-sdk/client-s3";
+import { Client } from "minio";
+
+export async function setupBucket(client: Client, bucketName: string): Promise<void> {
+ await client.makeBucket(bucketName);
+}
+
+export async function removeAllObject(client: Client, bucketName: string): Promise<void> {
+ const listObj = client.listObjectsV2(bucketName);
+
+ return new Promise((resolve, reject) => {
+ listObj.on("data", async (obj) => {
+ if (obj.name !== undefined) {
+ await client.removeObject(bucketName, obj.name);
+ }
+ });
+
+ listObj.on("error", (error) => {
+ reject(error);
+ });
+
+ listObj.on("end", () => {
+ resolve();
+ });
+ });
+}
+
+export async function destroyBucket(client: Client, bucketName: string): Promise<void> {
+ await client.removeBucket(bucketName);
+}

tests/s3/unit.test.ts

@@ -4,7 +4,7 @@ import { createHash } from "node:crypto";
 import { BlobFileNotExistError, BlobMismatchedMD5IntegrityError } from "../../src/errors";
 import { S3Client } from "@aws-sdk/client-s3";
 import { ConnectionString } from "../../src/connectionString";
-import { destroyBucket, removeAllObject, setupBucket } from "./util";
+import { destroyBucket, removeAllObject, setupBucket } from "./aws.util";
 import { S3Storage } from "../../src/s3/s3";
 
 describe("S3 Provider - Unit", () => {