Merge pull request #153 from DataCloud-project/fr-141

- Updated to skeletonlabs v2 (frontend)
- Updated to svelte-kit v2 (frontend)
- Setting up eslint correctly (frontend)
- Solving eslint errors (over 900 errors were fixed, and of these quite a few were silenced) (frontend)
- Artifacts API created and moved to controller (backend)
- Artifacts browser created (frontend)

controller/.eslintignore

@@ -1,3 +1,4 @@
 src/server/schema.ts
 src/argo/schema.d.ts
-src/argo/argo-schema.d.ts
+src/argo/argo-schema.d.ts
+src/server/resolvers.ts

controller/.eslintrc.cjs

@@ -1,3 +1,4 @@
+// TODO: this is legacy eslintrc file: See the typescript-eslint docs for more info: https://typescript-eslint.io/linting/troubleshooting#i-get-errors-telling-me-eslint-was-configured-to-run--however-that-tsconfig-does-not--none-of-those-tsconfigs-include-this-file
 module.exports = {
   env: {
     es2021: true,

controller/src/argo/dry-runs.ts

@@ -130,13 +130,17 @@ export function convertArgoWorkflowNode(node: ArgoNode, argoWorkflow: ArgoWorkfl
 
   if (type === DryRunNodeType.Pod) {
     podName = getPodName(node, argoWorkflow);
+    // node.inputs?.artifacts?.map(({ name, s3 }) => (console.log(`${name} s3`, s3)));
+    // console.log('node', node);
     inputArtifacts = node.inputs?.artifacts?.map(({ name, s3 }) => ({
       name,
       key: s3?.key,
+      bucketName: s3?.bucket,
     }));
     outputArtifacts = node.outputs?.artifacts?.map(({ name, s3 }) => ({
       name,
       key: s3?.key,
+      bucketName: s3?.bucket,
     }));
   }
 

controller/src/minio/minio.ts

@@ -1,5 +1,6 @@
 import { Client as MinioClient } from 'minio';
 import { URL } from 'node:url';
+import type { BucketItem, BucketItemFromList, BucketItemStat } from 'minio';
 
 import {
   minioAccessKey,
@@ -10,6 +11,8 @@ import {
   minioSecretKey,
 } from '../config.js';
 
+export type ArtifactItem = BucketItem & { bucketName: string };
+
 if (!minioAccessKey) {
   throw new Error('MINIO_ACCESS_KEY is not set');
 }
@@ -36,18 +39,139 @@ const minioInternalClient = new MinioClient({
   region: minioRegion,
 });
 
-export async function computePresignedPutUrl(objectName: string): Promise<string> {
-  const expire = 60 * 60 * 24 * 7; // 7 days
-  return await minioPublicClient.presignedPutObject(minioBucketName, objectName, expire);
+// get presigned put url - for uploading object (artifact)
+export async function computePresignedPutUrl(
+  objectName: string, _bucketName?: string): Promise<string> {
+  const bucketName = _bucketName || minioBucketName;
+  const expire = 60 * 10; // 10 minutes before url expires
+  return await minioPublicClient.presignedPutObject(bucketName, objectName, expire);
 }
 
-export async function computePresignedGetUrl(objectName: string): Promise<string> {
-  // One hour
-  const expire = 3600;
-  return await minioPublicClient.presignedGetObject(minioBucketName, objectName, expire);
+// get presigned get url - for downloading object (artifact)
+export async function computePresignedGetUrl(
+  objectName: string, _bucketName?: string): Promise<string> {
+  const bucketName = _bucketName || minioBucketName;
+  const expire = 60 * 10; // 10 minutes before url expires
+  return await minioPublicClient.presignedGetObject(bucketName, objectName, expire);
 }
 
-export async function getObjectSize(objectName: string): Promise<number> {
-  const stat = await minioInternalClient.statObject(minioBucketName, objectName);
+// get the object (artifact) size
+export async function getObjectSize(objectName: string, _bucketName: string): Promise<number> {
+  const bucketName = _bucketName || minioBucketName;
+  const stat = await minioInternalClient.statObject(bucketName, objectName);
   return stat.size;
 }
+
+// get all objects (artifacts) in a bucket -- default bucket is minioBucketName
+/*
+export async function listAllObjects(
+  _bucketName: string): Promise<BucketItem[]> {
+  const bucketName = _bucketName || minioBucketName;
+  // console.log("bucketName: ",bucketName);
+  const stream = minioInternalClient.listObjectsV2(bucketName, '', true);
+  // const stream = minioInternalClient.listObjects(bucketName, '', true);
+  const objects: BucketItem[] = [];
+  return new Promise((resolve, reject) => {
+    stream.on('data', (object) => {
+      objects.push(object);
+    });
+    stream.on('end', () => {
+      resolve(objects);
+    });
+    stream.on('error', (error) => {
+      // eslint-disable-next-line no-console
+      console.error(error);
+      reject(error);
+    });
+  });
+}
+*/
+// get all objects (artifacts) in a bucket -- default bucket is minioBucketName
+export async function listAllObjects(
+  _bucketName?: string): Promise<ArtifactItem[]> {
+  const bucketName = _bucketName || minioBucketName;
+  // console.log("bucketName: ",bucketName);
+  const stream = minioInternalClient.listObjectsV2(bucketName, '', true);
+  // const stream = minioInternalClient.listObjects(bucketName, '', true);
+  const objects: ArtifactItem[] = [];
+  return new Promise((resolve, reject) => {
+    stream.on('data', (object) => {
+      const artifactItem = object as ArtifactItem;
+      artifactItem.bucketName = bucketName;
+      objects.push(artifactItem);
+    });
+    stream.on('end', () => {
+      resolve(objects);
+    });
+    stream.on('error', (error) => {
+      // eslint-disable-next-line no-console
+      console.error(error);
+      reject(error);
+    });
+  });
+}
+
+// get metadata about single object (artifact)
+export async function getObjectMetadata(
+  objectName: string, _bucketName?: string): Promise<BucketItemStat> {
+  const bucketName = _bucketName || minioBucketName;
+  const metadata = await minioInternalClient.statObject(bucketName, objectName);
+  return metadata;
+}
+
+// get buckets
+export async function listAllBuckets(): Promise<BucketItemFromList[]> {
+  return await minioInternalClient.listBuckets();
+}
+
+// check if a bucket exists
+/* -- not currently used
+export async function bucketExists(bucketName: string): Promise<boolean> {
+  return await minioInternalClient.bucketExists(bucketName);
+}
+*/
+
+// create bucket
+export async function createBucket(bucketName: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    minioInternalClient.makeBucket(bucketName, minioRegion, (error) => {
+      if (error) {
+        // console.log('Error creating bucket.', error);
+        reject(error);
+      } else {
+        // console.log(`Bucket "${bucketName}" created successfully in "${minioRegion}".`);
+        resolve(bucketName);
+      }
+    });
+  });
+}
+
+// delete bucket
+export async function deleteBucket(bucketName: string): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    minioInternalClient.removeBucket(bucketName, (error) => {
+      if (error) {
+        // console.log('Error deleting bucket.', error);
+        reject(error);
+      } else {
+        // console.log(`Bucket "${bucketName}" deleted successfully.`);
+        resolve(true);
+      }
+    });
+  });
+}
+
+// delete objects (artifacts) in a bucket
+export async function deleteObjects(objects: string[], bucketName: string): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    minioInternalClient.removeObjects(bucketName, objects, (error) => {
+      if (error) {
+        // console.log('Error deleting objects.', error);
+        reject(error);
+      } else {
+        // console.log(`Objects "${objects}" deleted successfully.`);
+        resolve(true);
+      }
+    });
+  });
+}