Sync the security groups (#48)

nuclia · Apr 9, 2024 · 7cb07c6 · 7cb07c6
1 parent d921324
commit 7cb07c6
Show file tree

Hide file tree

Showing 13 changed files with 106 additions and 39 deletions.
diff --git a/electron-app/package-lock.json b/electron-app/package-lock.json
diff --git a/electron-app/package.json b/electron-app/package.json
@@ -22,7 +22,7 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@nuclia/core": "^1.14.4",
+    "@nuclia/core": "^1.14.5",
     "cheerio": "^1.0.0-rc.12",
     "commander": "^11.1.0",
     "compression": "^1.7.4",

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "nuclia-sync-agent-app",
-  "version": "1.2.15",
+  "version": "1.2.16",
   "description": "This is a Nuclia Sync Agent App",
   "main": "build/index.js",
   "scripts": {

diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md
@@ -1,3 +1,7 @@
+# 1.2.16 (2024-04-09)
+
+- Sync the security groups
+
 # 1.2.15 (2024-04-05)
 
 - Support XPath selector in sitemap and RSS connectors

diff --git a/server/package-lock.json b/server/package-lock.json
diff --git a/server/package.json b/server/package.json
@@ -25,7 +25,7 @@
     "start-server": "npm run compile && node build/start-file-system-server.js"
   },
   "dependencies": {
-    "@nuclia/core": "^1.14.4",
+    "@nuclia/core": "^1.14.5",
     "cheerio": "^1.0.0-rc.12",
     "commander": "^11.1.0",
     "compression": "^1.7.4",

diff --git a/server/src/logic/connector/domain/connector.ts b/server/src/logic/connector/domain/connector.ts
@@ -62,4 +62,5 @@ export interface IConnector {
   hasAuthData(): boolean;
   refreshAuthentication(): Observable<boolean>;
   isAccessTokenValid(): Observable<boolean>;
+  getGroups?: (resource: SyncItem) => Observable<string[]>;
 }
diff --git a/server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts b/server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts
@@ -132,7 +132,13 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
           Authorization: `Bearer ${this.params.token || ''}`,
         },
       }).then(
-        (res) => res.json(),
+        (res) => {
+          if (res.status === 401) {
+            return { error: { status: 'UNAUTHENTICATED' } };
+          } else {
+            return res.json();
+          }
+        },
         (err) => {
           console.error(`Error fetching about: ${err}`);
           throw new Error(err);
@@ -316,4 +322,23 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
         });
     });
   }
+
+  getGroups(resource: SyncItem): Observable<string[]> {
+    return from(
+      fetch(
+        `https://www.googleapis.com/drive/v3/files/${resource.originalId}?fields=permissions&corpora=allDrives&supportsAllDrives=true&includeItemsFromAllDrives=true`,
+        {
+          headers: { Authorization: `Bearer ${this.params.token}` },
+        },
+      ),
+    ).pipe(
+      switchMap((res) => res.json()),
+      map((res) => {
+        const groups = (res.permissions || [])
+          .filter((perm: { type: string }) => perm.type === 'group')
+          .map((perm: { emailAddress: string }) => perm.emailAddress);
+        return groups;
+      }),
+    );
+  }
 }
diff --git a/server/src/logic/sync/domain/dto/create-sync.dto.ts b/server/src/logic/sync/domain/dto/create-sync.dto.ts
@@ -15,6 +15,7 @@ export class CreateSyncDto {
       id: this.options.id,
       filters: this.options.filters,
       disabled: this.options.disabled,
+      syncSecurityGroups: this.options.syncSecurityGroups,
     };
 
     if (this.options.labels) returnObj.labels = this.options.labels;

diff --git a/server/src/logic/sync/domain/dto/update-sync.dto.ts b/server/src/logic/sync/domain/dto/update-sync.dto.ts
@@ -20,6 +20,7 @@ export class UpdateSyncDto {
     if (this.options.foldersToSync) returnObj.foldersToSync = this.options.foldersToSync;
     if (this.options.filters) returnObj.filters = this.options.filters;
     if (this.options.disabled !== undefined) returnObj.disabled = this.options.disabled;
+    if (this.options.syncSecurityGroups !== undefined) returnObj.syncSecurityGroups = this.options.syncSecurityGroups;
 
     return returnObj;
   }

diff --git a/server/src/logic/sync/domain/nuclia-cloud.ts b/server/src/logic/sync/domain/nuclia-cloud.ts
@@ -47,32 +47,34 @@ export class NucliaCloud {
     const slug = sha256(originalId);
     const text = data.text;
     const buffer = data.buffer;
+    const resourceData: Partial<ICreateResource> = { title: filename };
+    if (data.metadata.labels) {
+      resourceData.usermetadata = { classifications: data.metadata?.labels };
+    }
+    if (data.metadata.path) {
+      let path = data.metadata.path;
+      if (path && !path.startsWith('/')) {
+        path = `/${path}`;
+      }
+      resourceData.origin = { path };
+    }
+    if (data.metadata?.groups) {
+      resourceData.security = { access_groups: data.metadata.groups };
+    }
     if (buffer || text) {
       return this.getKb().pipe(
         switchMap((kb) =>
           kb.getResourceBySlug(slug, [], []).pipe(
             switchMap((resource) => {
               if (data.metadata?.labels) {
-                return resource
-                  .modify({ usermetadata: { classifications: data.metadata.labels } })
-                  .pipe(map(() => resource));
+                return resource.modify(resourceData).pipe(map(() => resource));
               } else {
                 return of(resource);
               }
             }),
             catchError((error) => {
               if (error.status === 404) {
-                const resourceData: ICreateResource = { slug, title: filename };
-                if (data.metadata.labels) {
-                  resourceData.usermetadata = { classifications: data.metadata?.labels };
-                }
-                if (data.metadata.path) {
-                  let path = data.metadata.path;
-                  if (path && !path.startsWith('/')) {
-                    path = `/${path}`;
-                  }
-                  resourceData.origin = { path };
-                }
+                resourceData.slug = slug;
                 return kb.createResource(resourceData, true).pipe(
                   retry(RETRY_CONFIG),
                   map((data) => kb.getResourceFromData({ id: data.uuid })),

diff --git a/server/src/logic/sync/domain/sync.entity.ts b/server/src/logic/sync/domain/sync.entity.ts
@@ -62,6 +62,12 @@ export const FiltersValidator = z.object({
 });
 export type Filters = z.infer<typeof FiltersValidator>;
 
+export enum ContentType {
+  blob = 'blob',
+  link = 'link',
+  text = 'text',
+}
+
 export interface ISyncEntity {
   connector: Connector;
   kb: NucliaOptions;
@@ -72,6 +78,7 @@ export interface ISyncEntity {
   foldersToSync?: SyncItem[];
   filters?: Filters;
   disabled?: boolean;
+  syncSecurityGroups?: boolean;
 }
 
 export class SyncEntity {
@@ -85,9 +92,11 @@ export class SyncEntity {
   public foldersToSync?: SyncItem[] = [];
   public filters?: Filters;
   public disabled?: boolean;
+  public syncSecurityGroups?: boolean;
 
   constructor(options: ISyncEntity) {
-    const { connector, kb, labels, title, id, lastSyncGMT, foldersToSync, filters, disabled } = options;
+    const { connector, kb, labels, title, id, lastSyncGMT, foldersToSync, filters, disabled, syncSecurityGroups } =
+      options;
     this.connector = connector;
     this.kb = kb;
     this.labels = labels;
@@ -97,6 +106,7 @@ export class SyncEntity {
     this.foldersToSync = foldersToSync;
     this.filters = filters;
     this.disabled = disabled;
+    this.syncSecurityGroups = syncSecurityGroups;
     this.setConnectorDefinition();
   }
 

diff --git a/server/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts b/server/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts
@@ -6,7 +6,7 @@ import { EVENTS } from '../../../../events/events';
 import { eventEmitter } from '../../../../server';
 import { SyncItem } from '../../../connector/domain/connector';
 import { NucliaCloud } from '../nuclia-cloud';
-import { SyncEntity, TO_BE_CHECKED } from '../sync.entity';
+import { ContentType, SyncEntity, TO_BE_CHECKED } from '../sync.entity';
 
 require('localstorage-polyfill');
 
@@ -17,14 +17,37 @@ export interface SyncSingleFileUseCase {
 function downloadFileOrLink(
   sync: SyncEntity,
   item: SyncItem,
-): Observable<{ type: 'blob' | 'link' | 'text'; blob?: Blob; link?: any; text?: TextField }> {
+): Observable<{
+  type: ContentType;
+  blob?: Blob;
+  link?: any;
+  text?: TextField;
+  extra?: { groups?: string[] };
+}> {
   const connector = sync.sourceConnector;
-  if (connector?.isExternal) {
-    return connector.getLink(item).pipe(map((link) => ({ type: 'link', link })));
+  if (!connector) {
+    throw new Error('No connector found');
   } else {
-    return connector!
-      .download(item)
-      .pipe(map((res) => (res instanceof Blob ? { type: 'blob', blob: res } : { type: 'text', text: res })));
+    if (connector?.isExternal) {
+      return connector.getLink(item).pipe(map((link) => ({ type: ContentType.link, link })));
+    } else {
+      return connector.download(item).pipe(
+        map((res) =>
+          res instanceof Blob ? { type: ContentType.blob, blob: res } : { type: ContentType.text, text: res },
+        ),
+        switchMap((res) => {
+          if (sync.syncSecurityGroups && connector.getGroups) {
+            return connector.getGroups(item).pipe(
+              map((groups) => {
+                return { ...res, extra: { groups } };
+              }),
+            );
+          } else {
+            return of(res);
+          }
+        }),
+      );
+    }
   }
 }
 
@@ -46,22 +69,22 @@ export class SyncSingleFile implements SyncSingleFileUseCase {
     return downloadFileOrLink(sync, item).pipe(
       switchMap((data) => {
         try {
-          if (data.type === 'blob' && data.blob) {
+          if (data.type === ContentType.blob && data.blob) {
             return from(data.blob.arrayBuffer()).pipe(
               switchMap((arrayBuffer) => {
                 return nucliaConnector.upload(item.originalId, item.title, {
                   buffer: arrayBuffer,
-                  metadata: { ...item.metadata, labels: sync.labels },
+                  metadata: { ...item.metadata, labels: sync.labels, groups: data.extra?.groups },
                   mimeType: item.mimeType,
                 });
               }),
             );
-          } else if (data.type === 'text' && data.text) {
+          } else if (data.type === ContentType.text && data.text) {
             return nucliaConnector.upload(item.originalId, item.title, {
               text: data.text,
-              metadata: { labels: sync.labels },
+              metadata: { labels: sync.labels, groups: data.extra?.groups },
             });
-          } else if (data.type === 'link' && data.link) {
+          } else if (data.type === ContentType.link && data.link) {
             const mimeType =
               item.mimeType !== TO_BE_CHECKED ? of(item.mimeType || 'text/html') : this.checkMimetype(data.link.uri);
             return mimeType.pipe(