Ericbrehault/sc 8739/store origin path for all sync connectors (#14)

* avoid failure if config file cannot be read * electron-app/package-lock.json use latest sdk * get file path from gdrive items * prefix path with / * get path from dropbox items * OneDrive: get files from subfolders + get path * set path for foldr and sitemap connectors * set path for Confluence items * lint * fix test * set nuclia sdk version explicitly * disable electron build for now (Forge makes an error)
nuclia · Feb 14, 2024 · 4dca82d · 4dca82d
1 parent 3266684
commit 4dca82d
Show file tree

Hide file tree

Showing 14 changed files with 179 additions and 47 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -51,7 +51,7 @@ jobs:
       run: npm run compile
       working-directory: server
 
-    - name: build
-      env:
-        CI: false
-      run: npm run make
+    # - name: build
+    #   env:
+    #     CI: false
+    #   run: npm run make
diff --git a/electron-app/package-lock.json b/electron-app/package-lock.json
diff --git a/electron-app/package.json b/electron-app/package.json
@@ -22,7 +22,7 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@nuclia/core": "1.9.0",
+    "@nuclia/core": "^1.11.6",
     "cheerio": "^1.0.0-rc.12",
     "commander": "^11.1.0",
     "compression": "^1.7.4",

diff --git a/server/package-lock.json b/server/package-lock.json
diff --git a/server/package.json b/server/package.json
@@ -22,7 +22,7 @@
     "start-server": "npm run compile && node build/start-file-system-server.js"
   },
   "dependencies": {
-    "@nuclia/core": "1.9.1",
+    "@nuclia/core": "^1.11.6",
     "cheerio": "^1.0.0-rc.12",
     "commander": "^11.1.0",
     "compression": "^1.7.4",

diff --git a/server/src/logic/connector/infrastructure/connectors/confluence.connector.ts b/server/src/logic/connector/infrastructure/connectors/confluence.connector.ts
@@ -163,7 +163,7 @@ export class ConfluenceImpl implements IConnector {
     return {
       title: (isFolder ? raw.name : raw.title) || '',
       originalId: (isFolder ? raw.key : itemOriginalId) || '',
-      metadata: { type: raw.type || '' },
+      metadata: { type: raw.type || '', path: raw._links?.webui || '' },
       status: FileStatus.PENDING,
       uuid: `${raw.id}` || '',
       isFolder: false,

diff --git a/server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts b/server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts
@@ -189,7 +189,9 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
     return {
       title: raw.name || '',
       originalId: (isFolder ? raw.path_lower : raw.id) || '',
-      metadata: {},
+      metadata: {
+        path: raw.path_lower.split('/').slice(0, -1).join('/'),
+      },
       status: FileStatus.PENDING,
       uuid: (isFolder ? raw.path_lower : raw.id) || '',
       modifiedGMT: raw.client_modified,
@@ -202,7 +204,9 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
     return {
       title: raw.metadata?.metadata?.['name'] || '',
       originalId: raw.metadata?.metadata?.['id'] || '',
-      metadata: {},
+      metadata: {
+        path: raw.path_lower.split('/').slice(0, -1).join('/'),
+      },
       status: FileStatus.PENDING,
       uuid: raw.metadata?.metadata?.['uuid'] || '',
       isFolder: raw.match_type?.['.tag'] === 'folder',

diff --git a/server/src/logic/connector/infrastructure/connectors/folder.connector.ts b/server/src/logic/connector/infrastructure/connectors/folder.connector.ts
@@ -145,7 +145,9 @@ class FolderImpl implements IConnector {
     return files.map((file) => ({
       title: file.split('/').pop() || '',
       originalId: file,
-      metadata: {},
+      metadata: {
+        path: file.split('/').slice(0, -1).join('/'),
+      },
       status: FileStatus.PENDING,
       uid: '',
     }));

diff --git a/server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts b/server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts
@@ -1,4 +1,4 @@
-import { catchError, concatMap, forkJoin, from, map, Observable, of } from 'rxjs';
+import { Observable, catchError, concatMap, forkJoin, from, map, of, switchMap } from 'rxjs';
 
 import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector';
 import { SourceConnectorDefinition } from '../factory';
@@ -48,7 +48,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
       });
     }
     try {
-      return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe(
+      return forkJoin((folders || []).map((folder) => this._getFileItems('', folder.uuid))).pipe(
         map((results) => {
           const items = results.reduce(
             (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)),
@@ -73,7 +73,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
       });
     }
     try {
-      return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe(
+      return forkJoin((folders || []).map((folder) => this._getFileItems('', folder.uuid))).pipe(
         map((results) => {
           const result: { items: SyncItem[] } = {
             items: [],
@@ -96,7 +96,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
   }
 
   getFiles(query?: string): Observable<SearchResults> {
-    return this._getItems(query);
+    return this._getFileItems(query);
   }
 
   isAccessTokenValid(): Observable<boolean> {
@@ -125,6 +125,77 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
     );
   }
 
+  private getSubFolders(folders: SearchResults, folderId: string): string[] {
+    const getChildren = (folderId: string) => {
+      return folders.items.filter((item) => item.parents?.includes(folderId)).map((item) => item.originalId);
+    };
+    const children = getChildren(folderId);
+    return children.reduce((acc, child) => [...acc, ...getChildren(child)], children);
+  }
+
+  private _getFileItems(query = '', folder = ''): Observable<SearchResults> {
+    return this.getFolders().pipe(
+      switchMap((folders) => {
+        if (folder) {
+          const allTargetedFolders = [folder, ...this.getSubFolders(folders, folder)];
+          return forkJoin(allTargetedFolders.map((folder) => this._getItems(query, folder, false))).pipe(
+            map((results) => {
+              const items = results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]);
+              return {
+                files: {
+                  items,
+                },
+                folders,
+              };
+            }),
+          );
+        } else {
+          return this._getItems(query, '', false).pipe(map((results) => ({ files: results, folders })));
+        }
+      }),
+      map(({ files, folders }) => {
+        const getFolder = (folderId: string) => {
+          return folders.items.find((folder) => folder.originalId === folderId);
+        };
+        const parents = folders.items.reduce(
+          (acc, folder) => {
+            if (folder.parents) {
+              acc[folder.originalId] = folder.parents[0];
+            }
+            return acc;
+          },
+          {} as { [key: string]: string },
+        );
+        const getFolderPath = (folderId: string | undefined) => {
+          if (!folderId) {
+            return [];
+          }
+          let path: string[] = [];
+          let currentFolder = getFolder(folderId);
+          while (currentFolder) {
+            path = [currentFolder.title, ...path];
+            if (!parents[currentFolder.originalId]) {
+              break;
+            }
+            currentFolder = getFolder(parents[currentFolder.originalId]);
+          }
+          return path;
+        };
+        const itemsWithPath = files.items.map((item) => ({
+          ...item,
+          metadata: {
+            ...item.metadata,
+            path: getFolderPath(item.parents?.[0]).join('/'),
+          },
+        }));
+        return {
+          ...files,
+          items: itemsWithPath,
+        };
+      }),
+    );
+  }
+
   // Script create the tree https://gist.github.com/tanaikech/97b336f04c739ae0181a606eab3dff42
   private _getItems(
     query = '',
@@ -148,7 +219,6 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
     if (nextPage) {
       path += `&pageToken=${nextPage}`;
     }
-
     return from(
       fetch(path, {
         headers: {

diff --git a/server/src/logic/connector/infrastructure/connectors/onedrive.connector.ts b/server/src/logic/connector/infrastructure/connectors/onedrive.connector.ts
@@ -124,13 +124,28 @@ export class OneDriveImpl extends OAuthBaseConnector implements IConnector {
     );
   }
 
-  private _getItems(
+  private _getItems(query = '', folder = '', foldersOnly = false, previous?: SearchResults): Observable<SearchResults> {
+    return this._getOneDriveItems(query, folder, foldersOnly, undefined, previous).pipe(
+      map((res) => {
+        const items = (res.value || [])
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          .filter((item: any) => foldersOnly || !!item.file)
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          .map((item: any) => (foldersOnly ? this.mapToSyncItemFolder(item) : this.mapToSyncItem(item)));
+        return { items };
+      }),
+    );
+  }
+
+  private _getOneDriveItems(
     query = '',
     folder = '',
     foldersOnly = false,
     nextPage?: string,
-    previous?: SearchResults,
-  ): Observable<SearchResults> {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    previous?: any,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  ): Observable<any> {
     let path = `https://graph.microsoft.com/v1.0/me/drive/${folder ? `items/${folder}` : 'root'}`;
     if (query) {
       path += `/search(q='${query}')`;
@@ -167,16 +182,27 @@ export class OneDriveImpl extends OAuthBaseConnector implements IConnector {
             res['@odata.nextLink'] && res['@odata.nextLink'].includes('&$skiptoken=')
               ? res?.['@odata.nextLink'].split('&$skiptoken=')[1].split('&')[0]
               : undefined;
-          const items = (res.value || [])
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            .filter((item: any) => foldersOnly || !!item.file)
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            .map((item: any) => (foldersOnly ? this.mapToSyncItemFolder(item) : this.mapToSyncItem(item)));
-          const results = {
-            items: [...(previous?.items || []), ...items],
-            nextPage,
-          };
-          return nextPage ? this._getItems(query, folder, foldersOnly, nextPage, results) : of(results);
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          const folders: string[] = (res.value || []).filter((item: any) => !!item.folder).map((item: any) => item.id);
+          const results = { ...res, value: [...(previous?.value || []), ...(res?.value || [])] };
+          const currentFolderResults = nextPage
+            ? this._getOneDriveItems(query, folder, foldersOnly, nextPage, results)
+            : of(results);
+          if (folders.length === 0) {
+            return currentFolderResults;
+          } else {
+            return forkJoin([
+              currentFolderResults,
+              ...folders.map((subfolder) => this._getOneDriveItems(query, subfolder, foldersOnly)),
+            ]).pipe(
+              map((subresults) =>
+                subresults.reduce(
+                  (acc, subresult) => ({ ...acc, value: [...(acc.value || []), ...(subresult.value || [])] }),
+                  {},
+                ),
+              ),
+            );
+          }
         }
       }),
     );
@@ -189,7 +215,11 @@ export class OneDriveImpl extends OAuthBaseConnector implements IConnector {
       title: item.name,
       originalId: item.id,
       modifiedGMT: item.lastModifiedDateTime,
-      metadata: { mimeType: item.file.mimeType, downloadLink: item['@microsoft.graph.downloadUrl'] },
+      metadata: {
+        mimeType: item.file.mimeType,
+        downloadLink: item['@microsoft.graph.downloadUrl'],
+        path: item.parentReference.path,
+      },
       status: FileStatus.PENDING,
     };
   }

diff --git a/server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts b/server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts
@@ -78,6 +78,7 @@ class SitemapImpl implements IConnector {
           originalId: parsedUrl.loc,
           metadata: {
             uri: parsedUrl.loc,
+            path: parsedUrl.loc.replace(/https?:\/\//, ''),
             lastModified: parsedUrl.lastmod,
           },
         })),

diff --git a/server/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js b/server/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js
@@ -148,6 +148,7 @@ describe('Test last modified', () => {
           metadata: {
             needsPdfConversion: 'yes',
             mimeType: 'application/pdf',
+            path: '',
           },
           status: FileStatus.PENDING,
         },