Support syncing of the root folder for the different connectors

nuclia · Apr 15, 2024 · ac75d00 · ac75d00
1 parent 91a6419
commit ac75d00
Show file tree

Hide file tree

Showing 7 changed files with 103 additions and 58 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "nuclia-sync-agent-app",
-  "version": "1.2.18",
+  "version": "1.2.19",
   "description": "This is a Nuclia Sync Agent App",
   "main": "build/index.js",
   "scripts": {

diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md
@@ -1,3 +1,7 @@
+# 1.2.19 (2024-04-15)
+
+- Support syncing of the root folder for the different connectors.
+
 # 1.2.18 (2024-04-11)
 
 - Fix: do not send empty headers

diff --git a/server/README.md b/server/README.md
@@ -10,3 +10,16 @@ To install and run the Nuclia Sync Agent server, run the following commands:
 npm install -g @nuclia/sync-agent
 nuclia-sync-agent
 ```
+
+## Note
+
+The Nuclia Sync Agent stores the configuration and the files in the `.nuclia/sync.json` file.
+
+To sync a root folder, we use a specific format:
+
+```json
+{
+  "uuid": "",
+  "originalId": "/"
+}
+```
diff --git a/server/src/logic/connector/infrastructure/connectors/confluence.connector.ts b/server/src/logic/connector/infrastructure/connectors/confluence.connector.ts
@@ -106,7 +106,7 @@ export class ConfluenceImpl implements IConnector {
     if (loadFolders) {
       endpoint += '/rest/api/space?';
     } else {
-      if (folder) {
+      if (folder && folder !== '/') {
         endpoint += `/rest/api/content/search?cql=space="${folder}" and lastModified > "${
           lastModified ? lastModified.slice(0, 16).replace('T', ' ') : '1970-01-01'
         }"`;

diff --git a/server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts b/server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts
@@ -119,6 +119,9 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
     nextPage?: string | number,
     previous?: SearchResults,
   ): Observable<SearchResults> {
+    if (path === '/') {
+      path = '';
+    }
     const success = (url: string) => {
       /* eslint-disable  @typescript-eslint/no-explicit-any */
       return (res: any) => {

diff --git a/server/src/logic/connector/infrastructure/connectors/sharepoint.connector.ts b/server/src/logic/connector/infrastructure/connectors/sharepoint.connector.ts
@@ -1,4 +1,4 @@
-import { catchError, concatMap, forkJoin, from, map, Observable, of } from 'rxjs';
+import { catchError, concatMap, forkJoin, from, map, Observable, of, switchMap } from 'rxjs';
 
 import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector';
 import { SourceConnectorDefinition } from '../factory';
@@ -115,62 +115,83 @@ export class SharepointImpl extends OAuthBaseConnector implements IConnector {
     previous?: SearchResults,
   ): Observable<SearchResults> {
     let path = '';
-    return (siteId ? of(siteId) : this.getSiteId()).pipe(
-      concatMap((_siteId) => {
-        siteId = _siteId;
-        if (foldersOnly) {
-          path = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists`;
-        } else if (folder) {
-          path = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${folder}/items?expand=fields`;
-          if (since) {
-            path += `&$filter=fields/Modified gt '${since}'`;
+    if (!foldersOnly && !folder) {
+      return this.getAllItems(since, siteId);
+    } else {
+      return (siteId ? of(siteId) : this.getSiteId()).pipe(
+        concatMap((_siteId) => {
+          siteId = _siteId;
+          if (foldersOnly) {
+            path = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists`;
+          } else if (folder) {
+            path = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${folder}/items?expand=fields`;
+            if (since) {
+              path += `&$filter=fields/Modified gt '${since}'`;
+            }
+          } else {
+            throw new Error('One-shot import not implemented for Sharepoint.');
           }
-        } else {
-          throw new Error('One-shot import not implemented for Sharepoint.');
-        }
-        if (nextPage) {
-          path += `&$skiptoken=${nextPage}`;
-        }
-        return from(
-          fetch(path, {
-            headers: {
-              Authorization: `Bearer ${this.params.token}`,
-              Prefer: 'HonorNonIndexedQueriesWarningMayFailRandomly',
-            },
-          }).then(
-            (res) => res.json(),
-            (err) => {
-              console.error(`Error fetching ${path}: ${err}`);
-            },
-          ),
-        );
-      }),
-      concatMap((res) => {
-        if (res.error) {
-          console.error(`Error fetching ${path}: ${res.error}`);
-          if (res.error.code === 'InvalidAuthenticationToken') {
-            throw new Error('Unauthorized');
+          if (nextPage) {
+            path += `&$skiptoken=${nextPage}`;
+          }
+          return from(
+            fetch(path, {
+              headers: {
+                Authorization: `Bearer ${this.params.token}`,
+                Prefer: 'HonorNonIndexedQueriesWarningMayFailRandomly',
+              },
+            }).then(
+              (res) => res.json(),
+              (err) => {
+                console.error(`Error fetching ${path}: ${err}`);
+              },
+            ),
+          );
+        }),
+        concatMap((res) => {
+          if (res.error) {
+            console.error(`Error fetching ${path}: ${res.error}`);
+            if (res.error.code === 'InvalidAuthenticationToken') {
+              throw new Error('Unauthorized');
+            } else {
+              throw new Error(res.error.message || 'Unknown error');
+            }
           } else {
-            throw new Error(res.error.message || 'Unknown error');
+            const nextPage =
+              res['@odata.nextLink'] && res['@odata.nextLink'].includes('&$skiptoken=')
+                ? res?.['@odata.nextLink'].split('&$skiptoken=')[1].split('&')[0]
+                : undefined;
+            const items = (res.value || [])
+              // eslint-disable-next-line @typescript-eslint/no-explicit-any
+              .filter((item: any) => foldersOnly || item.fields?.ContentType === 'Document')
+              // eslint-disable-next-line @typescript-eslint/no-explicit-any
+              .map((item: any) =>
+                foldersOnly ? this.mapToSyncItemFolder(item) : this.mapToSyncItem(item, siteId || '', folder),
+              );
+            const results = {
+              items: [...(previous?.items || []), ...items],
+              nextPage,
+            };
+            return nextPage ? this._getItems(folder, foldersOnly, since, siteId, nextPage, results) : of(results);
           }
-        } else {
-          const nextPage =
-            res['@odata.nextLink'] && res['@odata.nextLink'].includes('&$skiptoken=')
-              ? res?.['@odata.nextLink'].split('&$skiptoken=')[1].split('&')[0]
-              : undefined;
-          const items = (res.value || [])
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            .filter((item: any) => foldersOnly || item.fields?.ContentType === 'Document')
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            .map((item: any) =>
-              foldersOnly ? this.mapToSyncItemFolder(item) : this.mapToSyncItem(item, siteId || '', folder),
-            );
-          const results = {
-            items: [...(previous?.items || []), ...items],
-            nextPage,
-          };
-          return nextPage ? this._getItems(folder, foldersOnly, since, siteId, nextPage, results) : of(results);
-        }
+        }),
+      );
+    }
+  }
+
+  private getAllItems(since?: string, siteId?: string): Observable<SearchResults> {
+    return this._getItems('', true, undefined, siteId).pipe(
+      switchMap((res) =>
+        forkJoin((res.items || []).map((folder) => this._getItems(folder.uuid, false, since, siteId))),
+      ),
+      map((results) => {
+        const result: { items: SyncItem[] } = {
+          items: [],
+        };
+        results.forEach((res) => {
+          result.items = [...result.items, ...res.items];
+        });
+        return result;
       }),
     );
   }

diff --git a/server/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts b/server/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts
@@ -36,12 +36,16 @@ export class SyncAllFolders implements SyncAllFoldersUseCase {
       return folder;
     });
 
-    const [, updateSyncDto] = UpdateSyncDto.create({
+    const [message, updateSyncDto] = UpdateSyncDto.create({
       lastSyncGMT: new Date().toISOString(),
       id: syncEntity.id,
       foldersToSync: foldersToSyncCopy,
     });
-    new UpdateSync(this.repository).execute(updateSyncDto!);
+    if (updateSyncDto) {
+      new UpdateSync(this.repository).execute(updateSyncDto);
+    } else {
+      throw new Error(`Error updating sync: ${message}`);
+    }
   };
 
   processSyncEntity(syncEntity: SyncEntity) {