manage deletions in other connectors but sharepoint

nuclia · Apr 25, 2024 · 9db94f8 · 9db94f8
1 parent ef4ee51
commit 9db94f8
Show file tree

Hide file tree

Showing 9 changed files with 79 additions and 34 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "nuclia-sync-agent-app",
-  "version": "1.2.21",
+  "version": "1.3.0",
   "description": "This is a Nuclia Sync Agent App",
   "main": "build/index.js",
   "scripts": {

diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md
@@ -1,3 +1,7 @@
+# 1.3.0 (2024-04-25)
+
+- Support deletion syncing
+
 # 1.2.21 (2024-04-18)
 
 - Do not override title on existing resource

diff --git a/server/src/logic/connector/infrastructure/connectors/confluence.connector.ts b/server/src/logic/connector/infrastructure/connectors/confluence.connector.ts
@@ -60,15 +60,32 @@ export class ConfluenceImpl implements IConnector {
     );
   }
 
-  getLastModified(since: string, folders?: SyncItem[]): Observable<SearchResults> {
+  getLastModified(since: string, folders?: SyncItem[], existings?: string[]): Observable<SearchResults> {
     if (!folders || folders.length === 0) {
       return of({
         items: [],
       });
     } else {
-      return forkJoin((folders || []).map((folder) => this._getFiles('', false, folder.originalId, since))).pipe(
+      const newFiles = forkJoin(
+        (folders || []).map((folder) => this._getFiles('', false, folder.originalId, since)),
+      ).pipe(
         map((results) => ({ items: results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]) })),
       );
+      const deletedFiles = forkJoin(
+        (folders || []).map((folder) => this._getFiles('', false, folder.originalId, since, true)),
+      ).pipe(
+        map((results) => ({ items: results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]) })),
+      );
+      return forkJoin([newFiles, deletedFiles]).pipe(
+        map(([newFiles, deletedFiles]) => {
+          const deleted = deletedFiles.items
+            .filter((item) => existings?.includes(item.originalId))
+            .map((item) => ({ ...item, deleted: true }));
+          return {
+            items: [...newFiles.items, ...deleted],
+          };
+        }),
+      );
     }
   }
 
@@ -77,6 +94,7 @@ export class ConfluenceImpl implements IConnector {
     loadFolders = false,
     folder = '',
     lastModified?: string,
+    deleted = false,
     start?: number,
     previous?: SearchResults,
   ): Observable<SearchResults> {
@@ -115,6 +133,9 @@ export class ConfluenceImpl implements IConnector {
       } else {
         endpoint += '/rest/api/content?';
       }
+      if (deleted) {
+        endpoint += '&status=trashed';
+      }
     }
     return from(
       fetch(`${endpoint}&limit=${BATCH_SIZE}&start=${start || 0}`, {
@@ -132,7 +153,7 @@ export class ConfluenceImpl implements IConnector {
         const items = [...(previous?.items || []), ...newItems];
         const next = (start || 0) + BATCH_SIZE;
         return result._links.next
-          ? this._getFiles(query, loadFolders, folder, lastModified, next, { items })
+          ? this._getFiles(query, loadFolders, folder, lastModified, deleted, next, { items })
           : of({ items });
       }),
     );
@@ -141,7 +162,8 @@ export class ConfluenceImpl implements IConnector {
   /* eslint-disable  @typescript-eslint/no-explicit-any */
   private mapResults(raw: any, isFolder = false): SyncItem {
     const isAttachment = raw.type === 'attachment';
-    const itemOriginalId = isAttachment ? raw._links?.webui?.split('pageId=')[1]?.split('&')[0] || '' : raw.id;
+    const pageId = raw._links?.webui?.split('pageId=')[1]?.split('&')[0] || '';
+    const itemOriginalId = isAttachment ? `${pageId}|${raw.id}` || '' : raw.id;
     return {
       title: (isFolder ? raw.name : raw.title) || '',
       originalId: (isFolder ? raw.key : itemOriginalId) || '',
@@ -157,8 +179,9 @@ export class ConfluenceImpl implements IConnector {
   ): Observable<{ body: string; format?: 'PLAIN' | 'MARKDOWN' | 'HTML' } | Blob | undefined> {
     try {
       if (resource.metadata.type === 'attachment') {
+        const id = resource.originalId.split('|')[0];
         return from(
-          fetch(`${this.params.url}/download/attachments/${resource.originalId}/${resource.title}`, {
+          fetch(`${this.params.url}/download/attachments/${id}/${resource.title}`, {
             method: 'GET',
             headers: {
               Authorization: `Basic ${btoa(this.params.user + ':' + this.params.token)}`,

diff --git a/server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts b/server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts
@@ -40,20 +40,22 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
     return true;
   }
 
-  getLastModified(since: string, folders?: SyncItem[] | undefined): Observable<SearchResults> {
+  getLastModified(since: string, folders?: SyncItem[] | undefined, existings?: string[]): Observable<SearchResults> {
     if ((folders ?? []).length === 0) {
       return of({
         items: [],
       });
     }
     return forkJoin((folders || []).map((folder) => this._getFiles('', false, folder.originalId))).pipe(
       map((results) => {
-        const items = results.reduce(
-          (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)),
-          [] as SyncItem[],
-        );
+        const items = results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]);
+        const currentIds = items.map((item) => item.originalId);
+        const newItems = items.filter((item) => item.modifiedGMT && item.modifiedGMT > since);
+        const toDelete = (existings || [])
+          .filter((id) => !currentIds?.includes(id))
+          .map((id) => ({ uuid: id, originalId: id, title: '', metadata: {}, deleted: true }));
         return {
-          items,
+          items: [...newItems, ...toDelete],
         };
       }),
     );

diff --git a/server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts b/server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts
@@ -41,20 +41,22 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
     return true;
   }
 
-  getLastModified(since: string, folders?: SyncItem[] | undefined): Observable<SearchResults> {
+  getLastModified(since: string, folders?: SyncItem[] | undefined, existings?: string[]): Observable<SearchResults> {
     if ((folders ?? []).length === 0) {
       return of({
         items: [],
       });
     }
     return forkJoin((folders || []).map((folder) => this._getFileItems('', folder.uuid))).pipe(
       map((results) => {
-        const items = results.reduce(
-          (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)),
-          [] as SyncItem[],
-        );
+        const items = results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]);
+        const currentIds = items.map((item) => item.originalId);
+        const newItems = items.filter((item) => item.modifiedGMT && item.modifiedGMT > since);
+        const toDelete = (existings || [])
+          .filter((id) => !currentIds?.includes(id))
+          .map((id) => ({ uuid: id, originalId: id, title: '', metadata: {}, deleted: true }));
         return {
-          items,
+          items: [...newItems, ...toDelete],
         };
       }),
     );
@@ -237,7 +239,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
     previous?: SearchResults,
   ): Observable<SearchResults> {
     let path =
-      'https://www.googleapis.com/drive/v3/files?pageSize=50&fields=nextPageToken,files(id,name,mimeType,modifiedTime,parents)';
+      'https://www.googleapis.com/drive/v3/files?pageSize=50&fields=nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,trashed,parents)';
     const allDrives = '&corpora=allDrives&supportsAllDrives=true&includeItemsFromAllDrives=true';
     path += allDrives;
     if (query) {
@@ -272,7 +274,9 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
           }
         } else {
           const nextPage = res['nextPageToken'];
-          const items = (res.files || []).map((item: unknown) => this.mapToSyncItem(item));
+          const items = (res.files || [])
+            .filter((item: any) => !item.trashed)
+            .map((item: unknown) => this.mapToSyncItem(item));
           const results = {
             items: [...(previous?.items || []), ...items],
             nextPage,
@@ -290,7 +294,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
       uuid: item.id,
       title: item.name,
       originalId: item.id,
-      modifiedGMT: item.modifiedTime,
+      modifiedGMT: item.modifiedTime > item.createdTime ? item.modifiedTime : item.createdTime,
       parents: item.parents,
       mimeType: needsPdfConversion ? 'application/pdf' : item.mimeType,
       metadata: {

diff --git a/server/src/logic/connector/infrastructure/connectors/onedrive.connector.ts b/server/src/logic/connector/infrastructure/connectors/onedrive.connector.ts
@@ -40,20 +40,22 @@ export class OneDriveImpl extends OAuthBaseConnector implements IConnector {
     return true;
   }
 
-  getLastModified(since: string, folders?: SyncItem[] | undefined): Observable<SearchResults> {
+  getLastModified(since: string, folders?: SyncItem[] | undefined, existings?: string[]): Observable<SearchResults> {
     if ((folders ?? []).length === 0) {
       return of({
         items: [],
       });
     }
     return forkJoin((folders || []).map((folder) => this._getItems('', folder.uuid))).pipe(
       map((results) => {
-        const items = results.reduce(
-          (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)),
-          [] as SyncItem[],
-        );
+        const items = results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]);
+        const currentIds = items.map((item) => item.originalId);
+        const newItems = items.filter((item) => item.modifiedGMT && item.modifiedGMT > since);
+        const toDelete = (existings || [])
+          .filter((id) => !currentIds?.includes(id))
+          .map((id) => ({ uuid: id, originalId: id, title: '', metadata: {}, deleted: true }));
         return {
-          items,
+          items: [...newItems, ...toDelete],
         };
       }),
     );

diff --git a/server/src/logic/connector/infrastructure/connectors/rss.connector.ts b/server/src/logic/connector/infrastructure/connectors/rss.connector.ts
@@ -106,7 +106,8 @@ class RSSImpl implements IConnector {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  getLastModified(since: string, folders?: SyncItem[]): Observable<SearchResults> {
+  getLastModified(since: string, folders?: SyncItem[], existings?: string[]): Observable<SearchResults> {
+    // we do not manage deletions in this connectors as RSS will only push recent items
     return this._getFiles().pipe(
       map((searchResults) => ({
         ...searchResults,

diff --git a/server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts b/server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts
@@ -115,14 +115,22 @@ class SitemapImpl implements IConnector {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  getLastModified(since: string, folders?: SyncItem[]): Observable<SearchResults> {
+  getLastModified(since: string, folders?: SyncItem[], existings?: string[]): Observable<SearchResults> {
     return this._getFiles().pipe(
-      map((searchResults) => ({
-        ...searchResults,
-        items: searchResults.items.filter(
+      map((searchResults) => {
+        const currentIds = searchResults.items.map((item) => item.originalId);
+        const toSync = searchResults.items.filter(
           (item) => !item.metadata['lastModified'] || item.metadata['lastModified'] > since,
-        ),
-      })),
+        );
+        const toDelete = existings?.filter((id) => !currentIds.includes(id)) ?? [];
+        return {
+          ...searchResults,
+          items: [
+            ...toSync,
+            ...toDelete.map((id) => ({ uuid: id, originalId: id, title: '', metadata: {}, deleted: true })),
+          ],
+        };
+      }),
     );
   }
 

diff --git a/server/src/logic/sync/presentation/routes.ts b/server/src/logic/sync/presentation/routes.ts
@@ -55,6 +55,7 @@ export class SyncFileSystemRoutes {
             connector: sync.connector.name,
             lastSyncGMT: sync.lastSyncGMT,
             disabled: sync.disabled,
+            totalSyncedResources: sync.originalIds?.length || 0,
           }));
         res.status(200).send(kbSyncs);
       } catch (error) {