Ericbrehault/sc 10019/sync agent bugs (#62)

* Do not crash if oauth refresh is failing * fix filter on extension in dropbox
nuclia · May 28, 2024 · e3724ea · e3724ea
1 parent eb719bb
commit e3724ea
Show file tree

Hide file tree

Showing 5 changed files with 91 additions and 44 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "nuclia-sync-agent-app",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "description": "This is a Nuclia Sync Agent App",
   "main": "build/index.js",
   "scripts": {

diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md
@@ -1,3 +1,7 @@
+# 1.5.1 (2024-05-28)
+
+- Fix: do not crash if oauth refresh is failing
+
 # 1.5.0 (2024-05-07)
 
 - Supports local extraction of web pages

diff --git a/server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts b/server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts
@@ -180,6 +180,7 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
       originalId: (isFolder ? raw.path_lower : raw.id) || '',
       metadata: {
         path: raw.path_lower,
+        extension: raw.path_lower.split('.').pop() || '',
       },
       status: FileStatus.PENDING,
       uuid: (isFolder ? raw.path_lower : raw.id) || '',
@@ -194,6 +195,7 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
       originalId: raw.metadata?.metadata?.['id'] || '',
       metadata: {
         path: raw.path_lower.split('/').slice(0, -1).join('/'),
+        extension: raw.path_lower.split('.').pop() || '',
       },
       status: FileStatus.PENDING,
       uuid: raw.metadata?.metadata?.['uuid'] || '',

diff --git a/server/src/logic/connector/infrastructure/connectors/oauth.base.ts b/server/src/logic/connector/infrastructure/connectors/oauth.base.ts
@@ -1,27 +1,33 @@
-import { Observable, from, map } from 'rxjs';
+import { Observable, from, map, of } from 'rxjs';
 import { ConnectorParameters } from '../../domain/connector';
 
 export class OAuthBaseConnector {
   params: ConnectorParameters = {};
   refreshAuthentication(): Observable<boolean> {
-    return from(
-      fetch(`${this.params.refresh_endpoint}?refresh_token=${this.params.refresh}`, {
-        method: 'GET',
-        headers: {
-          origin: 'http://localhost:4200/',
-        },
-      }).then((res) => res.json()),
-    ).pipe(
-      map((res) => {
-        if (res.token) {
-          this.params.token = res.token;
-          return true;
-        } else {
-          this.params.token = '';
-          this.params.refresh = '';
-          return false;
-        }
-      }),
-    );
+    if (!this.params.refresh) {
+      this.params.token = '';
+      this.params.refresh = '';
+      return of(false);
+    } else {
+      return from(
+        fetch(`${this.params.refresh_endpoint}?refresh_token=${this.params.refresh}`, {
+          method: 'GET',
+          headers: {
+            origin: 'http://localhost:4200/',
+          },
+        }).then((res) => res.json()),
+      ).pipe(
+        map((res) => {
+          if (res.token) {
+            this.params.token = res.token;
+            return true;
+          } else {
+            this.params.token = '';
+            this.params.refresh = '';
+            return false;
+          }
+        }),
+      );
+    }
   }
 }
diff --git a/server/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts b/server/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts
@@ -1,5 +1,17 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { Observable, concatMap, delay, filter, from, lastValueFrom, map, of, switchMap, toArray } from 'rxjs';
+import {
+  Observable,
+  catchError,
+  concatMap,
+  delay,
+  filter,
+  from,
+  lastValueFrom,
+  map,
+  of,
+  switchMap,
+  toArray,
+} from 'rxjs';
 
 import { EVENTS } from '../../../../events/events';
 import { eventEmitter } from '../../../../server';
@@ -79,15 +91,22 @@ export class SyncAllFolders implements SyncAllFoldersUseCase {
   }
 
   processItems(syncEntity: SyncEntity, items: SyncItem[]) {
-    const filteredMimetypes = (syncEntity.filters?.fileExtensions?.extensions || '')
-      .split(',')
-      .map((ext) => lookup(ext.trim()) || '')
-      .filter((ext) => ext);
+    const filteredExtensions = (syncEntity.filters?.fileExtensions?.extensions || '').split(',').filter((ext) => ext);
+    const filteredMimetypes = filteredExtensions.map((ext) => lookup(ext.trim()) || '').filter((ext) => ext);
     return of(...items).pipe(
       filter((item) => {
         let isExtensionOk = true;
         if (filteredMimetypes.length > 0) {
-          const isFiltered = filteredMimetypes.includes(item.mimeType || '');
+          let isFiltered = false;
+          if (item.mimeType) {
+            isFiltered = filteredMimetypes.includes(item.mimeType || '');
+          } else if (item.metadata['extension']) {
+            let extension = item.metadata['extension'];
+            if (!extension.startsWith('.')) {
+              extension = `.${extension}`;
+            }
+            isFiltered = filteredExtensions.includes(extension);
+          }
           isExtensionOk = syncEntity.filters?.fileExtensions?.exclude ? !isFiltered : isFiltered;
         }
         let isDateOk = true;
@@ -128,24 +147,40 @@ export class SyncAllFolders implements SyncAllFoldersUseCase {
   private _execute(syncObj: ISyncEntity): Observable<void> {
     console.log(`Syncing ${syncObj.id}`);
     return new RefreshAccessToken(this.repository).execute(new SyncEntity(syncObj)).pipe(
-      concatMap((syncEntity) =>
-        this.processSyncEntity(syncEntity).pipe(
-          concatMap((result) => {
-            if (result) {
-              const processed = result.filter((res) => res.success && res.action === 'upload').map((res) => res.id);
-              const deleted = result.filter((res) => res.success && res.action === 'delete').map((res) => res.id);
-              const successCount = result.filter((res) => res.success).length;
+      catchError(() => {
+        console.error(`Cannot refresh token for ${syncObj.id}`);
+        return of(undefined);
+      }),
+      concatMap((syncEntity) => {
+        if (!syncEntity) {
+          eventEmitter.emit(EVENTS.FINISH_SYNCHRONIZATION_SYNC_OBJECT, {
+            from: syncObj.id,
+            to: 'Unknown kb',
+            date: new Date().toISOString(),
+            processed: [],
+            successCount: 0,
+            error: 'Cannot refresh OAuth token',
+          });
+          return of(undefined);
+        } else {
+          return this.processSyncEntity(syncEntity).pipe(
+            concatMap((result) => {
+              if (result) {
+                const processed = result.filter((res) => res.success && res.action === 'upload').map((res) => res.id);
+                const deleted = result.filter((res) => res.success && res.action === 'delete').map((res) => res.id);
+                const successCount = result.filter((res) => res.success).length;
 
-              console.log('processed', processed);
-              console.log('deleted', deleted);
-              console.log('successCount', successCount);
-              return this.callbackFinishSync(syncEntity, processed, deleted, successCount, '');
-            } else {
-              return of(undefined);
-            }
-          }),
-        ),
-      ),
+                console.log('processed', processed);
+                console.log('deleted', deleted);
+                console.log('successCount', successCount);
+                return this.callbackFinishSync(syncEntity, processed, deleted, successCount, '');
+              } else {
+                return of(undefined);
+              }
+            }),
+          );
+        }
+      }),
     );
   }
 }