Ericbrehault/sc 9423/folder sync check if folder exists

.github/workflows/build.yml

@@ -16,7 +16,7 @@ jobs:
           node-version: 20
 
       - name: Get package version
-        run: node -p -e '`PACKAGE_VERSION=${require("./package.json").version}`' >> $GITHUB_ENV
+        run: node utils/get-version.js >> $GITHUB_ENV
 
       - name: install dependencies electron-app
         run: npm install
@@ -90,7 +90,7 @@ jobs:
           node-version: 20
 
       - name: Get package version
-        run: node -p -e '`PACKAGE_VERSION=${require("./package.json").version}`' >> $GITHUB_ENV
+        run: node utils/get-version.js >> $GITHUB_ENV
 
       - name: Set version in source
         run: |-
@@ -161,7 +161,7 @@ jobs:
           node-version: 20
 
       - name: Get package version
-        run: node -p -e "`PACKAGE_VERSION=${require('./package.json').version}`" >> $GITHUB_ENV
+        run: node utils/get-version.js >> $GITHUB_ENV
 
       - name: install dependencies electron-app
         run: npm install

package.json

@@ -1,6 +1,6 @@
 {
   "name": "nuclia-sync-agent-app",
-  "version": "1.2.12",
+  "version": "1.2.13",
   "description": "This is a Nuclia Sync Agent App",
   "main": "build/index.js",
   "scripts": {

server/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 1.2.13 (2024-04-04)
+
+- Report errors from the different connectors
+
 # 1.2.7 (2024-03-28)
 
 - Fix file type filtering

server/src/logic/connector/domain/connector.ts

@@ -34,6 +34,7 @@ export type SyncItem = z.infer<typeof SyncItemValidator>;
 export interface SearchResults {
   items: SyncItem[];
   nextPage?: Observable<SearchResults>;
+  error?: string;
 }
 
 export interface Field {
@@ -52,7 +53,6 @@ export interface IConnector {
   areParametersValid(params: ConnectorParameters): boolean;
   getParameters(): ConnectorParameters;
   getFolders(query?: string): Observable<SearchResults>;
-  getFiles(query?: string): Observable<SearchResults>;
   getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults>;
   getLastModified(since: string, folders?: SyncItem[]): Observable<SearchResults>;
   // we cannot use the TextField from the SDK because we want to keep connectors independant

server/src/logic/connector/infrastructure/connectors/confluence.connector.ts

@@ -41,10 +41,6 @@ export class ConfluenceImpl implements IConnector {
     return this._getFiles('', true);
   }
 
-  getFiles(query?: string): Observable<SearchResults> {
-    return this._getFiles(query);
-  }
-
   getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults> {
     if ((folders ?? []).length === 0) {
       return of({
@@ -67,6 +63,7 @@ export class ConfluenceImpl implements IConnector {
       console.error(err);
       return of({
         items: [],
+        error: `Error fetching files: ${err}`,
       });
     }
   }
@@ -85,6 +82,7 @@ export class ConfluenceImpl implements IConnector {
         console.error(err);
         return of({
           items: [],
+          error: `Error fetching last modified files: ${err}`,
         });
       }
     }

server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts

@@ -61,6 +61,7 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
     } catch (err) {
       return of({
         items: [],
+        error: `Error fetching last modified files: ${err}`,
       });
     }
   }
@@ -86,6 +87,7 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
     } catch (err) {
       return of({
         items: [],
+        error: `Error fetching files: ${err}`,
       });
     }
   }
@@ -94,10 +96,6 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
     return this._getFiles(query, true, '');
   }
 
-  getFiles(query?: string): Observable<SearchResults> {
-    return this._getFiles(query);
-  }
-
   isAccessTokenValid(): Observable<boolean> {
     return from(
       fetch('https://api.dropboxapi.com/2/users/get_current_account ', {

server/src/logic/connector/infrastructure/connectors/folder.connector.ts

@@ -1,7 +1,7 @@
 import { Blob as FSBlob } from 'buffer';
 import * as fs from 'fs';
 import path from 'path';
-import { forkJoin, map, Observable, of, switchMap } from 'rxjs';
+import { forkJoin, from, map, Observable, of, switchMap } from 'rxjs';
 import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector';
 import { SourceConnectorDefinition } from '../factory';
 import { lookup } from 'mime-types';
@@ -40,10 +40,6 @@ class FolderImpl implements IConnector {
     throw new Error('Method not supported by Folder connector.');
   }
 
-  getFiles(query?: string): Observable<SearchResults> {
-    return this._getFiles(this.params.path, query);
-  }
-
   getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults> {
     if ((folders ?? []).length === 0) {
       return of({
@@ -80,25 +76,42 @@ class FolderImpl implements IConnector {
       return forkJoin(
         (folders || []).map((folder) =>
           this._getFiles(folder.originalId).pipe(
-            switchMap((results) => this.getFilesModifiedSince(results.items, since)),
+            switchMap((results) =>
+              from(this.getFilesModifiedSince(results.items, since)).pipe(
+                map((items) => ({ items, error: results.error })),
+              ),
+            ),
           ),
         ),
       ).pipe(
         map((results) => {
-          const items = results.reduce((acc, result) => acc.concat(result), [] as SyncItem[]);
+          const items = results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]);
+          const errors = results
+            .map((result) => result.error)
+            .filter((error) => !!error)
+            .join('. ');
           return {
             items,
+            error: errors,
           };
         }),
       );
     } catch (err) {
       return of({
         items: [],
+        error: 'Error getting last modified files.',
       });
     }
   }
 
   private _getFiles(path: string, query?: string): Observable<SearchResults> {
+    if (!fs.existsSync(path)) {
+      console.error(`Folder ${path} does not exist.`);
+      return of({
+        items: [],
+        error: `Folder ${path} does not exist.`,
+      });
+    }
     return of({
       items: this.mapFSFiles(this.listAllFiles(path)).filter((item) =>
         query ? item.title.toLocaleLowerCase().includes(query?.toLocaleLowerCase()) : true,

server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts

@@ -62,6 +62,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
     } catch (err) {
       return of({
         items: [],
+        error: `Error fetching last modified files: ${err}`,
       });
     }
   }
@@ -87,6 +88,7 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
     } catch (err) {
       return of({
         items: [],
+        error: `Error fetching files: ${err}`,
       });
     }
   }
@@ -137,10 +139,6 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
     );
   }
 
-  getFiles(query?: string): Observable<SearchResults> {
-    return this._getFileItems(query);
-  }
-
   isAccessTokenValid(): Observable<boolean> {
     return from(
       fetch('https://www.googleapis.com/drive/v3/about?fields=user', {

server/src/logic/connector/infrastructure/connectors/onedrive.connector.ts

@@ -61,6 +61,7 @@ export class OneDriveImpl extends OAuthBaseConnector implements IConnector {
     } catch (err) {
       return of({
         items: [],
+        error: `Error fetching last modified files: ${err}`,
       });
     }
   }
@@ -86,6 +87,7 @@ export class OneDriveImpl extends OAuthBaseConnector implements IConnector {
     } catch (err) {
       return of({
         items: [],
+        error: `Error fetching files: ${err}`,
       });
     }
   }
@@ -94,10 +96,6 @@ export class OneDriveImpl extends OAuthBaseConnector implements IConnector {
     return this._getItems(query, '', true);
   }
 
-  getFiles(query?: string): Observable<SearchResults> {
-    return this._getItems(query);
-  }
-
   isAccessTokenValid(): Observable<boolean> {
     return from(
       fetch('https://graph.microsoft.com/v1.0/me/drive', {

server/src/logic/connector/infrastructure/connectors/rss.connector.ts

@@ -1,4 +1,4 @@
-import { from, map, Observable, of } from 'rxjs';
+import { catchError, from, map, Observable, of } from 'rxjs';
 import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector';
 import { SourceConnectorDefinition } from '../factory';
 import * as cheerio from 'cheerio';
@@ -64,7 +64,7 @@ class RSSImpl implements IConnector {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  getFiles(query?: string): Observable<SearchResults> {
+  private _getFiles(query?: string): Observable<SearchResults> {
     const url = this.params['url'];
 
     return from(fetchRSS(url)).pipe(
@@ -82,6 +82,7 @@ class RSSImpl implements IConnector {
           },
         })),
       })),
+      catchError((err) => of({ items: [], error: `Error fetching RSS feed: ${err}` })),
     );
   }
 
@@ -100,12 +101,12 @@ class RSSImpl implements IConnector {
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults> {
-    return this.getFiles();
+    return this._getFiles();
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   getLastModified(since: string, folders?: SyncItem[]): Observable<SearchResults> {
-    return this.getFiles().pipe(
+    return this._getFiles().pipe(
       map((searchResults) => ({
         ...searchResults,
         items: searchResults.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since),

server/src/logic/connector/infrastructure/connectors/sharepoint.connector.ts

@@ -58,6 +58,7 @@ export class SharepointImpl extends OAuthBaseConnector implements IConnector {
     } catch (err) {
       return of({
         items: [],
+        error: `Error fetching last modified files: ${err}`,
       });
     }
   }
@@ -67,11 +68,6 @@ export class SharepointImpl extends OAuthBaseConnector implements IConnector {
     return this._getItems('', true);
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  getFiles(query?: string): Observable<SearchResults> {
-    return this._getItems();
-  }
-
   getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults> {
     if ((folders ?? []).length === 0) {
       return of({
@@ -93,6 +89,7 @@ export class SharepointImpl extends OAuthBaseConnector implements IConnector {
     } catch (err) {
       return of({
         items: [],
+        error: `Error fetching files: ${err}`,
       });
     }
   }

server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts

@@ -1,4 +1,4 @@
-import { from, map, Observable, of, switchMap } from 'rxjs';
+import { catchError, from, map, Observable, of, switchMap } from 'rxjs';
 import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector';
 import { SourceConnectorDefinition } from '../factory';
 import * as cheerio from 'cheerio';
@@ -16,7 +16,7 @@ async function fetchSitemap(url: string): Promise<string> {
     return response.text();
   } catch (error) {
     console.error('Error fetching sitemap', error);
-    return Promise.resolve('');
+    return Promise.reject('Error fetching sitemap');
   }
 }
 
@@ -72,7 +72,7 @@ class SitemapImpl implements IConnector {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  getFiles(query?: string | undefined): Observable<SearchResults> {
+  private _getFiles(query?: string | undefined): Observable<SearchResults> {
     const sitemapUrl = this.params['sitemap'];
 
     return this._getSiteMap(sitemapUrl).pipe(
@@ -90,6 +90,7 @@ class SitemapImpl implements IConnector {
           },
         })),
       })),
+      catchError((err) => of({ items: [], error: `${err}` })),
     );
   }
 
@@ -109,12 +110,12 @@ class SitemapImpl implements IConnector {
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults> {
-    return this.getFiles();
+    return this._getFiles();
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   getLastModified(since: string, folders?: SyncItem[]): Observable<SearchResults> {
-    return this.getFiles().pipe(
+    return this._getFiles().pipe(
       map((searchResults) => ({
         ...searchResults,
         items: searchResults.items.filter(

server/src/logic/sync/domain/sync.entity.ts

@@ -118,15 +118,6 @@ export class SyncEntity {
     return this.sourceConnector.getFolders();
   }
 
-  get files(): Observable<SearchResults> {
-    if (!this.sourceConnector) {
-      return of({
-        items: [],
-      });
-    }
-    return this.sourceConnector.getFiles();
-  }
-
   getLastModified(): Observable<{ success: boolean; results: SyncItem[]; error?: string }> {
     const foldersToSyncPending: SyncItem[] = (this.foldersToSync ?? []).filter(
       (folder) => folder.status === FileStatus.PENDING || folder.status === undefined,
@@ -137,15 +128,19 @@ export class SyncEntity {
     const getFilesFoldersUpdated =
       foldersToSyncUpdated.length > 0
         ? this.sourceConnector!.getLastModified(this.lastSyncGMT || '2000-01-01T00:00:00.000Z', foldersToSyncUpdated)
-        : of({ items: [] });
+        : of({ items: [] } as SearchResults);
 
     const getFilesFolderPending =
       foldersToSyncPending.length > 0
         ? this.sourceConnector!.getFilesFromFolders(foldersToSyncPending)
-        : of({ items: [] });
+        : of({ items: [] } as SearchResults);
     return forkJoin([getFilesFoldersUpdated, getFilesFolderPending]).pipe(
       map(([updated, pending]) => {
-        return { success: true, results: [...updated.items, ...pending.items] };
+        return {
+          success: !updated.error && !pending.error,
+          results: [...updated.items, ...pending.items],
+          error: [updated.error, pending.error].filter((err) => err).join('. '),
+        };
       }),
       catchError((err) => {
         console.error(`Error on ${this.id}: ${err.message}`);

server/src/subscribers.ts

@@ -56,8 +56,8 @@ export function initFileSystemSubscribers(basePath: string) {
     const saveLog = new SaveLogs(new LogRepository(new FileSystemLogDatasource(basePath)));
     saveLog.execute(
       new LogEntity({
-        message: 'Synchronization finished',
-        level: LogSeverityLevel.low,
+        message: payload['error'] || 'Synchronization finished',
+        level: payload['error'] ? LogSeverityLevel.high : LogSeverityLevel.low,
         action: EVENTS.FINISH_SYNCHRONIZATION_SYNC_OBJECT,
         payload,
       }),

utils/get-version.js

@@ -0,0 +1,2 @@
+const { version } = require('../package.json');
+console.log(`PACKAGE_VERSION=${version}`);