Ericbrehault/sc 9585/support xpath selector when uploading links (#47)

* support xpath selector * clean up useless try/catch
nuclia · Apr 5, 2024 · d921324 · d921324
1 parent 9685d87
commit d921324
Show file tree

Hide file tree

Showing 16 changed files with 151 additions and 227 deletions.
diff --git a/electron-app/package-lock.json b/electron-app/package-lock.json
diff --git a/electron-app/package.json b/electron-app/package.json
@@ -22,7 +22,7 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@nuclia/core": "^1.14.2",
+    "@nuclia/core": "^1.14.4",
     "cheerio": "^1.0.0-rc.12",
     "commander": "^11.1.0",
     "compression": "^1.7.4",

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "nuclia-sync-agent-app",
-  "version": "1.2.14",
+  "version": "1.2.15",
   "description": "This is a Nuclia Sync Agent App",
   "main": "build/index.js",
   "scripts": {

diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md
@@ -1,3 +1,7 @@
+# 1.2.15 (2024-04-05)
+
+- Support XPath selector in sitemap and RSS connectors
+
 # 1.2.13 (2024-04-04)
 
 - Report errors from the different connectors

diff --git a/server/package-lock.json b/server/package-lock.json
diff --git a/server/package.json b/server/package.json
@@ -25,7 +25,7 @@
     "start-server": "npm run compile && node build/start-file-system-server.js"
   },
   "dependencies": {
-    "@nuclia/core": "^1.14.2",
+    "@nuclia/core": "^1.14.4",
     "cheerio": "^1.0.0-rc.12",
     "commander": "^11.1.0",
     "compression": "^1.7.4",

diff --git a/server/src/logic/connector/domain/connector.ts b/server/src/logic/connector/domain/connector.ts
@@ -12,6 +12,7 @@ export interface Link {
   uri: string;
   extra_headers: { [key: string]: string };
   cssSelector?: string;
+  xpathSelector?: string;
 }
 
 export interface ConnectorParameters {

diff --git a/server/src/logic/connector/infrastructure/connectors/confluence.connector.ts b/server/src/logic/connector/infrastructure/connectors/confluence.connector.ts
@@ -47,25 +47,17 @@ export class ConfluenceImpl implements IConnector {
         items: [],
       });
     }
-    try {
-      return forkJoin((folders || []).map((folder) => this._getFiles('', false, folder.originalId))).pipe(
-        map((results) => {
-          const result: { items: SyncItem[] } = {
-            items: [],
-          };
-          results.forEach((res) => {
-            result.items = [...result.items, ...res.items];
-          });
-          return result;
-        }),
-      );
-    } catch (err) {
-      console.error(err);
-      return of({
-        items: [],
-        error: `Error fetching files: ${err}`,
-      });
-    }
+    return forkJoin((folders || []).map((folder) => this._getFiles('', false, folder.originalId))).pipe(
+      map((results) => {
+        const result: { items: SyncItem[] } = {
+          items: [],
+        };
+        results.forEach((res) => {
+          result.items = [...result.items, ...res.items];
+        });
+        return result;
+      }),
+    );
   }
 
   getLastModified(since: string, folders?: SyncItem[]): Observable<SearchResults> {
@@ -74,17 +66,9 @@ export class ConfluenceImpl implements IConnector {
         items: [],
       });
     } else {
-      try {
-        return forkJoin((folders || []).map((folder) => this._getFiles('', false, folder.originalId, since))).pipe(
-          map((results) => ({ items: results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]) })),
-        );
-      } catch (err) {
-        console.error(err);
-        return of({
-          items: [],
-          error: `Error fetching last modified files: ${err}`,
-        });
-      }
+      return forkJoin((folders || []).map((folder) => this._getFiles('', false, folder.originalId, since))).pipe(
+        map((results) => ({ items: results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]) })),
+      );
     }
   }
 

diff --git a/server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts b/server/src/logic/connector/infrastructure/connectors/dropbox.connector.ts
@@ -46,24 +46,17 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
         items: [],
       });
     }
-    try {
-      return forkJoin((folders || []).map((folder) => this._getFiles('', false, folder.originalId))).pipe(
-        map((results) => {
-          const items = results.reduce(
-            (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)),
-            [] as SyncItem[],
-          );
-          return {
-            items,
-          };
-        }),
-      );
-    } catch (err) {
-      return of({
-        items: [],
-        error: `Error fetching last modified files: ${err}`,
-      });
-    }
+    return forkJoin((folders || []).map((folder) => this._getFiles('', false, folder.originalId))).pipe(
+      map((results) => {
+        const items = results.reduce(
+          (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)),
+          [] as SyncItem[],
+        );
+        return {
+          items,
+        };
+      }),
+    );
   }
 
   getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults> {
@@ -72,24 +65,17 @@ export class DropboxImpl extends OAuthBaseConnector implements IConnector {
         items: [],
       });
     }
-    try {
-      return forkJoin((folders || []).map((folder) => this._getFiles('', false, folder.originalId))).pipe(
-        map((results) => {
-          const result: { items: SyncItem[] } = {
-            items: [],
-          };
-          results.forEach((res) => {
-            result.items = [...result.items, ...res.items];
-          });
-          return result;
-        }),
-      );
-    } catch (err) {
-      return of({
-        items: [],
-        error: `Error fetching files: ${err}`,
-      });
-    }
+    return forkJoin((folders || []).map((folder) => this._getFiles('', false, folder.originalId))).pipe(
+      map((results) => {
+        const result: { items: SyncItem[] } = {
+          items: [],
+        };
+        results.forEach((res) => {
+          result.items = [...result.items, ...res.items];
+        });
+        return result;
+      }),
+    );
   }
 
   getFolders(query?: string | undefined): Observable<SearchResults> {

diff --git a/server/src/logic/connector/infrastructure/connectors/folder.connector.ts b/server/src/logic/connector/infrastructure/connectors/folder.connector.ts
@@ -46,23 +46,17 @@ class FolderImpl implements IConnector {
         items: [],
       });
     }
-    try {
-      return forkJoin((folders || []).map((folder) => this._getFiles(folder.originalId))).pipe(
-        map((results) => {
-          const result: { items: SyncItem[] } = {
-            items: [],
-          };
-          results.forEach((res) => {
-            result.items = [...result.items, ...res.items];
-          });
-          return result;
-        }),
-      );
-    } catch (err) {
-      return of({
-        items: [],
-      });
-    }
+    return forkJoin((folders || []).map((folder) => this._getFiles(folder.originalId))).pipe(
+      map((results) => {
+        const result: { items: SyncItem[] } = {
+          items: [],
+        };
+        results.forEach((res) => {
+          result.items = [...result.items, ...res.items];
+        });
+        return result;
+      }),
+    );
   }
 
   getLastModified(since: string, folders?: SyncItem[]): Observable<SearchResults> {
@@ -72,36 +66,29 @@ class FolderImpl implements IConnector {
       });
     }
 
-    try {
-      return forkJoin(
-        (folders || []).map((folder) =>
-          this._getFiles(folder.originalId).pipe(
-            switchMap((results) =>
-              from(this.getFilesModifiedSince(results.items, since)).pipe(
-                map((items) => ({ items, error: results.error })),
-              ),
+    return forkJoin(
+      (folders || []).map((folder) =>
+        this._getFiles(folder.originalId).pipe(
+          switchMap((results) =>
+            from(this.getFilesModifiedSince(results.items, since)).pipe(
+              map((items) => ({ items, error: results.error })),
             ),
           ),
         ),
-      ).pipe(
-        map((results) => {
-          const items = results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]);
-          const errors = results
-            .map((result) => result.error)
-            .filter((error) => !!error)
-            .join('. ');
-          return {
-            items,
-            error: errors,
-          };
-        }),
-      );
-    } catch (err) {
-      return of({
-        items: [],
-        error: 'Error getting last modified files.',
-      });
-    }
+      ),
+    ).pipe(
+      map((results) => {
+        const items = results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]);
+        const errors = results
+          .map((result) => result.error)
+          .filter((error) => !!error)
+          .join('. ');
+        return {
+          items,
+          error: errors,
+        };
+      }),
+    );
   }
 
   private _getFiles(path: string, query?: string): Observable<SearchResults> {

diff --git a/server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts b/server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts
@@ -47,24 +47,17 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
         items: [],
       });
     }
-    try {
-      return forkJoin((folders || []).map((folder) => this._getFileItems('', folder.uuid))).pipe(
-        map((results) => {
-          const items = results.reduce(
-            (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)),
-            [] as SyncItem[],
-          );
-          return {
-            items,
-          };
-        }),
-      );
-    } catch (err) {
-      return of({
-        items: [],
-        error: `Error fetching last modified files: ${err}`,
-      });
-    }
+    return forkJoin((folders || []).map((folder) => this._getFileItems('', folder.uuid))).pipe(
+      map((results) => {
+        const items = results.reduce(
+          (acc, result) => acc.concat(result.items.filter((item) => item.modifiedGMT && item.modifiedGMT > since)),
+          [] as SyncItem[],
+        );
+        return {
+          items,
+        };
+      }),
+    );
   }
 
   getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults> {
@@ -73,24 +66,17 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
         items: [],
       });
     }
-    try {
-      return forkJoin((folders || []).map((folder) => this._getFileItems('', folder.uuid))).pipe(
-        map((results) => {
-          const result: { items: SyncItem[] } = {
-            items: [],
-          };
-          results.forEach((res) => {
-            result.items = [...result.items, ...res.items];
-          });
-          return result;
-        }),
-      );
-    } catch (err) {
-      return of({
-        items: [],
-        error: `Error fetching files: ${err}`,
-      });
-    }
+    return forkJoin((folders || []).map((folder) => this._getFileItems('', folder.uuid))).pipe(
+      map((results) => {
+        const result: { items: SyncItem[] } = {
+          items: [],
+        };
+        results.forEach((res) => {
+          result.items = [...result.items, ...res.items];
+        });
+        return result;
+      }),
+    );
   }
 
   getFolders(query?: string | undefined): Observable<SearchResults> {