implement filtering

nuclia · Mar 11, 2024 · 117d7c8 · 117d7c8
1 parent 45649af
commit 117d7c8
Show file tree

Hide file tree

Showing 13 changed files with 52 additions and 17 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "nuclia-sync-agent-app",
-  "version": "1.0.11",
+  "version": "1.1.0",
   "description": "\"This is a Nuclia Sync Agent App\"",
   "main": "build/index.js",
   "scripts": {

diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md
@@ -1,3 +1,7 @@
+# 1.1.0 (2024-03-11)
+
+- Support filtering options
+
 # 1.0.2 (2024-02-26)
 
 - Fix npm package

diff --git a/server/src/logic/connector/domain/connector.ts b/server/src/logic/connector/domain/connector.ts
@@ -24,6 +24,7 @@ export const SyncItemValidator = z.object({
   metadata: z.record(z.string()),
   status: z.nativeEnum(FileStatus).optional(),
   modifiedGMT: z.string().optional(),
+  mimeType: z.string().optional(),
   isFolder: z.boolean().optional(),
   parents: z.array(z.string()).optional(),
 });

diff --git a/server/src/logic/connector/infrastructure/connectors/folder.connector.ts b/server/src/logic/connector/infrastructure/connectors/folder.connector.ts
@@ -4,6 +4,7 @@ import path from 'path';
 import { forkJoin, map, Observable, of, switchMap } from 'rxjs';
 import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector';
 import { SourceConnectorDefinition } from '../factory';
+import { lookup } from 'mime-types';
 
 const FILES_TO_IGNORE = ['.DS_Store', 'Thumbs.db'];
 
@@ -145,6 +146,7 @@ class FolderImpl implements IConnector {
     return files.map((file) => ({
       title: file.split('/').pop() || '',
       originalId: file,
+      mimeType: lookup(file) || 'application/octet-stream',
       metadata: {
         path: file.split('/').slice(0, -1).join('/'),
       },

diff --git a/server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts b/server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts
@@ -260,9 +260,9 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
       originalId: item.id,
       modifiedGMT: item.modifiedTime,
       parents: item.parents,
+      mimeType: needsPdfConversion ? 'application/pdf' : item.mimeType,
       metadata: {
         needsPdfConversion: needsPdfConversion ? 'yes' : 'no',
-        mimeType: needsPdfConversion ? 'application/pdf' : item.mimeType,
       },
       status: FileStatus.PENDING,
     };

diff --git a/server/src/logic/connector/infrastructure/connectors/onedrive.connector.ts b/server/src/logic/connector/infrastructure/connectors/onedrive.connector.ts
@@ -215,8 +215,8 @@ export class OneDriveImpl extends OAuthBaseConnector implements IConnector {
       title: item.name,
       originalId: item.id,
       modifiedGMT: item.lastModifiedDateTime,
+      mimeType: item.file.mimeType,
       metadata: {
-        mimeType: item.file.mimeType,
         downloadLink: item['@microsoft.graph.downloadUrl'],
         path: item.parentReference.path,
       },

diff --git a/server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts b/server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts
@@ -81,6 +81,7 @@ class SitemapImpl implements IConnector {
           status: FileStatus.PENDING,
           uuid: `${new Date().getTime()}`,
           originalId: parsedUrl.loc,
+          mimeType: 'text/html',
           metadata: {
             uri: parsedUrl.loc,
             path: parsedUrl.loc.replace(/https?:\/\//, ''),

diff --git a/server/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js b/server/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js
@@ -9,9 +9,9 @@ const data = [
     title: 'PO6300590983',
     originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
     modifiedGMT: '2023-11-29T12:49:27.539Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -20,9 +20,9 @@ const data = [
     title: 'PO6300604892',
     originalId: '19QJOCaOY4R8EQZ7VDrmmu2FBkeOlRAxJ',
     modifiedGMT: '2023-11-27T12:48:06.061Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -31,9 +31,9 @@ const data = [
     title: 'PO4550970006',
     originalId: '1-5mIXJuiLTFxTO4mmVdXGNdf-Da-EzgA',
     modifiedGMT: '2023-11-27T12:46:08.712Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -145,9 +145,9 @@ describe('Test last modified', () => {
           title: 'PO6300590983',
           originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
           modifiedGMT: '2023-11-29T12:49:27.539Z',
+          mimeType: 'application/pdf',
           metadata: {
             needsPdfConversion: 'yes',
-            mimeType: 'application/pdf',
             path: '',
           },
           status: FileStatus.PENDING,

diff --git a/server/src/logic/connector/infrastructure/connectors/tests/onedrive.connector.spec.js b/server/src/logic/connector/infrastructure/connectors/tests/onedrive.connector.spec.js
@@ -9,9 +9,9 @@ const data = [
     title: 'PO6300590983',
     originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
     modifiedGMT: '2023-11-29T12:49:27.539Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -20,9 +20,9 @@ const data = [
     title: 'PO6300604892',
     originalId: '19QJOCaOY4R8EQZ7VDrmmu2FBkeOlRAxJ',
     modifiedGMT: '2023-11-27T12:48:06.061Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -31,9 +31,9 @@ const data = [
     title: 'PO4550970006',
     originalId: '1-5mIXJuiLTFxTO4mmVdXGNdf-Da-EzgA',
     modifiedGMT: '2023-11-27T12:46:08.712Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -145,9 +145,9 @@ describe('Test last modified', () => {
           title: 'PO6300590983',
           originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
           modifiedGMT: '2023-11-29T12:49:27.539Z',
+          mimeType: 'application/pdf',
           metadata: {
             needsPdfConversion: 'yes',
-            mimeType: 'application/pdf',
           },
           status: FileStatus.PENDING,
         },

diff --git a/server/src/logic/connector/infrastructure/connectors/tests/sharepoint.connector.spec.js b/server/src/logic/connector/infrastructure/connectors/tests/sharepoint.connector.spec.js
@@ -9,9 +9,9 @@ const data = [
     title: 'PO6300590983',
     originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
     modifiedGMT: '2023-11-29T12:49:27.539Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -20,9 +20,9 @@ const data = [
     title: 'PO6300604892',
     originalId: '19QJOCaOY4R8EQZ7VDrmmu2FBkeOlRAxJ',
     modifiedGMT: '2023-11-27T12:48:06.061Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -31,9 +31,9 @@ const data = [
     title: 'PO4550970006',
     originalId: '1-5mIXJuiLTFxTO4mmVdXGNdf-Da-EzgA',
     modifiedGMT: '2023-11-27T12:46:08.712Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },

diff --git a/server/src/logic/sync/domain/nuclia-cloud.ts b/server/src/logic/sync/domain/nuclia-cloud.ts
@@ -42,7 +42,7 @@ export class NucliaCloud {
   upload(
     originalId: string,
     filename: string,
-    data: { buffer?: ArrayBuffer; text?: TextField; metadata?: any },
+    data: { buffer?: ArrayBuffer; text?: TextField; metadata?: any; mimeType?: string },
   ): Observable<{ success: boolean; message?: string }> {
     const slug = sha256(originalId);
     const text = data.text;
@@ -94,7 +94,7 @@ export class NucliaCloud {
             try {
               return resource
                 .upload('file', buffer, false, {
-                  contentType: data.metadata.mimeType || lookup(filename) || 'application/octet-stream',
+                  contentType: data.mimeType || lookup(filename) || 'application/octet-stream',
                   filename,
                 })
                 .pipe(

diff --git a/server/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts b/server/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts
@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { concatMap, delay, lastValueFrom, map, of, switchMap, tap, toArray } from 'rxjs';
+import { concatMap, delay, filter, lastValueFrom, map, of, switchMap, tap, toArray } from 'rxjs';
 
 import { EVENTS } from '../../../../events/events';
 import { eventEmitter } from '../../../../server';
@@ -10,6 +10,7 @@ import { ISyncRepository } from '../sync.repository';
 import { RefreshAccessToken } from './refresh-access-token.use-case';
 import { SyncSingleFile } from './sync-single-file.use-case';
 import { UpdateSync } from './update-sync.use-case';
+import { lookup } from 'mime-types';
 
 require('localstorage-polyfill');
 
@@ -66,7 +67,32 @@ export class SyncAllFolders implements SyncAllFoldersUseCase {
   }
 
   processItems(syncEntity: SyncEntity, items: SyncItem[]) {
+    const filteredMimetypes = (syncEntity.filters?.fileExtensions?.extensions || '')
+      .split(',')
+      .map((ext) => lookup(ext.trim()) || '')
+      .filter((ext) => ext);
     return of(...items).pipe(
+      filter((item) => {
+        let isExtensionOk = true;
+        if (!item.mimeType) {
+          isExtensionOk = false;
+        } else {
+          if (filteredMimetypes.length > 0) {
+            const isFiltered = filteredMimetypes.includes(item.mimeType);
+            isExtensionOk = syncEntity.filters?.fileExtensions?.exclude ? !isFiltered : isFiltered;
+          }
+        }
+        let isDateOk = true;
+        if (item.modifiedGMT && syncEntity.filters?.modified) {
+          if (syncEntity.filters.modified.from) {
+            isDateOk = item.modifiedGMT >= syncEntity.filters.modified.from;
+          }
+          if (syncEntity.filters.modified.to) {
+            isDateOk = isDateOk && item.modifiedGMT <= syncEntity.filters.modified.to;
+          }
+        }
+        return isDateOk && isExtensionOk;
+      }),
       concatMap((item) =>
         new SyncSingleFile(syncEntity, item).execute().pipe(
           map((res) => ({ id: item.originalId, success: res.success })),

diff --git a/server/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts b/server/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts
@@ -52,6 +52,7 @@ export class SyncSingleFile implements SyncSingleFileUseCase {
                 return nucliaConnector.upload(item.originalId, item.title, {
                   buffer: arrayBuffer,
                   metadata: { ...item.metadata, labels: sync.labels },
+                  mimeType: item.mimeType,
                 });
               }),
             );