Ericbrehault/sc 8740/implement filtering in synchronizer

package.json

@@ -1,6 +1,6 @@
 {
   "name": "nuclia-sync-agent-app",
-  "version": "1.0.11",
+  "version": "1.1.0",
   "description": "\"This is a Nuclia Sync Agent App\"",
   "main": "build/index.js",
   "scripts": {

server/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 1.1.0 (2024-03-11)
+
+- Support filtering options
+
 # 1.0.2 (2024-02-26)
 
 - Fix npm package

server/src/logic/connector/domain/connector.ts

@@ -24,6 +24,7 @@ export const SyncItemValidator = z.object({
   metadata: z.record(z.string()),
   status: z.nativeEnum(FileStatus).optional(),
   modifiedGMT: z.string().optional(),
+  mimeType: z.string().optional(),
   isFolder: z.boolean().optional(),
   parents: z.array(z.string()).optional(),
 });

server/src/logic/connector/infrastructure/connectors/folder.connector.ts

@@ -4,6 +4,7 @@ import path from 'path';
 import { forkJoin, map, Observable, of, switchMap } from 'rxjs';
 import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector';
 import { SourceConnectorDefinition } from '../factory';
+import { lookup } from 'mime-types';
 
 const FILES_TO_IGNORE = ['.DS_Store', 'Thumbs.db'];
 
@@ -145,6 +146,7 @@ class FolderImpl implements IConnector {
     return files.map((file) => ({
       title: file.split('/').pop() || '',
       originalId: file,
+      mimeType: lookup(file) || 'application/octet-stream',
       metadata: {
         path: file.split('/').slice(0, -1).join('/'),
       },

server/src/logic/connector/infrastructure/connectors/gdrive.connector.ts

@@ -260,9 +260,9 @@ export class GDriveImpl extends OAuthBaseConnector implements IConnector {
       originalId: item.id,
       modifiedGMT: item.modifiedTime,
       parents: item.parents,
+      mimeType: needsPdfConversion ? 'application/pdf' : item.mimeType,
       metadata: {
         needsPdfConversion: needsPdfConversion ? 'yes' : 'no',
-        mimeType: needsPdfConversion ? 'application/pdf' : item.mimeType,
       },
       status: FileStatus.PENDING,
     };

server/src/logic/connector/infrastructure/connectors/onedrive.connector.ts

@@ -215,8 +215,8 @@ export class OneDriveImpl extends OAuthBaseConnector implements IConnector {
       title: item.name,
       originalId: item.id,
       modifiedGMT: item.lastModifiedDateTime,
+      mimeType: item.file.mimeType,
       metadata: {
-        mimeType: item.file.mimeType,
         downloadLink: item['@microsoft.graph.downloadUrl'],
         path: item.parentReference.path,
       },

server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts

@@ -81,6 +81,7 @@ class SitemapImpl implements IConnector {
           status: FileStatus.PENDING,
           uuid: `${new Date().getTime()}`,
           originalId: parsedUrl.loc,
+          mimeType: 'text/html',
           metadata: {
             uri: parsedUrl.loc,
             path: parsedUrl.loc.replace(/https?:\/\//, ''),

server/src/logic/connector/infrastructure/connectors/tests/gdrive.connector.spec.js

@@ -9,9 +9,9 @@ const data = [
     title: 'PO6300590983',
     originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
     modifiedGMT: '2023-11-29T12:49:27.539Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -20,9 +20,9 @@ const data = [
     title: 'PO6300604892',
     originalId: '19QJOCaOY4R8EQZ7VDrmmu2FBkeOlRAxJ',
     modifiedGMT: '2023-11-27T12:48:06.061Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -31,9 +31,9 @@ const data = [
     title: 'PO4550970006',
     originalId: '1-5mIXJuiLTFxTO4mmVdXGNdf-Da-EzgA',
     modifiedGMT: '2023-11-27T12:46:08.712Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -145,9 +145,9 @@ describe('Test last modified', () => {
           title: 'PO6300590983',
           originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
           modifiedGMT: '2023-11-29T12:49:27.539Z',
+          mimeType: 'application/pdf',
           metadata: {
             needsPdfConversion: 'yes',
-            mimeType: 'application/pdf',
             path: '',
           },
           status: FileStatus.PENDING,

server/src/logic/connector/infrastructure/connectors/tests/onedrive.connector.spec.js

@@ -9,9 +9,9 @@ const data = [
     title: 'PO6300590983',
     originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
     modifiedGMT: '2023-11-29T12:49:27.539Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -20,9 +20,9 @@ const data = [
     title: 'PO6300604892',
     originalId: '19QJOCaOY4R8EQZ7VDrmmu2FBkeOlRAxJ',
     modifiedGMT: '2023-11-27T12:48:06.061Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -31,9 +31,9 @@ const data = [
     title: 'PO4550970006',
     originalId: '1-5mIXJuiLTFxTO4mmVdXGNdf-Da-EzgA',
     modifiedGMT: '2023-11-27T12:46:08.712Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -145,9 +145,9 @@ describe('Test last modified', () => {
           title: 'PO6300590983',
           originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
           modifiedGMT: '2023-11-29T12:49:27.539Z',
+          mimeType: 'application/pdf',
           metadata: {
             needsPdfConversion: 'yes',
-            mimeType: 'application/pdf',
           },
           status: FileStatus.PENDING,
         },

server/src/logic/connector/infrastructure/connectors/tests/sharepoint.connector.spec.js

@@ -9,9 +9,9 @@ const data = [
     title: 'PO6300590983',
     originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
     modifiedGMT: '2023-11-29T12:49:27.539Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -20,9 +20,9 @@ const data = [
     title: 'PO6300604892',
     originalId: '19QJOCaOY4R8EQZ7VDrmmu2FBkeOlRAxJ',
     modifiedGMT: '2023-11-27T12:48:06.061Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },
@@ -31,9 +31,9 @@ const data = [
     title: 'PO4550970006',
     originalId: '1-5mIXJuiLTFxTO4mmVdXGNdf-Da-EzgA',
     modifiedGMT: '2023-11-27T12:46:08.712Z',
+    mimeType: 'application/pdf',
     metadata: {
       needsPdfConversion: 'yes',
-      mimeType: 'application/pdf',
     },
     status: 'PENDING',
   },

server/src/logic/sync/domain/dto/create-sync.dto.ts

@@ -1,7 +1,7 @@
 import { v4 as uuidv4 } from 'uuid';
 import { MakeOptional } from '../../../../types/server';
 import { getConnector } from '../../../connector/infrastructure/factory';
-import { ISyncEntity, NucliaOptionsValidator } from '../sync.entity';
+import { FiltersValidator, ISyncEntity, NucliaOptionsValidator } from '../sync.entity';
 import { validateZodSchema } from './validate';
 
 export class CreateSyncDto {
@@ -13,6 +13,7 @@ export class CreateSyncDto {
       kb: this.options.kb,
       title: this.options.title,
       id: this.options.id,
+      filters: this.options.filters,
     };
 
     if (this.options.labels) returnObj.labels = this.options.labels;
@@ -46,6 +47,14 @@ export class CreateSyncDto {
       return ['You can not create a sync with foldersToSync'];
     }
 
+    if (props.filters) {
+      try {
+        validateZodSchema(FiltersValidator, props.filters);
+      } catch (error) {
+        return [`Invalid format for filters: ${error}`];
+      }
+    }
+
     return [undefined, new CreateSyncDto({ ...props, id })];
   }
 }

server/src/logic/sync/domain/dto/update-sync.dto.ts

@@ -1,6 +1,6 @@
 import { SyncItemValidator } from '../../../connector/domain/connector';
 import { getConnector } from '../../../connector/infrastructure/factory';
-import { ISyncEntity } from '../sync.entity';
+import { FiltersValidator, ISyncEntity } from '../sync.entity';
 import { validateZodSchema } from './validate';
 
 type Values = Partial<ISyncEntity> & { id: string };
@@ -18,12 +18,13 @@ export class UpdateSyncDto {
     if (this.options.title) returnObj.title = this.options.title;
     if (this.options.lastSyncGMT) returnObj.lastSyncGMT = this.options.lastSyncGMT;
     if (this.options.foldersToSync) returnObj.foldersToSync = this.options.foldersToSync;
+    if (this.options.filters) returnObj.filters = this.options.filters;
 
     return returnObj;
   }
 
   static create(props: Values): [string?, UpdateSyncDto?] {
-    const { id, connector, kb, foldersToSync } = props;
+    const { id, connector, kb, foldersToSync, filters } = props;
 
     if (!id) {
       return ['id is mandatory'];
@@ -51,6 +52,14 @@ export class UpdateSyncDto {
       }
     }
 
+    if (filters) {
+      try {
+        validateZodSchema(FiltersValidator, filters);
+      } catch (error) {
+        return [`Invalid format for filters: ${error}`];
+      }
+    }
+
     if (foldersToSync && foldersToSync.length > 0) {
       let errorMsg = '';
       const valid = foldersToSync.some((folder) => {

server/src/logic/sync/domain/nuclia-cloud.ts

@@ -42,7 +42,7 @@ export class NucliaCloud {
   upload(
     originalId: string,
     filename: string,
-    data: { buffer?: ArrayBuffer; text?: TextField; metadata?: any },
+    data: { buffer?: ArrayBuffer; text?: TextField; metadata?: any; mimeType?: string },
   ): Observable<{ success: boolean; message?: string }> {
     const slug = sha256(originalId);
     const text = data.text;
@@ -94,7 +94,7 @@ export class NucliaCloud {
             try {
               return resource
                 .upload('file', buffer, false, {
-                  contentType: data.metadata.mimeType || lookup(filename) || 'application/octet-stream',
+                  contentType: data.mimeType || lookup(filename) || 'application/octet-stream',
                   filename,
                 })
                 .pipe(

server/src/logic/sync/domain/sync.entity.ts

@@ -42,6 +42,22 @@ export type NucliaOptions = z.infer<typeof NucliaOptionsValidator>;
 
 export type Classification = {};
 
+export const FiltersValidator = z.object({
+  fileExtensions: z
+    .object({
+      extensions: z.string(),
+      exclude: z.boolean().optional(),
+    })
+    .optional(),
+  modified: z
+    .object({
+      from: z.string().optional(),
+      to: z.string().optional(),
+    })
+    .optional(),
+});
+export type Filters = z.infer<typeof FiltersValidator>;
+
 export interface ISyncEntity {
   connector: Connector;
   kb: NucliaOptions;
@@ -50,6 +66,7 @@ export interface ISyncEntity {
   id: string;
   lastSyncGMT?: string;
   foldersToSync?: SyncItem[];
+  filters?: Filters;
 }
 
 export class SyncEntity {
@@ -61,16 +78,18 @@ export class SyncEntity {
   public sourceConnector?: IConnector;
   public lastSyncGMT?: string;
   public foldersToSync?: SyncItem[] = [];
+  public filters?: Filters;
 
   constructor(options: ISyncEntity) {
-    const { connector, kb, labels, title, id, lastSyncGMT, foldersToSync } = options;
+    const { connector, kb, labels, title, id, lastSyncGMT, foldersToSync, filters } = options;
     this.connector = connector;
     this.kb = kb;
     this.labels = labels;
     this.title = title;
     this.id = id;
     this.lastSyncGMT = lastSyncGMT;
     this.foldersToSync = foldersToSync;
+    this.filters = filters;
     this.setConnectorDefinition();
   }
 

server/src/logic/sync/domain/use-cases/sync-all-folders-data.use-case.ts

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { concatMap, delay, lastValueFrom, map, of, switchMap, tap, toArray } from 'rxjs';
+import { concatMap, delay, filter, lastValueFrom, map, of, switchMap, tap, toArray } from 'rxjs';
 
 import { EVENTS } from '../../../../events/events';
 import { eventEmitter } from '../../../../server';
@@ -10,6 +10,7 @@ import { ISyncRepository } from '../sync.repository';
 import { RefreshAccessToken } from './refresh-access-token.use-case';
 import { SyncSingleFile } from './sync-single-file.use-case';
 import { UpdateSync } from './update-sync.use-case';
+import { lookup } from 'mime-types';
 
 require('localstorage-polyfill');
 
@@ -66,7 +67,32 @@ export class SyncAllFolders implements SyncAllFoldersUseCase {
   }
 
   processItems(syncEntity: SyncEntity, items: SyncItem[]) {
+    const filteredMimetypes = (syncEntity.filters?.fileExtensions?.extensions || '')
+      .split(',')
+      .map((ext) => lookup(ext.trim()) || '')
+      .filter((ext) => ext);
     return of(...items).pipe(
+      filter((item) => {
+        let isExtensionOk = true;
+        if (!item.mimeType) {
+          isExtensionOk = false;
+        } else {
+          if (filteredMimetypes.length > 0) {
+            const isFiltered = filteredMimetypes.includes(item.mimeType);
+            isExtensionOk = syncEntity.filters?.fileExtensions?.exclude ? !isFiltered : isFiltered;
+          }
+        }
+        let isDateOk = true;
+        if (item.modifiedGMT && syncEntity.filters?.modified) {
+          if (syncEntity.filters.modified.from) {
+            isDateOk = item.modifiedGMT >= syncEntity.filters.modified.from;
+          }
+          if (syncEntity.filters.modified.to) {
+            isDateOk = isDateOk && item.modifiedGMT <= syncEntity.filters.modified.to;
+          }
+        }
+        return isDateOk && isExtensionOk;
+      }),
       concatMap((item) =>
         new SyncSingleFile(syncEntity, item).execute().pipe(
           map((res) => ({ id: item.originalId, success: res.success })),

server/src/logic/sync/domain/use-cases/sync-single-file.use-case.ts

@@ -52,6 +52,7 @@ export class SyncSingleFile implements SyncSingleFileUseCase {
                 return nucliaConnector.upload(item.originalId, item.title, {
                   buffer: arrayBuffer,
                   metadata: { ...item.metadata, labels: sync.labels },
+                  mimeType: item.mimeType,
                 });
               }),
             );