sharepoint

server/src/logic/connector/infrastructure/connectors/sharepoint.connector.ts

@@ -0,0 +1,246 @@
+import { Observable, catchError, concatMap, forkJoin, from, map, of } from 'rxjs';
+
+import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector';
+import { SourceConnectorDefinition } from '../factory';
+import { OAuthBaseConnector } from './oauth.base';
+
+export const SharepointConnector: SourceConnectorDefinition = {
+  id: 'sharepoint',
+  factory: () => new SharepointImpl(),
+};
+
+export class SharepointImpl extends OAuthBaseConnector implements IConnector {
+  params: ConnectorParameters = {};
+  isExternal = false;
+
+  hasAuthData() {
+    return !!this.params.token;
+  }
+
+  setParameters(params: ConnectorParameters) {
+    this.params = params;
+  }
+
+  getParameters(): ConnectorParameters {
+    return this.params;
+  }
+
+  areParametersValid(params: ConnectorParameters) {
+    if (!params?.token) {
+      return false;
+    }
+    if (!params?.refresh) {
+      return false;
+    }
+    return true;
+  }
+
+  getLastModified(since: string, folders?: SyncItem[] | undefined): Observable<SearchResults> {
+    if ((folders ?? []).length === 0) {
+      return of({
+        items: [],
+      });
+    }
+    try {
+      return forkJoin((folders || []).map((folder) => this._getItems(folder.uuid, false, since))).pipe(
+        map((results) => {
+          const items = results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]);
+          return {
+            items,
+          };
+        }),
+      );
+    } catch (err) {
+      return of({
+        items: [],
+      });
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  getFolders(query?: string | undefined): Observable<SearchResults> {
+    return this._getItems('', true);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  getFiles(query?: string): Observable<SearchResults> {
+    return this._getItems();
+  }
+
+  getFilesFromFolders(folders: SyncItem[]): Observable<SearchResults> {
+    if ((folders ?? []).length === 0) {
+      return of({
+        items: [],
+      });
+    }
+    try {
+      return forkJoin((folders || []).map((folder) => this._getItems(folder.uuid))).pipe(
+        map((results) => {
+          const result: { items: SyncItem[] } = {
+            items: [],
+          };
+          results.forEach((res) => {
+            result.items = [...result.items, ...res.items];
+          });
+          return result;
+        }),
+      );
+    } catch (err) {
+      return of({
+        items: [],
+      });
+    }
+  }
+
+  isAccesTokenValid(): Observable<boolean> {
+    return from(
+      fetch('https://graph.microsoft.com/v1.0/sites', {
+        headers: {
+          Authorization: `Bearer ${this.params.token || ''}`,
+        },
+      }).then(
+        (res) => res.json(),
+        (err) => {
+          console.error(`Error fetching about: ${JSON.stringify(err)}`);
+          throw new Error(err);
+        },
+      ),
+    ).pipe(
+      concatMap((res) => {
+        if (res.error && res.error.code === 'InvalidAuthenticationToken') {
+          return of(false);
+        }
+        return of(true);
+      }),
+      catchError(() => {
+        return of(true);
+      }),
+    );
+  }
+
+  private _getItems(
+    folder = '',
+    foldersOnly = false,
+    since?: string,
+    siteId?: string,
+    nextPage?: string,
+    previous?: SearchResults,
+  ): Observable<SearchResults> {
+    let path = '';
+    return (siteId ? of(siteId) : this.getSiteId()).pipe(
+      concatMap((_siteId) => {
+        siteId = _siteId;
+        if (foldersOnly) {
+          path = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists`;
+        } else if (folder) {
+          path = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${folder}/items?expand=fields`;
+          if (since) {
+            path += `&$filter=fields/Modified gt '${since}'`;
+          }
+        } else {
+          throw new Error('One-shot import not implemented for Sharepoint.');
+        }
+        if (nextPage) {
+          path += `&$skiptoken=${nextPage}`;
+        }
+        return from(
+          fetch(path, {
+            headers: {
+              Authorization: `Bearer ${this.params.token}`,
+              Prefer: 'HonorNonIndexedQueriesWarningMayFailRandomly',
+            },
+          }).then(
+            (res) => res.json(),
+            (err) => {
+              console.error(`Error fetching ${path}: ${err}`);
+            },
+          ),
+        );
+      }),
+      concatMap((res) => {
+        if (res.error) {
+          console.error(`Error fetching ${path}: ${res.error}`);
+          if (res.error.code === 'InvalidAuthenticationToken') {
+            throw new Error('Unauthorized');
+          } else {
+            throw new Error(res.error.message || 'Unknown error');
+          }
+        } else {
+          const nextPage =
+            res['@odata.nextLink'] && res['@odata.nextLink'].includes('&$skiptoken=')
+              ? res?.['@odata.nextLink'].split('&$skiptoken=')[1].split('&')[0]
+              : undefined;
+          const items = (res.value || [])
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            .filter((item: any) => foldersOnly || item.fields?.ContentType === 'Document')
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            .map((item: any) =>
+              foldersOnly ? this.mapToSyncItemFolder(item) : this.mapToSyncItem(item, siteId || '', folder),
+            );
+          const results = {
+            items: [...(previous?.items || []), ...items],
+            nextPage,
+          };
+          return nextPage ? this._getItems(folder, foldersOnly, since, siteId, nextPage, results) : of(results);
+        }
+      }),
+    );
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private mapToSyncItem(item: any, siteId: string, listId: string): SyncItem {
+    return {
+      uuid: item.webUrl,
+      title: item.fields?.FileLeafRef || item.webUrl,
+      originalId: item.webUrl,
+      modifiedGMT: item.lastModifiedDateTime,
+      metadata: {
+        downloadLink: `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${listId}/items/${item.id}/driveitem/content`,
+      },
+      status: FileStatus.PENDING,
+    };
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private mapToSyncItemFolder(item: any): SyncItem {
+    return {
+      uuid: item.id,
+      title: item.name,
+      originalId: item.id,
+      metadata: {},
+      status: FileStatus.PENDING,
+      isFolder: true,
+    };
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  getLink(resource: SyncItem): Observable<Link> {
+    throw new Error('Method not implemented.');
+  }
+
+  download(resource: SyncItem): Observable<Blob> {
+    return from(
+      fetch(resource.metadata.downloadLink, { headers: { Authorization: `Bearer ${this.params.token || ''}` } }).then(
+        (res) => res.blob(),
+      ),
+    );
+  }
+
+  private getSiteId(): Observable<string> {
+    const path = `https://graph.microsoft.com/v1.0/sites?search=${this.params.site_name}}`;
+    return from(
+      fetch(path, {
+        headers: {
+          Authorization: `Bearer ${this.params.token || ''}`,
+        },
+      }).then((res) => res.json()),
+    ).pipe(
+      map((res) => {
+        if (res.error) {
+          throw new Error(`Error fetching Sharepoint site id for "${this.params.site_name}": ${res.error.message}`);
+        }
+        return res.value[0]?.id || '';
+      }),
+    );
+  }
+}

server/src/logic/connector/infrastructure/connectors/tests/sharepoint.connector.spec.js

@@ -0,0 +1,136 @@
+import { firstValueFrom, of } from 'rxjs';
+import { describe, expect, test, vi } from 'vitest';
+import { FileStatus } from '../../../domain/connector';
+import { SharepointImpl } from '../sharepoint.connector';
+
+const data = [
+  {
+    uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
+    title: 'PO6300590983',
+    originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
+    modifiedGMT: '2023-11-29T12:49:27.539Z',
+    metadata: {
+      needsPdfConversion: 'yes',
+      mimeType: 'application/pdf',
+    },
+    status: 'PENDING',
+  },
+  {
+    uuid: '19QJOCaOY4R8EQZ7VDrmmu2FBkeOlRAxJ',
+    title: 'PO6300604892',
+    originalId: '19QJOCaOY4R8EQZ7VDrmmu2FBkeOlRAxJ',
+    modifiedGMT: '2023-11-27T12:48:06.061Z',
+    metadata: {
+      needsPdfConversion: 'yes',
+      mimeType: 'application/pdf',
+    },
+    status: 'PENDING',
+  },
+  {
+    uuid: '1-5mIXJuiLTFxTO4mmVdXGNdf-Da-EzgA',
+    title: 'PO4550970006',
+    originalId: '1-5mIXJuiLTFxTO4mmVdXGNdf-Da-EzgA',
+    modifiedGMT: '2023-11-27T12:46:08.712Z',
+    metadata: {
+      needsPdfConversion: 'yes',
+      mimeType: 'application/pdf',
+    },
+    status: 'PENDING',
+  },
+];
+
+const sharepointTest = test.extend({
+  // eslint-disable-next-line no-empty-pattern
+  sourceConnector: async ({}, use) => {
+    const mock = vi.spyOn(SharepointImpl.prototype, '_getItems').mockImplementation(() => {
+      return of({
+        items: data,
+        nextPage: null,
+      });
+    });
+    await use(new SharepointImpl());
+    mock.mockRestore();
+  },
+});
+
+describe('Test validate sharepoint params', () => {
+  sharepointTest('Incorrect - Without params', ({ sourceConnector }) => {
+    expect(sourceConnector.areParametersValid({})).toBe(false);
+  });
+
+  sharepointTest('Incorrect - With wrong params', ({ sourceConnector }) => {
+    expect(
+      sourceConnector.areParametersValid({
+        incorrect: 'test',
+      }),
+    ).toBe(false);
+  });
+
+  sharepointTest('Incorrect - With wrong params - one valid', ({ sourceConnector }) => {
+    expect(
+      sourceConnector.areParametersValid({
+        incorrect: 'test',
+        token: 'test',
+      }),
+    ).toBe(false);
+  });
+
+  sharepointTest('Incorrect - With wrong params - one valid', ({ sourceConnector }) => {
+    expect(
+      sourceConnector.areParametersValid({
+        incorrect: 'test',
+        refresh: 'test',
+      }),
+    ).toBe(false);
+  });
+
+  sharepointTest('Incorrect - With empty params', ({ sourceConnector }) => {
+    expect(
+      sourceConnector.areParametersValid({
+        token: '',
+        refresh: '',
+      }),
+    ).toBe(false);
+  });
+
+  sharepointTest('Correct - With correct params', ({ sourceConnector }) => {
+    expect(
+      sourceConnector.areParametersValid({
+        token: 'test',
+        refresh: 'test',
+      }),
+    ).toBe(true);
+  });
+});
+
+describe('Test last modified', () => {
+  sharepointTest('Get last modified', async ({ sourceConnector }) => {
+    const lastModified = await firstValueFrom(
+      sourceConnector.getLastModified('2023-11-28T00:00:00.000Z', [
+        {
+          uuid: 'test_uuid',
+          title: 'Test folder',
+          originalId: 'test_uuid',
+          metadata: {},
+          status: FileStatus.PENDING,
+        },
+      ]),
+    );
+
+    expect(lastModified).toEqual({
+      items: [
+        {
+          uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
+          title: 'PO6300590983',
+          originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo',
+          modifiedGMT: '2023-11-29T12:49:27.539Z',
+          metadata: {
+            needsPdfConversion: 'yes',
+            mimeType: 'application/pdf',
+          },
+          status: FileStatus.PENDING,
+        },
+      ],
+    });
+  });
+});

server/src/logic/connector/infrastructure/factory.ts

@@ -3,6 +3,7 @@ import { DropboxConnector } from './connectors/dropbox.connector';
 import { FolderConnector } from './connectors/folder.connector';
 import { GDriveConnector } from './connectors/gdrive.connector';
 import { OneDriveConnector } from './connectors/onedrive.connector';
+import { SharepointConnector } from './connectors/sharepoint.connector';
 
 export interface ConnectorDefinition {
   id: string;
@@ -21,6 +22,7 @@ const connectors: { [id: string]: SourceConnectorDefinition } = {
   gdrive: GDriveConnector,
   dropbox: DropboxConnector,
   onedrive: OneDriveConnector,
+  sharepoint: SharepointConnector,
 };
 
 // TODO: add the dynamic connectors