From 6c8788f19246467241094789f5ef745a516254fc Mon Sep 17 00:00:00 2001 From: Eric BREHAULT Date: Mon, 15 Jan 2024 14:08:11 +0100 Subject: [PATCH] sharepoint --- .../connectors/sharepoint.connector.ts | 246 ++++++++++++++++++ .../tests/sharepoint.connector.spec.js | 136 ++++++++++ .../logic/connector/infrastructure/factory.ts | 2 + 3 files changed, 384 insertions(+) create mode 100644 server/src/logic/connector/infrastructure/connectors/sharepoint.connector.ts create mode 100644 server/src/logic/connector/infrastructure/connectors/tests/sharepoint.connector.spec.js diff --git a/server/src/logic/connector/infrastructure/connectors/sharepoint.connector.ts b/server/src/logic/connector/infrastructure/connectors/sharepoint.connector.ts new file mode 100644 index 0000000..8ea39e6 --- /dev/null +++ b/server/src/logic/connector/infrastructure/connectors/sharepoint.connector.ts @@ -0,0 +1,246 @@ +import { Observable, catchError, concatMap, forkJoin, from, map, of } from 'rxjs'; + +import { ConnectorParameters, FileStatus, IConnector, Link, SearchResults, SyncItem } from '../../domain/connector'; +import { SourceConnectorDefinition } from '../factory'; +import { OAuthBaseConnector } from './oauth.base'; + +export const SharepointConnector: SourceConnectorDefinition = { + id: 'sharepoint', + factory: () => new SharepointImpl(), +}; + +export class SharepointImpl extends OAuthBaseConnector implements IConnector { + params: ConnectorParameters = {}; + isExternal = false; + + hasAuthData() { + return !!this.params.token; + } + + setParameters(params: ConnectorParameters) { + this.params = params; + } + + getParameters(): ConnectorParameters { + return this.params; + } + + areParametersValid(params: ConnectorParameters) { + if (!params?.token) { + return false; + } + if (!params?.refresh) { + return false; + } + return true; + } + + getLastModified(since: string, folders?: SyncItem[] | undefined): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { + return forkJoin((folders || []).map((folder) => this._getItems(folder.uuid, false, since))).pipe( + map((results) => { + const items = results.reduce((acc, result) => acc.concat(result.items), [] as SyncItem[]); + return { + items, + }; + }), + ); + } catch (err) { + return of({ + items: [], + }); + } + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + getFolders(query?: string | undefined): Observable { + return this._getItems('', true); + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + getFiles(query?: string): Observable { + return this._getItems(); + } + + getFilesFromFolders(folders: SyncItem[]): Observable { + if ((folders ?? []).length === 0) { + return of({ + items: [], + }); + } + try { + return forkJoin((folders || []).map((folder) => this._getItems(folder.uuid))).pipe( + map((results) => { + const result: { items: SyncItem[] } = { + items: [], + }; + results.forEach((res) => { + result.items = [...result.items, ...res.items]; + }); + return result; + }), + ); + } catch (err) { + return of({ + items: [], + }); + } + } + + isAccesTokenValid(): Observable { + return from( + fetch('https://graph.microsoft.com/v1.0/sites', { + headers: { + Authorization: `Bearer ${this.params.token || ''}`, + }, + }).then( + (res) => res.json(), + (err) => { + console.error(`Error fetching about: ${JSON.stringify(err)}`); + throw new Error(err); + }, + ), + ).pipe( + concatMap((res) => { + if (res.error && res.error.code === 'InvalidAuthenticationToken') { + return of(false); + } + return of(true); + }), + catchError(() => { + return of(true); + }), + ); + } + + private _getItems( + folder = '', + foldersOnly = false, + since?: string, + siteId?: string, + nextPage?: string, + previous?: SearchResults, + ): Observable { + let path = ''; + return (siteId ? of(siteId) : this.getSiteId()).pipe( + concatMap((_siteId) => { + siteId = _siteId; + if (foldersOnly) { + path = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists`; + } else if (folder) { + path = `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${folder}/items?expand=fields`; + if (since) { + path += `&$filter=fields/Modified gt '${since}'`; + } + } else { + throw new Error('One-shot import not implemented for Sharepoint.'); + } + if (nextPage) { + path += `&$skiptoken=${nextPage}`; + } + return from( + fetch(path, { + headers: { + Authorization: `Bearer ${this.params.token}`, + Prefer: 'HonorNonIndexedQueriesWarningMayFailRandomly', + }, + }).then( + (res) => res.json(), + (err) => { + console.error(`Error fetching ${path}: ${err}`); + }, + ), + ); + }), + concatMap((res) => { + if (res.error) { + console.error(`Error fetching ${path}: ${res.error}`); + if (res.error.code === 'InvalidAuthenticationToken') { + throw new Error('Unauthorized'); + } else { + throw new Error(res.error.message || 'Unknown error'); + } + } else { + const nextPage = + res['@odata.nextLink'] && res['@odata.nextLink'].includes('&$skiptoken=') + ? res?.['@odata.nextLink'].split('&$skiptoken=')[1].split('&')[0] + : undefined; + const items = (res.value || []) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .filter((item: any) => foldersOnly || item.fields?.ContentType === 'Document') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .map((item: any) => + foldersOnly ? this.mapToSyncItemFolder(item) : this.mapToSyncItem(item, siteId || '', folder), + ); + const results = { + items: [...(previous?.items || []), ...items], + nextPage, + }; + return nextPage ? this._getItems(folder, foldersOnly, since, siteId, nextPage, results) : of(results); + } + }), + ); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private mapToSyncItem(item: any, siteId: string, listId: string): SyncItem { + return { + uuid: item.webUrl, + title: item.fields?.FileLeafRef || item.webUrl, + originalId: item.webUrl, + modifiedGMT: item.lastModifiedDateTime, + metadata: { + downloadLink: `https://graph.microsoft.com/v1.0/sites/${siteId}/lists/${listId}/items/${item.id}/driveitem/content`, + }, + status: FileStatus.PENDING, + }; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private mapToSyncItemFolder(item: any): SyncItem { + return { + uuid: item.id, + title: item.name, + originalId: item.id, + metadata: {}, + status: FileStatus.PENDING, + isFolder: true, + }; + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + getLink(resource: SyncItem): Observable { + throw new Error('Method not implemented.'); + } + + download(resource: SyncItem): Observable { + return from( + fetch(resource.metadata.downloadLink, { headers: { Authorization: `Bearer ${this.params.token || ''}` } }).then( + (res) => res.blob(), + ), + ); + } + + private getSiteId(): Observable { + const path = `https://graph.microsoft.com/v1.0/sites?search=${this.params.site_name}}`; + return from( + fetch(path, { + headers: { + Authorization: `Bearer ${this.params.token || ''}`, + }, + }).then((res) => res.json()), + ).pipe( + map((res) => { + if (res.error) { + throw new Error(`Error fetching Sharepoint site id for "${this.params.site_name}": ${res.error.message}`); + } + return res.value[0]?.id || ''; + }), + ); + } +} diff --git a/server/src/logic/connector/infrastructure/connectors/tests/sharepoint.connector.spec.js b/server/src/logic/connector/infrastructure/connectors/tests/sharepoint.connector.spec.js new file mode 100644 index 0000000..ff4f374 --- /dev/null +++ b/server/src/logic/connector/infrastructure/connectors/tests/sharepoint.connector.spec.js @@ -0,0 +1,136 @@ +import { firstValueFrom, of } from 'rxjs'; +import { describe, expect, test, vi } from 'vitest'; +import { FileStatus } from '../../../domain/connector'; +import { SharepointImpl } from '../sharepoint.connector'; + +const data = [ + { + uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', + title: 'PO6300590983', + originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', + modifiedGMT: '2023-11-29T12:49:27.539Z', + metadata: { + needsPdfConversion: 'yes', + mimeType: 'application/pdf', + }, + status: 'PENDING', + }, + { + uuid: '19QJOCaOY4R8EQZ7VDrmmu2FBkeOlRAxJ', + title: 'PO6300604892', + originalId: '19QJOCaOY4R8EQZ7VDrmmu2FBkeOlRAxJ', + modifiedGMT: '2023-11-27T12:48:06.061Z', + metadata: { + needsPdfConversion: 'yes', + mimeType: 'application/pdf', + }, + status: 'PENDING', + }, + { + uuid: '1-5mIXJuiLTFxTO4mmVdXGNdf-Da-EzgA', + title: 'PO4550970006', + originalId: '1-5mIXJuiLTFxTO4mmVdXGNdf-Da-EzgA', + modifiedGMT: '2023-11-27T12:46:08.712Z', + metadata: { + needsPdfConversion: 'yes', + mimeType: 'application/pdf', + }, + status: 'PENDING', + }, +]; + +const sharepointTest = test.extend({ + // eslint-disable-next-line no-empty-pattern + sourceConnector: async ({}, use) => { + const mock = vi.spyOn(SharepointImpl.prototype, '_getItems').mockImplementation(() => { + return of({ + items: data, + nextPage: null, + }); + }); + await use(new SharepointImpl()); + mock.mockRestore(); + }, +}); + +describe('Test validate sharepoint params', () => { + sharepointTest('Incorrect - Without params', ({ sourceConnector }) => { + expect(sourceConnector.areParametersValid({})).toBe(false); + }); + + sharepointTest('Incorrect - With wrong params', ({ sourceConnector }) => { + expect( + sourceConnector.areParametersValid({ + incorrect: 'test', + }), + ).toBe(false); + }); + + sharepointTest('Incorrect - With wrong params - one valid', ({ sourceConnector }) => { + expect( + sourceConnector.areParametersValid({ + incorrect: 'test', + token: 'test', + }), + ).toBe(false); + }); + + sharepointTest('Incorrect - With wrong params - one valid', ({ sourceConnector }) => { + expect( + sourceConnector.areParametersValid({ + incorrect: 'test', + refresh: 'test', + }), + ).toBe(false); + }); + + sharepointTest('Incorrect - With empty params', ({ sourceConnector }) => { + expect( + sourceConnector.areParametersValid({ + token: '', + refresh: '', + }), + ).toBe(false); + }); + + sharepointTest('Correct - With correct params', ({ sourceConnector }) => { + expect( + sourceConnector.areParametersValid({ + token: 'test', + refresh: 'test', + }), + ).toBe(true); + }); +}); + +describe('Test last modified', () => { + sharepointTest('Get last modified', async ({ sourceConnector }) => { + const lastModified = await firstValueFrom( + sourceConnector.getLastModified('2023-11-28T00:00:00.000Z', [ + { + uuid: 'test_uuid', + title: 'Test folder', + originalId: 'test_uuid', + metadata: {}, + status: FileStatus.PENDING, + }, + ]), + ); + + expect(lastModified).toEqual({ + items: [ + { + uuid: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', + title: 'PO6300590983', + originalId: '1v8WV_aNM5qB_642saVlPhOkN1xI0NtQo', + modifiedGMT: '2023-11-29T12:49:27.539Z', + metadata: { + needsPdfConversion: 'yes', + mimeType: 'application/pdf', + }, + status: FileStatus.PENDING, + }, + ], + }); + }); +}); diff --git a/server/src/logic/connector/infrastructure/factory.ts b/server/src/logic/connector/infrastructure/factory.ts index 9dadb18..71d0beb 100644 --- a/server/src/logic/connector/infrastructure/factory.ts +++ b/server/src/logic/connector/infrastructure/factory.ts @@ -3,6 +3,7 @@ import { DropboxConnector } from './connectors/dropbox.connector'; import { FolderConnector } from './connectors/folder.connector'; import { GDriveConnector } from './connectors/gdrive.connector'; import { OneDriveConnector } from './connectors/onedrive.connector'; +import { SharepointConnector } from './connectors/sharepoint.connector'; export interface ConnectorDefinition { id: string; @@ -21,6 +22,7 @@ const connectors: { [id: string]: SourceConnectorDefinition } = { gdrive: GDriveConnector, dropbox: DropboxConnector, onedrive: OneDriveConnector, + sharepoint: SharepointConnector, }; // TODO: add the dynamic connectors