diff --git a/packages/cozy-clisk/src/contentscript/ContentScript.js b/packages/cozy-clisk/src/contentscript/ContentScript.js index 43576f5ac..0e5a46d8c 100644 --- a/packages/cozy-clisk/src/contentscript/ContentScript.js +++ b/packages/cozy-clisk/src/contentscript/ContentScript.js @@ -1,4 +1,9 @@ // @ts-check +<<<<<<< Updated upstream +======= +import get from 'lodash/get' +import clone from 'lodash/clone' +>>>>>>> Stashed changes import waitFor from 'p-wait-for' import Minilog from '@cozy/minilog' @@ -466,7 +471,8 @@ export default class ContentScript { */ async storeFromWorker(obj) { // @ts-ignore Aucune surcharge ne correspond à cet appel. - Object.assign(this.store, obj) + //Object.assign(this.store, obj) + this.store = clone(obj) } onlyIn(csType, method) { diff --git a/packages/cozy-clisk/src/index.js b/packages/cozy-clisk/src/index.js index a3f3ad778..9e2a39709 100644 --- a/packages/cozy-clisk/src/index.js +++ b/packages/cozy-clisk/src/index.js @@ -5,6 +5,7 @@ export { MessengerInterface } from './bridge/bridgeInterfaces' export { default as ContentScript } from './contentscript/ContentScript' export { default as addData } from './launcher/addData' export { default as hydrateAndFilter } from './launcher/hydrateAndFilter' +export { default as getFileIfExists } from './launcher/getFileIfExists' export { default as saveFiles } from './launcher/saveFiles' export { default as saveBills } from './launcher/saveBills' export { default as saveIdentity } from './launcher/saveIdentity' diff --git a/packages/cozy-clisk/src/launcher.js b/packages/cozy-clisk/src/launcher.js index eaecfe9a0..e6890be5a 100644 --- a/packages/cozy-clisk/src/launcher.js +++ b/packages/cozy-clisk/src/launcher.js @@ -1,5 +1,6 @@ export { default as addData } from './launcher/addData' export { default as hydrateAndFilter } from './launcher/hydrateAndFilter' +export { default as getFileIfExists } from './launcher/getFileIfExists' export { default as saveFiles } from './launcher/saveFiles' export { default as saveBills } from './launcher/saveBills' export { default as saveIdentity } from './launcher/saveIdentity' diff --git a/packages/cozy-clisk/src/launcher/getFileIfExists.js b/packages/cozy-clisk/src/launcher/getFileIfExists.js new file mode 100644 index 000000000..eef8ae044 --- /dev/null +++ b/packages/cozy-clisk/src/launcher/getFileIfExists.js @@ -0,0 +1,128 @@ +import Minilog from '@cozy/minilog' +import { Q } from 'cozy-client' +import get from 'lodash/get' + +const log = Minilog('getFileIfExists') + +module.exports = getFileIfExists + +async function getFileIfExists({ client, entry, options, folderPath, slug }) { + const fileIdAttributes = options.fileIdAttributes + const sourceAccountIdentifier = options.sourceAccountIdentifier + + const isReadyForFileMetadata = + fileIdAttributes && slug && sourceAccountIdentifier + if (isReadyForFileMetadata) { + log.debug('Detecting file with metadata') + const file = await getFileFromMetaData( + client, + entry, + fileIdAttributes, + sourceAccountIdentifier, + slug + ) + if (!file) { + // no file with correct metadata, maybe the corresponding file already exist in the default + // path from a previous version of the connector + log.debug('Rolling back on detection by filename') + return getFileFromPath(client, entry, folderPath) + } else { + return file + } + } else { + log.debug('Rolling back on detection by filename') + return getFileFromPath(client, entry, folderPath) + } +} + +async function getFileFromMetaData( + client, + entry, + fileIdAttributes, + sourceAccountIdentifier, + slug +) { + log.debug( + `Checking existence of ${calculateFileKey(entry, fileIdAttributes)}` + ) + const files = await client.queryAll( + Q('io.cozy.files') + .where({ + metadata: { + fileIdAttributes: calculateFileKey(entry, fileIdAttributes) + }, + trashed: false, + cozyMetadata: { + sourceAccountIdentifier, + createdByApp: slug + } + }) + .indexFields([ + 'metadata.fileIdAttributes', + 'trashed', + 'cozyMetadata.sourceAccountIdentifier', + 'cozyMetadata.createdByApp' + ]) + ) + if (files && files[0]) { + if (files.length > 1) { + log.warn( + `Found ${files.length} files corresponding to ${calculateFileKey( + entry, + fileIdAttributes + )}` + ) + } + return files[0] + } else { + log.debug('File not found') + return false + } +} + +async function getFileFromPath(client, entry, folderPath) { + try { + log.debug(`Checking existence of ${getFilePath({ entry, folderPath })}`) + const result = await client + .collection('io.cozy.files') + .statByPath(getFilePath({ entry, folderPath })) + return result.data + } catch (err) { + log.debug(err.message) + return false + } +} + +function getFilePath({ file, entry, folderPath }) { + if (file) { + return folderPath + '/' + getAttribute(file, 'name') + } else if (entry) { + return folderPath + '/' + getFileName(entry) + } +} + +function getAttribute(obj, attribute) { + return get(obj, `attributes.${attribute}`, get(obj, attribute)) +} + +function calculateFileKey(entry, fileIdAttributes) { + return fileIdAttributes + .sort() + .map(key => get(entry, key)) + .join('####') +} + +function getFileName(entry) { + let filename + if (entry.filename) { + filename = entry.filename + } else { + log.error('Could not get a file name for the entry') + return false + } + return sanitizeFileName(filename) +} + +function sanitizeFileName(filename) { + return filename.replace(/^\.+$/, '').replace(/[/?<>\\:*|":]/g, '') +} diff --git a/packages/cozy-clisk/src/launcher/getFileIfExists.spec.js b/packages/cozy-clisk/src/launcher/getFileIfExists.spec.js new file mode 100644 index 000000000..39c07a8a6 --- /dev/null +++ b/packages/cozy-clisk/src/launcher/getFileIfExists.spec.js @@ -0,0 +1,129 @@ +import getFileIfExists from './getFileIfExists' + +describe('getFileIfExists', function () { + function setup() { + const statByPath = jest.fn() + const queryAll = jest.fn() + const client = { + queryAll, + collection: () => ({ + statByPath + }) + } + return { + statByPath, + queryAll, + client + } + } + + it('should use metadata if fileIdAttributes, slug and sourceAccountIdentifier are present', async () => { + const { client, queryAll } = setup() + queryAll.mockResolvedValue([{ _id: 'abcd' }]) // Mocking the return with a simple file obj + const file = await getFileIfExists({ + client, + entry: { + filename: 'filename' + }, + options: { + fileIdAttributes: ['filename'], + sourceAccountIdentifier: 'Identifier' + }, + folderPath: '/fakepath', + slug: 'slug' + }) + // Test that the Q request cotains our parameters + expect(queryAll).toHaveBeenCalledWith( + expect.objectContaining({ + selector: expect.objectContaining({ + cozyMetadata: expect.objectContaining({ + createdByApp: 'slug', + sourceAccountIdentifier: 'Identifier' + }), + metadata: expect.objectContaining({ fileIdAttributes: 'filename' }) + }) + }) + ) + expect(file).toStrictEqual({ _id: 'abcd' }) + }) + + it('should roll back to filename when with metadata return nothing', async () => { + const { client, queryAll, statByPath } = setup() + queryAll.mockResolvedValue(undefined) // Mocking an empty return + statByPath.mockResolvedValue({ data: { _id: 'abcd' } }) + const file = await getFileIfExists({ + client, + entry: { + filename: 'filename' + }, + options: { + fileIdAttributes: ['filename'], + sourceAccountIdentifier: 'Identifier' + }, + folderPath: '/fakepath', + slug: 'slug' + }) + expect(queryAll).toHaveBeenCalledWith( + expect.objectContaining({ + selector: expect.objectContaining({ + cozyMetadata: expect.objectContaining({ + createdByApp: 'slug', + sourceAccountIdentifier: 'Identifier' + }), + metadata: expect.objectContaining({ fileIdAttributes: 'filename' }) + }) + }) + ) + expect(statByPath).toHaveBeenCalledWith('/fakepath/filename') + expect(file).toStrictEqual({ _id: 'abcd' }) + }) + + it('shoud use filename if sourceAccountIdentifier is missing', async () => { + const { statByPath, client } = setup() + statByPath.mockResolvedValue({ data: { _id: 'abcd' } }) + const file = await getFileIfExists({ + client, + entry: { + filename: 'filename' + }, + options: { fileIdAttributes: ['filename'] }, + folderPath: '/fakepath', + slug: 'slug' + }) + expect(statByPath).toHaveBeenCalledWith('/fakepath/filename') + expect(file).toStrictEqual({ _id: 'abcd' }) + }) + + it('shoud use filename if slug is missing', async () => { + const { statByPath, client } = setup() + statByPath.mockResolvedValue({ data: { _id: 'abcd' } }) + const file = await getFileIfExists({ + client, + entry: { + filename: 'filename' + }, + options: { + fileIdAttributes: ['filename'], + sourceAccountIdentifier: 'Identifier' + }, + folderPath: '/fakepath' + }) + expect(statByPath).toHaveBeenCalledWith('/fakepath/filename') + expect(file).toStrictEqual({ _id: 'abcd' }) + }) + it('shoud use filename if fileIdAttributes is missing', async () => { + const { statByPath, client } = setup() + statByPath.mockResolvedValue({ data: { _id: 'abcd' } }) + const file = await getFileIfExists({ + client, + entry: { + filename: 'filename' + }, + options: { sourceAccountIdentifier: 'Identifier' }, + folderPath: '/fakepath', + slug: 'slug' + }) + expect(statByPath).toHaveBeenCalledWith('/fakepath/filename') + expect(file).toStrictEqual({ _id: 'abcd' }) + }) +}) diff --git a/packages/cozy-clisk/src/launcher/saveBills.js b/packages/cozy-clisk/src/launcher/saveBills.js index db10d62b1..d417dc2b5 100644 --- a/packages/cozy-clisk/src/launcher/saveBills.js +++ b/packages/cozy-clisk/src/launcher/saveBills.js @@ -2,7 +2,7 @@ import hydrateAndFilter from './hydrateAndFilter' import addData from './addData' import Minilog from '@cozy/minilog' -import _ from 'lodash' +import _, { lowerCase } from 'lodash' const log = Minilog('saveBills') const DOCTYPE = 'io.cozy.bills' @@ -137,6 +137,7 @@ function convertCurrency(currency) { } function checkRequiredAttributes(entries) { + log.warn('Checking AAAABttributes') for (let entry of entries) { for (let attr in requiredAttributes) { if (entry[attr] == null) { @@ -144,9 +145,14 @@ function checkRequiredAttributes(entries) { `saveBills: an entry is missing the required ${attr} attribute` ) } + log.warn(attr) const checkFunction = requiredAttributes[attr] const isExpectedType = _(entry[attr])[checkFunction]() if (isExpectedType === false) { + log.warn(entry.date) + log.warn(typeof entry.date) + log.warn(_.isDate(entry.date)) + log.warn(entry) throw new Error( `saveBills: an entry has a ${attr} which does not respect ${checkFunction}` ) diff --git a/packages/cozy-clisk/src/launcher/saveBills.spec.js b/packages/cozy-clisk/src/launcher/saveBills.spec.js index 7a69f2cc7..f17508a1c 100644 --- a/packages/cozy-clisk/src/launcher/saveBills.spec.js +++ b/packages/cozy-clisk/src/launcher/saveBills.spec.js @@ -51,7 +51,7 @@ describe('saveBills', function () { ) }) it('should check dates in bills', async () => { - expect.assertions(1) + //expect.assertions(1) await expect(async () => { return saveBills( [ @@ -59,9 +59,14 @@ describe('saveBills', function () { filename: 'filename', fileurl: 'fileurl', amount: 12, - date: 'nodate', + // date: 'nodate', + date: Date.now(), vendor: 'vendor', +<<<<<<< Updated upstream fileDocument: { _id: 'testfileid' } +======= + //fileDocument: {_id: 'testfileid'} +>>>>>>> Stashed changes } ], { @@ -72,13 +77,19 @@ describe('saveBills', function () { 'saveBills: an entry has a date which does not respect isDate' ) }) +<<<<<<< Updated upstream it('should convert valid date strings in bills', async () => { const result = await saveBills( +======= + it('should save a bills with required attribute', async () => { + saveBills( +>>>>>>> Stashed changes [ { filename: 'filename', fileurl: 'fileurl', amount: 12, +<<<<<<< Updated upstream date: '2012-09-02', vendor: 'vendor', fileDocument: { _id: 'testfileid' } @@ -100,5 +111,15 @@ describe('saveBills', function () { vendor: 'vendor' } ]) +======= + // date: 'nodate', + date: Date.now(), + vendor: 'vendor', + fileDocument: {_id: 'testfileid'} + } + ] + ) + +>>>>>>> Stashed changes }) }) diff --git a/packages/cozy-clisk/src/launcher/saveFiles.js b/packages/cozy-clisk/src/launcher/saveFiles.js index f2eab1857..874ebfc54 100644 --- a/packages/cozy-clisk/src/launcher/saveFiles.js +++ b/packages/cozy-clisk/src/launcher/saveFiles.js @@ -45,7 +45,7 @@ const saveFiles = async (client, entries, folderPath, options = {}) => { noMetadataDeduplicationWarning(saveOptions) const canBeSaved = entry => entry.filestream - + const shouldBeSaved = entry => !entry.fileDocument let savedFiles = 0 const savedEntries = [] for (let entry of entries) { @@ -58,7 +58,8 @@ const saveFiles = async (client, entries, folderPath, options = {}) => { ) return } - if (canBeSaved(entry)) { + + if (canBeSaved(entry) && shouldBeSaved(entry)) { const resultFolderPath = await getOrCreateDestinationPath( entry, saveOptions @@ -84,11 +85,14 @@ const saveFiles = async (client, entries, folderPath, options = {}) => { } const saveEntry = async function (client, entry, options) { - let file = await getFileIfExists(client, entry, options) let shouldReplace = false - if (file) { + if (entry.fileDocument) { try { - shouldReplace = await shouldReplaceFile(file, entry, options) + shouldReplace = await shouldReplaceFile( + entry.fileDocument, + entry, + options + ) } catch (err) { log.info(`Error in shouldReplaceFile : ${err.message}`) shouldReplace = true @@ -97,13 +101,13 @@ const saveEntry = async function (client, entry, options) { let method = 'create' - if (shouldReplace && file) { + if (shouldReplace && entry.fileDocument) { method = 'updateById' - log.debug(`Will replace ${getFilePath({ options, file })}...`) + log.debug(`Will replace ${getFilePath({ options, entry })}...`) } try { - if (!file || method === 'updateById') { + if (!entry.fileDocument || method === 'updateById') { log.debug(omit(entry, 'filestream')) logFileStream(entry.filestream) log.debug( @@ -113,11 +117,17 @@ const saveEntry = async function (client, entry, options) { })} does not exist yet or is not valid` ) entry._cozy_file_to_create = true - file = await retry(createFile, { + entry.fileDocument = await retry(createFile, { interval: 1000, throw_original: true, max_tries: options.retry, - args: [client, entry, options, method, file ? file : undefined] + args: [ + client, + entry, + options, + method, + entry.fileDocument ? entry.fileDocument : undefined + ] }).catch(err => { if (err.message === 'BAD_DOWNLOADED_FILE') { log.warn( @@ -130,8 +140,6 @@ const saveEntry = async function (client, entry, options) { }) } - attachFileToEntry(entry, file) - sanitizeEntry(entry) if (options.postProcess) { await options.postProcess(entry) @@ -178,95 +186,6 @@ function noMetadataDeduplicationWarning(options) { } } -async function getFileIfExists(client, entry, options) { - const fileIdAttributes = options.fileIdAttributes - const slug = options.manifest.slug - const sourceAccountIdentifier = get( - options, - 'sourceAccountOptions.sourceAccountIdentifier' - ) - - const isReadyForFileMetadata = - fileIdAttributes && slug && sourceAccountIdentifier - if (isReadyForFileMetadata) { - const file = await getFileFromMetaData( - client, - entry, - fileIdAttributes, - sourceAccountIdentifier, - slug - ) - if (!file) { - // no file with correct metadata, maybe the corresponding file already exist in the default - // path from a previous version of the connector - return getFileFromPath(client, entry, options) - } else { - return file - } - } else { - return getFileFromPath(client, entry, options) - } -} - -async function getFileFromMetaData( - client, - entry, - fileIdAttributes, - sourceAccountIdentifier, - slug -) { - log.debug( - `Checking existence of ${calculateFileKey(entry, fileIdAttributes)}` - ) - const { data: files } = await client.queryAll( - Q('io.cozy.files') - .where({ - metadata: { - fileIdAttributes: calculateFileKey(entry, fileIdAttributes) - }, - trashed: false, - cozyMetadata: { - sourceAccountIdentifier, - createdByApp: slug - } - }) - .indexFields([ - 'metadata.fileIdAttributes', - 'trashed', - 'cozyMetadata.sourceAccountIdentifier', - 'cozyMetadata.createdByApp' - ]) - ) - - if (files && files[0]) { - if (files.length > 1) { - log.warn( - `Found ${files.length} files corresponding to ${calculateFileKey( - entry, - fileIdAttributes - )}` - ) - } - return files[0] - } else { - log.debug('not found') - return false - } -} - -async function getFileFromPath(client, entry, options) { - try { - log.debug(`Checking existence of ${getFilePath({ entry, options })}`) - const result = await client - .collection('io.cozy.files') - .statByPath(getFilePath({ entry, options })) - return result.data - } catch (err) { - log.debug(err.message) - return false - } -} - async function createFile(client, entry, options, method, file) { const folder = await client .collection('io.cozy.files') @@ -404,7 +323,6 @@ const removeFile = async function (client, file) { } module.exports = saveFiles -module.exports.getFileIfExists = getFileIfExists function getFileName(entry) { let filename @@ -495,11 +413,6 @@ function sanitizeEntry(entry) { return entry } -function attachFileToEntry(entry, fileDocument) { - entry.fileDocument = fileDocument - return entry -} - function getFilePath({ file, entry, options }) { const folderPath = options.folderPath if (file) {