feat(cozy-clisk): Implementing selective file dowload and fixing meta…

…data deduplication

packages/cozy-clisk/src/index.js

@@ -5,6 +5,7 @@ export { MessengerInterface } from './bridge/bridgeInterfaces'
 export { default as ContentScript } from './contentscript/ContentScript'
 export { default as addData } from './launcher/addData'
 export { default as hydrateAndFilter } from './launcher/hydrateAndFilter'
+export { default as getFileIfExists } from './launcher/getFileIfExists'
 export { default as saveFiles } from './launcher/saveFiles'
 export { default as saveBills } from './launcher/saveBills'
 export { default as saveIdentity } from './launcher/saveIdentity'

packages/cozy-clisk/src/launcher.js

@@ -1,5 +1,6 @@
 export { default as addData } from './launcher/addData'
 export { default as hydrateAndFilter } from './launcher/hydrateAndFilter'
+export { default as getFileIfExists } from './launcher/getFileIfExists'
 export { default as saveFiles } from './launcher/saveFiles'
 export { default as saveBills } from './launcher/saveBills'
 export { default as saveIdentity } from './launcher/saveIdentity'

packages/cozy-clisk/src/launcher/getFileIfExists.js

@@ -0,0 +1,128 @@
+import Minilog from '@cozy/minilog'
+import { Q } from 'cozy-client'
+import get from 'lodash/get'
+
+const log = Minilog('getFileIfExists')
+
+module.exports = getFileIfExists
+
+async function getFileIfExists({ client, entry, options, folderPath, slug }) {
+  const fileIdAttributes = options.fileIdAttributes
+  const sourceAccountIdentifier = options.sourceAccountIdentifier
+
+  const isReadyForFileMetadata =
+    fileIdAttributes && slug && sourceAccountIdentifier
+  if (isReadyForFileMetadata) {
+    log.debug('Detecting file with metadata')
+    const file = await getFileFromMetaData(
+      client,
+      entry,
+      fileIdAttributes,
+      sourceAccountIdentifier,
+      slug
+    )
+    if (!file) {
+      // no file with correct metadata, maybe the corresponding file already exist in the default
+      // path from a previous version of the connector
+      log.debug('Rolling back on detection by filename')
+      return getFileFromPath(client, entry, folderPath)
+    } else {
+      return file
+    }
+  } else {
+    log.debug('Rolling back on detection by filename')
+    return getFileFromPath(client, entry, folderPath)
+  }
+}
+
+async function getFileFromMetaData(
+  client,
+  entry,
+  fileIdAttributes,
+  sourceAccountIdentifier,
+  slug
+) {
+  log.debug(
+    `Checking existence of ${calculateFileKey(entry, fileIdAttributes)}`
+  )
+  const files = await client.queryAll(
+    Q('io.cozy.files')
+      .where({
+        metadata: {
+          fileIdAttributes: calculateFileKey(entry, fileIdAttributes)
+        },
+        trashed: false,
+        cozyMetadata: {
+          sourceAccountIdentifier,
+          createdByApp: slug
+        }
+      })
+      .indexFields([
+        'metadata.fileIdAttributes',
+        'trashed',
+        'cozyMetadata.sourceAccountIdentifier',
+        'cozyMetadata.createdByApp'
+      ])
+  )
+  if (files && files[0]) {
+    if (files.length > 1) {
+      log.warn(
+        `Found ${files.length} files corresponding to ${calculateFileKey(
+          entry,
+          fileIdAttributes
+        )}`
+      )
+    }
+    return files[0]
+  } else {
+    log.debug('File not found')
+    return false
+  }
+}
+
+async function getFileFromPath(client, entry, folderPath) {
+  try {
+    log.debug(`Checking existence of ${getFilePath({ entry, folderPath })}`)
+    const result = await client
+      .collection('io.cozy.files')
+      .statByPath(getFilePath({ entry, folderPath }))
+    return result.data
+  } catch (err) {
+    log.debug(err.message)
+    return false
+  }
+}
+
+function getFilePath({ file, entry, folderPath }) {
+  if (file) {
+    return folderPath + '/' + getAttribute(file, 'name')
+  } else if (entry) {
+    return folderPath + '/' + getFileName(entry)
+  }
+}
+
+function getAttribute(obj, attribute) {
+  return get(obj, `attributes.${attribute}`, get(obj, attribute))
+}
+
+function calculateFileKey(entry, fileIdAttributes) {
+  return fileIdAttributes
+    .sort()
+    .map(key => get(entry, key))
+    .join('####')
+}
+
+function getFileName(entry) {
+  let filename
+  if (entry.filename) {
+    filename = entry.filename
+  } else {
+    log.error('Could not get a file name for the entry')
+    return false
+  }
+  return sanitizeFileName(filename)
+}
+
+function sanitizeFileName(filename) {
+  return filename.replace(/^\.+$/, '').replace(/[/?<>\\:*|":]/g, '')
+}

packages/cozy-clisk/src/launcher/getFileIfExists.spec.js

@@ -0,0 +1,129 @@
+import getFileIfExists from './getFileIfExists'
+
+describe('getFileIfExists', function () {
+  function setup() {
+    const statByPath = jest.fn()
+    const queryAll = jest.fn()
+    const client = {
+      queryAll,
+      collection: () => ({
+        statByPath
+      })
+    }
+    return {
+      statByPath,
+      queryAll,
+      client
+    }
+  }
+
+  it('should use metadata if fileIdAttributes, slug and sourceAccountIdentifier are present', async () => {
+    const { client, queryAll } = setup()
+    queryAll.mockResolvedValue([{ _id: 'abcd' }]) // Mocking the return with a simple file obj
+    const file = await getFileIfExists({
+      client,
+      entry: {
+        filename: 'filename'
+      },
+      options: {
+        fileIdAttributes: ['filename'],
+        sourceAccountIdentifier: 'Identifier'
+      },
+      folderPath: '/fakepath',
+      slug: 'slug'
+    })
+    // Test that the Q request cotains our parameters
+    expect(queryAll).toHaveBeenCalledWith(
+      expect.objectContaining({
+        selector: expect.objectContaining({
+          cozyMetadata: expect.objectContaining({
+            createdByApp: 'slug',
+            sourceAccountIdentifier: 'Identifier'
+          }),
+          metadata: expect.objectContaining({ fileIdAttributes: 'filename' })
+        })
+      })
+    )
+    expect(file).toStrictEqual({ _id: 'abcd' })
+  })
+
+  it('should roll back to filename when with metadata return nothing', async () => {
+    const { client, queryAll, statByPath } = setup()
+    queryAll.mockResolvedValue(undefined) // Mocking an empty return
+    statByPath.mockResolvedValue({ data: { _id: 'abcd' } })
+    const file = await getFileIfExists({
+      client,
+      entry: {
+        filename: 'filename'
+      },
+      options: {
+        fileIdAttributes: ['filename'],
+        sourceAccountIdentifier: 'Identifier'
+      },
+      folderPath: '/fakepath',
+      slug: 'slug'
+    })
+    expect(queryAll).toHaveBeenCalledWith(
+      expect.objectContaining({
+        selector: expect.objectContaining({
+          cozyMetadata: expect.objectContaining({
+            createdByApp: 'slug',
+            sourceAccountIdentifier: 'Identifier'
+          }),
+          metadata: expect.objectContaining({ fileIdAttributes: 'filename' })
+        })
+      })
+    )
+    expect(statByPath).toHaveBeenCalledWith('/fakepath/filename')
+    expect(file).toStrictEqual({ _id: 'abcd' })
+  })
+
+  it('shoud use filename if sourceAccountIdentifier is missing', async () => {
+    const { statByPath, client } = setup()
+    statByPath.mockResolvedValue({ data: { _id: 'abcd' } })
+    const file = await getFileIfExists({
+      client,
+      entry: {
+        filename: 'filename'
+      },
+      options: { fileIdAttributes: ['filename'] },
+      folderPath: '/fakepath',
+      slug: 'slug'
+    })
+    expect(statByPath).toHaveBeenCalledWith('/fakepath/filename')
+    expect(file).toStrictEqual({ _id: 'abcd' })
+  })
+
+  it('shoud use filename if slug is missing', async () => {
+    const { statByPath, client } = setup()
+    statByPath.mockResolvedValue({ data: { _id: 'abcd' } })
+    const file = await getFileIfExists({
+      client,
+      entry: {
+        filename: 'filename'
+      },
+      options: {
+        fileIdAttributes: ['filename'],
+        sourceAccountIdentifier: 'Identifier'
+      },
+      folderPath: '/fakepath'
+    })
+    expect(statByPath).toHaveBeenCalledWith('/fakepath/filename')
+    expect(file).toStrictEqual({ _id: 'abcd' })
+  })
+  it('shoud use filename if fileIdAttributes is missing', async () => {
+    const { statByPath, client } = setup()
+    statByPath.mockResolvedValue({ data: { _id: 'abcd' } })
+    const file = await getFileIfExists({
+      client,
+      entry: {
+        filename: 'filename'
+      },
+      options: { sourceAccountIdentifier: 'Identifier' },
+      folderPath: '/fakepath',
+      slug: 'slug'
+    })
+    expect(statByPath).toHaveBeenCalledWith('/fakepath/filename')
+    expect(file).toStrictEqual({ _id: 'abcd' })
+  })
+})