diff --git a/spec/FilesRouter.spec.js b/spec/FilesRouter.spec.js new file mode 100644 index 0000000000..bbb1ad728b --- /dev/null +++ b/spec/FilesRouter.spec.js @@ -0,0 +1,79 @@ +const fs = require('fs'); +const path = require('path'); + +describe('FilesRouter', () => { + describe_only_db('mongo')('File Uploads', () => { + + beforeEach(async () => { + // Set the maxUploadSize to 1GB + await reconfigureServer({ + maxUploadSize: '1GB', + }); + }); + + const V8_STRING_LIMIT_BYTES = 536_870_912; + + /** + * Quick helper function to upload the file to the server via the REST API + * We do this because creating a Parse.File object with a file over 512MB + * will try to use the Web FileReader API, which will fail the test + * + * @param {string} fileName the name of the file + * @param {string} filePath the path to the file locally + * @returns + */ + const postFile = async (fileName, filePath) => { + const url = `${Parse.serverURL}/files/${fileName}`; + const headers = { + 'X-Parse-Application-Id': Parse.applicationId, + 'X-Parse-Master-Key': Parse.masterKey, + 'Content-Type': 'multipart/form-data', + }; + + const fileStream = fs.createReadStream(filePath); + + // Send the request + const response = await fetch(url, { + method: 'POST', + headers, + body: fileStream, + duplex: 'half' // This is required to send a stream + }); + + return response; + }; + + it('should allow Parse.File uploads under 512MB', async done => { + const filePath = path.join(__dirname, 'file.txt'); + await fs.promises.writeFile(filePath, Buffer.alloc(1024 * 1024)); + + const response = await postFile('file.txt', filePath); + expect(response.ok).toBe(true); + + fs.unlinkSync(filePath); + done(); + }); + + it('should allow Parse.File uploads exactly 512MB', async done => { + const filePath = path.join(__dirname, 'file.txt'); + await fs.promises.writeFile(filePath, Buffer.alloc(V8_STRING_LIMIT_BYTES)); + + const response = await postFile('file.txt', filePath); + expect(response.ok).toBe(true); + + fs.unlinkSync(filePath); + done(); + }); + + it('should allow Parse.File uploads over 512MB', async done => { + const filePath = path.join(__dirname, 'file.txt'); + await fs.promises.writeFile(filePath, Buffer.alloc(V8_STRING_LIMIT_BYTES + 50 * 1024 * 1024)); + + const response = await postFile('file.txt', filePath); + expect(response.ok).toBe(true); + + fs.unlinkSync(filePath); + done(); + }); + }); +}); diff --git a/spec/GridFSBucketStorageAdapter.spec.js b/spec/GridFSBucketStorageAdapter.spec.js index 7e9c84a59e..9ae3140340 100644 --- a/spec/GridFSBucketStorageAdapter.spec.js +++ b/spec/GridFSBucketStorageAdapter.spec.js @@ -9,7 +9,33 @@ async function expectMissingFile(gfsAdapter, name) { await gfsAdapter.getFileData(name); fail('should have thrown'); } catch (e) { - expect(e.message).toEqual('FileNotFound: file myFileName was not found'); + expect(e.message).toEqual(`FileNotFound: file ${name} was not found`); + } +} + +const TYPES = ['string', 'blob']; + +const createData = (type, data) => { + switch (type) { + case 'string': + return data; + case 'blob': + return new Blob([data]); + default: + throw new Error(`Invalid type: ${type}`); + } +} + +const getDataAsString = async (type, data, encoding = 'utf8') => { + switch (type) { + case 'string': + return data.toString(encoding); + case 'blob': + return (data instanceof Blob) + ? await data.text(encoding) : + data.toString(encoding); + default: + throw new Error(`Invalid type: ${type}`); } } @@ -43,13 +69,21 @@ describe_only_db('mongo')('GridFSBucket', () => { {}, '89E4AFF1-DFE4-4603-9574-BFA16BB446FD' ); - await expectMissingFile(encryptedAdapter, 'myFileName'); - const originalString = 'abcdefghi'; - await encryptedAdapter.createFile('myFileName', originalString); - const unencryptedResult = await unencryptedAdapter.getFileData('myFileName'); - expect(unencryptedResult.toString('utf8')).not.toBe(originalString); - const encryptedResult = await encryptedAdapter.getFileData('myFileName'); - expect(encryptedResult.toString('utf8')).toBe(originalString); + + for (const type of TYPES) { + const fileName = `myFileName-${type}`; + await expectMissingFile(encryptedAdapter, fileName); + const rawData = 'abcdefghi'; + + const originalData = createData(type, rawData); + await encryptedAdapter.createFile(fileName, originalData); + + const unencryptedResult = await unencryptedAdapter.getFileData(fileName); + expect(unencryptedResult.toString('utf8')).not.toBe(rawData); + + const encryptedResult = await encryptedAdapter.getFileData(fileName); + expect(encryptedResult.toString('utf8')).toBe(rawData); + } }); it('should rotate key of all unencrypted GridFS files to encrypted files', async () => { @@ -59,155 +93,144 @@ describe_only_db('mongo')('GridFSBucket', () => { {}, '89E4AFF1-DFE4-4603-9574-BFA16BB446FD' ); - const fileName1 = 'file1.txt'; - const data1 = 'hello world'; - const fileName2 = 'file2.txt'; - const data2 = 'hello new world'; - //Store unecrypted files - await unencryptedAdapter.createFile(fileName1, data1); - const unencryptedResult1 = await unencryptedAdapter.getFileData(fileName1); - expect(unencryptedResult1.toString('utf8')).toBe(data1); - await unencryptedAdapter.createFile(fileName2, data2); - const unencryptedResult2 = await unencryptedAdapter.getFileData(fileName2); - expect(unencryptedResult2.toString('utf8')).toBe(data2); - //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter - const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey(); - expect(rotated.length).toEqual(2); - expect( - rotated.filter(function (value) { - return value === fileName1; - }).length - ).toEqual(1); - expect( - rotated.filter(function (value) { - return value === fileName2; - }).length - ).toEqual(1); - expect(notRotated.length).toEqual(0); - let result = await encryptedAdapter.getFileData(fileName1); - expect(result instanceof Buffer).toBe(true); - expect(result.toString('utf-8')).toEqual(data1); - const encryptedData1 = await unencryptedAdapter.getFileData(fileName1); - expect(encryptedData1.toString('utf-8')).not.toEqual(unencryptedResult1); - result = await encryptedAdapter.getFileData(fileName2); - expect(result instanceof Buffer).toBe(true); - expect(result.toString('utf-8')).toEqual(data2); - const encryptedData2 = await unencryptedAdapter.getFileData(fileName2); - expect(encryptedData2.toString('utf-8')).not.toEqual(unencryptedResult2); + + for (const type of TYPES) { + const rawData = [`hello world ${type}`, `hello new world ${type}`]; + const fileNames = ['file1.txt', 'file2.txt']; + + // Store unencrypted files and verify + for (let i = 0; i < fileNames.length; i++) { + const data = createData(type, rawData[i]); + await unencryptedAdapter.createFile(fileNames[i], data); + const unencryptedResult = await unencryptedAdapter.getFileData(fileNames[i]); + expect(await getDataAsString(type, unencryptedResult)).toBe(rawData[i]); + } + + // Rotate encryption key and verify + const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey(); + expect(rotated.length).toEqual(fileNames.length); + fileNames.forEach(fileName => { + expect(rotated.includes(fileName)).toBe(true); + }); + expect(notRotated.length).toEqual(0); + + // clear files for next iteration + for (let i = 0; i < fileNames.length; i++) { + await unencryptedAdapter.deleteFile(fileNames[i]); + expectMissingFile(unencryptedAdapter, fileNames[i]); + } + } }); it('should rotate key of all old encrypted GridFS files to encrypted files', async () => { const oldEncryptionKey = 'oldKeyThatILoved'; const oldEncryptedAdapter = new GridFSBucketAdapter(databaseURI, {}, oldEncryptionKey); const encryptedAdapter = new GridFSBucketAdapter(databaseURI, {}, 'newKeyThatILove'); - const fileName1 = 'file1.txt'; - const data1 = 'hello world'; - const fileName2 = 'file2.txt'; - const data2 = 'hello new world'; - //Store unecrypted files - await oldEncryptedAdapter.createFile(fileName1, data1); - const oldEncryptedResult1 = await oldEncryptedAdapter.getFileData(fileName1); - expect(oldEncryptedResult1.toString('utf8')).toBe(data1); - await oldEncryptedAdapter.createFile(fileName2, data2); - const oldEncryptedResult2 = await oldEncryptedAdapter.getFileData(fileName2); - expect(oldEncryptedResult2.toString('utf8')).toBe(data2); - //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter - const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey({ - oldKey: oldEncryptionKey, - }); - expect(rotated.length).toEqual(2); - expect( - rotated.filter(function (value) { - return value === fileName1; - }).length - ).toEqual(1); - expect( - rotated.filter(function (value) { - return value === fileName2; - }).length - ).toEqual(1); - expect(notRotated.length).toEqual(0); - let result = await encryptedAdapter.getFileData(fileName1); - expect(result instanceof Buffer).toBe(true); - expect(result.toString('utf-8')).toEqual(data1); - let decryptionError1; - let encryptedData1; - try { - encryptedData1 = await oldEncryptedAdapter.getFileData(fileName1); - } catch (err) { - decryptionError1 = err; - } - expect(decryptionError1).toMatch('Error'); - expect(encryptedData1).toBeUndefined(); - result = await encryptedAdapter.getFileData(fileName2); - expect(result instanceof Buffer).toBe(true); - expect(result.toString('utf-8')).toEqual(data2); - let decryptionError2; - let encryptedData2; - try { - encryptedData2 = await oldEncryptedAdapter.getFileData(fileName2); - } catch (err) { - decryptionError2 = err; + + for (const type of TYPES) { + const rawData = [`hello world ${type}`, `hello new world ${type}`]; + const fileNames = ['file1.txt', 'file2.txt']; + + //Store unecrypted files + for (let i = 0; i < fileNames.length; i++) { + await oldEncryptedAdapter.createFile(fileNames[i], createData(type, rawData[i])); + const oldEncryptedResult = await oldEncryptedAdapter.getFileData(fileNames[i]); + expect(await getDataAsString(type, oldEncryptedResult)).toBe(rawData[i]); + } + + //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter + const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey({ + oldKey: oldEncryptionKey, + }); + expect(rotated.length).toEqual(2); + expect( + rotated.filter(function (value) { + return value === fileNames[0]; + }).length + ).toEqual(1); + expect( + rotated.filter(function (value) { + return value === fileNames[1]; + }).length + ).toEqual(1); + expect(notRotated.length).toEqual(0); + + // make sure old encrypted files can't be decrypted + for (let i = 0; i < fileNames.length; i++) { + const result = await encryptedAdapter.getFileData(fileNames[i]); + expect(result instanceof Buffer).toBe(true); + expect(await getDataAsString(type, result)).toEqual(rawData[i]); + + let decryptionError; + let encryptedData; + try { + encryptedData = await oldEncryptedAdapter.getFileData(fileNames[i]); + } catch (err) { + decryptionError = err; + } + expect(decryptionError).toMatch('Error'); + expect(encryptedData).toBeUndefined(); + + // clear files for next iteration + await oldEncryptedAdapter.deleteFile(fileNames[i]); + expectMissingFile(oldEncryptedAdapter, fileNames[i]); + } } - expect(decryptionError2).toMatch('Error'); - expect(encryptedData2).toBeUndefined(); }); it('should rotate key of all old encrypted GridFS files to unencrypted files', async () => { const oldEncryptionKey = 'oldKeyThatILoved'; const oldEncryptedAdapter = new GridFSBucketAdapter(databaseURI, {}, oldEncryptionKey); const unEncryptedAdapter = new GridFSBucketAdapter(databaseURI); - const fileName1 = 'file1.txt'; - const data1 = 'hello world'; - const fileName2 = 'file2.txt'; - const data2 = 'hello new world'; - //Store unecrypted files - await oldEncryptedAdapter.createFile(fileName1, data1); - const oldEncryptedResult1 = await oldEncryptedAdapter.getFileData(fileName1); - expect(oldEncryptedResult1.toString('utf8')).toBe(data1); - await oldEncryptedAdapter.createFile(fileName2, data2); - const oldEncryptedResult2 = await oldEncryptedAdapter.getFileData(fileName2); - expect(oldEncryptedResult2.toString('utf8')).toBe(data2); - //Check if unEncrypted adapter can read data and make sure it's not the same as oldEncrypted adapter - const { rotated, notRotated } = await unEncryptedAdapter.rotateEncryptionKey({ - oldKey: oldEncryptionKey, - }); - expect(rotated.length).toEqual(2); - expect( - rotated.filter(function (value) { - return value === fileName1; - }).length - ).toEqual(1); - expect( - rotated.filter(function (value) { - return value === fileName2; - }).length - ).toEqual(1); - expect(notRotated.length).toEqual(0); - let result = await unEncryptedAdapter.getFileData(fileName1); - expect(result instanceof Buffer).toBe(true); - expect(result.toString('utf-8')).toEqual(data1); - let decryptionError1; - let encryptedData1; - try { - encryptedData1 = await oldEncryptedAdapter.getFileData(fileName1); - } catch (err) { - decryptionError1 = err; - } - expect(decryptionError1).toMatch('Error'); - expect(encryptedData1).toBeUndefined(); - result = await unEncryptedAdapter.getFileData(fileName2); - expect(result instanceof Buffer).toBe(true); - expect(result.toString('utf-8')).toEqual(data2); - let decryptionError2; - let encryptedData2; - try { - encryptedData2 = await oldEncryptedAdapter.getFileData(fileName2); - } catch (err) { - decryptionError2 = err; + for (const type of TYPES) { + const rawData = [`hello world ${type}`, `hello new world ${type}`]; + const fileNames = ['file1.txt', 'file2.txt']; + + //Store unecrypted files + for (let i = 0; i < fileNames.length; i++) { + await oldEncryptedAdapter.createFile(fileNames[i], createData(type, rawData[i])); + const oldEncryptedResult = await oldEncryptedAdapter.getFileData(fileNames[i]); + expect(await getDataAsString(type, oldEncryptedResult)).toBe(rawData[i]); + } + + //Check if unEncrypted adapter can read data and make sure it's not the same as oldEncrypted adapter + const { rotated, notRotated } = await unEncryptedAdapter.rotateEncryptionKey({ + oldKey: oldEncryptionKey, + }); + expect(rotated.length).toEqual(2); + expect( + rotated.filter(function (value) { + return value === fileNames[0]; + }).length + ).toEqual(1); + expect( + rotated.filter(function (value) { + return value === fileNames[1]; + }).length + ).toEqual(1); + expect(notRotated.length).toEqual(0); + + // make sure the files can be decrypted by the new adapter + for (let i = 0; i < fileNames.length; i++) { + const result = await unEncryptedAdapter.getFileData(fileNames[i]); + expect(result instanceof Buffer).toBe(true); + expect(await getDataAsString(type, result)).toEqual(rawData[i]); + let decryptionError; + let encryptedData; + try { + encryptedData = await oldEncryptedAdapter.getFileData(fileNames[i]); + } catch (err) { + decryptionError = err; + } + expect(decryptionError).toMatch('Error'); + expect(encryptedData).toBeUndefined(); + + // clear files for next iteration + await oldEncryptedAdapter.deleteFile(fileNames[i]); + expectMissingFile(oldEncryptedAdapter, fileNames[i]); + } + } - expect(decryptionError2).toMatch('Error'); - expect(encryptedData2).toBeUndefined(); }); it('should only encrypt specified fileNames', async () => { @@ -215,67 +238,70 @@ describe_only_db('mongo')('GridFSBucket', () => { const oldEncryptedAdapter = new GridFSBucketAdapter(databaseURI, {}, oldEncryptionKey); const encryptedAdapter = new GridFSBucketAdapter(databaseURI, {}, 'newKeyThatILove'); const unEncryptedAdapter = new GridFSBucketAdapter(databaseURI); - const fileName1 = 'file1.txt'; - const data1 = 'hello world'; - const fileName2 = 'file2.txt'; - const data2 = 'hello new world'; - //Store unecrypted files - await oldEncryptedAdapter.createFile(fileName1, data1); - const oldEncryptedResult1 = await oldEncryptedAdapter.getFileData(fileName1); - expect(oldEncryptedResult1.toString('utf8')).toBe(data1); - await oldEncryptedAdapter.createFile(fileName2, data2); - const oldEncryptedResult2 = await oldEncryptedAdapter.getFileData(fileName2); - expect(oldEncryptedResult2.toString('utf8')).toBe(data2); - //Inject unecrypted file to see if causes an issue - const fileName3 = 'file3.txt'; - const data3 = 'hello past world'; - await unEncryptedAdapter.createFile(fileName3, data3, 'text/utf8'); - //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter - const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey({ - oldKey: oldEncryptionKey, - fileNames: [fileName1, fileName2], - }); - expect(rotated.length).toEqual(2); - expect( - rotated.filter(function (value) { - return value === fileName1; - }).length - ).toEqual(1); - expect( - rotated.filter(function (value) { - return value === fileName2; - }).length - ).toEqual(1); - expect(notRotated.length).toEqual(0); - expect( - rotated.filter(function (value) { - return value === fileName3; - }).length - ).toEqual(0); - let result = await encryptedAdapter.getFileData(fileName1); - expect(result instanceof Buffer).toBe(true); - expect(result.toString('utf-8')).toEqual(data1); - let decryptionError1; - let encryptedData1; - try { - encryptedData1 = await oldEncryptedAdapter.getFileData(fileName1); - } catch (err) { - decryptionError1 = err; - } - expect(decryptionError1).toMatch('Error'); - expect(encryptedData1).toBeUndefined(); - result = await encryptedAdapter.getFileData(fileName2); - expect(result instanceof Buffer).toBe(true); - expect(result.toString('utf-8')).toEqual(data2); - let decryptionError2; - let encryptedData2; - try { - encryptedData2 = await oldEncryptedAdapter.getFileData(fileName2); - } catch (err) { - decryptionError2 = err; + + for (const type of TYPES) { + const rawData = [`hello world ${type}`, `hello new world ${type}`]; + const fileNames = ['file1.txt', 'file2.txt']; + + //Store unecrypted files + for (let i = 0; i < fileNames.length; i++) { + await oldEncryptedAdapter.createFile(fileNames[i], createData(type, rawData[i])); + const oldEncryptedResult = await oldEncryptedAdapter.getFileData(fileNames[i]); + expect(await getDataAsString(type, oldEncryptedResult)).toBe(rawData[i]); + } + + + //Inject unecrypted file to see if causes an issue + const fileName3 = 'file3.txt'; + const data3 = 'hello past world'; + await unEncryptedAdapter.createFile(fileName3, data3, 'text/utf8'); + + //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter + const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey({ + oldKey: oldEncryptionKey, + fileNames, + }); + expect(rotated.length).toEqual(2); + expect( + rotated.filter(function (value) { + return value === fileNames[0]; + }).length + ).toEqual(1); + expect( + rotated.filter(function (value) { + return value === fileNames[1]; + }).length + ).toEqual(1); + expect(notRotated.length).toEqual(0); + expect( + rotated.filter(function (value) { + return value === fileName3; + }).length + ).toEqual(0); + + for (let i = 0; i < fileNames.length; i++) { + const result = await encryptedAdapter.getFileData(fileNames[i]); + expect(result instanceof Buffer).toBe(true); + expect(await getDataAsString(type, result)).toEqual(rawData[i]); + let decryptionError; + let encryptedData; + try { + encryptedData = await oldEncryptedAdapter.getFileData(fileNames[i]); + } catch (err) { + decryptionError = err; + } + expect(decryptionError).toMatch('Error'); + expect(encryptedData).toBeUndefined(); + + // clear files for next iteration + await oldEncryptedAdapter.deleteFile(fileNames[i]); + expectMissingFile(oldEncryptedAdapter, fileNames[i]); + } + + // clear file3 for next iteration + await unEncryptedAdapter.deleteFile(fileName3); + expectMissingFile(unEncryptedAdapter, fileName3); } - expect(decryptionError2).toMatch('Error'); - expect(encryptedData2).toBeUndefined(); }); it("should return fileNames of those it can't encrypt with the new key", async () => { @@ -283,66 +309,69 @@ describe_only_db('mongo')('GridFSBucket', () => { const oldEncryptedAdapter = new GridFSBucketAdapter(databaseURI, {}, oldEncryptionKey); const encryptedAdapter = new GridFSBucketAdapter(databaseURI, {}, 'newKeyThatILove'); const unEncryptedAdapter = new GridFSBucketAdapter(databaseURI); - const fileName1 = 'file1.txt'; - const data1 = 'hello world'; - const fileName2 = 'file2.txt'; - const data2 = 'hello new world'; - //Store unecrypted files - await oldEncryptedAdapter.createFile(fileName1, data1); - const oldEncryptedResult1 = await oldEncryptedAdapter.getFileData(fileName1); - expect(oldEncryptedResult1.toString('utf8')).toBe(data1); - await oldEncryptedAdapter.createFile(fileName2, data2); - const oldEncryptedResult2 = await oldEncryptedAdapter.getFileData(fileName2); - expect(oldEncryptedResult2.toString('utf8')).toBe(data2); - //Inject unecrypted file to see if causes an issue - const fileName3 = 'file3.txt'; - const data3 = 'hello past world'; - await unEncryptedAdapter.createFile(fileName3, data3, 'text/utf8'); - //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter - const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey({ - oldKey: oldEncryptionKey, - }); - expect(rotated.length).toEqual(2); - expect( - rotated.filter(function (value) { - return value === fileName1; - }).length - ).toEqual(1); - expect( - rotated.filter(function (value) { - return value === fileName2; - }).length - ).toEqual(1); - expect(notRotated.length).toEqual(1); - expect( - notRotated.filter(function (value) { - return value === fileName3; - }).length - ).toEqual(1); - let result = await encryptedAdapter.getFileData(fileName1); - expect(result instanceof Buffer).toBe(true); - expect(result.toString('utf-8')).toEqual(data1); - let decryptionError1; - let encryptedData1; - try { - encryptedData1 = await oldEncryptedAdapter.getFileData(fileName1); - } catch (err) { - decryptionError1 = err; - } - expect(decryptionError1).toMatch('Error'); - expect(encryptedData1).toBeUndefined(); - result = await encryptedAdapter.getFileData(fileName2); - expect(result instanceof Buffer).toBe(true); - expect(result.toString('utf-8')).toEqual(data2); - let decryptionError2; - let encryptedData2; - try { - encryptedData2 = await oldEncryptedAdapter.getFileData(fileName2); - } catch (err) { - decryptionError2 = err; + + for (const type of TYPES) { + const rawData = [`hello world ${type}`, `hello new world ${type}`]; + const fileNames = ['file1.txt', 'file2.txt']; + + //Store unecrypted files + for (let i = 0; i < fileNames.length; i++) { + await oldEncryptedAdapter.createFile(fileNames[i], createData(type, rawData[i])); + const oldEncryptedResult = await oldEncryptedAdapter.getFileData(fileNames[i]); + expect(await getDataAsString(type, oldEncryptedResult)).toBe(rawData[i]); + } + + //Inject unecrypted file to see if causes an issue + const fileName3 = 'file3.txt'; + const data3 = 'hello past world'; + await unEncryptedAdapter.createFile(fileName3, data3, 'text/utf8'); + + //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter + const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey({ + oldKey: oldEncryptionKey, + }); + expect(rotated.length).toEqual(2); + expect( + rotated.filter(function (value) { + return value === fileNames[0]; + }).length + ).toEqual(1); + expect( + rotated.filter(function (value) { + return value === fileNames[1]; + }).length + ).toEqual(1); + expect(notRotated.length).toEqual(1); + expect( + notRotated.filter(function (value) { + return value === fileName3; + }).length + ).toEqual(1); + + // make sure the files can be decrypted by the new adapter + for (let i = 0; i < fileNames.length; i++) { + const result = await encryptedAdapter.getFileData(fileNames[i]); + expect(result instanceof Buffer).toBe(true); + expect(await getDataAsString(type, result)).toEqual(rawData[i]); + let decryptionError; + let encryptedData; + try { + encryptedData = await oldEncryptedAdapter.getFileData(fileNames[i]); + } catch (err) { + decryptionError = err; + } + expect(decryptionError).toMatch('Error'); + expect(encryptedData).toBeUndefined(); + + // clear files for next iteration + await oldEncryptedAdapter.deleteFile(fileNames[i]); + expectMissingFile(oldEncryptedAdapter, fileNames[i]); + + } + // clear file3 for next iteration + await unEncryptedAdapter.deleteFile(fileName3); + expectMissingFile(unEncryptedAdapter, fileName3); } - expect(decryptionError2).toMatch('Error'); - expect(encryptedData2).toBeUndefined(); }); it('should save metadata', async () => { @@ -360,6 +389,20 @@ describe_only_db('mongo')('GridFSBucket', () => { // Empty json for file not found gfsMetadata = await gfsAdapter.getMetadata('myUnknownFile'); expect(gfsMetadata).toEqual({}); + + // now do the same for blob + const originalBlob = new Blob([originalString]); + await gfsAdapter.createFile('myFileNameBlob', originalBlob, null, { + metadata, + }); + const gfsResultBlob = await gfsAdapter.getFileData('myFileNameBlob'); + expect(await getDataAsString('blob', gfsResultBlob)).toBe(originalString); + gfsMetadata = await gfsAdapter.getMetadata('myFileNameBlob'); + expect(gfsMetadata.metadata).toEqual(metadata); + + // Empty json for file not found + gfsMetadata = await gfsAdapter.getMetadata('myUnknownFileBlob'); + expect(gfsMetadata).toEqual({}); }); it('should save metadata with file', async () => { @@ -441,6 +484,84 @@ describe_only_db('mongo')('GridFSBucket', () => { await expectMissingFile(gfsAdapter, 'myFileName'); }); + + it('should reject if there is an error in cipher update', async () => { + const gfsAdapter = new GridFSBucketAdapter(databaseURI, {}, 'encryptionKey'); + const error = new Error('Cipher error'); + const crypto = require('crypto'); + + // Mock the createCipheriv method to return a mocked cipher object + spyOn(crypto, 'createCipheriv').and.returnValue({ + // eslint-disable-next-line no-unused-vars + update: (_chunk) => { + throw error; + }, + final: () => { + return Buffer.from('encryptedData'); + }, + }); + + for (const type of TYPES) { + try { + await gfsAdapter.createFile(`testfile-${type}.txt`, createData(type, 'testdata')); + fail('Expected error not thrown'); + } catch (err) { + expect(err).toEqual(jasmine.any(Error)); + expect(err.message).toBe(error.message); + } + } + // Restore the original method + crypto.createCipheriv.and.callThrough(); + }); + + + it('should reject if there is an error in cipher final', async () => { + const gfsAdapter = new GridFSBucketAdapter(databaseURI, {}, 'encryptionKey'); + const error = new Error('Cipher error'); + const crypto = require('crypto'); + + // Mock the createCipheriv method to return a mocked cipher object + spyOn(crypto, 'createCipheriv').and.returnValue({ + // eslint-disable-next-line no-unused-vars + update: (_chunk) => { + return Buffer.from('encryptedData'); + }, + final: () => { + throw error; + }, + }); + + for (const type of TYPES) { + try { + await gfsAdapter.createFile(`testfile-${type}.txt`, createData(type, 'testdata')); + fail('Expected error not thrown'); + } catch (err) { + expect(err).toEqual(jasmine.any(Error)); + expect(err.message).toBe(error.message); + } + } + // Restore the original method + crypto.createCipheriv.and.callThrough(); + }); + + it ('should handle error in createFile when _getBucket is called', async () => { + const error = new Error('Error in createFile'); + const gfsAdapter = new GridFSBucketAdapter(databaseURI); + spyOn(gfsAdapter, '_getBucket').and.throwError(error); + + for (const type of TYPES) { + try { + await gfsAdapter.createFile(`testfile-${type}.txt`, createData(type, 'testdata')); + fail('Expected error not thrown'); + } catch (err) { + expect(err).toEqual(jasmine.any(Error)); + expect(err.message).toBe(error.message); + } + } + // Restore the original method + gfsAdapter._getBucket.and.callThrough(); + }); + it('handleShutdown, close connection', async () => { const databaseURI = 'mongodb://localhost:27017/parse'; const gfsAdapter = new GridFSBucketAdapter(databaseURI); diff --git a/src/Adapters/Files/FilesAdapter.js b/src/Adapters/Files/FilesAdapter.js index f06c52df89..334e753803 100644 --- a/src/Adapters/Files/FilesAdapter.js +++ b/src/Adapters/Files/FilesAdapter.js @@ -26,7 +26,7 @@ export class FilesAdapter { /** Responsible for storing the file in order to be retrieved later by its filename * * @param {string} filename - the filename to save - * @param {*} data - the buffer of data from the file + * @param {*} data - the representation of data from the file as Buffer or Blob * @param {string} contentType - the supposed contentType * @discussion the contentType can be undefined if the controller was not able to determine it * @param {object} options - (Optional) options to be passed to file adapter (S3 File Adapter Only) diff --git a/src/Adapters/Files/GridFSBucketAdapter.js b/src/Adapters/Files/GridFSBucketAdapter.js index 242fc08a0d..2204ed1680 100644 --- a/src/Adapters/Files/GridFSBucketAdapter.js +++ b/src/Adapters/Files/GridFSBucketAdapter.js @@ -11,6 +11,7 @@ import { MongoClient, GridFSBucket, Db } from 'mongodb'; import { FilesAdapter, validateFilename } from './FilesAdapter'; import defaults from '../../defaults'; const crypto = require('crypto'); +const { Transform, Readable } = require('stream'); export class GridFSBucketAdapter extends FilesAdapter { _databaseURI: string; @@ -66,29 +67,81 @@ export class GridFSBucketAdapter extends FilesAdapter { const stream = await bucket.openUploadStream(filename, { metadata: options.metadata, }); - if (this._encryptionKey !== null) { + + return new Promise((resolve, reject) => { try { - const iv = crypto.randomBytes(16); - const cipher = crypto.createCipheriv(this._algorithm, this._encryptionKey, iv); - const encryptedResult = Buffer.concat([ - cipher.update(data), - cipher.final(), - iv, - cipher.getAuthTag(), - ]); - await stream.write(encryptedResult); - } catch (err) { - return new Promise((resolve, reject) => { - return reject(err); - }); + const iv = this._encryptionKey !== null + ? crypto.randomBytes(16) + : null; + + const cipher = this._encryptionKey !== null && iv + ? crypto.createCipheriv(this._algorithm, this._encryptionKey, iv) + : null; + + // when working with a Blob, it could be over the max size of a buffer, so we need to stream it + if (data instanceof Blob) { + let readableStream = data.stream(); + + // may come in as a web stream, so we need to convert it to a node stream + if (readableStream instanceof ReadableStream) { + readableStream = Readable.fromWeb(readableStream); + } + + if (cipher && iv) { + // we need to stream the data through the cipher + const cipherTransform = new Transform({ + transform(chunk, encoding, callback) { + try { + const encryptedChunk = cipher.update(chunk); + callback(null, encryptedChunk); + } catch (err) { + callback(err); + } + }, + // at the end we need to push the final cipher text, iv, and auth tag + flush(callback) { + try { + this.push(cipher.final()); + this.push(iv); + this.push(cipher.getAuthTag()); + callback(); + } catch (err) { + callback(err); + } + } + }); + // pipe the stream through the cipher and then to the gridfs stream + readableStream + .pipe(cipherTransform) + .on('error', reject) + .pipe(stream) + .on('error', reject); + } else { + // if we don't have a cipher, we can just pipe the stream to the gridfs stream + readableStream.pipe(stream) + .on('error', reject) + } + } else { + if (cipher && iv) { + const encryptedResult = Buffer.concat([ + cipher.update(data), + cipher.final(), + iv, + cipher.getAuthTag(), + ]); + stream.write(encryptedResult); + + } else { + stream.write(data); + } + stream.end(); + } + + stream.on('finish', resolve); + stream.on('error', reject); + } catch (e) { + reject(e); } - } else { - await stream.write(data); - } - stream.end(); - return new Promise((resolve, reject) => { - stream.on('finish', resolve); - stream.on('error', reject); }); } diff --git a/src/Routers/FilesRouter.js b/src/Routers/FilesRouter.js index 13aab81548..94fc89b451 100644 --- a/src/Routers/FilesRouter.js +++ b/src/Routers/FilesRouter.js @@ -172,8 +172,18 @@ export class FilesRouter { } } - const base64 = req.body.toString('base64'); - const file = new Parse.File(filename, { base64 }, contentType); + // If the request body is a buffer and it's size is greater than the V8 string size limit + // we need to use a Blob to avoid the V8 string size limit + const MAX_V8_STRING_SIZE_BYTES = 536_870_912; + + let file; + + if (Buffer.isBuffer(req.body) && req.body?.length >= MAX_V8_STRING_SIZE_BYTES) { + file = new Parse.File(filename, new Blob([req.body]), contentType); + } else { + file = new Parse.File(filename, { base64: req.body.toString('base64') }, contentType); + } + const { metadata = {}, tags = {} } = req.fileData || {}; try { // Scan request data for denied keywords @@ -213,8 +223,18 @@ export class FilesRouter { // if the ParseFile returned is type uri, download the file before saving it await addFileDataIfNeeded(fileObject.file); // update fileSize - const bufferData = Buffer.from(fileObject.file._data, 'base64'); - fileObject.fileSize = Buffer.byteLength(bufferData); + let fileData; + // if the file is a blob, get the size from the blob + if (fileObject.file._source?.file instanceof Blob) { + // get the size of the blob + fileObject.fileSize = fileObject.file._source.file.size; + // set the file data + fileData = fileObject.file._source?.file; + } else { + const bufferData = Buffer.from(fileObject.file._data, 'base64'); + fileObject.fileSize = Buffer.byteLength(bufferData); + fileData = bufferData; + } // prepare file options const fileOptions = { metadata: fileObject.file._metadata, @@ -228,7 +248,7 @@ export class FilesRouter { const createFileResult = await filesController.createFile( config, fileObject.file._name, - bufferData, + fileData, fileObject.file._source.type, fileOptions );