diff --git a/package-lock.json b/package-lock.json index 3663cc6..742d631 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ffc-pay-etl-framework", - "version": "0.1.13", + "version": "1.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ffc-pay-etl-framework", - "version": "0.1.13", + "version": "1.1.2", "license": "OGL-UK-3.0", "dependencies": { "@faker-js/faker": "^8.4.1", @@ -2042,9 +2042,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "dependencies": { "path-key": "^3.1.0", diff --git a/package.json b/package.json index 00b796f..8eae875 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ffc-pay-etl-framework", - "version": "1.1.1", + "version": "1.1.2", "publisher": "Defra", "main": "dist/cjs/index.js", "private": false, diff --git a/src/loaders/csvloader.js b/src/loaders/csvloader.js index 6404fa8..08e10fa 100644 --- a/src/loaders/csvloader.js +++ b/src/loaders/csvloader.js @@ -16,7 +16,7 @@ function CSVLoader(options){ csvLoader._columns = options.columns csvLoader.pump = (csvLoader) => { return csvLoader - .pipe(parse({ delimiter: ",", from_line: 2 })) + .pipe(parse({ delimiter: ",", from_line: 3 })) .pipe(new Transform({ readableObjectMode: true, writableObjectMode: true, diff --git a/test/etl/etl.test.js b/test/etl/etl.test.js index 5c423b6..1f3d12a 100644 --- a/test/etl/etl.test.js +++ b/test/etl/etl.test.js @@ -11,6 +11,7 @@ describe('ETL tests', () => { }) it('should fire finish event', (done) => { const testData = [ + "2\n", "column1, column2, column3\n", "1,2,3\n", "4,5,6\n" @@ -34,7 +35,9 @@ describe('ETL tests', () => { }) }) it('should fire result event', (done) => { + jest.setTimeout(10000) const testData = [ + "2\n", "column1, column2, column3\n", "1,2,3\n", "4,5,6\n" diff --git a/test/loaders/csvLoader.test.js b/test/loaders/csvLoader.test.js index 2a1cea4..c04f59a 100644 --- a/test/loaders/csvLoader.test.js +++ b/test/loaders/csvLoader.test.js @@ -8,11 +8,12 @@ jest.mock('fs') describe('csvLoader tests', () => { it('should load a csv file', (done) => { const testData = [ + "2\n", "column1, column2, column3\n", "1,2,3\n", "4,5,6\n" ] - let lineCount = 1 + let lineCount = 2 const testPath = "someRandomPath" fs.__setMockFileContent(testPath, testData) const uut = CSVLoader({ path: testPath, columns: ["a","b","c"]}) @@ -22,7 +23,7 @@ describe('csvLoader tests', () => { objectMode: true, transform(chunk, _, callback){ expect(chunk.join(",")).toEqual(testData[lineCount].replace(/\n/,"")) - if(lineCount === testData.length - 1) //Ignore header row + if(lineCount === testData.length - 1) done() lineCount +=1 callback(null, chunk) @@ -30,7 +31,9 @@ describe('csvLoader tests', () => { })) }) it('should count csv file lines', (done) => { + jest.setTimeout(10000) const testData = [ + "2\n", "column1, column2, column3\n", "1,2,3\n", "4,5,6\n" @@ -45,7 +48,7 @@ describe('csvLoader tests', () => { objectMode: true, transform(chunk, _, callback){ expect(chunk._linecount).toEqual(lineCount) - if(lineCount === testData.length - 1) //Ignore header row + if(lineCount === testData.length - 2) //Ignore header row done() lineCount +=1 callback(null, chunk)