adjust loader to consider row count on row 1 (#50)

DEFRA · Dec 6, 2024 · efd0f91 · efd0f91
1 parent 394979a
commit efd0f91
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 10 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-etl-framework",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "publisher": "Defra",
   "main": "dist/cjs/index.js",
   "private": false,

diff --git a/src/loaders/csvloader.js b/src/loaders/csvloader.js
@@ -16,7 +16,7 @@ function CSVLoader(options){
     csvLoader._columns = options.columns
     csvLoader.pump = (csvLoader) => {
         return csvLoader
-            .pipe(parse({ delimiter: ",", from_line: 2 }))
+            .pipe(parse({ delimiter: ",", from_line: 3 }))
             .pipe(new Transform({
                 readableObjectMode: true,
                 writableObjectMode: true,

diff --git a/test/etl/etl.test.js b/test/etl/etl.test.js
@@ -11,6 +11,7 @@ describe('ETL tests', () => {
   })
   it('should fire finish event', (done) => {
     const testData = [
+      "2\n",
       "column1, column2, column3\n",
       "1,2,3\n",
       "4,5,6\n"
@@ -34,7 +35,9 @@ describe('ETL tests', () => {
     })
   })
   it('should fire result event', (done) => {
+    jest.setTimeout(10000)
     const testData = [
+      "2\n",
       "column1, column2, column3\n",
       "1,2,3\n",
       "4,5,6\n"

diff --git a/test/loaders/csvLoader.test.js b/test/loaders/csvLoader.test.js
@@ -8,11 +8,12 @@ jest.mock('fs')
 describe('csvLoader tests', () => {
     it('should load a csv file', (done) => {
         const testData = [
+            "2\n",
             "column1, column2, column3\n",
             "1,2,3\n",
             "4,5,6\n"
         ]
-        let lineCount = 1
+        let lineCount = 2
         const testPath = "someRandomPath"
         fs.__setMockFileContent(testPath, testData)
         const uut = CSVLoader({ path: testPath, columns: ["a","b","c"]})
@@ -22,15 +23,17 @@ describe('csvLoader tests', () => {
                 objectMode: true,
                 transform(chunk, _, callback){
                     expect(chunk.join(",")).toEqual(testData[lineCount].replace(/\n/,""))
-                    if(lineCount === testData.length - 1) //Ignore header row
+                    if(lineCount === testData.length - 1)
                         done()
                     lineCount +=1
                     callback(null, chunk)
                 }
             }))
     })
     it('should count csv file lines', (done) => {
+        jest.setTimeout(10000)
         const testData = [
+            "2\n",
             "column1, column2, column3\n",
             "1,2,3\n",
             "4,5,6\n"
@@ -45,7 +48,7 @@ describe('csvLoader tests', () => {
                 objectMode: true,
                 transform(chunk, _, callback){
                     expect(chunk._linecount).toEqual(lineCount)
-                    if(lineCount === testData.length - 1) //Ignore header row
+                    if(lineCount === testData.length - 2) //Ignore header row
                         done()
                     lineCount +=1
                     callback(null, chunk)