use to_timestamp instead of to_date (#41)

DEFRA · Oct 15, 2024 · 394979a · 394979a
1 parent 54a05bb
commit 394979a
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 11 deletions.
diff --git a/.all-contributorsrc b/.all-contributorsrc
@@ -21,12 +21,21 @@
     },
     {
       "login": "GodsonLeigh",
-      "name": "GodsonLeigh",
+      "name": "Leigh Godson",
       "avatar_url": "https://avatars.githubusercontent.com/u/139965284?v=4",
       "profile": "https://github.com/GodsonLeigh",
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "samplackett",
+      "name": "Sam Plackett",
+      "avatar_url": "https://avatars.githubusercontent.com/u/60177449?v=4",
+      "profile": "https://github.com/samplackett",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
 
-[![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-3-orange.svg?style=flat-square)](#contributors-)
 
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
@@ -98,8 +98,8 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
   <tbody>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/suityou01"><img src="https://avatars.githubusercontent.com/u/7422644?v=4?s=100" width="100px;" alt="Charlie Benger-Stevenson"/><br /><sub><b>Charlie Benger-Stevenson</b></sub></a><br /><a href="https://github.com/DEFRA/ffc-pay-etl-framework/commits?author=suityou01" title="Code">💻</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/GodsonLeigh"><img src="https://avatars.githubusercontent.com/u/139965284?v=4?s=100" width="100px;" alt="GodsonLeigh"/><br /><sub><b>GodsonLeigh</b></sub></a><br /><a href="https://github.com/DEFRA/ffc-pay-etl-framework/commits?author=GodsonLeigh" title="Code">💻</a></td>
-    </tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/GodsonLeigh"><img src="https://avatars.githubusercontent.com/u/139965284?v=4?s=100" width="100px;" alt="Leigh Godson"/><br /><sub><b>Leigh Godson</b></sub></a><br /><a href="https://github.com/DEFRA/ffc-pay-etl-framework/commits?author=GodsonLeigh" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/samplackett"><img src="https://avatars.githubusercontent.com/u/60177449?v=4?s=100" width="100px;" alt="Sam Plackett"/><br /><sub><b>Sam Plackett</b></sub></a><br /><a href="https://github.com/DEFRA/ffc-pay-etl-framework/commits?author=samplackett" title="Code">💻</a></td>    </tr>
   </tbody>
   <tfoot>
     <tr>

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-etl-framework",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "publisher": "Defra",
   "main": "dist/cjs/index.js",
   "private": false,

diff --git a/src/destinations/postgresDestination.js b/src/destinations/postgresDestination.js
@@ -42,8 +42,11 @@ function writeInsertStatement(columnMapping, table, chunk){
         if(mapping?.targetType === "varchar" || mapping?.targetType === "char"){
             return `'${chunk[index]}'`
         }
-        if(mapping?.targetType === "date"){
-            return `to_date('${chunk[index]}','${mapping?.format}')`
+        if (mapping?.targetType === "date") {
+            if (!chunk[index]) {
+                return `''`
+            }
+            return `to_timestamp('${chunk[index]}','${mapping?.format}')`
         }
         return chunk[index] ? chunk[index] : 'null'
     })})`

diff --git a/test/destinations/postgresDestination.test.js b/test/destinations/postgresDestination.test.js
@@ -133,7 +133,7 @@ describe('postgresDestination tests', () => {
         const readable = Readable.from([testData])
         readable
             .on('close', (result) => {
-                expect(mockConnection.db.query).toHaveBeenLastCalledWith("INSERT INTO target (target_column1,target_column2,target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
+                expect(mockConnection.db.query).toHaveBeenLastCalledWith("INSERT INTO target (target_column1,target_column2,target_column3) VALUES ('a',to_timestamp('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
                 done()
             })
             .pipe(uut)
@@ -157,7 +157,19 @@ describe('postgresDestination tests', () => {
         mockChunk.rowId = 1
         mockChunk._columns = ["column1", "column2", "column3"]
         const result = writeInsertStatement(newMapping, mockTable, mockChunk)
-        expect(result).toEqual("INSERT INTO MockTable (target_column1,target_column2,target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
+        expect(result).toEqual("INSERT INTO MockTable (target_column1,target_column2,target_column3) VALUES ('a',to_timestamp('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
+    })
+    it('should write a sql statement correctly if date but no value', () => {
+        const newMapping = JSON.parse(JSON.stringify(config.mapping))
+        newMapping[1].targetType = "date"
+        newMapping[1].format = "DD-MM-YYYY HH24:MI:SS"
+        const mockTable = "MockTable"
+        const mockChunk = ["a", "", "c"]
+        mockChunk.errors = []
+        mockChunk.rowId = 1
+        mockChunk._columns = ["column1", "column2", "column3"]
+        const result = writeInsertStatement(newMapping, mockTable, mockChunk)
+        expect(result).toEqual("INSERT INTO MockTable (target_column1,target_column2,target_column3) VALUES ('a','','c')")
     })
     it('should write a sql statement when a target column is a keyword', () => {
         const newMapping = JSON.parse(JSON.stringify(config.mapping))
@@ -170,7 +182,7 @@ describe('postgresDestination tests', () => {
         mockChunk.rowId = 1
         mockChunk._columns = ["column1", "column2", "column3"]
         const result = writeInsertStatement(newMapping, mockTable, mockChunk)
-        expect(result).toEqual("INSERT INTO MockTable (target_column1,\"User\",target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
+        expect(result).toEqual("INSERT INTO MockTable (target_column1,\"User\",target_column3) VALUES ('a',to_timestamp('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
     })
     it('should write a sql statement when a source column is a keyword and there is no target column', () => {
         const newMapping = JSON.parse(JSON.stringify(config.mapping))
@@ -184,7 +196,7 @@ describe('postgresDestination tests', () => {
         mockChunk.rowId = 1
         mockChunk._columns = ["column1", "User", "column3"]
         const result = writeInsertStatement(newMapping, mockTable, mockChunk)
-        expect(result).toEqual("INSERT INTO MockTable (target_column1,\"User\",target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
+        expect(result).toEqual("INSERT INTO MockTable (target_column1,\"User\",target_column3) VALUES ('a',to_timestamp('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
     })
     it('should write a sql statement when a target column type is a number', () => {
         const newMapping = JSON.parse(JSON.stringify(config.mapping))