From 394979a076a28decfde6890fa8bcfc18ed815533 Mon Sep 17 00:00:00 2001 From: Sam Plackett <60177449+samplackett@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:50:24 +0100 Subject: [PATCH] use to_timestamp instead of to_date (#41) --- .all-contributorsrc | 11 +++++++++- README.md | 6 +++--- package.json | 2 +- src/destinations/postgresDestination.js | 7 +++++-- test/destinations/postgresDestination.test.js | 20 +++++++++++++++---- 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index e63189a..9cd74f4 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -21,12 +21,21 @@ }, { "login": "GodsonLeigh", - "name": "GodsonLeigh", + "name": "Leigh Godson", "avatar_url": "https://avatars.githubusercontent.com/u/139965284?v=4", "profile": "https://github.com/GodsonLeigh", "contributions": [ "code" ] + }, + { + "login": "samplackett", + "name": "Sam Plackett", + "avatar_url": "https://avatars.githubusercontent.com/u/60177449?v=4", + "profile": "https://github.com/samplackett", + "contributions": [ + "code" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 455d4e8..0fa54bb 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ -[![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors-) +[![All Contributors](https://img.shields.io/badge/all_contributors-3-orange.svg?style=flat-square)](#contributors-) @@ -98,8 +98,8 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d Charlie Benger-Stevenson
Charlie Benger-Stevenson
💻 - GodsonLeigh
GodsonLeigh
💻 - + Leigh Godson
Leigh Godson
💻 + Sam Plackett
Sam Plackett
💻 diff --git a/package.json b/package.json index e25cba8..00b796f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ffc-pay-etl-framework", - "version": "1.1.0", + "version": "1.1.1", "publisher": "Defra", "main": "dist/cjs/index.js", "private": false, diff --git a/src/destinations/postgresDestination.js b/src/destinations/postgresDestination.js index 49a6bbb..86af56b 100644 --- a/src/destinations/postgresDestination.js +++ b/src/destinations/postgresDestination.js @@ -42,8 +42,11 @@ function writeInsertStatement(columnMapping, table, chunk){ if(mapping?.targetType === "varchar" || mapping?.targetType === "char"){ return `'${chunk[index]}'` } - if(mapping?.targetType === "date"){ - return `to_date('${chunk[index]}','${mapping?.format}')` + if (mapping?.targetType === "date") { + if (!chunk[index]) { + return `''` + } + return `to_timestamp('${chunk[index]}','${mapping?.format}')` } return chunk[index] ? chunk[index] : 'null' })})` diff --git a/test/destinations/postgresDestination.test.js b/test/destinations/postgresDestination.test.js index 562c6c0..5ac6396 100644 --- a/test/destinations/postgresDestination.test.js +++ b/test/destinations/postgresDestination.test.js @@ -133,7 +133,7 @@ describe('postgresDestination tests', () => { const readable = Readable.from([testData]) readable .on('close', (result) => { - expect(mockConnection.db.query).toHaveBeenLastCalledWith("INSERT INTO target (target_column1,target_column2,target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')") + expect(mockConnection.db.query).toHaveBeenLastCalledWith("INSERT INTO target (target_column1,target_column2,target_column3) VALUES ('a',to_timestamp('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')") done() }) .pipe(uut) @@ -157,7 +157,19 @@ describe('postgresDestination tests', () => { mockChunk.rowId = 1 mockChunk._columns = ["column1", "column2", "column3"] const result = writeInsertStatement(newMapping, mockTable, mockChunk) - expect(result).toEqual("INSERT INTO MockTable (target_column1,target_column2,target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')") + expect(result).toEqual("INSERT INTO MockTable (target_column1,target_column2,target_column3) VALUES ('a',to_timestamp('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')") + }) + it('should write a sql statement correctly if date but no value', () => { + const newMapping = JSON.parse(JSON.stringify(config.mapping)) + newMapping[1].targetType = "date" + newMapping[1].format = "DD-MM-YYYY HH24:MI:SS" + const mockTable = "MockTable" + const mockChunk = ["a", "", "c"] + mockChunk.errors = [] + mockChunk.rowId = 1 + mockChunk._columns = ["column1", "column2", "column3"] + const result = writeInsertStatement(newMapping, mockTable, mockChunk) + expect(result).toEqual("INSERT INTO MockTable (target_column1,target_column2,target_column3) VALUES ('a','','c')") }) it('should write a sql statement when a target column is a keyword', () => { const newMapping = JSON.parse(JSON.stringify(config.mapping)) @@ -170,7 +182,7 @@ describe('postgresDestination tests', () => { mockChunk.rowId = 1 mockChunk._columns = ["column1", "column2", "column3"] const result = writeInsertStatement(newMapping, mockTable, mockChunk) - expect(result).toEqual("INSERT INTO MockTable (target_column1,\"User\",target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')") + expect(result).toEqual("INSERT INTO MockTable (target_column1,\"User\",target_column3) VALUES ('a',to_timestamp('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')") }) it('should write a sql statement when a source column is a keyword and there is no target column', () => { const newMapping = JSON.parse(JSON.stringify(config.mapping)) @@ -184,7 +196,7 @@ describe('postgresDestination tests', () => { mockChunk.rowId = 1 mockChunk._columns = ["column1", "User", "column3"] const result = writeInsertStatement(newMapping, mockTable, mockChunk) - expect(result).toEqual("INSERT INTO MockTable (target_column1,\"User\",target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')") + expect(result).toEqual("INSERT INTO MockTable (target_column1,\"User\",target_column3) VALUES ('a',to_timestamp('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')") }) it('should write a sql statement when a target column type is a number', () => { const newMapping = JSON.parse(JSON.stringify(config.mapping))