Merge pull request #35 from DEFRA/fix/date-formatting

Fix/date formatting
DEFRA · Oct 1, 2024 · 74a1648 · 74a1648
2 parents 5236c0a + b112996
commit 74a1648
Show file tree

Hide file tree

Showing 6 changed files with 181 additions and 59 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -26,15 +26,17 @@
 				"chris-noring.node-snippets",
 				"cweijan.vscode-postgresql-client2",
 				"mechatroner.rainbow-csv",
-				"firsttris.vscode-jest-runner",
 				"esbenp.prettier-vscode"
 			],
 			"settings": {
-				"jest.runMode": "on-demand"
+				"jest.runMode": {
+					"type": "on-demand",
+					"coverage": false
+				}
 			}
 		}
 	},
-	"overrideCommand": true
+	// "overrideCommand": true
 	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
 	// "remoteUser": "root"
 }
diff --git a/__mocks__/sequelize.js b/__mocks__/sequelize.js
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-etl-framework",
-  "version": "0.1.12",
+  "version": "0.1.13",
   "publisher": "Defra",
   "main": "dist/cjs/index.js",
   "private": false,
@@ -63,6 +63,7 @@
     "jest-junit": "^16.0.0",
     "nodemon": "^3.1.4",
     "prettier": "3.3.3",
+    "sequelize-mock": "^0.10.2",
     "supports-color": "^9.4.0"
   },
   "dependencies": {

diff --git a/src/destinations/postgresDestination.js b/src/destinations/postgresDestination.js
@@ -1,9 +1,48 @@
 const EventEmitter = require('node:events')
-const util = require('node:util')
 const { Transform } = require('node:stream')
 const { Sequelize } = require('sequelize')
 const debug = require('debug')('destination')
 const DEFAULT_PORT = 5432
+
+function isKeyWord(column) {
+    return ['USER'].includes(column.toUpperCase())
+}
+
+function getMappingForColumn(mapping, column){
+    const [map] = mapping.filter(m => m.column === column)
+    return map
+}
+
+function writeInsertStatement(columnMapping, table, chunk){
+    let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
+        const mapping = getMappingForColumn(columnMapping, column)
+        return mapping.targetColumn
+            ?
+            isKeyWord(mapping.targetColumn)
+                ? `"${mapping.targetColumn}"`
+                : mapping.targetColumn
+            : isKeyWord(column)
+                ? `"${mapping.column}"`
+                : mapping.column
+    })
+    .join(",")}) VALUES (${chunk._columns.map((column,index) => {
+        const mapping = getMappingForColumn(columnMapping, column)
+        if(!mapping) debug('Mapping not found for column %s', column)
+        if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
+            debug('Source data is not a number')
+            return 0
+        }
+        if(mapping.targetType === "varchar" || mapping.targetType === "char"){
+            return `'${chunk[index]}'`
+        }
+        if(mapping.targetType === "date"){
+            return `to_date('${chunk[index]}','${mapping.format}')`
+        }
+        return chunk[index] ? chunk[index] : 'null'
+    })})`
+    return statement
+}
+
 /**
  * 
  * @param {Object} options 
@@ -44,39 +83,7 @@ function PostgresDestination(options){
         debug(e)
         throw e
     }
-
-    function getMappingForColumn(column){
-        const [map] = mapping.filter(m => m.column === column)
-        return map
-    }
-    function isKeyWord(column) {
-        return ['USER'].includes(column.toUpperCase())
-    }
-    function writeInsertStatement(chunk){
-        let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
-            let mapping = getMappingForColumn(column)
-            return mapping
-                ?
-                isKeyWord(mapping.targetColumn)
-                    ? `"${mapping.targetColumn}"`
-                    : mapping.targetColumn
-                : isKeyWord(column)
-                    ? `"${mapping.column}"`
-                    : mapping.column
-        })
-        .join(",")}) VALUES (${chunk._columns.map((column,index) => {
-            let mapping = getMappingForColumn(column)
-            if(!mapping) debug('Mapping not found for column %s', column)
-            if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
-                debug('Source data is not a number')
-                return 0
-            }
-            if(mapping.targetType === "varchar" || mapping.targetType === "char")
-                return `'${chunk[index]}'`
-            return chunk[index] ? chunk[index] : 'null'
-        })})`
-        return statement
-    }
+
     return new Transform({
         objectMode: true,
         emitClose: true,
@@ -89,7 +96,7 @@ function PostgresDestination(options){
             let insertStatement
             // @ts-ignore
             if(chunk.errors.length === 0 | options.includeErrors){
-                insertStatement = writeInsertStatement(chunk)
+                insertStatement = writeInsertStatement(mapping, table, chunk)
                 debug('Insert statement: [%s]', insertStatement)
                 // @ts-ignore
                 this.sequelize.query(insertStatement)
@@ -118,5 +125,8 @@ function PostgresDestination(options){
 }
 
 module.exports = {
-    PostgresDestination
+    PostgresDestination,
+    writeInsertStatement,
+    isKeyWord,
+    getMappingForColumn
 }
diff --git a/test/destinations/postgresDestination.test.js b/test/destinations/postgresDestination.test.js
@@ -1,8 +1,21 @@
 const { expect } = require('@jest/globals')
-const { PostgresDestination } = require('../../src/destinations')
+const { 
+    PostgresDestination, writeInsertStatement, isKeyWord, 
+    getMappingForColumn 
+} = require('../../src/destinations/postgresDestination')
 const { Readable } = require('node:stream')
 const { Sequelize } = require('sequelize')
-jest.mock('sequelize')
+
+jest.mock('sequelize', () => {
+    const mockQuery = jest.fn().mockResolvedValue([[], 1])
+    return {
+        Sequelize: jest.fn(() =>({
+                authenticate: jest.fn(),
+                query: mockQuery
+            })
+        )
+    }
+})
 
 jest.mock('fs', () => ({
     writeFileSync: jest.fn(),
@@ -38,7 +51,7 @@ const config = {
 }
 
 describe('postgresDestination tests', () => {
-    afterEach(() => {
+    beforeEach(() => {
         jest.clearAllMocks()
     })
     it('should write a row', (done) => {
@@ -89,7 +102,7 @@ describe('postgresDestination tests', () => {
             .pipe(uut)
     })
     it('should connect to different port', () => {
-        PostgresDestination({
+        new PostgresDestination({
             username: "postgres",
             password : "abc",
             database: "etl_db",
@@ -128,4 +141,93 @@ describe('postgresDestination tests', () => {
             )
         }
     )
+    it('should format a date as specified', (done) => {
+        const newConfig = JSON.parse(JSON.stringify(config))
+        newConfig.mapping[1].targetType = "date"
+        newConfig.mapping[1].format = "DD-MM-YYYY HH24:MI:SS"
+        const uut = new PostgresDestination(newConfig)
+        const testData =["a", "19-06-2024 00:00", "c"]
+        testData.errors = []
+        testData.rowId = 1
+        testData._columns = ["column1", "column2", "column3"]
+        const readable = Readable.from([testData])
+        readable
+            .on('close', (result) => {
+                expect(Sequelize().query).toHaveBeenLastCalledWith("INSERT INTO target (target_column1,target_column2,target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
+                done()
+            })
+            .pipe(uut)
+    })
+    it('should write a sql statement', () => {
+        const mockTable = "MockTable"
+        const mockChunk = ["a", "19-06-2024 00:00", "c"]
+        mockChunk.errors = []
+        mockChunk.rowId = 1
+        mockChunk._columns = ["column1", "column2", "column3"]
+        const result = writeInsertStatement(config.mapping, mockTable, mockChunk)
+        expect(result).toEqual("INSERT INTO MockTable (target_column1,target_column2,target_column3) VALUES ('a','19-06-2024 00:00','c')")
+    })
+    it('should write a sql statement with a date format', () => {
+        const newMapping = [...config.mapping]
+        newMapping[1].targetType = "date"
+        newMapping[1].format = "DD-MM-YYYY HH24:MI:SS"
+        const mockTable = "MockTable"
+        const mockChunk = ["a", "19-06-2024 00:00", "c"]
+        mockChunk.errors = []
+        mockChunk.rowId = 1
+        mockChunk._columns = ["column1", "column2", "column3"]
+        const result = writeInsertStatement(newMapping, mockTable, mockChunk)
+        expect(result).toEqual("INSERT INTO MockTable (target_column1,target_column2,target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
+    })
+    it('should write a sql statement when a target column is a keyword', () => {
+        const newMapping = JSON.parse(JSON.stringify(config.mapping))
+        newMapping[1].targetColumn = "User"
+        newMapping[1].targetType = "date"
+        newMapping[1].format = "DD-MM-YYYY HH24:MI:SS"
+        const mockTable = "MockTable"
+        const mockChunk = ["a", "19-06-2024 00:00", "c"]
+        mockChunk.errors = []
+        mockChunk.rowId = 1
+        mockChunk._columns = ["column1", "column2", "column3"]
+        const result = writeInsertStatement(newMapping, mockTable, mockChunk)
+        expect(result).toEqual("INSERT INTO MockTable (target_column1,\"User\",target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
+    })
+    it('should write a sql statement when a source column is a keyword and there is no target column', () => {
+        const newMapping = JSON.parse(JSON.stringify(config.mapping))
+        newMapping[1].column = "User"
+        newMapping[1].targetType = "date"
+        delete newMapping[1].targetColumn
+        newMapping[1].format = "DD-MM-YYYY HH24:MI:SS"
+        const mockTable = "MockTable"
+        const mockChunk = ["a", "19-06-2024 00:00", "c"]
+        mockChunk.errors = []
+        mockChunk.rowId = 1
+        mockChunk._columns = ["column1", "User", "column3"]
+        const result = writeInsertStatement(newMapping, mockTable, mockChunk)
+        expect(result).toEqual("INSERT INTO MockTable (target_column1,\"User\",target_column3) VALUES ('a',to_date('19-06-2024 00:00','DD-MM-YYYY HH24:MI:SS'),'c')")
+    })
+    it('should write a sql statement when a target column type is a number', () => {
+        const newMapping = [...config.mapping]
+        newMapping[1].targetType = "number"
+        newMapping[1].format = "DD-MM-YYYY HH24:MI:SS"
+        const mockTable = "MockTable"
+        const mockChunk = ["a", 999, "c"]
+        mockChunk.errors = []
+        mockChunk.rowId = 1
+        mockChunk._columns = ["column1", "column2", "column3"]
+        const result = writeInsertStatement(newMapping, mockTable, mockChunk)
+        expect(result).toEqual("INSERT INTO MockTable (target_column1,target_column2,target_column3) VALUES ('a',999,'c')")
+    })
+    it('should write a sql statement when a target column type is a number but the value is NaN', () => {
+        const newMapping = [...config.mapping]
+        newMapping[1].targetType = "number"
+        newMapping[1].format = "DD-MM-YYYY HH24:MI:SS"
+        const mockTable = "MockTable"
+        const mockChunk = ["a", "a999", "c"]
+        mockChunk.errors = []
+        mockChunk.rowId = 1
+        mockChunk._columns = ["column1", "column2", "column3"]
+        const result = writeInsertStatement(newMapping, mockTable, mockChunk)
+        expect(result).toEqual("INSERT INTO MockTable (target_column1,target_column2,target_column3) VALUES ('a',0,'c')")
+    })
 })