Merge pull request #26 from DEFRA/feature/string-replace-transformer

Added string replacement transformer

.snyk

@@ -0,0 +1,5 @@
+# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
+
+version: v1.25.0
+ignore: {}
+patch: {}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-etl-framework",
-  "version": "0.0.11",
+  "version": "0.1.12",
   "publisher": "Defra",
   "main": "dist/cjs/index.js",
   "private": false,
@@ -25,12 +25,42 @@
   "jest": {
     "setupFiles": [
       "<rootDir>/jest/setEnvVars.js"
+    ],
+    "collectCoverage": true,
+    "collectCoverageFrom": [
+      "**/*.js",
+      "!**/*.test.js"
+    ],
+    "coverageDirectory": "test-output",
+    "coverageReporters": [
+      "text-summary",
+      "lcov"
+    ],
+    "coveragePathIgnorePatterns": [
+      "<rootDir>/node_modules/",
+      "<rootDir>/test-output/",
+      "<rootDir>/test/"
+    ],
+    "modulePathIgnorePatterns": [
+      "node_modules"
+    ],
+    "reporters": [
+      "default",
+      [
+        "jest-junit",
+        {
+          "suiteName": "jest tests",
+          "outputDirectory": "test-output",
+          "outputName": "junit.xml"
+        }
+      ]
     ]
   },
   "devDependencies": {
     "all-contributors-cli": "^6.26.1",
     "debug": "^4.3.6",
     "jest": "^29.7.0",
+    "jest-junit": "^16.0.0",
     "nodemon": "^3.1.4",
     "prettier": "3.3.3",
     "supports-color": "^9.4.0"
@@ -44,9 +74,8 @@
     "pg": "^8.12.0",
     "pg-hstore": "^2.3.4",
     "sequelize": "^6.37.3",
-    "sqlite3": "^5.1.7",
     "tedious": "^18.2.4",
-    "typescript": "^5.5.3",
+    "typescript": "^4.0.0",
     "validator": "^13.12.0"
   },
   "files": [

src/destinations/postgresDestination.js

@@ -104,7 +104,7 @@ function PostgresDestination(options){
                         chunk.errors.push(error)
                         lastChunk = chunk
                         // @ts-ignore
-                        callback(null, chunk)
+                        callback(error, chunk)
                     })
             } else {
                 debug('Chunk has errors %o', chunk)

src/examples/csv-find-replace.js

@@ -0,0 +1,85 @@
+const { Etl, Loaders, Transformers, Destinations } = require("ffc-pay-etl-framework")
+
+let csvFile = `${process.cwd()}/test/fixtures/Monthly_Rainfall.csv`
+
+const columns = [
+    "Dist Code","Year",
+    "State Code",
+    "State Name",
+    "Dist Name",
+    "JANUARY RAINFALL (Millimeters)",
+    "FEBRUARY RAINFALL (Millimeters)",
+    "MARCH RAINFALL (Millimeters)",
+    "APRIL RAINFALL (Millimeters)",
+    "MAY RAINFALL (Millimeters)",
+    "JUNE RAINFALL (Millimeters)",
+    "JULY RAINFALL (Millimeters)",
+    "AUGUST RAINFALL (Millimeters)",
+    "SEPTEMBER RAINFALL (Millimeters)",
+    "OCTOBER RAINFALL (Millimeters)",
+    "NOVEMBER RAINFALL (Millimeters)",
+    "DECEMBER RAINFALL (Millimeters)",
+    "ANNUAL RAINFALL (Millimeters)"
+]
+
+const etl = new Etl.Etl()
+
+etl
+.loader(Loaders.CSVLoader({path: csvFile, columns: columns}))
+.transform(new Transformers.StringReplaceTransformer([
+    { column: "JANUARY RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "FEBRUARY RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "MARCH RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "APRIL RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "MAY RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "JUNE RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "JULY RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "AUGUST RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "SEPTEMBER RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "OCTOBER RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "NOVEMBER RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+    { column: "DECEMBER RAINFALL (Millimeters)",
+        find: ".",
+        replace: ","
+    },
+]))
+.destination(Destinations.CSVFileDestination({ 
+    fileName: "SoilType_Output.csv", 
+    headers: true, 
+    includeErrors: false, 
+    quotationMarks: true
+}))
+.pump()

src/transformers/README.md

@@ -74,3 +74,29 @@ ToUpperCaseTransformer({
 ### Example
 
 src/examples/csv-to-upper-case.js
+
+## StringReplaceTransformer
+
+### Options
+
+| option  | description                             |
+| ------- | --------------------------------------- |
+| column  | source column to be transformed         |
+| find    | search string to be replaced            |
+| replace | value to replace the search string with |
+
+### Usage
+
+```js
+StringReplaceTransformer([
+  {
+    column: "column2",
+    find: "'",
+    replace: "''",
+  },
+]);
+```
+
+### Example
+
+src/examples/csv-find-replace.js

src/transformers/index.js

@@ -1,7 +1,9 @@
 const { ToUpperCaseTransformer } = require("./toUpperCaseTransformer")
 const { FakerTransformer } = require("./fakerTransformer")
+const { StringReplaceTransformer } = require("./stringReplaceTransformer")
 
 module.exports = {
+    StringReplaceTransformer,
     ToUpperCaseTransformer,
     FakerTransformer
 }

src/transformers/stringReplaceTransformer.js

@@ -0,0 +1,31 @@
+const { Transform } = require("node:stream")
+
+/**
+ * 
+ * @param {Object} options 
+ * @param {String} options.column
+ * @returns Writable
+ */
+function StringReplaceTransformer(options){
+    let self = this
+    self.replacements = options
+
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        transform(chunk, _, callback){
+            const { _columns } = chunk
+            // @ts-ignore
+            self.replacements.forEach(r => {
+              const colIndex = _columns.indexOf(r.column)
+              chunk[colIndex] = chunk[colIndex].replace(r.find, r.replace)
+            })
+
+            callback(null, chunk)
+        }
+    })
+}
+
+module.exports = {
+    StringReplaceTransformer
+}

test/transformers/stringReplaceTransformer.test.js

@@ -0,0 +1,30 @@
+const { expect } = require("@jest/globals")
+const { StringReplaceTransformer } = require("../../src/transformers")
+const { Readable, PassThrough } = require("node:stream")
+
+describe('stringReplaceTransformer tests', () => {
+    it('should replace a search string with a replacement string', (done) => {
+        const uut = StringReplaceTransformer([{ 
+          column: "column2",
+          find: "'",
+          replace: "''"
+        }])
+        const testData =["a", "o'b", "c"]
+        testData.errors = []
+        testData.rowId = 1
+        testData._columns = ["columm1", "column2", "column3"]
+        const readable = Readable.from([testData])
+        readable
+            .pipe(uut)
+            .pipe(new PassThrough({
+                objectMode: true,
+                transform(chunk,_,callback){
+                    expect(chunk.errors.length).toEqual(0)
+                    expect(chunk[1]).toEqual("o''b")
+                    done()
+                    callback(null, chunk)
+                }
+            })
+        )
+    })
+})