First commit

.devcontainer/devcontainer.json

@@ -0,0 +1,35 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/universal
+{
+	"name": "Default Linux Universal",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	// "image": "mcr.microsoft.com/devcontainers/universal:2-linux",
+	"dockerComposeFile": "./docker-compose.yaml",
+	"service": "etl",
+	"workspaceFolder": "/workspace",
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "uname -a",
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-azuretools.vscode-docker",
+				"orta.vscode-jest",
+				"andys8.jest-snippets",
+				"chris-noring.node-snippets",
+				"cweijan.vscode-postgresql-client2",
+				"mechatroner.rainbow-csv"
+			]
+		}
+	},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,15 @@
+services:
+  etl:
+    image: mcr.microsoft.com/devcontainers/universal:2-linux
+    volumes:
+      - ../:/workspace:cached
+  postgres:
+    image: postgres:latest
+    ports:
+      - 5432:5432
+    volumes:
+      - ./databases/postgres:/var/lib/postgresql
+      - ./postgres_init.sql:/docker-entrypoint-initdb.d/init.sql
+    environment:
+      - POSTGRES_PASSWORD=ppp
+      - POSTGRES_DB=etl_db

.devcontainer/postgres_init.sql

@@ -0,0 +1,5 @@
+CREATE TABLE target (
+    target_column1 VARCHAR(100),
+    target_column2 VARCHAR(100),
+    target_column3 VARCHAR(100)
+);

.dockerignore

@@ -0,0 +1,10 @@
+.gitignore
+.pre-commit-config.yaml
+CODE_OF_CONDUCT.md
+CONTRIBUTING.md
+ISSUE_TEMPLATE.md
+LICENSE.md
+PULL_REQUEST_TEMPLATE.md
+README.md
+Jenkinsfile
+

.github/dependabot.yml

@@ -0,0 +1,16 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for more information:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://containers.dev/guide/dependabot
+
+version: 2
+updates:
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly
+ - package-ecosystem: "npm"
+   directory: "/"
+   schedule:
+     interval: daily

.github/workflows/pr-merge.yaml

@@ -0,0 +1,21 @@
+name: pr-merge-action
+run-name: increment package.json verion on pr merge
+on:
+  pull-request:
+    types: [closed]
+
+jobs:
+  version_bump:
+    # this job will only run if the PR has been merged
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    steps:
+    - name: ⬇️ Checkout repo
+      uses: actions/checkout@v4
+    - name: ⏫ Bumping version
+      uses: jpb06/bump-package@latest
+      with:
+        major-keywords: BREAKING CHANGE
+        minor-keywords: feat,minor
+        patch-keywords: fix,chore
+        should-default-to-patch: true

.gitignore

@@ -0,0 +1,14 @@
+./devcontainer/databases
+.config/
+.DS_Store
+.npm/
+dist
+.standard-v12-cache/
+.vscode
+.env
+node_modules
+npm-debug.log
+test-output
+output.sql
+SoilType_Output.csv
+ffc-pay-etl-framework*.tgz

CODE_OF_CONDUCT.md

@@ -0,0 +1,51 @@
+# Contributor Code of Conduct
+
+Our [values](https://www.gov.uk/government/organisations/department-for-environment-food-rural-affairs/about#:~:text=We%20are%20responsible%20for%20improving,food%2C%20farming%20and%20fishing%20industries.) guide us in our day-to-day interactions and decision-making. Our open source projects are no exception. Trust, respect, collaboration and transparency are core values we believe should live and breathe within our projects. Our community welcomes participants from around the world with different experiences, unique perspectives, and great ideas to share.
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+- Using welcoming and inclusive language
+- Being respectful of differing viewpoints and experiences
+- Gracefully accepting constructive criticism
+- Attempting collaboration before conflict
+- Focusing on what is best for the community
+- Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+- Violence, threats of violence, or inciting others to commit self-harm
+- The use of sexualized language or imagery and unwelcome sexual attention or advances
+- Trolling, intentionally spreading misinformation, insulting/derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or electronic address, without explicit permission
+- Abuse of the reporting process to intentionally harass or exclude others
+- Advocating for, or encouraging, any of the above behavior
+- Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting us anonymously through [this form](https://www.gov.uk/government/organisations/department-for-environment-food-rural-affairs/about/complaints-procedure). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+If you are unsure whether an incident is a violation, or whether the space where the incident took place is covered by our Code of Conduct, **we encourage you to still report it**. We would prefer to have a few extra reports where we decide to take no action, than to leave an incident go unnoticed and unresolved that may result in an individual or group to feel like they can no longer participate in the community. Reports deemed as not a violation will also allow us to improve our Code of Conduct and processes surrounding it. If you witness a dangerous situation or someone in distress, we encourage you to report even if you are only an observer.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), [version 1.4](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html).

CONTRIBUTING.md

@@ -0,0 +1,45 @@
+This is still a fairly new package. While it delivers some basic ETL functionality there is still a lot of work to be done.
+
+There are still a lot of things to be done, so all contributions are welcome!
+
+## Code of Conduct
+
+We take our open source community seriously and hold ourselves and other contributors to high standards of communication. By participating and contributing to this project, you agree to uphold our [Code of Conduct](https://github.com/ffc-pay-etl-framework/CODE-OF-CONDUCT.md).
+
+## Getting Started
+
+Contributions are made to this repo via Issues and Pull Requests (PRs). A few general guidelines that cover both:
+
+- Search for existing Issues and PRs before creating your own.
+- We work hard to makes sure issues are handled in a timely manner but, depending on the impact, it could take a while to investigate the root cause. A friendly ping in the comment thread to the submitter or a contributor can help draw attention if your issue is blocking.
+
+### Issues
+
+Issues should be used to report problems with the library, request a new feature, or to discuss potential changes before a PR is created. When you create a new Issue, a template will be loaded that will guide you through collecting and providing the information we need to investigate.
+
+If you find an Issue that addresses the problem you're having, please add your own reproduction information to the existing issue rather than creating a new one. Adding a [reaction](https://github.blog/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) can also help be indicating to our maintainers that a particular problem is affecting more than just the reporter.
+
+### Pull Requests
+
+PRs to our libraries are always welcome and can be a quick way to get your fix or improvement slated for the next release. In general, PRs should:
+
+- Only fix/add the functionality in question **OR** address wide-spread whitespace/style issues, not both.
+- Add unit or integration tests for fixed or changed functionality (if a test suite already exists).
+- Address a single concern in the least number of changed lines as possible.
+- Include documentation in the repo or on our [docs site](https://auth0.com/docs).
+- Be accompanied by a complete Pull Request template (loaded automatically when a PR is created).
+
+For changes that address core functionality or would require breaking changes (e.g. a major release), it's best to open an Issue to discuss your proposal first. This is not required but can save time creating and reviewing changes.
+
+In general, we follow the ["fork-and-pull" Git workflow](https://github.com/susam/gitpr)
+
+1. Fork the repository to your own Github account
+2. Clone the project to your machine
+3. Create a branch locally with a succinct but descriptive name
+4. Commit changes to the branch
+5. Following any formatting and testing guidelines specific to this repo
+6. Push changes to your fork
+7. Open a PR in our repository and follow the PR template so that we can efficiently review the changes.
+
+
+

ISSUE_TEMPLATE.md

@@ -0,0 +1,39 @@
+# Prerequisites
+
+Please answer the following questions for yourself before submitting an issue. 
+
+- [ ] I am running the latest version
+- [ ] I checked the documentation and found no answer
+- [ ] I checked to make sure that this issue has not already been filed
+
+# Expected Behavior
+
+Please describe the behavior you are expecting
+
+# Current Behavior
+
+What is the current behavior?
+
+# Failure Information (for bugs)
+
+Please help provide information about the failure if this is a bug. If it is not a bug, please remove the rest of this template.
+
+## Steps to Reproduce
+
+Please provide detailed steps for reproducing the issue.
+
+1. step 1
+2. step 2
+3. you get it...
+
+## Context
+
+Please provide any relevant information about your setup. This is important in case the issue is not reproducible except for under certain conditions.
+
+* Operating System:
+* NPM version:
+* Node version:
+
+## Failure Logs
+
+Please include any relevant log snippets or files here.

Jenkinsfile

@@ -0,0 +1,3 @@
+@Library('defra-library@v-9') _
+
+buildNpm()

LICENSE.md

@@ -0,0 +1,24 @@
+Defra - Copyright (c) 2024
+
+This software consists of voluntary contributions made by many individuals.
+For exact contribution history, see the revision history
+available at https://github.com/DEFRA/ffc-pay-etl-framework
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,32 @@
+# Description
+
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+Fixes # (issue)
+
+## Type of change
+
+Please delete options that are not relevant.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+# How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+- [ ] Test A
+- [ ] Test B
+
+# Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules

README.md

@@ -0,0 +1,55 @@
+## 🚀 Features
+- 💾 CSV Source and Destination
+- 👮 Validators
+- 🤖 Transformers
+- 🚨 Error Checking
+- 🐘 Write directly to Postgres
+- 👨‍⚕️ Intellisense 
+
+## 📦 Install
+```bash
+npm install --save-dev ffc-pay-etl-framework
+```
+
+## 🪄 Usage
+
+```js
+// ESM
+import { Etl, Loaders, Validators, Transformers, Destinations } from "ffc-pay-etl-framework"
+
+// CJS
+const { Etl, Loaders, Validators, Transformers, Destinations } = require("ffc-pay-etl-framework")
+
+let csvFile = `${process.cwd()}/test/fixtures/SoilType.csv`
+
+const etl = new Etl.Etl()
+
+etl
+.loader(Loaders.CSVLoader({path: csvFile, columns: columns}))
+.transform(Transformers.FakerTransformer({
+    columns: [{
+        name: "Dist Name",
+        faker: "location.city"
+    }]
+}))
+.destination(Destinations.CSVFileDestination({ 
+    fileName: "SoilType_Output.csv", 
+    headers: true, 
+    includeErrors: false, 
+    quotationMarks: true
+}))
+.pump()
+```
+
+## 📢 Shout outs
+
+[Faker JS](https://fakerjs.dev/guide/)
+
+[Validator](https://github.com/validatorjs/validator.js)
+
+[Kaggle](https://www.kaggle.com/datasets/anushkahedaoo/farming-factors)
+
+## ✨ Contributing
+
+Please make sure to read the [Contributing Guide](https://github.com/DEFRA/ffc-pay-etl-framework/blob/next/CONTRIBUTING.md) before making a pull request.
+