Add new clean process

This copies what we are doing in 3 other jobs to either mark as deleted or actually delete records from the DB. Bringing them into a process called clean makes it clearer what the steps are doing.

src/modules/clean/controllers.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const ProcessSteps = require('./process-steps.js')
+
+async function clean (_request, h) {
+ ProcessSteps.go()
+
+ return h.response().code(204)
+}
+
+module.exports = {
+ clean
+}

src/modules/clean/process-steps.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const DocumentsStep = require('./steps/documents.js')
+const DocumentHeadersStep = require('./steps/document-headers.js')
+const ReturnVersionsStep = require('./steps/return-versions.js')
+
+const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../lib/general.js')
+
+async function go () {
+ let processComplete = false
+
+ try {
+ global.GlobalNotifier.omg('clean started')
+
+ const startTime = currentTimeInNanoseconds()
+
+ await DocumentsStep.go()
+ await DocumentHeadersStep.go()
+ await ReturnVersionsStep.go()
+
+ processComplete = true
+
+ calculateAndLogTimeTaken(startTime, 'clean complete')
+ } catch (error) {
+ global.GlobalNotifier.oops('clean failed')
+ }
+
+ return processComplete
+}
+
+module.exports = {
+ go
+}

src/modules/clean/routes.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const controller = require('./controllers.js')
+
+const routes = [
+ {
+ method: 'post',
+ handler: controller.clean,
+ path: '/clean'
+ }
+]
+
+module.exports = routes

src/modules/clean/steps/document-headers.js

@@ -0,0 +1,40 @@
+'use strict'
+
+const db = require('../../../lib/connectors/db.js')
+const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../../lib/general.js')
+
+async function go () {
+ try {
+ global.GlobalNotifier.omg('clean.document-headers started')
+
+ const startTime = currentTimeInNanoseconds()
+
+ await _clean()
+
+ calculateAndLogTimeTaken(startTime, 'clean.document-headers complete')
+ } catch (error) {
+ global.GlobalNotifier.omfg('clean.document-headers errored', error)
+ throw error
+ }
+}
+
+async function _clean () {
+ return db.query(`
+ UPDATE
+ crm.document_header
+ SET
+ date_deleted = now()
+ WHERE
+ system_external_id NOT IN (
+ SELECT
+ l."LIC_NO"
+ FROM
+ "import"."NALD_ABS_LICENCES" l
+ )
+ AND date_deleted IS NULL;
+ `)
+}
+
+module.exports = {
+ go
+}

src/modules/clean/steps/documents.js

@@ -0,0 +1,39 @@
+'use strict'
+
+const db = require('../../../lib/connectors/db.js')
+const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../../lib/general.js')
+
+async function go () {
+ try {
+ global.GlobalNotifier.omg('clean.documents started')
+
+ const startTime = currentTimeInNanoseconds()
+
+ await _clean()
+
+ calculateAndLogTimeTaken(startTime, 'clean.documents complete')
+ } catch (error) {
+ global.GlobalNotifier.omfg('clean.documents errored', error)
+ throw error
+ }
+}
+
+async function _clean () {
+ return db.query(`
+ UPDATE
+ crm_v2.documents
+ SET
+ date_deleted = now()
+ WHERE
+ document_ref NOT IN (
+ SELECT
+ l."LIC_NO"
+ FROM import."NALD_ABS_LICENCES" l
+ )
+ AND date_deleted IS NULL;
+ `)
+}
+
+module.exports = {
+ go
+}

src/modules/clean/steps/return-versions.js

@@ -0,0 +1,165 @@
+'use strict'
+
+const db = require('../../../lib/connectors/db.js')
+const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../../lib/general.js')
+
+async function go () {
+ try {
+ global.GlobalNotifier.omg('clean.return-versions started')
+
+ const startTime = currentTimeInNanoseconds()
+
+ // Delete any return requirement points linked to deleted NALD return requirements
+ await _cleanPoints()
+
+ // Delete any return requirement purposes linked to deleted NALD return requirements
+ await _cleanPurposes()
+
+ // Delete any return requirements linked to deleted NALD return requirements
+ await _cleanRequirements()
+
+ // Delete any return versions that have no return requirements and that are linked to deleted return versions
+ await _cleanVersions()
+
+ // Update the mod logs to remove the return version ID for where the return version has now been deleted
+ await _cleanModLogs()
+
+ calculateAndLogTimeTaken(startTime, 'clean.return-versions complete')
+ } catch (error) {
+ global.GlobalNotifier.omfg('clean.return-versions errored', error)
+ throw error
+ }
+}
+
+async function _cleanPoints () {
+ return db.query(`
+ WITH nald_return_requirements AS (
+ SELECT concat_ws(':', nrf."FGAC_REGION_CODE", nrf."ID") AS nald_id
+ FROM "import"."NALD_RET_FORMATS" nrf
+ )
+ DELETE FROM water.return_requirement_points rrp WHERE rrp.return_requirement_id IN (
+ SELECT
+ rr.return_requirement_id
+ FROM
+ water.return_requirements rr
+ WHERE
+ NOT EXISTS (
+ SELECT 1
+ FROM nald_return_requirements nrr
+ WHERE rr.external_id = nrr.nald_id
+ )
+ AND NOT EXISTS (
+ SELECT 1
+ FROM "returns"."returns" rl
+ WHERE
+ rl.return_requirement = rr.legacy_id::varchar
+ LIMIT 1
+ )
+ );
+ `)
+}
+
+async function _cleanPurposes () {
+ return db.query(`
+ WITH nald_return_requirements AS (
+ SELECT concat_ws(':', nrf."FGAC_REGION_CODE", nrf."ID") AS nald_id
+ FROM "import"."NALD_RET_FORMATS" nrf
+ )
+ DELETE FROM water.return_requirement_purposes rrp WHERE rrp.return_requirement_id IN (
+ SELECT
+ rr.return_requirement_id
+ FROM
+ water.return_requirements rr
+ WHERE
+ NOT EXISTS (
+ SELECT 1
+ FROM nald_return_requirements nrr
+ WHERE rr.external_id = nrr.nald_id
+ )
+ AND NOT EXISTS (
+ SELECT 1
+ FROM "returns"."returns" rl
+ WHERE
+ rl.return_requirement = rr.legacy_id::varchar
+ LIMIT 1
+ )
+ );
+ `)
+}
+
+async function _cleanRequirements () {
+ return db.query(`
+ WITH nald_return_requirements AS (
+ SELECT concat_ws(':', nrf."FGAC_REGION_CODE", nrf."ID") AS nald_id
+ FROM "import"."NALD_RET_FORMATS" nrf
+ )
+ DELETE FROM water.return_requirements WHERE return_requirement_id IN (
+ SELECT
+ rr.return_requirement_id
+ FROM
+ water.return_requirements rr
+ WHERE
+ NOT EXISTS (
+ SELECT 1
+ FROM nald_return_requirements nrr
+ WHERE rr.external_id = nrr.nald_id
+ )
+ AND NOT EXISTS (
+ SELECT 1
+ FROM "returns"."returns" rl
+ WHERE
+ rl.return_requirement = rr.legacy_id::varchar
+ LIMIT 1
+ )
+ );
+ `)
+}
+
+async function _cleanVersions () {
+ return db.query(`
+ WITH nald_return_versions AS (
+ SELECT concat_ws(':', nv."FGAC_REGION_CODE", nv."AABL_ID", nv."VERS_NO") AS nald_id
+ FROM "import"."NALD_RET_VERSIONS" nv
+ )
+ DELETE FROM water.return_versions WHERE return_version_id IN (
+ SELECT
+ rv.return_version_id
+ FROM
+ water.return_versions rv
+ WHERE
+ NOT EXISTS (
+ SELECT 1
+ FROM nald_return_versions nrv
+ WHERE rv.external_id = nrv.nald_id
+ )
+ AND NOT EXISTS (
+ SELECT 1
+ FROM water.return_requirements rr
+ WHERE
+ rr.return_version_id = rv.return_version_id
+ LIMIT 1
+ )
+ );
+ `)
+}
+
+async function _cleanModLogs () {
+ return db.query(`
+ UPDATE
+ water.mod_logs ml
+ SET
+ return_version_id = NULL
+ WHERE
+ ml.return_version_id IS NOT NULL
+ AND ml.return_version_id NOT IN (
+ SELECT
+ rv.return_version_id
+ FROM
+ water.return_versions rv
+ );
+ `)
+}
+
+module.exports = {
+ go
+}