Split charging data job; rtns, mod logs and other (#1018)

https://eaflood.atlassian.net/browse/WATER-4654

> Part of the work to migrate management of return requirements from NALD to WRLS

For reasons detailed in [Add return requirements clean up step to import](#1017), we need to add a new step to the existing 'charging-data' job. But the job itself is already a bit of a mess.

- it is called 'charging-data' but doesn't do that any more
- it brings in fixed reference data, return versions and mod logs all in one go

We know this will soon be ditched, but it is a right-old mess. We could stop some of the stuff right now and split out the rest to make it clearer which bit is doing what. This wouldn't be much more effort than slotting a new step into the mess.

So, in this change, we perform some much-needed tidying up of the 'charging-data job'. This should make the change in PR #1017 much easier!

.env.example

@@ -46,7 +46,8 @@ IMPORT_LICENCE_AGREEMENTS=false
 WRLS_LOG_LEVEL=debug
 
 # Use Cron type syntax to set timings for these background processes
-WRLS_CRON_LICENCES='0 16 * * 1,2,3,4,5'
-WRLS_CRON_CHARGING='0 14 * * 1,2,3,4,5'
-WRLS_CRON_NALD='0 */1 * * *'
-WRLS_CRON_TRACKER='0 15 * * 1,2,3,4,5'
+WRLS_CRON_NALD='0 1 * * *'
+WRLS_CRON_LICENCES='0 4 * * 1,2,3,4,5'
+WRLS_CRON_RETURN_VERSIONS='0 6 * * 1,2,3,4,5'
+WRLS_CRON_MOD_LOGS='0 7 * * 1,2,3,4,5'
+WRLS_CRON_TRACKER='0 10 * * 1,2,3,4,5'

config.js

@@ -103,8 +103,11 @@ module.exports = {
  // was calculated from NALD data
  isBillingDocumentRoleImportEnabled: false
  },
- charging: {
- schedule: process.env.WRLS_CRON_CHARGING || '0 1 * * 1,2,3,4,5'
+ modLogs: {
+ schedule: process.env.WRLS_CRON_MOD_LOGS || '0 7 * * 1,2,3,4,5'
+ },
+ returnVersions: {
+ schedule: process.env.WRLS_CRON_RETURN_VERSIONS || '0 6 * * 1,2,3,4,5'
  },
  tracker: {
  schedule: process.env.WRLS_CRON_TRACKER || '0 10 * * 1,2,3,4,5'

index.js

@@ -28,6 +28,8 @@ const plugins = [
  require('./src/plugins/pg-boss.plugin'),
  require('./src/modules/licence-import/plugin'),
  require('./src/modules/charging-import/plugin'),
+ require('./src/modules/mod-logs/plugin'),
+ require('./src/modules/return-versions/plugin.js'),
  require('./src/modules/nald-import/plugin'),
  require('./src/modules/bill-runs-import/plugin'),
  require('./src/modules/core/plugin')

src/modules/charging-import/jobs/charging-data.js

@@ -6,9 +6,7 @@
 
 const job = require('../lib/job')
 const queryLoader = require('../lib/query-loader')
-const modLogQueries = require('../lib/queries/mod-logs.js')
 const purposesQueries = require('../lib/queries/purposes')
-const returnVersionQueries = require('../lib/queries/return-versions')
 const financialAgreementTypeQueries = require('../lib/queries/financial-agreement-types')
 
 const jobName = 'import.charging-data'
@@ -24,22 +22,7 @@ const handler = async () => {
  purposesQueries.importPrimaryPurposes,
  purposesQueries.importSecondaryPurposes,
  purposesQueries.importUses,
- purposesQueries.importValidPurposeCombinations,
- returnVersionQueries.importReturnVersions,
- returnVersionQueries.importReturnRequirements,
- returnVersionQueries.importReturnRequirementPoints,
- returnVersionQueries.importReturnRequirementPurposes,
- returnVersionQueries.importReturnVersionsMultipleUpload,
- returnVersionQueries.importReturnVersionsCreateNotesFromDescriptions,
- returnVersionQueries.importReturnVersionsCorrectStatusForWrls,
- returnVersionQueries.importReturnVersionsSetToDraftMissingReturnRequirements,
- returnVersionQueries.importReturnVersionsAddMissingReturnVersionEndDates,
- modLogQueries.importModLogs,
- modLogQueries.linkLicencesToModLogs,
- modLogQueries.linkChargeVersionsToModLogs,
- modLogQueries.linkLicenceVersionsToModLogs,
- modLogQueries.linkReturnVersionsToModLogs,
- modLogQueries.updateReturnVersionReasons
+ purposesQueries.importValidPurposeCombinations
  ])
 
  global.GlobalNotifier.omg('import.charging-data: finished')

src/modules/charging-import/plugin.js

@@ -1,21 +1,12 @@
 'use strict'
 
-const cron = require('node-cron')
-
 const chargeVersionsJob = require('./jobs/charge-versions')
 const chargingDataJob = require('./jobs/charging-data')
 
-const config = require('../../../config')
-
 const registerSubscribers = async server => {
  // Register handlers
  await server.messageQueue.subscribe(chargeVersionsJob.jobName, chargeVersionsJob.handler)
  await server.messageQueue.subscribe(chargingDataJob.jobName, chargingDataJob.handler)
-
- // Set up import of charge data on cron job
- cron.schedule(config.import.charging.schedule,
- () => server.messageQueue.publish(chargingDataJob.createMessage())
- )
 }
 
 const plugin = {

src/modules/mod-logs/controller.js

@@ -0,0 +1,14 @@
+'use strict'
+
+const ImportJob = require('./jobs/import.js')
+
+async function importModLogs (request, h) {
+ await request.messageQueue.deleteQueue(ImportJob.JOB_NAME)
+ await request.messageQueue.publish(ImportJob.createMessage())
+
+ return h.response().code(204)
+}
+
+module.exports = {
+ importModLogs
+}

src/modules/mod-logs/jobs/import.js

@@ -0,0 +1,46 @@
+'use strict'
+
+const { pool } = require('../../../lib/connectors/db.js')
+const Queries = require('../lib/queries.js')
+
+const JOB_NAME = 'mod-logs.import'
+
+function createMessage () {
+ return {
+ name: JOB_NAME,
+ options: {
+ singletonKey: JOB_NAME
+ }
+ }
+}
+
+async function handler () {
+ try {
+ global.GlobalNotifier.omg(`${JOB_NAME}: started`)
+
+ await pool.query(Queries.importModLogs)
+ await pool.query(Queries.linkLicencesToModLogs)
+ await pool.query(Queries.linkChargeVersionsToModLogs)
+ await pool.query(Queries.linkLicenceVersionsToModLogs)
+ await pool.query(Queries.linkReturnVersionsToModLogs)
+ await pool.query(Queries.updateReturnVersionReasons)
+ } catch (error) {
+ global.GlobalNotifier.omfg(`${JOB_NAME}: errored`, error)
+ throw error
+ }
+}
+
+async function onComplete (job) {
+ if (!job.failed) {
+ global.GlobalNotifier.omg(`${JOB_NAME}: finished`)
+ } else {
+ global.GlobalNotifier.omg(`${JOB_NAME}: failed`)
+ }
+}
+
+module.exports = {
+ JOB_NAME,
+ createMessage,
+ handler,
+ onComplete
+}

src/modules/mod-logs/plugin.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const cron = require('node-cron')
+
+const ImportJob = require('./jobs/import.js')
+
+const config = require('../../../config')
+
+async function register (server, _options) {
+ // Register import mod logs job
+ await server.messageQueue.subscribe(ImportJob.JOB_NAME, ImportJob.handler)
+ await server.messageQueue.onComplete(ImportJob.JOB_NAME, (executedJob) => {
+ return ImportJob.onComplete(executedJob)
+ })
+
+ // Set up import on cron job
+ cron.schedule(config.import.modLogs.schedule, async () => {
+ await server.messageQueue.publish(ImportJob.createMessage())
+ })
+}
+
+module.exports = {
+ plugin: {
+ name: 'importModLogs',
+ dependencies: ['pgBoss'],
+ register
+ }
+}

src/modules/mod-logs/routes.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const controller = require('./controller')
+
+const routes = [
+ {
+ method: 'post',
+ handler: controller.importModLogs,
+ path: '/import/mod-logs'
+ }
+]
+
+module.exports = routes

src/modules/return-versions/controller.js

@@ -0,0 +1,14 @@
+'use strict'
+
+const ImportJob = require('./jobs/import.js')
+
+async function importReturnVersions (request, h) {
+ await request.messageQueue.deleteQueue(ImportJob.JOB_NAME)
+ await request.messageQueue.publish(ImportJob.createMessage())
+
+ return h.response().code(204)
+}
+
+module.exports = {
+ importReturnVersions
+}

src/modules/return-versions/jobs/import.js

@@ -0,0 +1,49 @@
+'use strict'
+
+const { pool } = require('../../../lib/connectors/db.js')
+const Queries = require('../lib/queries.js')
+
+const JOB_NAME = 'return-versions.import'
+
+function createMessage () {
+ return {
+ name: JOB_NAME,
+ options: {
+ singletonKey: JOB_NAME
+ }
+ }
+}
+
+async function handler () {
+ try {
+ global.GlobalNotifier.omg(`${JOB_NAME}: started`)
+
+ await pool.query(Queries.importReturnVersions)
+ await pool.query(Queries.importReturnRequirements)
+ await pool.query(Queries.importReturnRequirementPoints)
+ await pool.query(Queries.importReturnRequirementPurposes)
+ await pool.query(Queries.importReturnVersionsMultipleUpload)
+ await pool.query(Queries.importReturnVersionsCreateNotesFromDescriptions)
+ await pool.query(Queries.importReturnVersionsCorrectStatusForWrls)
+ await pool.query(Queries.importReturnVersionsSetToDraftMissingReturnRequirements)
+ await pool.query(Queries.importReturnVersionsAddMissingReturnVersionEndDates)
+ } catch (error) {
+ global.GlobalNotifier.omfg(`${JOB_NAME}: errored`, error)
+ throw error
+ }
+}
+
+async function onComplete (job) {
+ if (!job.failed) {
+ global.GlobalNotifier.omg(`${JOB_NAME}: finished`)
+ } else {
+ global.GlobalNotifier.omg(`${JOB_NAME}: failed`)
+ }
+}
+
+module.exports = {
+ JOB_NAME,
+ createMessage,
+ handler,
+ onComplete
+}

src/modules/return-versions/plugin.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const cron = require('node-cron')
+
+const ImportJob = require('./jobs/import.js')
+
+const config = require('../../../config')
+
+async function register (server, _options) {
+ // Register import return versions job
+ await server.messageQueue.subscribe(ImportJob.JOB_NAME, ImportJob.handler)
+ await server.messageQueue.onComplete(ImportJob.JOB_NAME, (executedJob) => {
+ return ImportJob.onComplete(executedJob)
+ })
+
+ // Set up import on cron job
+ cron.schedule(config.import.returnVersions.schedule, async () => {
+ await server.messageQueue.publish(ImportJob.createMessage())
+ })
+}
+
+module.exports = {
+ plugin: {
+ name: 'importReturnVersions',
+ dependencies: ['pgBoss'],
+ register
+ }
+}

src/modules/return-versions/routes.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const controller = require('./controller')
+
+const routes = [
+ {
+ method: 'post',
+ handler: controller.importReturnVersions,
+ path: '/import/return-versions'
+ }
+]
+
+module.exports = routes

src/routes.js

@@ -5,6 +5,8 @@ const jobSummaryRoutes = require('./modules/jobs/routes')
 const licenceImportRoutes = require('./modules/licence-import/routes')
 const naldImportRoutes = require('./modules/nald-import/routes')
 const returnsRoutes = require('./modules/returns/routes')
+const returnVersionsRoutes = require('./modules/return-versions/routes.js')
+const modLogsRoutes = require('./modules/mod-logs/routes.js')
 
 module.exports = [
  ...chargingImportRoutes,
@@ -13,5 +15,7 @@ module.exports = [
  ...jobSummaryRoutes,
  ...licenceImportRoutes,
  ...naldImportRoutes,
- ...returnsRoutes
+ ...returnsRoutes,
+ ...returnVersionsRoutes,
+ ...modLogsRoutes
 ]

test/modules/charging-import/jobs/charging-data.test.js

@@ -9,9 +9,7 @@ const { experiment, test, beforeEach, afterEach } = exports.lab = Lab.script()
 const { expect } = Code
 
 // Test helpers
-const modLogQueries = require('../../../../src/modules/charging-import/lib/queries/mod-logs.js')
 const purposesQueries = require('../../../../src/modules/charging-import/lib/queries/purposes')
-const returnVersionQueries = require('../../../../src/modules/charging-import/lib/queries/return-versions')
 const financialAgreementTypeQueries = require('../../../../src/modules/charging-import/lib/queries/financial-agreement-types')
 
 // Things we need to stub
@@ -55,22 +53,7 @@ experiment('modules/charging-import/jobs/charging-data.js', () => {
  purposesQueries.importPrimaryPurposes,
  purposesQueries.importSecondaryPurposes,
  purposesQueries.importUses,
- purposesQueries.importValidPurposeCombinations,
- returnVersionQueries.importReturnVersions,
- returnVersionQueries.importReturnRequirements,
- returnVersionQueries.importReturnRequirementPoints,
- returnVersionQueries.importReturnRequirementPurposes,
- returnVersionQueries.importReturnVersionsMultipleUpload,
- returnVersionQueries.importReturnVersionsCreateNotesFromDescriptions,
- returnVersionQueries.importReturnVersionsCorrectStatusForWrls,
- returnVersionQueries.importReturnVersionsSetToDraftMissingReturnRequirements,
- returnVersionQueries.importReturnVersionsAddMissingReturnVersionEndDates,
- modLogQueries.importModLogs,
- modLogQueries.linkLicencesToModLogs,
- modLogQueries.linkChargeVersionsToModLogs,
- modLogQueries.linkLicenceVersionsToModLogs,
- modLogQueries.linkReturnVersionsToModLogs,
- modLogQueries.updateReturnVersionReasons
+ purposesQueries.importValidPurposeCombinations
  ]
  )).to.be.true()
  })