Remove system jobs from licence-import (#1026)

https://eaflood.atlassian.net/browse/WATER-4670

We've been making changes to the import service to support our work migrating return version management to WRLS, and replacing the current licence import with [one ready for ReSP](https://eaflood.atlassian.net/browse/WATER-4535).

We've also attempted to clarify what is actually happening during the various 'jobs' and to arrange them in our environments in a more logical order.

But the latest version that shipped is failing to complete. We believe the issues are:

- Each 'job' is taking longer to complete than expected
- Based on current timings, the 'jobs' are overlapping
- Our attempts to connect the new ReSP import with the existing one are adding to the burden
- Based on investigating the logs, we think downstream jobs are no longer being triggered

We're [working on simplifying the project](#1023) and removing elements that we think decrease performance, such as the use of [pg-boss](https://github.com/timgit/pg-boss).

But in the meantime, we need to get the import back up and running. This change removes the extra jobs we put in for connecting to the [water-abstraction-system](https://github.com/DEFRA/water-abstraction-system) until we can develop a better alternative. We also update the default schedule to allow each part of the import more time to complete to avoid overlapping.

.env.example

@@ -46,8 +46,8 @@ IMPORT_LICENCE_AGREEMENTS=false
 WRLS_LOG_LEVEL=debug
 
 # Use Cron type syntax to set timings for these background processes
-WRLS_CRON_NALD='0 1 * * *'
-WRLS_CRON_LICENCES='0 4 * * 1,2,3,4,5'
-WRLS_CRON_RETURN_VERSIONS='0 6 * * 1,2,3,4,5'
-WRLS_CRON_MOD_LOGS='0 7 * * 1,2,3,4,5'
+WRLS_CRON_NALD='15 23 * * *'
+WRLS_CRON_LICENCES='15 3 * * 1,2,3,4,5'
+WRLS_CRON_RETURN_VERSIONS='15 7 * * 1,2,3,4,5'
+WRLS_CRON_MOD_LOGS='30 7 * * 1,2,3,4,5'
 WRLS_CRON_TRACKER='0 10 * * 1,2,3,4,5'

config.js

@@ -84,10 +84,10 @@ module.exports = {
  nald: {
  zipPassword: process.env.NALD_ZIP_PASSWORD,
  path: process.env.S3_NALD_IMPORT_PATH || 'wal_nald_data_release',
- schedule: process.env.WRLS_CRON_NALD || '0 1 * * *'
+ schedule: process.env.WRLS_CRON_NALD || '15 23 * * *'
  },
  licences: {
- schedule: process.env.WRLS_CRON_LICENCES || '0 4 * * 1,2,3,4,5',
+ schedule: process.env.WRLS_CRON_LICENCES || '15 3 * * 1,2,3,4,',
  // Note: these 2 flags need to be set to false for charging go-live
  // to suspend the import of invoice accounts and licence agreements
  // Update: I've changed those values to false ahead of the v2.0 charging
@@ -104,10 +104,10 @@ module.exports = {
  isBillingDocumentRoleImportEnabled: false
  },
  modLogs: {
- schedule: process.env.WRLS_CRON_MOD_LOGS || '0 7 * * 1,2,3,4,5'
+ schedule: process.env.WRLS_CRON_MOD_LOGS || '30 7 * * 1,2,3,4,5'
  },
  returnVersions: {
- schedule: process.env.WRLS_CRON_RETURN_VERSIONS || '0 6 * * 1,2,3,4,5'
+ schedule: process.env.WRLS_CRON_RETURN_VERSIONS || '15 7 * * 1,2,3,4,5'
  },
  tracker: {
  schedule: process.env.WRLS_CRON_TRACKER || '0 10 * * 1,2,3,4,5'

src/modules/licence-import/jobs/import-company.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const QueueLicencesSystemJob = require('./queue-licences-system')
+const QueueLicencesJob = require('./queue-licences.js')
 const extract = require('../extract')
 const importCompanies = require('../connectors/import-companies')
 const load = require('../load')
@@ -81,7 +81,7 @@ async function onComplete (messageQueue) {
 
  if (count === 0) {
  await messageQueue.deleteQueue('__state__completed__licence-import.import-company')
- await messageQueue.publish(QueueLicencesSystemJob.createMessage())
+ await messageQueue.publish(QueueLicencesJob.createMessage())
 
  global.GlobalNotifier.omg(`${JOB_NAME}: finished`)
  }

src/modules/licence-import/plugin.js

@@ -5,12 +5,10 @@ const cron = require('node-cron')
 const DeleteRemovedDocumentsJob = require('./jobs/delete-removed-documents.js')
 const ImportCompanyJob = require('./jobs/import-company.js')
 const ImportLicenceJob = require('./jobs/import-licence.js')
-const ImportLicenceSystemJob = require('./jobs/import-licence-system.js')
 const ImportPointsJob = require('./jobs/import-points.js')
 const ImportPurposeConditionTypesJob = require('./jobs/import-purpose-condition-types.js')
 const QueueCompaniesJob = require('./jobs/queue-companies.js')
 const QueueLicencesJob = require('./jobs/queue-licences.js')
-const QueueLicencesSystemJob = require('./jobs/queue-licences-system.js')
 
 const config = require('../../../config')
 
@@ -39,16 +37,6 @@ async function register (server, _options) {
  return ImportCompanyJob.onComplete(server.messageQueue)
  })
 
- await server.messageQueue.subscribe(QueueLicencesSystemJob.name, QueueLicencesSystemJob.handler)
- await server.messageQueue.onComplete(QueueLicencesSystemJob.name, (executedJob) => {
- return QueueLicencesSystemJob.onComplete(server.messageQueue, executedJob)
- })
-
- await server.messageQueue.subscribe(ImportLicenceSystemJob.name, ImportLicenceSystemJob.options, ImportLicenceSystemJob.handler)
- await server.messageQueue.onComplete(ImportLicenceSystemJob.name, (executedJob) => {
- return ImportLicenceSystemJob.onComplete(server.messageQueue, executedJob)
- })
-
  await server.messageQueue.subscribe(QueueLicencesJob.name, QueueLicencesJob.handler)
  await server.messageQueue.onComplete(QueueLicencesJob.name, (executedJob) => {
  return QueueLicencesJob.onComplete(server.messageQueue, executedJob)

test/modules/licence-import/jobs/import-company.test.js

@@ -183,12 +183,12 @@ experiment('Licence Import: Import Company job', () => {
  expect(queueName).to.equal('__state__completed__licence-import.import-company')
  })
 
- test('the import licences job is published to the queue', async () => {
+ test('the queue licences job is published to the queue', async () => {
  await ImportCompanyJob.onComplete(messageQueue)
 
  const jobMessage = messageQueue.publish.lastCall.args[0]
 
- expect(jobMessage.name).to.equal('licence-import.queue-licences-system')
+ expect(jobMessage.name).to.equal('licence-import.queue-licences')
  })
  })