Move all reference data imports to a single process

We shouldn't be importing this data on a nightly basis. We have functionality that depends on this reference so we should be in control of when and how it gets populated.

But just in case we move all the old reference data import jobs to a new 'reference' process which will only ever be manually triggered from here on.

src/modules/charging-import/controller.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const chargeVersionsJob = require('./jobs/charge-versions')
-const chargingDataJob = require('./jobs/charging-data.js')
 
 /**
  * Run SQL queries to import charge versions / elements into
@@ -15,14 +14,6 @@ const postImportChargeVersions = async (request, h) => {
  return h.response().code(204)
 }
 
-const postImportChargingData = async (request, h) => {
- await request.messageQueue.deleteQueue(chargingDataJob.jobName)
- await request.messageQueue.publish(chargingDataJob.createMessage())
-
- return h.response().code(204)
-}
-
 module.exports = {
- postImportChargeVersions,
- postImportChargingData
+ postImportChargeVersions
 }

src/modules/charging-import/routes.js

@@ -7,10 +7,5 @@ module.exports = [
  method: 'post',
  handler: controller.postImportChargeVersions,
  path: '/import/charge-versions'
- },
- {
- method: 'post',
- handler: controller.postImportChargingData,
- path: '/import/charging-data'
  }
 ]

src/modules/licence-import/jobs/delete-removed-documents.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const documentsConnector = require('../connectors/documents')
-const ImportPurposeConditionTypesJob = require('./import-purpose-condition-types.js')
+const QueueCompaniesJob = require('../jobs/queue-companies.js')
 
 const JOB_NAME = 'licence-import.delete-removed-documents'
 
@@ -28,7 +28,7 @@ async function handler () {
 
 async function onComplete (messageQueue, job) {
  if (!job.failed) {
- await messageQueue.publish(ImportPurposeConditionTypesJob.createMessage())
+ await messageQueue.publish(QueueCompaniesJob.createMessage())
  }
 
  global.GlobalNotifier.omg(`${JOB_NAME}: finished`)

src/modules/licence-import/plugin.js

@@ -7,7 +7,6 @@ const ImportCompanyJob = require('./jobs/import-company.js')
 const ImportLicenceJob = require('./jobs/import-licence.js')
 const ImportLicenceSystemJob = require('./jobs/import-licence-system.js')
 const ImportPointsJob = require('./jobs/import-points.js')
-const ImportPurposeConditionTypesJob = require('./jobs/import-purpose-condition-types.js')
 const QueueCompaniesJob = require('./jobs/queue-companies.js')
 const QueueLicencesJob = require('./jobs/queue-licences.js')
 const QueueLicencesSystemJob = require('./jobs/queue-licences-system.js')
@@ -21,13 +20,6 @@ async function register (server, _options) {
  return DeleteRemovedDocumentsJob.onComplete(server.messageQueue, executedJob)
  })
 
- // When the documents have been marked as deleted import a list of all companies into the
- // water_import.company_import table
- await server.messageQueue.subscribe(ImportPurposeConditionTypesJob.name, ImportPurposeConditionTypesJob.handler)
- await server.messageQueue.onComplete(ImportPurposeConditionTypesJob.name, (executedJob) => {
- return ImportPurposeConditionTypesJob.onComplete(server.messageQueue, executedJob)
- })
-
  // When the water_import.company_import table is ready, jobs are scheduled to import each company
  await server.messageQueue.subscribe(QueueCompaniesJob.name, QueueCompaniesJob.handler)
  await server.messageQueue.onComplete(QueueCompaniesJob.name, (executedJob) => {

src/modules/reference/controllers.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const ProcessSteps = require('./process-steps.js')
+
+async function reference (_request, h) {
+ ProcessSteps.go()
+
+ return h.response().code(204)
+}
+
+module.exports = {
+ reference
+}

src/modules/licence-import/connectors/queries/purpose-condition-types.js → src/modules/reference/lib/condition-type-queries.js

@@ -1,12 +1,14 @@
-const createPurposeConditionTypes = `
+'use strict'
+
+const purposeConditionTypes = `
 INSERT INTO water.licence_version_purpose_condition_types (
  code,
  subcode,
  description,
  subcode_description
- ) 
- SELECT "CODE", "SUBCODE", "DESCR", "SUBCODE_DESC" FROM import."NALD_LIC_COND_TYPES" 
- WHERE "AFFECTS_ABS" = 'Y' 
+ )
+ SELECT "CODE", "SUBCODE", "DESCR", "SUBCODE_DESC" FROM import."NALD_LIC_COND_TYPES"
+ WHERE "AFFECTS_ABS" = 'Y'
  ON CONFLICT (code, subcode)
  DO UPDATE SET
  description = excluded.description,
@@ -15,5 +17,5 @@ INSERT INTO water.licence_version_purpose_condition_types (
 `
 
 module.exports = {
- createPurposeConditionTypes
+ purposeConditionTypes
 }

src/modules/charging-import/lib/queries/financial-agreement-types.js → src/modules/reference/lib/financial-agreement-type-queries.js

@@ -1,4 +1,6 @@
-const importFinancialAgreementTypes = `
+'use strict'
+
+const financialAgreementTypes = `
 INSERT INTO water.financial_agreement_types (financial_agreement_code, description, disabled, date_created, date_updated)
 SELECT a."CODE", a."DESCR", a."DISABLED"::boolean, now(), now()
 FROM import."NALD_FIN_AGRMNT_TYPES" a
@@ -12,5 +14,5 @@ DO
 `
 
 module.exports = {
- importFinancialAgreementTypes
+ financialAgreementTypes
 }

src/modules/charging-import/lib/queries/purposes.js → src/modules/reference/lib/purpose-queries.js

@@ -1,18 +1,12 @@
 'use strict'
 
-const importPrimaryPurposes = `insert into water.purposes_primary (legacy_id, description, date_created, date_updated)
+const primaryPurposes = `insert into water.purposes_primary (legacy_id, description, date_created, date_updated)
  select p."CODE", p."DESCR", now(), now()
  from import."NALD_PURP_PRIMS" p on conflict (legacy_id) do update set
  description= excluded.description,
  date_updated = now();`
 
-const importSecondaryPurposes = `
- insert into water.purposes_secondary (legacy_id, description, date_created, date_updated)
- select p."CODE", p."DESCR", now(), now()
- from import."NALD_PURP_SECS" p on conflict (legacy_id) do update set
- description= excluded.description, date_updated = now();`
-
-const importUses = `insert into water.purposes_uses (
+const purposes = `insert into water.purposes_uses (
  legacy_id,
  description,
  date_created,
@@ -40,20 +34,26 @@ const importUses = `insert into water.purposes_uses (
  loss_factor = excluded.loss_factor,
  is_two_part_tariff = excluded.is_two_part_tariff;`
 
-const importValidPurposeCombinations = `INSERT INTO water.purposes as prps (purpose_primary_id, purpose_secondary_id, purpose_use_id, date_created)
-SELECT pp.purpose_primary_id,
-ps.purpose_secondary_id,
-pu.purpose_use_id,
-now()
-FROM import."NALD_PURPOSES" as NALD_P
-JOIN water.purposes_primary as pp ON NALD_P."APPR_CODE" = pp."legacy_id"
-JOIN water.purposes_secondary as ps ON NALD_P."APSE_CODE" = ps."legacy_id"
-JOIN water.purposes_uses as pu ON NALD_P."APUS_CODE" = pu."legacy_id"
-WHERE NALD_P."DISABLED" = 'N' ON CONFLICT DO NOTHING;`
+const secondaryPurposes = `
+ insert into water.purposes_secondary (legacy_id, description, date_created, date_updated)
+ select p."CODE", p."DESCR", now(), now()
+ from import."NALD_PURP_SECS" p on conflict (legacy_id) do update set
+ description= excluded.description, date_updated = now();`
+
+const validPurposeCombinations = `INSERT INTO water.purposes as prps (purpose_primary_id, purpose_secondary_id, purpose_use_id, date_created)
+ SELECT pp.purpose_primary_id,
+ ps.purpose_secondary_id,
+ pu.purpose_use_id,
+ now()
+ FROM import."NALD_PURPOSES" as NALD_P
+ JOIN water.purposes_primary as pp ON NALD_P."APPR_CODE" = pp."legacy_id"
+ JOIN water.purposes_secondary as ps ON NALD_P."APSE_CODE" = ps."legacy_id"
+ JOIN water.purposes_uses as pu ON NALD_P."APUS_CODE" = pu."legacy_id"
+ WHERE NALD_P."DISABLED" = 'N' ON CONFLICT DO NOTHING;`
 
 module.exports = {
- importPrimaryPurposes,
- importSecondaryPurposes,
- importUses,
- importValidPurposeCombinations
+ primaryPurposes,
+ purposes,
+ secondaryPurposes,
+ validPurposeCombinations
 }

src/modules/reference/process-steps.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const ImportStep = require('./steps/import.js')
+
+const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../lib/general.js')
+
+async function go () {
+ let processComplete = false
+
+ try {
+ global.GlobalNotifier.omg('reference started')
+
+ const startTime = currentTimeInNanoseconds()
+
+ await ImportStep.go()
+
+ processComplete = true
+
+ calculateAndLogTimeTaken(startTime, 'reference complete')
+ } catch (error) {
+ global.GlobalNotifier.oops('reference failed')
+ }
+
+ return processComplete
+}
+
+module.exports = {
+ go
+}

src/modules/reference/routes.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const controllers = require('./controllers')
+
+const routes = [
+ {
+ method: 'post',
+ handler: controllers.reference,
+ path: '/reference'
+ }
+]
+
+module.exports = routes

src/modules/reference/steps/import.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const db = require('../../../lib/connectors/db.js')
+const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../../lib/general.js')
+const ConditionTypeQueries = require('../lib/condition-type-queries.js')
+const FinancialAgreementTypeQueries = require('../lib/financial-agreement-type-queries.js')
+const PurposeQueries = require('../lib/purpose-queries.js')
+
+async function go () {
+ try {
+ global.GlobalNotifier.omg('reference.import started')
+
+ const startTime = currentTimeInNanoseconds()
+
+ await db.query(ConditionTypeQueries.purposeConditionTypes)
+
+ await db.query(FinancialAgreementTypeQueries.financialAgreementTypes)
+
+ await db.query(PurposeQueries.primaryPurposes)
+ await db.query(PurposeQueries.secondaryPurposes)
+ await db.query(PurposeQueries.purposes)
+ await db.query(PurposeQueries.validPurposeCombinations)
+
+ calculateAndLogTimeTaken(startTime, 'reference.import complete')
+ } catch (error) {
+ global.GlobalNotifier.omfg('reference.import errored', error)
+ throw error
+ }
+}
+
+module.exports = {
+ go
+}

src/routes.js

@@ -11,6 +11,7 @@ const PermitRoutes = require('./modules/permit/routes.js')
 const CrmRoutes = require('./modules/crm/routes.js')
 const ReturnVersionsRoutes = require('./modules/return-versions/routes.js')
 const ModLogsRoutes = require('./modules/mod-logs/routes.js')
+const ReferenceRoutes = require('./modules/reference/routes.js')
 const NightlyImportRoutes = require('./modules/nightly-import/routes.js')
 
 module.exports = [
@@ -26,5 +27,6 @@ module.exports = [
  ...CrmRoutes,
  ...ReturnVersionsRoutes,
  ...ModLogsRoutes,
+ ...ReferenceRoutes,
  ...NightlyImportRoutes
 ]