From 8840f9150a70d41017a52b7d4eca0f4625c09399 Mon Sep 17 00:00:00 2001 From: Alan Cruikshanks Date: Fri, 13 Sep 2024 17:05:15 +0100 Subject: [PATCH] Move all reference data imports to a single process We shouldn't be importing this data on a nightly basis. We have functionality that depends on this reference so we should be in control of when and how it gets populated. But just in case we move all the old reference data import jobs to a new 'reference' process which will only ever be manually triggered from here on. --- src/modules/charging-import/controller.js | 11 +---- .../charging-import/jobs/charging-data.js | 39 ---------------- src/modules/charging-import/routes.js | 5 --- .../connectors/purpose-conditions-types.js | 14 ------ .../jobs/delete-removed-documents.js | 4 +- .../jobs/import-purpose-condition-types.js | 42 ------------------ src/modules/licence-import/plugin.js | 8 ---- src/modules/reference/controllers.js | 13 ++++++ .../lib/condition-type-queries.js} | 12 ++--- .../lib/financial-agreement-type-queries.js} | 6 ++- .../lib/purpose-queries.js} | 44 +++++++++---------- src/modules/reference/process-steps.js | 29 ++++++++++++ src/modules/reference/routes.js | 13 ++++++ src/modules/reference/steps/import.js | 33 ++++++++++++++ src/routes.js | 2 + 15 files changed, 126 insertions(+), 149 deletions(-) delete mode 100644 src/modules/charging-import/jobs/charging-data.js delete mode 100644 src/modules/licence-import/connectors/purpose-conditions-types.js delete mode 100644 src/modules/licence-import/jobs/import-purpose-condition-types.js create mode 100644 src/modules/reference/controllers.js rename src/modules/{licence-import/connectors/queries/purpose-condition-types.js => reference/lib/condition-type-queries.js} (74%) rename src/modules/{charging-import/lib/queries/financial-agreement-types.js => reference/lib/financial-agreement-type-queries.js} (84%) rename src/modules/{charging-import/lib/queries/purposes.js => reference/lib/purpose-queries.js} (62%) create mode 100644 src/modules/reference/process-steps.js create mode 100644 src/modules/reference/routes.js create mode 100644 src/modules/reference/steps/import.js diff --git a/src/modules/charging-import/controller.js b/src/modules/charging-import/controller.js index bcc71698..14d41891 100644 --- a/src/modules/charging-import/controller.js +++ b/src/modules/charging-import/controller.js @@ -1,7 +1,6 @@ 'use strict' const chargeVersionsJob = require('./jobs/charge-versions') -const chargingDataJob = require('./jobs/charging-data.js') /** * Run SQL queries to import charge versions / elements into @@ -15,14 +14,6 @@ const postImportChargeVersions = async (request, h) => { return h.response().code(204) } -const postImportChargingData = async (request, h) => { - await request.messageQueue.deleteQueue(chargingDataJob.jobName) - await request.messageQueue.publish(chargingDataJob.createMessage()) - - return h.response().code(204) -} - module.exports = { - postImportChargeVersions, - postImportChargingData + postImportChargeVersions } diff --git a/src/modules/charging-import/jobs/charging-data.js b/src/modules/charging-import/jobs/charging-data.js deleted file mode 100644 index cf32182f..00000000 --- a/src/modules/charging-import/jobs/charging-data.js +++ /dev/null @@ -1,39 +0,0 @@ -'use strict' - -/** - * @note: this needs to remain and should not be deleted - */ - -const job = require('../lib/job') -const queryLoader = require('../lib/query-loader') -const purposesQueries = require('../lib/queries/purposes') -const financialAgreementTypeQueries = require('../lib/queries/financial-agreement-types') - -const jobName = 'import.charging-data' - -const createMessage = () => job.createMessage(jobName) - -const handler = async () => { - try { - global.GlobalNotifier.omg('import.charging-data: started') - - await queryLoader.loadQueries([ - financialAgreementTypeQueries.importFinancialAgreementTypes, - purposesQueries.importPrimaryPurposes, - purposesQueries.importSecondaryPurposes, - purposesQueries.importUses, - purposesQueries.importValidPurposeCombinations - ]) - - global.GlobalNotifier.omg('import.charging-data: finished') - } catch (error) { - global.GlobalNotifier.omfg('import.charging-data: errored', error) - throw error - } -} - -module.exports = { - jobName, - createMessage, - handler -} diff --git a/src/modules/charging-import/routes.js b/src/modules/charging-import/routes.js index 62f8ab9f..666c0cba 100644 --- a/src/modules/charging-import/routes.js +++ b/src/modules/charging-import/routes.js @@ -7,10 +7,5 @@ module.exports = [ method: 'post', handler: controller.postImportChargeVersions, path: '/import/charge-versions' - }, - { - method: 'post', - handler: controller.postImportChargingData, - path: '/import/charging-data' } ] diff --git a/src/modules/licence-import/connectors/purpose-conditions-types.js b/src/modules/licence-import/connectors/purpose-conditions-types.js deleted file mode 100644 index 9a5e2e36..00000000 --- a/src/modules/licence-import/connectors/purpose-conditions-types.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict' - -const { pool } = require('../../../lib/connectors/db') -const queries = require('./queries/purpose-condition-types') - -/** - * Create purpose condition types - * If they exist update the existing records - */ -const createPurposeConditionTypes = async () => pool.query(queries.createPurposeConditionTypes) - -module.exports = { - createPurposeConditionTypes -} diff --git a/src/modules/licence-import/jobs/delete-removed-documents.js b/src/modules/licence-import/jobs/delete-removed-documents.js index 045f1241..73c7a04c 100644 --- a/src/modules/licence-import/jobs/delete-removed-documents.js +++ b/src/modules/licence-import/jobs/delete-removed-documents.js @@ -1,7 +1,7 @@ 'use strict' const documentsConnector = require('../connectors/documents') -const ImportPurposeConditionTypesJob = require('./import-purpose-condition-types.js') +const QueueCompaniesJob = require('../jobs/queue-companies.js') const JOB_NAME = 'licence-import.delete-removed-documents' @@ -28,7 +28,7 @@ async function handler () { async function onComplete (messageQueue, job) { if (!job.failed) { - await messageQueue.publish(ImportPurposeConditionTypesJob.createMessage()) + await messageQueue.publish(QueueCompaniesJob.createMessage()) } global.GlobalNotifier.omg(`${JOB_NAME}: finished`) diff --git a/src/modules/licence-import/jobs/import-purpose-condition-types.js b/src/modules/licence-import/jobs/import-purpose-condition-types.js deleted file mode 100644 index f19a94a0..00000000 --- a/src/modules/licence-import/jobs/import-purpose-condition-types.js +++ /dev/null @@ -1,42 +0,0 @@ -'use strict' - -const QueueCompaniesJob = require('./queue-companies.js') -const purposeConditionsConnector = require('../connectors/purpose-conditions-types') - -const JOB_NAME = 'licence-import.import-purpose-condition-types' - -function createMessage () { - return { - name: JOB_NAME, - options: { - singletonKey: JOB_NAME, - expireIn: '1 hours' - } - } -} - -async function handler () { - try { - global.GlobalNotifier.omg(`${JOB_NAME}: started`) - - return purposeConditionsConnector.createPurposeConditionTypes() - } catch (error) { - global.GlobalNotifier.omfg(`${JOB_NAME}: errored`, error) - throw error - } -} - -async function onComplete (messageQueue, job) { - if (!job.failed) { - await messageQueue.publish(QueueCompaniesJob.createMessage()) - } - - global.GlobalNotifier.omg(`${JOB_NAME}: finished`) -} - -module.exports = { - createMessage, - handler, - onComplete, - name: JOB_NAME -} diff --git a/src/modules/licence-import/plugin.js b/src/modules/licence-import/plugin.js index 11bf33d5..2b050789 100644 --- a/src/modules/licence-import/plugin.js +++ b/src/modules/licence-import/plugin.js @@ -7,7 +7,6 @@ const ImportCompanyJob = require('./jobs/import-company.js') const ImportLicenceJob = require('./jobs/import-licence.js') const ImportLicenceSystemJob = require('./jobs/import-licence-system.js') const ImportPointsJob = require('./jobs/import-points.js') -const ImportPurposeConditionTypesJob = require('./jobs/import-purpose-condition-types.js') const QueueCompaniesJob = require('./jobs/queue-companies.js') const QueueLicencesJob = require('./jobs/queue-licences.js') const QueueLicencesSystemJob = require('./jobs/queue-licences-system.js') @@ -21,13 +20,6 @@ async function register (server, _options) { return DeleteRemovedDocumentsJob.onComplete(server.messageQueue, executedJob) }) - // When the documents have been marked as deleted import a list of all companies into the - // water_import.company_import table - await server.messageQueue.subscribe(ImportPurposeConditionTypesJob.name, ImportPurposeConditionTypesJob.handler) - await server.messageQueue.onComplete(ImportPurposeConditionTypesJob.name, (executedJob) => { - return ImportPurposeConditionTypesJob.onComplete(server.messageQueue, executedJob) - }) - // When the water_import.company_import table is ready, jobs are scheduled to import each company await server.messageQueue.subscribe(QueueCompaniesJob.name, QueueCompaniesJob.handler) await server.messageQueue.onComplete(QueueCompaniesJob.name, (executedJob) => { diff --git a/src/modules/reference/controllers.js b/src/modules/reference/controllers.js new file mode 100644 index 00000000..2922955d --- /dev/null +++ b/src/modules/reference/controllers.js @@ -0,0 +1,13 @@ +'use strict' + +const ProcessSteps = require('./process-steps.js') + +async function reference (_request, h) { + ProcessSteps.go() + + return h.response().code(204) +} + +module.exports = { + reference +} diff --git a/src/modules/licence-import/connectors/queries/purpose-condition-types.js b/src/modules/reference/lib/condition-type-queries.js similarity index 74% rename from src/modules/licence-import/connectors/queries/purpose-condition-types.js rename to src/modules/reference/lib/condition-type-queries.js index 8fc249e0..04cde1fb 100644 --- a/src/modules/licence-import/connectors/queries/purpose-condition-types.js +++ b/src/modules/reference/lib/condition-type-queries.js @@ -1,12 +1,14 @@ -const createPurposeConditionTypes = ` +'use strict' + +const purposeConditionTypes = ` INSERT INTO water.licence_version_purpose_condition_types ( code, subcode, description, subcode_description - ) - SELECT "CODE", "SUBCODE", "DESCR", "SUBCODE_DESC" FROM import."NALD_LIC_COND_TYPES" - WHERE "AFFECTS_ABS" = 'Y' + ) + SELECT "CODE", "SUBCODE", "DESCR", "SUBCODE_DESC" FROM import."NALD_LIC_COND_TYPES" + WHERE "AFFECTS_ABS" = 'Y' ON CONFLICT (code, subcode) DO UPDATE SET description = excluded.description, @@ -15,5 +17,5 @@ INSERT INTO water.licence_version_purpose_condition_types ( ` module.exports = { - createPurposeConditionTypes + purposeConditionTypes } diff --git a/src/modules/charging-import/lib/queries/financial-agreement-types.js b/src/modules/reference/lib/financial-agreement-type-queries.js similarity index 84% rename from src/modules/charging-import/lib/queries/financial-agreement-types.js rename to src/modules/reference/lib/financial-agreement-type-queries.js index 974e15e0..ebb5f4aa 100644 --- a/src/modules/charging-import/lib/queries/financial-agreement-types.js +++ b/src/modules/reference/lib/financial-agreement-type-queries.js @@ -1,4 +1,6 @@ -const importFinancialAgreementTypes = ` +'use strict' + +const financialAgreementTypes = ` INSERT INTO water.financial_agreement_types (financial_agreement_code, description, disabled, date_created, date_updated) SELECT a."CODE", a."DESCR", a."DISABLED"::boolean, now(), now() FROM import."NALD_FIN_AGRMNT_TYPES" a @@ -12,5 +14,5 @@ DO ` module.exports = { - importFinancialAgreementTypes + financialAgreementTypes } diff --git a/src/modules/charging-import/lib/queries/purposes.js b/src/modules/reference/lib/purpose-queries.js similarity index 62% rename from src/modules/charging-import/lib/queries/purposes.js rename to src/modules/reference/lib/purpose-queries.js index 71477f13..357040ae 100644 --- a/src/modules/charging-import/lib/queries/purposes.js +++ b/src/modules/reference/lib/purpose-queries.js @@ -1,18 +1,12 @@ 'use strict' -const importPrimaryPurposes = `insert into water.purposes_primary (legacy_id, description, date_created, date_updated) +const primaryPurposes = `insert into water.purposes_primary (legacy_id, description, date_created, date_updated) select p."CODE", p."DESCR", now(), now() from import."NALD_PURP_PRIMS" p on conflict (legacy_id) do update set description= excluded.description, date_updated = now();` -const importSecondaryPurposes = ` - insert into water.purposes_secondary (legacy_id, description, date_created, date_updated) - select p."CODE", p."DESCR", now(), now() - from import."NALD_PURP_SECS" p on conflict (legacy_id) do update set - description= excluded.description, date_updated = now();` - -const importUses = `insert into water.purposes_uses ( +const purposes = `insert into water.purposes_uses ( legacy_id, description, date_created, @@ -40,20 +34,26 @@ const importUses = `insert into water.purposes_uses ( loss_factor = excluded.loss_factor, is_two_part_tariff = excluded.is_two_part_tariff;` -const importValidPurposeCombinations = `INSERT INTO water.purposes as prps (purpose_primary_id, purpose_secondary_id, purpose_use_id, date_created) -SELECT pp.purpose_primary_id, -ps.purpose_secondary_id, -pu.purpose_use_id, -now() -FROM import."NALD_PURPOSES" as NALD_P -JOIN water.purposes_primary as pp ON NALD_P."APPR_CODE" = pp."legacy_id" -JOIN water.purposes_secondary as ps ON NALD_P."APSE_CODE" = ps."legacy_id" -JOIN water.purposes_uses as pu ON NALD_P."APUS_CODE" = pu."legacy_id" -WHERE NALD_P."DISABLED" = 'N' ON CONFLICT DO NOTHING;` +const secondaryPurposes = ` + insert into water.purposes_secondary (legacy_id, description, date_created, date_updated) + select p."CODE", p."DESCR", now(), now() + from import."NALD_PURP_SECS" p on conflict (legacy_id) do update set + description= excluded.description, date_updated = now();` + +const validPurposeCombinations = `INSERT INTO water.purposes as prps (purpose_primary_id, purpose_secondary_id, purpose_use_id, date_created) + SELECT pp.purpose_primary_id, + ps.purpose_secondary_id, + pu.purpose_use_id, + now() + FROM import."NALD_PURPOSES" as NALD_P + JOIN water.purposes_primary as pp ON NALD_P."APPR_CODE" = pp."legacy_id" + JOIN water.purposes_secondary as ps ON NALD_P."APSE_CODE" = ps."legacy_id" + JOIN water.purposes_uses as pu ON NALD_P."APUS_CODE" = pu."legacy_id" + WHERE NALD_P."DISABLED" = 'N' ON CONFLICT DO NOTHING;` module.exports = { - importPrimaryPurposes, - importSecondaryPurposes, - importUses, - importValidPurposeCombinations + primaryPurposes, + purposes, + secondaryPurposes, + validPurposeCombinations } diff --git a/src/modules/reference/process-steps.js b/src/modules/reference/process-steps.js new file mode 100644 index 00000000..2653666d --- /dev/null +++ b/src/modules/reference/process-steps.js @@ -0,0 +1,29 @@ +'use strict' + +const ImportStep = require('./steps/import.js') + +const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../lib/general.js') + +async function go () { + let processComplete = false + + try { + global.GlobalNotifier.omg('reference started') + + const startTime = currentTimeInNanoseconds() + + await ImportStep.go() + + processComplete = true + + calculateAndLogTimeTaken(startTime, 'reference complete') + } catch (error) { + global.GlobalNotifier.oops('reference failed') + } + + return processComplete +} + +module.exports = { + go +} diff --git a/src/modules/reference/routes.js b/src/modules/reference/routes.js new file mode 100644 index 00000000..46ab3ffd --- /dev/null +++ b/src/modules/reference/routes.js @@ -0,0 +1,13 @@ +'use strict' + +const controllers = require('./controllers') + +const routes = [ + { + method: 'post', + handler: controllers.reference, + path: '/reference' + } +] + +module.exports = routes diff --git a/src/modules/reference/steps/import.js b/src/modules/reference/steps/import.js new file mode 100644 index 00000000..ba7c29f5 --- /dev/null +++ b/src/modules/reference/steps/import.js @@ -0,0 +1,33 @@ +'use strict' + +const db = require('../../../lib/connectors/db.js') +const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../../lib/general.js') +const ConditionTypeQueries = require('../lib/condition-type-queries.js') +const FinancialAgreementTypeQueries = require('../lib/financial-agreement-type-queries.js') +const PurposeQueries = require('../lib/purpose-queries.js') + +async function go () { + try { + global.GlobalNotifier.omg('reference.import started') + + const startTime = currentTimeInNanoseconds() + + await db.query(ConditionTypeQueries.purposeConditionTypes) + + await db.query(FinancialAgreementTypeQueries.financialAgreementTypes) + + await db.query(PurposeQueries.primaryPurposes) + await db.query(PurposeQueries.secondaryPurposes) + await db.query(PurposeQueries.purposes) + await db.query(PurposeQueries.validPurposeCombinations) + + calculateAndLogTimeTaken(startTime, 'reference.import complete') + } catch (error) { + global.GlobalNotifier.omfg('reference.import errored', error) + throw error + } +} + +module.exports = { + go +} diff --git a/src/routes.js b/src/routes.js index 01cc53ad..a2bedc22 100644 --- a/src/routes.js +++ b/src/routes.js @@ -11,6 +11,7 @@ const PermitRoutes = require('./modules/permit/routes.js') const CrmRoutes = require('./modules/crm/routes.js') const ReturnVersionsRoutes = require('./modules/return-versions/routes.js') const ModLogsRoutes = require('./modules/mod-logs/routes.js') +const ReferenceRoutes = require('./modules/reference/routes.js') const NightlyImportRoutes = require('./modules/nightly-import/routes.js') module.exports = [ @@ -26,5 +27,6 @@ module.exports = [ ...CrmRoutes, ...ReturnVersionsRoutes, ...ModLogsRoutes, + ...ReferenceRoutes, ...NightlyImportRoutes ]