Reduce DB calls in water import

By getting the data we need for the licences when fetching the licences to import in `import.js`, we save approximately 73K calls to the DB. In our tests over 5000 licences this saved 5 secs. Not massive, but a noticeable improvement so worth doing.
DEFRA · Sep 16, 2024 · 5cc4390 · 5cc4390
1 parent 32e787e
commit 5cc4390
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 47 deletions.
diff --git a/src/modules/water/lib/fetcher.js b/src/modules/water/lib/fetcher.js
@@ -2,9 +2,8 @@
 
 const db = require('../../../lib/connectors/db.js')
 
-async function go (licenceRef) {
- const naldLicence = await _licence(licenceRef)
- const { ID: id, FGAC_REGION_CODE: regionCode } = naldLicence
+async function go (naldLicence) {
+ const { ID: id, FGAC_REGION_CODE: regionCode, LIC_NO: licenceRef } = naldLicence
 
  const naldLicenceVersions = await _licenceVersions(regionCode, id)
  const naldLicenceVersionPurposes = await _licenceVersionPurposes(regionCode, id)
@@ -65,30 +64,6 @@ function _addressAndPartyIds (licenceVersions, licenceRoles) {
  return { addressIds, partyIds }
 }
 
-async function _licence (licenceRef) {
- const query = `
- SELECT
- "ID",
- "LIC_NO",
- "FGAC_REGION_CODE",
- "ORIG_EFF_DATE",
- "EXPIRY_DATE",
- "LAPSED_DATE",
- "REV_DATE",
- "AREP_EIUC_CODE",
- "AREP_AREA_CODE",
- "AREP_SUC_CODE",
- "AREP_LEAP_CODE"
- FROM
- "import"."NALD_ABS_LICENCES"
- WHERE "LIC_NO"=$1;
- `
-
- const results = await db.query(query, [licenceRef])
-
- return results[0]
-}
-
 async function _licencePriorToImport (licenceRef) {
  const query = `
  SELECT

diff --git a/src/modules/water/lib/loader.js b/src/modules/water/lib/loader.js
@@ -4,18 +4,19 @@ const db = require('../../../lib/connectors/db.js')
 const Fetcher = require('./fetcher.js')
 const Transformer = require('./transformer.js')
 
-async function go (licenceRef) {
+async function go (naldLicence) {
+ const { LIC_NO: licenceRef } = naldLicence
+
  try {
  const {
  licencePriorToImport,
  naldAddresses,
- naldLicence,
  naldLicenceRoles,
  naldLicenceVersions,
  naldLicenceVersionPurposes,
  naldLicenceVersionPurposeConditions,
  naldParties
- } = await Fetcher.go(licenceRef)
+ } = await Fetcher.go(naldLicence)
 
  const transformedLicenceData = Transformer.go(
  naldAddresses,

diff --git a/src/modules/water/steps/import.js b/src/modules/water/steps/import.js
@@ -36,21 +36,6 @@ async function _import (licences, count) {
  let progress = PROGRESS_TICK
  let rejected = 0
 
- // for (let i = 0; i < count; i++) {
- // if (i === progress) {
- // progress = progress + PROGRESS_TICK
- // global.GlobalNotifier.omg(`water.import progress (${i} of ${count})`)
- // }
-
- // const licenceToProcess = licences[i]
-
- // try {
- // await Loader.go(licenceToProcess.LIC_NO)
- // } catch (error) {
- // rejected += 1
- // }
- // }
-
  for (let i = 0; i < count; i += batchSize) {
  if (i === progress) {
  progress = progress + PROGRESS_TICK
@@ -60,7 +45,8 @@ async function _import (licences, count) {
  const licencesToProcess = licences.slice(i, i + batchSize)
 
  const processes = licencesToProcess.map((licenceToProcess) => {
- return Loader.go(licenceToProcess.LIC_NO)
+ // return Loader.go(licenceToProcess.LIC_NO)
+ return Loader.go(licenceToProcess)
  })
 
  const results = await Promise.allSettled(processes)
@@ -79,7 +65,24 @@ async function _import (licences, count) {
 }
 
 async function _licences () {
- return db.query('SELECT "LIC_NO" FROM "import"."NALD_ABS_LICENCES";')
+ const query = `
+ SELECT
+ "ID",
+ "LIC_NO",
+ "FGAC_REGION_CODE",
+ "ORIG_EFF_DATE",
+ "EXPIRY_DATE",
+ "LAPSED_DATE",
+ "REV_DATE",
+ "AREP_EIUC_CODE",
+ "AREP_AREA_CODE",
+ "AREP_SUC_CODE",
+ "AREP_LEAP_CODE"
+ FROM
+ "import"."NALD_ABS_LICENCES";
+ `
+
+ return db.query(query)
 }
 
 module.exports = {