Make batch sizes configurable

DEFRA · Sep 17, 2024 · 5917c4b · 5917c4b
1 parent d2f9e0a
commit 5917c4b
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 3 deletions.
diff --git a/.env.example b/.env.example
@@ -31,6 +31,9 @@ S3_NALD_IMPORT_PATH=
 
 WATER_SERVICE_MAILBOX=
 
+# In processes like licence-details, configure how many 'things' to import within each batch
+IMPORT_PROCESS_BATCH_SIZE=
+
 # For development only! See https://eaflood.atlassian.net/browse/WATER-3201
 # Enables import of licence agreements during the licence import process. This was a one time import
 # run in production that we often need to re-run in local and non-production environments

diff --git a/config.js b/config.js
@@ -67,6 +67,9 @@ module.exports = {
 
  proxy: process.env.PROXY,
 
+ // In processes like licence-details, configure how many 'things' to import within each batch
+ processBatchSize: parseInt(process.env.IMPORT_PROCESS_BATCH_SIZE) || 10,
+
  import: {
  nald: {
  zipPassword: process.env.NALD_ZIP_PASSWORD,

diff --git a/src/modules/company-details/steps/import.js b/src/modules/company-details/steps/import.js
@@ -4,6 +4,8 @@ const db = require('../../../lib/connectors/db.js')
 const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../../lib/general.js')
 const Loader = require('../lib/loader.js')
 
+const config = require('../../../../config.js')
+
 const PROGRESS_TICK = 1000
 
 async function go () {
@@ -31,7 +33,7 @@ async function go () {
 }
 
 async function _import (parties, count) {
- const batchSize = 10
+ const batchSize = config.processBatchSize
 
  let progress = PROGRESS_TICK
  let rejected = 0

diff --git a/src/modules/licence-details/steps/licence-import.js b/src/modules/licence-details/steps/licence-import.js
@@ -4,6 +4,8 @@ const db = require('../../../lib/connectors/db.js')
 const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../../lib/general.js')
 const Loader = require('../lib/loader.js')
 
+const config = require('../../../../config.js')
+
 const PROGRESS_TICK = 1000
 
 async function go () {
@@ -31,7 +33,7 @@ async function go () {
 }
 
 async function _import (licences, count) {
- const batchSize = 10
+ const batchSize = config.processBatchSize
 
  let progress = PROGRESS_TICK
  let rejected = 0

diff --git a/src/modules/permit/steps/import.js b/src/modules/permit/steps/import.js
@@ -4,6 +4,8 @@ const db = require('../../../lib/connectors/db.js')
 const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../../lib/general.js')
 const Loader = require('../lib/loader.js')
 
+const config = require('../../../../config.js')
+
 const PROGRESS_TICK = 1000
 
 async function go () {
@@ -30,7 +32,7 @@ async function go () {
 }
 
 async function _import (licenceReferences, count) {
- const batchSize = 10
+ const batchSize = config.processBatchSize
 
  let progress = PROGRESS_TICK
  let rejected = 0