Reorganise into a single 'import process' spike

.env.example

@@ -1,8 +1,5 @@
 SERVICE_NAME=water-abstraction-import
 
-JWT_TOKEN=
-JWT_SECRET=
-
 RETURNS_URI=
 WATER_URI=
 CRM_URI=
@@ -20,7 +17,6 @@ PROXY=
 # Airbrake config
 AIRBRAKE_HOST=https://my-errbit-instance.com
 AIRBRAKE_KEY=longvaluefullofnumbersandlettersinlowercase
-ENVIRONMENT=dev
 
 ERRBIT_KEY=
 ERRBIT_SERVER=
@@ -31,23 +27,17 @@ ENVIRONMENT=pre
 IMPORT_RETURNS_YEARS=
 NALD_ZIP_PASSWORD=
 
-REDIS_URI=
-
 S3_NALD_IMPORT_PATH=
 
 WATER_SERVICE_MAILBOX=
 
+# In processes like licence-details, configure how many 'things' to import within each batch
+IMPORT_PROCESS_BATCH_SIZE=
+
 # For development only! See https://eaflood.atlassian.net/browse/WATER-3201
 # Enables import of licence agreements during the licence import process. This was a one time import
 # run in production that we often need to re-run in local and non-production environments
 IMPORT_LICENCE_AGREEMENTS=false
 
 # Set log level for app. Default is 'info'
-WRLS_LOG_LEVEL=debug
-
-# Use Cron type syntax to set timings for these background processes
-WRLS_CRON_NALD='0 1 * * *'
-WRLS_CRON_LICENCES='0 4 * * 1,2,3,4,5'
-WRLS_CRON_RETURN_VERSIONS='0 6 * * 1,2,3,4,5'
-WRLS_CRON_MOD_LOGS='0 7 * * 1,2,3,4,5'
-WRLS_CRON_TRACKER='0 10 * * 1,2,3,4,5'
+WRLS_LOG_LEVEL=info

config.js

@@ -5,19 +5,12 @@ require('dotenv').config()
 const environment = process.env.ENVIRONMENT
 const isProduction = environment === 'prd'
 
-const isTlsConnection = (process.env.REDIS_HOST || '').includes('aws')
-
 module.exports = {
 
  blipp: {
  showAuth: true
  },
 
- jwt: {
- key: process.env.JWT_SECRET,
- verifyOptions: { algorithms: ['HS256'] }
- },
-
  // This config is specifically for hapi-pino which was added to replace the deprecated (and noisy!) hapi/good. At
  // some point all logging would go through this. But for now, it just covers requests & responses
  log: {
@@ -48,12 +41,6 @@ module.exports = {
  max: 20
  },
 
- pgBoss: {
- schema: 'water_import',
- application_name: process.env.SERVICE_NAME,
- newJobCheckIntervalSeconds: 10
- },
-
  server: {
  port: 8007,
  router: {
@@ -80,14 +67,15 @@ module.exports = {
 
  proxy: process.env.PROXY,
 
+ // In processes like licence-details, configure how many 'things' to import within each batch
+ processBatchSize: parseInt(process.env.IMPORT_PROCESS_BATCH_SIZE) || 10,
+
  import: {
  nald: {
  zipPassword: process.env.NALD_ZIP_PASSWORD,
- path: process.env.S3_NALD_IMPORT_PATH || 'wal_nald_data_release',
- schedule: process.env.WRLS_CRON_NALD || '0 1 * * *'
+ path: process.env.S3_NALD_IMPORT_PATH || 'wal_nald_data_release'
  },
  licences: {
- schedule: process.env.WRLS_CRON_LICENCES || '0 4 * * 1,2,3,4,5',
  // Note: these 2 flags need to be set to false for charging go-live
  // to suspend the import of invoice accounts and licence agreements
  // Update: I've changed those values to false ahead of the v2.0 charging
@@ -102,28 +90,10 @@ module.exports = {
  // This will supersede the implementation here where the billing contact history
  // was calculated from NALD data
  isBillingDocumentRoleImportEnabled: false
- },
- modLogs: {
- schedule: process.env.WRLS_CRON_MOD_LOGS || '0 7 * * 1,2,3,4,5'
- },
- returnVersions: {
- schedule: process.env.WRLS_CRON_RETURN_VERSIONS || '0 6 * * 1,2,3,4,5'
- },
- tracker: {
- schedule: process.env.WRLS_CRON_TRACKER || '0 10 * * 1,2,3,4,5'
  }
  },
-
- redis: {
- host: process.env.REDIS_HOST || '127.0.0.1',
- port: process.env.REDIS_PORT || 6379,
- password: process.env.REDIS_PASSWORD || '',
- ...(isTlsConnection) && { tls: {} },
- db: 0
- },
  notify: {
- templates: {
- service_status_alert: 'c34d1b16-694b-4364-8e7e-83e9dbd34a62'
- }
+ mailbox: process.env.WATER_SERVICE_MAILBOX,
+ templateId: 'c34d1b16-694b-4364-8e7e-83e9dbd34a62'
  }
 }

index.js

@@ -4,7 +4,6 @@ require('dotenv').config()
 // -------------- Require vendor code -----------------
 const Blipp = require('blipp')
 
-const HapiAuthJwt2 = require('hapi-auth-jwt2')
 const moment = require('moment')
 
 moment.locale('en-gb')
@@ -23,26 +22,9 @@ const plugins = [
  {
  plugin: Blipp,
  options: config.blipp
- },
- HapiAuthJwt2,
- require('./src/plugins/pg-boss.plugin'),
- require('./src/modules/licence-import/plugin'),
- require('./src/modules/charging-import/plugin'),
- require('./src/modules/mod-logs/plugin'),
- require('./src/modules/return-versions/plugin.js'),
- require('./src/modules/nald-import/plugin'),
- require('./src/modules/bill-runs-import/plugin'),
- require('./src/modules/core/plugin')
+ }
 ]
 
-const configureServerAuthStrategy = (server) => {
- server.auth.strategy('jwt', 'jwt', {
- ...config.jwt,
- validate: async (decoded) => ({ isValid: !!decoded.id })
- })
- server.auth.default('jwt')
-}
-
 const start = async function () {
  await server.register(plugins)
 
@@ -54,8 +36,6 @@ const start = async function () {
 
  server.validator(require('@hapi/joi'))
 
- configureServerAuthStrategy(server)
-
  server.route(routes)
 
  if (!module.parent) {
@@ -83,7 +63,6 @@ process
  // If there are no in-flight requests Hapi will immediately stop. If there are they get 25 seconds to finish
  // before Hapi terminates them
  await server.stop(options)
- await server.messageQueue.stop()
 
  // Log we're shut down using the same log format as the rest of our log output
  server.logger.info("That's all folks!")