Warnings for mapped data.

lib/importer/govuk-prototype-kit.config.json

@@ -81,6 +81,14 @@
     {
       "macroName": "importerRangeSelector",
       "importFrom": "importer/macros/range_selector.njk"
+    },
+    {
+      "macroName": "importerMappingWarningList",
+      "importFrom": "importer/macros/mapping_warning_list.njk"
+    },
+    {
+      "macroName": "importerMappingErrorList",
+      "importFrom": "importer/macros/mapping_error_list.njk"
     }
   ]
-}
+}

lib/importer/index.js

@@ -9,6 +9,8 @@ const os = require('node:os');
 
 const IMPORTER_SESSION_KEY = "importer.session"
 const IMPORTER_ERROR_KEY = "importer.error"
+const MAPPING_ERROR_KEY = "mapping.errors"
+const MAPPING_WARNING_KEY = "mapping.warnings"
 
 //--------------------------------------------------------------------
 // The endpoints used to receive POST requests from importer macros,
@@ -30,28 +32,40 @@ exports.Initialise = (config, router, prototypeKit) => {
   // one created by us.
   config.uploadPath ??= path.join(os.tmpdir(), 'reg-dyn-importer'); ;
 
+
   //--------------------------------------------------------------------
   // Removes any previous importer error from the session. When we set
   // an error we redirect to the referer and we expect that page to show
   // the error. Calling this in each POST endpoint ensures that we don't
   // remember errors after they have been shown,
   //--------------------------------------------------------------------
-  const cleanRequest = (request) => {
+  router.all("*", (request, res, next) => {
     delete request.session.data[IMPORTER_ERROR_KEY]
-  }
+    delete request.session.data[MAPPING_ERROR_KEY]
+    delete request.session.data[MAPPING_WARNING_KEY]
+    next();
+  });
+
 
   //--------------------------------------------------------------------
   // Make the route functions available in the templates. These functions
   // allow users to find the path that they should use to submit data,
   // and also allows them to specify which url to redirect to after
-  // the POST data has been processed.
+  // the POST data has been processed. Where an extra error url is provided
+  // it will be used if the POST request does not succeed (e.g. when
+  // mapping the data this can be used to redirect to the review page
+  // to view warnings)
   //--------------------------------------------------------------------
   for ([key, value] of IMPORTER_ROUTE_MAP) {
     const k = key;
     const v = value;
 
-    prototypeKit.views.addFunction(k, (next)=>{
-      return `${v}?next=${encodeURIComponent(next)}`;
+    prototypeKit.views.addFunction(k, (next, errorPage=null)=>{
+      let url = `${v}?next=${encodeURIComponent(next)}`;
+      if (errorPage) {
+        url = url + `&error=${encodeURIComponent(errorPage)}`
+      }
+      return url
     }, {})
   }
 
@@ -84,6 +98,27 @@ exports.Initialise = (config, router, prototypeKit) => {
     return false;
   }, {})
 
+
+  //--------------------------------------------------------------------
+  // Returns warnings or errors arising from applying a mapping.
+  //--------------------------------------------------------------------
+  prototypeKit.views.addFunction("importerMappingErrors", (data) => {
+    if (data[MAPPING_ERROR_KEY]) {
+        return data[MAPPING_ERROR_KEY]
+    }
+
+    return false;
+  }, {})
+
+  prototypeKit.views.addFunction("importerMappingWarnings", (data) => {
+    if (data[MAPPING_WARNING_KEY]) {
+        return data[MAPPING_WARNING_KEY]
+    }
+
+    return false;
+  }, {})
+
+
   //--------------------------------------------------------------------
   // Allows a template to obtain `count` rows from the start of the data
   // range.
@@ -169,8 +204,13 @@ exports.Initialise = (config, router, prototypeKit) => {
   // Redirects the current request to the 'next' URL after decoding the
   // encoded URI.
   //--------------------------------------------------------------------
-  const redirectOnwards = (request, response) => {
-    response.redirect(decodeURIComponent(request.query.next));
+  const redirectOnwards = (request, response, failed=false) => {
+    console.log(request.query)
+    if (failed && request.query.error) {
+      response.redirect(decodeURIComponent(request.query.error));
+    } else {
+      response.redirect(decodeURIComponent(request.query.next));
+    }
   }
 
   //--------------------------------------------------------------------
@@ -181,7 +221,6 @@ exports.Initialise = (config, router, prototypeKit) => {
   // time we run through the process.
   //--------------------------------------------------------------------
   router.all(IMPORTER_ROUTE_MAP.get("importerStartPath"),(request, response) => {
-    cleanRequest(request);
     redirectOnwards(request, response);
   });
 
@@ -192,8 +231,6 @@ exports.Initialise = (config, router, prototypeKit) => {
   const uploader = getUploader(config);
 
   router.post(IMPORTER_ROUTE_MAP.get("importerUploadPath"), uploader.single("file"), (request, response) => {
-      cleanRequest(request);
-
       let createResponse = session_lib.CreateSession(config, request);
 
       if (createResponse.error) {
@@ -320,6 +357,24 @@ exports.Initialise = (config, router, prototypeKit) => {
     }
 
     session.mapping = request.body;
+    if (Object.values(session.mapping).every((v) => v=='') ) {
+      request.session.data[IMPORTER_ERROR_KEY] = "No columns were mapped to the expected fields"
+      if (!request.query.error) {
+        response.redirect(request.get('Referrer'))
+      } else {
+        redirectOnwards(request, response, failed=true);
+      }
+      return;
+    }
+
+    const mapResults = sheets_lib.MapData(session);
+    if (mapResults.warningCount >= 0 || mapResults.errorCount >= 0) {
+      request.session.data[MAPPING_ERROR_KEY] = ["a test error"]
+      request.session.data[MAPPING_WARNING_KEY] = ["a test warning", "another warning"]
+
+      redirectOnwards(request, response, failed=true);
+      return
+    }
 
     // Ensure the session is persisted. Currently in session, eventually another way
     request.session.data[IMPORTER_SESSION_KEY] = session;

lib/importer/nunjucks/importer/macros/field_mapper.njk

@@ -17,13 +17,25 @@
 {% macro importerFieldMapper(data, caption='', columnTitle='Column', examplesTitle='Example values', fieldsTitle='Fields') %}
     {% set fields = data['importer.session']['fields'] %}
     {% set headings = importerGetHeaders(data) %}
-    {% set error = headings.error %}
+    {% set error = importerError(data) %}
 
-    {% if error %}
-    <p id="mapping-error" class="govuk-error-message">
-      <span class="govuk-visually-hidden">Error:</span> {{ error.text }}
-    </p>
-    {% endif %}
+
+  {% if error %}
+  <div class="govuk-error-summary" data-module="govuk-error-summary">
+    <div role="alert">
+      <h2 class="govuk-error-summary__title">
+        There was a problem submitting this data
+      </h2>
+      <div class="govuk-error-summary__body">
+        <ul class="govuk-list govuk-error-summary__list">
+          <li>
+            {{ error.text }}
+          </li>
+        </ul>
+      </div>
+    </div>
+  </div>
+  {% endif %}
 
 
 <table class="govuk-table">

lib/importer/nunjucks/importer/macros/mapping_error_list.njk

@@ -0,0 +1,19 @@
+
+
+{#
+    Shows errors generated when attempting to
+    perform a mapping
+#}
+{% macro importerMappingErrorList(data) %}
+    {% set errors = importerMappingErrors(data) %}
+    {% if errors %}
+    <div>
+        <h2 class="govuk-heading-m">Errors</h2>
+        <ul>
+            {% for error in errors %}
+            <li>{{ error }}</li>
+            {% endfor %}
+        </ul>
+    <div>
+    {% endif %}
+{% endmacro %}

lib/importer/nunjucks/importer/macros/mapping_warning_list.njk

@@ -0,0 +1,19 @@
+
+
+{#
+    Shows warnings generated when attempting to
+    perform a mapping
+#}
+{% macro importerMappingWarningList(data) %}
+    {% set warnings = importerMappingWarnings(data) %}
+    {% if warnings %}
+    <div>
+        <h2 class="govuk-heading-m">Warnings</h2>
+        <ul>
+            {% for warning in warnings %}
+            <li>{{ warning }}</li>
+            {% endfor %}
+        </ul>
+    <div>
+    {% endif %}
+{% endmacro %}

lib/importer/sheets.js

@@ -218,10 +218,10 @@ exports.GetColumnValues = (session, columnIndex, cellWidth=30, count=10) => {
 // Uses the session provided, which must contain a sheet name and a
 // mapping to perform the mapping of the data across the remaining
 // rows in the sheet to return an array of objects.
-exports.MapData = (session, previewLimit = DEFAULT_PREVIEW_LIMIT) => {
-  // Construct the range to be mapped - everything but the first row
-  const sheetDimensions = backend.SessionGetInputDimensions(session.backendSid)
-        .sheetDimensions.get(session.sheet);
+exports.MapData = (session, limit=0) => {
+    // Construct the range to be mapped - everything but the first row
+    const sheetDimensions = backend.SessionGetInputDimensions(session.backendSid)
+          .sheetDimensions.get(session.sheet);
 
   const hRange = session.headerRange;
   const rowRange = backend.SessionSuggestDataRange(session.backendSid, session.headerRange, session.footerRange);
@@ -246,8 +246,8 @@ exports.MapData = (session, previewLimit = DEFAULT_PREVIEW_LIMIT) => {
   // Read all the results into memory
   const resultSummary = backend.JobGetSummary(backendJid);
 
-  const recordsToPreview = Math.min(resultSummary.recordCount, previewLimit);
-  const results = backend.JobGetRecords(backendJid, 0, recordsToPreview);
+  const recordsToView = limit > 0 ? Math.min(resultSummary.recordCount, limit) : resultSummary.recordCount;
+  const results = backend.JobGetRecords(backendJid, 0, recordsToView);
 
   // Rewrite the results to be arrays in session.fields order, rather than objects
   const fields = session.fields;
@@ -274,7 +274,7 @@ exports.MapData = (session, previewLimit = DEFAULT_PREVIEW_LIMIT) => {
   return {
     resultRecords: resultsAsArrays,
     totalCount: resultSummary.recordCount,
-    extraRecordCount: resultSummary.recordCount-recordsToPreview,
+    extraRecordCount: resultSummary.recordCount-recordsToView,
     warningCount: resultSummary.warningCount,
     errorCount: resultSummary.errorCount,
     warnings: warnings,

lib/importer/templates/review.html

@@ -1,5 +1,6 @@
 {% extends "layouts/main.html" %}
-{% from "importer/macros/table_view.njk" import importerTableView %}
+{% from "importer/macros/mapping_warning_list.njk" import importerMappingWarningList %}
+{% from "importer/macros/mapping_error_list.njk" import importerMappingErrorList %}
 
 
 {% block pageTitle %} {{ serviceName }} – GOV.UK Prototype Kit {% endblock %}
@@ -12,11 +13,10 @@
     <div class="govuk-grid-column-two-thirds">
         <h1 class="govuk-heading-l">Review your data</h1>
 
-        <form action="{{ importerReviewDataPath('/success') }}" method="post">
-            <div class="govuk-button-group">
-                {{ govukButton({ text: "Submit" }) }}
-            </div>
-        </form>
+        {{ importerMappingWarningList(data) }}
+
+        {{ importerMappingErrorList(data) }}
+
     </div>
 </div>
 {% endblock %}

prototypes/basic/app/views/mapping.html

@@ -14,7 +14,7 @@ <h1 class="govuk-heading-l">Map columns</h1>
 
         <h2 class="govuk-heading-m">{{sheet}}</h2>
 
-        <form action="{{ importerMapDataPath('/success') }}" method="post">
+        <form action="{{ importerMapDataPath('/success', '/review') }}" method="post">
 
             {{ importerFieldMapper(data) }}
 

prototypes/basic/app/views/review.html

@@ -1,5 +1,6 @@
 {% extends "layouts/main.html" %}
-{% from "importer/macros/table_view.njk" import importerTableView %}
+{% from "importer/macros/mapping_warning_list.njk" import importerMappingWarningList %}
+{% from "importer/macros/mapping_error_list.njk" import importerMappingErrorList %}
 
 
 {% block pageTitle %} {{ serviceName }} – GOV.UK Prototype Kit {% endblock %}
@@ -12,11 +13,10 @@
     <div class="govuk-grid-column-two-thirds">
         <h1 class="govuk-heading-l">Review your data</h1>
 
-        <form action="{{ importerReviewDataPath('/success') }}" method="post">
-            <div class="govuk-button-group">
-                {{ govukButton({ text: "Submit" }) }}
-            </div>
-        </form>
+        {{ importerMappingWarningList(data) }}
+
+        {{ importerMappingErrorList(data) }}
+
     </div>
 </div>
 {% endblock %}