Changed respondent data to log warning messages instead of print stat…

…ements

If respondent data is empty is is not zipped up in reports
added feature tests

pyproject.toml

@@ -32,6 +32,7 @@ flake8 = "^4.0.1"
 mypy = "^0.942"
 types-requests = "^2.28.7"
 types-python-dateutil = "^2.8.19"
+behave = "^1.2.6"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

reports/mi_hub_respondent_data_report.py

@@ -34,5 +34,6 @@ def get_mi_hub_respondent_data(config, questionnaire_name):
             date_of_birth=item.get("qHousehold.QHHold.Person[1].tmpDoB"),
             age=item.get("qHousehold.QHHold.Person[1].DVAge"),
         )
-        mi_hub_respondent_data_list.append(mi_hub_respondent_data_record)
+        if not all(value in ("", None) for value in vars(mi_hub_respondent_data_record).values()):
+            mi_hub_respondent_data_list.append(mi_hub_respondent_data_record)
     return mi_hub_respondent_data_list

services/deliver_mi_hub_reports_service.py

@@ -1,5 +1,5 @@
 import datetime
-
+import logging
 from functions.csv_functions import write_csv
 from functions.zip_functions import create_zip
 
@@ -19,22 +19,23 @@ def upload_mi_hub_reports_to_gcp(
                 {"filename": "call_history.csv", "content": call_history_csv}
             )
         else:
-            print(f"No call history for {questionnaire_name}")
+            logging.warning(f"No call history for {questionnaire_name}")
 
         if mi_hub_respondent_data:
             respondent_data_csv = write_csv(mi_hub_respondent_data)
             zip_data.append(
                 {"filename": "respondent_data.csv", "content": respondent_data_csv}
             )
         else:
-            print(f"No respondent data for {questionnaire_name}")
+            logging.warning(f"No respondent data for {questionnaire_name}")
 
         if zip_data:
             zipped_data = create_zip(zip_data)
             datetime_string = datetime.datetime.now().strftime("%d%m%Y_%H%M%S")
             mi_filename = f"mi_{questionnaire_name}_{datetime_string}"
             google_storage.upload_zip(f"{mi_filename}.zip", zipped_data)
         else:
-            print(f"No data for {questionnaire_name}")
+            logging.warning(f"No data for {questionnaire_name}")
 
+        logging.info(f"Done - {questionnaire_name}")
         return f"Done - {questionnaire_name}"

tests/cloud_functions/test_deliver_mi_hub_processor.py

@@ -287,7 +287,7 @@ def mock_mi_hub_respondent_data_partially_complete():
 @mock.patch(
     "cloud_functions.deliver_mi_hub_reports.DeliverMiHubReportsService.upload_mi_hub_reports_to_gcp"
 )
-def test_deliver_mi_hub_reports_cloud_function_processor_calls_upload_mi_hub_reports_to_gcp_with_empty_data(
+def test_deliver_mi_hub_reports_cloud_function_processor_calls_upload_mi_hub_reports_to_gcp_partially_complete(
     _mock_upload_mi_hub_reports_to_gcp,
     _mock_get_mi_hub_respondent_data,
     _mock_get_mi_hub_call_history,

tests/conftest.py

@@ -114,6 +114,15 @@ def api_reporting_data_response():
     }
 
 
+@pytest.fixture
+def api_reporting_data_response_empty():
+    return {
+        "instrumentName": "",
+        "instrumentId": "",
+        "reportingData": [],
+    }
+
+
 @pytest.fixture
 def interviewer_name():
     return "ricer"

tests/data_sources/test_questionnaire_data.py

@@ -109,3 +109,20 @@ def test_get_questionnaire_name_from_id_returns_empty_string_when_questionnaire_
 
     # act & assert
     assert get_questionnaire_name(config, "12345-12345-12345-12345-ZZZZZ") == ""
+
+
+def test_get_questionnaire_data_empty(
+    questionnaire_name,
+    config,
+    questionnaire_fields_to_get,
+    requests_mock,
+    api_reporting_data_response_empty,
+):
+    requests_mock.get(
+        f"http://{config.blaise_api_url}/api/v2/serverparks/gusty/questionnaires/{questionnaire_name}/report",
+        json=api_reporting_data_response_empty,
+    )
+    reporting_data = get_questionnaire_data(
+        questionnaire_name, config, questionnaire_fields_to_get
+    )
+    assert reporting_data == []

tests/features/environment.py

@@ -0,0 +1,61 @@
+from typing import List
+import pytest
+from werkzeug.security import generate_password_hash
+from app.app import setup_app
+from models.mi_hub_call_history_model import MiHubCallHistory
+from unittest.mock import create_autospec
+from functions.google_storage_functions import GoogleStorage
+
+
+def mock_mi_hub_call_history() -> List[MiHubCallHistory]:
+    return [
+        MiHubCallHistory(
+            questionnaire_name="LMS2222Z",
+            questionnaire_id="s0me-r7nd0m-gu1d",
+            serial_number="900001",
+            dial_date="20221017",
+            dial_time="10:31:36",
+            end_time="10:32:06",
+            seconds_interview=30,
+            call_number=1,
+            dial_number=1,
+            interviewer="thorne1",
+            dial_line_number=None,
+            outcome_code="461",
+        ),
+        MiHubCallHistory(
+            questionnaire_name="LMS2222Z",
+            questionnaire_id="s0me-r7nd0m-gu1d",
+            serial_number="900002",
+            dial_date="20221017",
+            dial_time="10:33:36",
+            end_time="10:34:36",
+            seconds_interview=60,
+            call_number=1,
+            dial_number=1,
+            interviewer="thorne1",
+            dial_line_number=None,
+            outcome_code="461",
+        ),
+        MiHubCallHistory(
+            questionnaire_name="LMS2222Z",
+            questionnaire_id="s0me-r7nd0m-gu1d",
+            serial_number="900003",
+            dial_date="20221017",
+            dial_time="10:35:36",
+            end_time="10:35:46",
+            seconds_interview=10,
+            call_number=1,
+            dial_number=1,
+            interviewer="thorne1",
+            dial_line_number=None,
+            outcome_code="461",
+        ),
+    ]
+
+
+def before_scenario(context, _scenario):
+    context.questionnaire_name = "LMS2222Z"
+    context.mi_hub_respondent_data = []
+    context.mi_hub_call_history = mock_mi_hub_call_history()
+    context.mock_google_storage = create_autospec(GoogleStorage)

tests/features/run_mi_hub_reports.feature

@@ -0,0 +1,63 @@
+Feature: Generate Reports
+
+  Scenario Outline: All fields and data are available
+    Given all of the following fields are available for the Respondent Data report
+      | field_name                        | value           |
+      | QID.Serial_Number                 | 900001          |
+      | QHAdmin.HOut                      | 310             |
+      | QHAdmin.Interviewer[1]            | testuser        |
+      | Mode                              | testmode        |
+      | QDataBag.PostCode                 | PO57 2OD        |
+      | QHousehold.QHHold.Person[1].Sex   | Male            |
+      | QHousehold.QHHold.Person[1].tmpDoB| 2-11-2022_3:08  |
+      | QHousehold.QHHold.Person[1].DVAge | 2               |
+      | DateTimeStamp                     | 2-11-2022_9:20  |
+    Then data is present in all fields
+    When the report generation is triggered
+    Then the report should be generated and delivered
+    And no warnings should be logged
+    Examples:
+      | field_name   |
+      | 0            |
+
+Scenario Outline: Some fields are missing and all data is available
+    Given all of the following fields are available for the Respondent Data report
+      | field_name                        | value           |
+      | QID.Serial_Number                 | 900001          |
+      | QHAdmin.HOut                      | 310             |
+      | QHAdmin.Interviewer[1]            | testuser        |
+      | Mode                              | testmode        |
+      | QDataBag.PostCode                 |                 |
+      | QHousehold.QHHold.Person[1].Sex   | Male            |
+      | QHousehold.QHHold.Person[1].tmpDoB|                 |
+      | QHousehold.QHHold.Person[1].DVAge | 2               |
+      | DateTimeStamp                     | 2-11-2022_9:20  |
+    Then data is present in all fields
+    Then the field "postcode" is missing
+    And the field "date_of_birth" is missing
+    When the report generation is triggered
+    Then the report should be generated and delivered
+    And "Done - LMS2222Z" is logged as an information message
+    Examples:
+      | field_name   |
+      | 0            |
+
+
+  Scenario Outline: All fields are missing
+    Given none of the following fields are available for the report
+      | field_name                        |
+      | QID.Serial_Number                 |
+      | QHAdmin.HOut                      |
+      | QHAdmin.Interviewer[1]            |
+      | Mode                              |
+      | QDataBag.PostCode                 |
+      | QHousehold.QHHold.Person[1].Sex   |
+      | QHousehold.QHHold.Person[1].tmpDoB|
+      | QHousehold.QHHold.Person[1].DVAge |
+      | DateTimeStamp                     |
+    When the report generation is triggered
+    Then the report should be generated and delivered
+    And "No respondent data for LMS2222Z" is logged as an warning message
+    Examples:
+      | field_name   |
+      | 0            |

tests/features/steps/steps.py

@@ -0,0 +1,83 @@
+# type: ignore[no-redef]
+
+import logging
+from behave import given, then, when
+from models.mi_hub_respondent_data_model import MiHubRespondentData
+from services.deliver_mi_hub_reports_service import DeliverMiHubReportsService
+
+
+@given('all of the following fields are available for the Respondent Data report')
+def step_impl(context):
+    if context.table:
+        fields = {row["field_name"]: row["value"] for row in context.table}
+
+        context.mi_hub_respondent_data = [MiHubRespondentData(
+            serial_number=str(fields["QID.Serial_Number"]),
+            outcome_code=str(fields["QHAdmin.HOut"]),
+            date_completed=str(fields["DateTimeStamp"]),
+            interviewer=str(fields["QHAdmin.Interviewer[1]"]),
+            mode=str(fields["Mode"]),
+            postcode=str(fields["QDataBag.PostCode"]),
+            gender=str(fields["QHousehold.QHHold.Person[1].Sex"]),
+            date_of_birth=str(fields["QHousehold.QHHold.Person[1].tmpDoB"]),
+            age=str(fields["QHousehold.QHHold.Person[1].DVAge"]),
+        )]
+
+
+@when('the report generation is triggered')
+def step_impl(context):
+    response = DeliverMiHubReportsService.upload_mi_hub_reports_to_gcp(
+        questionnaire_name=context.questionnaire_name,
+        mi_hub_call_history=context.mi_hub_call_history,
+        mi_hub_respondent_data=context.mi_hub_respondent_data,
+        google_storage=context.mock_google_storage,
+    )
+
+    context.response = response
+
+
+@then('data is present in all fields')
+def step_impl(context):
+    for respondent_data in context.mi_hub_respondent_data:
+        assert (
+                not all(value in ("", None) for value in vars(respondent_data).values())
+        ), f"Fields are not all present"
+
+
+@then('the field "{field}" is missing')
+def step_impl(context, field):
+    for respondent_data in context.mi_hub_respondent_data:
+        assert (
+            getattr(respondent_data, field) == ""
+        ), f"Field {field} is present"
+
+
+@given('none of the following fields are available for the report')
+def step_impl(context):
+    context.mi_hub_respondent_data = []
+
+
+@then('the report should be generated and delivered')
+def step_impl(context):
+    assert (
+        context.response == "Done - " + context.questionnaire_name
+    ), f"No report is generated"
+
+
+@then('no warnings should be logged')
+def step_impl(context):
+    assert (
+            True
+    ), f"Warnings should not be generated"
+
+
+@then('"{message}" is logged as an {error_level} message')
+def step_impl(context, message, error_level):
+    messages = [
+        (record.levelno, record.getMessage()) for record in context.log_capture.buffer
+    ]
+    mappings = {"information": logging.INFO, "error": logging.ERROR, "warning": logging.WARNING}
+    assert (
+               mappings[error_level],
+               message,
+           ) in messages, f"Could not find {mappings[error_level]}, {message} in {messages}"