Create a new orchestration config to save bundles to ecr-viewer (#2853)

* Create a new orchestration config to save bundles to ecr-viewer

* Remove old config

* Remove some subfolders

* Update tests to handle updated save call.

* Update tests to handle new logic in save endpoint.

* Fix how to determine if a message is a bundle

* Fix tests for new logic in save endpoint

* Make converting eicr the default behaviour

containers/ecr-viewer/seed-scripts/create-seed-data.py

@@ -1,3 +1,4 @@
+import argparse
 import json
 import os
 import traceback
@@ -8,7 +9,22 @@
 BASEDIR = os.path.dirname(os.path.abspath(__file__))
 
 
-def convert_files():
+def _get_args():
+    parser = argparse.ArgumentParser(
+        prog="Create Seed Data",
+        description="Convert eICR and RR files to FHIR bundles and insert them into the database.",
+        epilog="For each directory in baseECR/LA the script will look for a `CDA_eICR.xml` and `CDA_RR.xml` file. If they are found, it will convert them into a FHIR bundle (saved as `bundle.json`) and insert that into the database using the Orchestration service.",
+    )
+    parser.add_argument(
+        "-s",
+        "--skip_convert",
+        action="store_true",
+        help="If this is set, if `bundle.json` already exists, the script will not look for `CDA_eICR.xml` and `CDA_RR.xml` files to convert them again, and use the existing `bundle.json`.",
+    )
+    return parser.parse_args()
+
+
+def _process_files(args):
     """
     Convert eICR and RR into FHIR bundles using the FHIR converter.
 
@@ -28,7 +44,24 @@ def convert_files():
 
                 # Check if it's a directory
                 if os.path.isdir(folder_path):
-                    try:
+                    # If `bundle.json` exists and ski_convert is set just upload the bundle
+                    if (
+                        os.path.exists(os.path.join(folder_path, "bundle.json"))
+                        and args.skip_convert
+                    ):
+                        # Just upload the bundle
+                        with open(
+                            os.path.join(folder_path, "bundle.json")
+                        ) as fhir_file:
+                            payload = {
+                                "message_type": "fhir",
+                                "data_type": "fhir",
+                                "config_file_name": "save-bundle-to-ecr-viewer.json",
+                                "message": json.load(fhir_file),
+                            }
+                            _process_eicrs(subfolder, folder, folder_path, payload)
+                    # If we are not just inserting the bundle, check for the necessary files
+                    elif os.path.exists(os.path.join(folder_path, "CDA_eICR.xml")):
                         # Open the necessary files in the folder
                         with (
                             open(
@@ -41,49 +74,52 @@ def convert_files():
                             payload = {
                                 "message_type": "ecr",
                                 "data_type": "ecr",
-                                "config_file_name": "seed-ecr-viewer-config.json",
+                                "config_file_name": "save-eicr-to-ecr-viewer-config.json",
                                 "message": eicr_file.read(),
                                 "rr_data": rr_file.read(),
                             }
-
-                            print(f"{URL}/process-message for {subfolder}/{folder}")
-
-                            response = requests.post(
-                                f"{URL}/process-message", json=payload
-                            )
-                            if response.status_code == 200:
-                                responses_json = response.json()["processed_values"][
-                                    "responses"
-                                ]
-                                for response in responses_json:
-                                    if "stamped_ecr" in response:
-                                        with open(
-                                            os.path.join(folder_path, "bundle.json"),
-                                            "w",
-                                        ) as fhir_file:
-                                            json.dump(
-                                                response["stamped_ecr"][
-                                                    "extended_bundle"
-                                                ],
-                                                fhir_file,
-                                                indent=4,
-                                            )
-                                print(
-                                    f"Converted {folder} in {subfolder} successfully."
-                                )
-                            # Handle the case where the response fails
-                            else:
-                                print(f"Failed to convert {folder} in {subfolder}.")
-                    # Handle file not found or other potential errors
-                    except FileNotFoundError as e:
-                        print(f"Required file not found in {folder_path}: {e}")
-                    except Exception as e:
+                            _process_eicrs(subfolder, folder, folder_path, payload)
+                    # If neither `bundle.json` nor `CDA_eICR.xml` exists, skip processing
+                    else:
                         print(
-                            f"An error occurred processing {folder} in {subfolder}: {e}\n\n{traceback.format_exc()}"
+                            f"Neither `bundle.json` nor `CDA_eICR.xml` found in {folder_path}. Skipping."
                         )
+                        continue
+
                 # If the subfolder is not a directory, print a message
                 else:
                     print(f"{subfolder_path} is not a valid directory.")
 
 
-convert_files()
+def _process_eicrs(subfolder, folder, folder_path, payload):
+    try:
+        print(f"{URL}/process-message for {subfolder}/{folder}")
+        response = requests.post(f"{URL}/process-message", json=payload)
+        if response.status_code == 200:
+            responses_json = response.json()["processed_values"]["responses"]
+            for response in responses_json:
+                if "stamped_ecr" in response:
+                    with open(
+                        os.path.join(folder_path, "bundle.json"), "w"
+                    ) as fhir_file:
+                        json.dump(
+                            response["stamped_ecr"]["extended_bundle"],
+                            fhir_file,
+                            indent=4,
+                        )
+            print(f"Converted {folder} in {subfolder} successfully.")
+        # Handle the case where the response fails
+        else:
+            print(f"Failed to convert {folder} in {subfolder}.")
+    # Handle file not found or other potential errors
+    except FileNotFoundError as e:
+        print(f"Required file not found in {folder_path}: {e}")
+    except Exception as e:
+        print(
+            f"An error occurred processing {folder} in {subfolder}: {e}\n\n{traceback.format_exc()}"
+        )
+
+
+if __name__ == "__main__":
+    args = _get_args()
+    _process_files(args)

containers/orchestration/app/custom_configs/save-bundle-to-ecr-viewer.json

@@ -0,0 +1,28 @@
+{
+  "workflow": [
+    {
+      "name": "metadata_values",
+      "service": "message_parser",
+      "endpoint": "/parse_message",
+      "params": {
+        "message_format": "fhir",
+        "parsing_schema_name": "ecr_viewer_metadata.json"
+      }
+    },
+    {
+      "name:": "save_bundle",
+      "service": "save_bundle",
+      "endpoint": "/api/save-fhir-data",
+      "params": {
+        "saveSource": "postgres"
+      },
+      "previous_response_to_param_mapping": {
+        "metadata_values": "metadata"
+      }
+    }
+  ],
+  "outputs": [
+    "metadata_values"
+  ],
+  "default-response": false
+}

containers/orchestration/app/handlers/request_builders/ecr_viewer.py

@@ -35,7 +35,15 @@ def build_save_fhir_data_body(
         if workflow_params.get("fhirBundle"):
             fhirBundle = workflow_params["fhirBundle"].json()["extended_bundle"]
         else:
-            fhirBundle = input_msg
+            # If the message from the last step is a FHIR bundle, use it.
+            if input_msg.get("resourceType") == "Bundle":
+                fhirBundle = input_msg
+            # If the message from the last step is not a FHIR bundle, use the orginal message from the request.
+            elif orchestration_request.get("message_type") == "fhir":
+                fhirBundle = orchestration_request.get("message")
+            # If neither the message from the last step nor the original message from the request is a FHIR bundle, raise an error.
+            else:
+                raise ValueError("Invalid message type for FHIR data.")
 
         request = {
             "fhirBundle": fhirBundle,

containers/orchestration/tests/integration/test_orchestration.py

@@ -189,7 +189,7 @@ def test_previous_response_mapping_for_ecr_viewer(setup, clean_up_db):
         form_data = {
             "message_type": "ecr",
             "data_type": "zip",
-            "config_file_name": "seed-ecr-viewer-config.json",
+            "config_file_name": "save-eicr-to-ecr-viewer-config.json",
         }
         files = {"upload_file": ("file.zip", file)}
         orchestration_response = httpx.post(

containers/orchestration/tests/test_process_message_endpoint.py

@@ -84,6 +84,7 @@ def test_process_message_success(patched_post_request):
         "bundle": {
             "bundle_type": "batch",
             "placeholder_id": "abcdefg",
+            "resourceType": "Bundle",
             "entry": [{"resource": {"id": "foo"}}],
         }
     }
@@ -94,6 +95,7 @@ def test_process_message_success(patched_post_request):
         "extended_bundle": {
             "bundle_type": "batch",
             "placeholder_id": "abcdefg",
+            "resourceType": "Bundle",
             "entry": [{"resource": {"id": "foo"}}],
         }
     }
@@ -289,6 +291,7 @@ def test_process_zip_success(patched_post_request):
                     "bundle": {
                         "bundle_type": "batch",
                         "placeholder_id": "abcdefg",
+                        "resourceType": "Bundle",
                         "entry": [],
                     }
                 }
@@ -309,6 +312,7 @@ def test_process_zip_success(patched_post_request):
             "extended_bundle": {
                 "bundle_type": "batch",
                 "placeholder_id": "abcdefg",
+                "resourceType": "Bundle",
                 "entry": [{"resource": {"id": "foo"}}],
             }
         }