Add metadata to save fhir data api (#2455)

* Add metadata to save fhir data api * More detailed error handling * Add metadata saving to S3 saving * Move metadata to new endpoint * Revert changes to save fhir data route * Update error messages to include Azure * Remove S3, Azure saving for metadata. * Add metadata endpoint to the fhir data save endpoint * Update a couple of tests * Update status code in test * Revert config change and fix documentation --------- Co-authored-by: Josh Nygaard <[email protected]>
CDCgov · Sep 13, 2024 · d155933 · d155933
1 parent 8a46342
commit d155933
Show file tree

Hide file tree

Showing 4 changed files with 156 additions and 28 deletions.
diff --git a/containers/ecr-viewer/src/app/api/save-fhir-data/route.ts b/containers/ecr-viewer/src/app/api/save-fhir-data/route.ts
@@ -1,15 +1,10 @@
 import { NextRequest, NextResponse } from "next/server";
-import {
-  saveToS3,
-  saveToAzure,
-  saveToPostgres,
-} from "./save-fhir-data-service";
-import { S3_SOURCE, AZURE_SOURCE, POSTGRES_SOURCE } from "@/app/api/utils";
+import { saveFhirData, saveWithMetadata } from "./save-fhir-data-service";
 
 /**
  * Handles POST requests and saves the FHIR Bundle to the database.
- * @param request - The incoming request object. Expected to have a JSON body in the format `{"fhirBundle":{}, "saveSource": "postgres|s3""}`. FHIR bundle must include the ecr ID under entry[0].resource.id.
- * @returns A `NextResponse` object with a JSON payload indicating the success message and the status code set to 200. The response content type is set to `application/json`.
+ * @param request - The incoming request object. Expected to have a JSON body in the format `{"fhirBundle":{}, "saveSource": "postgres|s3|azure""}`. FHIR bundle must include the ecr ID under entry[0].resource.id.
+ * @returns A `NextResponse` object with a JSON payload indicating the success message. The response content type is set to `application/json`.
  */
 export async function POST(request: NextRequest) {
   let requestBody;
@@ -44,25 +39,20 @@ export async function POST(request: NextRequest) {
     return NextResponse.json(
       {
         message:
-          "Save location is undefined. Please provide a valid value for 'saveSource' (postgres or s3).",
+          'Save location is undefined. Please provide a valid value for \'saveSource\' ("postgres", "s3", or "azure").',
       },
       { status: 400 },
     );
   }
 
-  if (saveSource === S3_SOURCE) {
-    return saveToS3(fhirBundle, ecrId);
-  } else if (saveSource === AZURE_SOURCE) {
-    return saveToAzure(fhirBundle, ecrId);
-  } else if (saveSource === POSTGRES_SOURCE) {
-    return await saveToPostgres(fhirBundle, ecrId);
-  } else {
-    return NextResponse.json(
-      {
-        message:
-          "Invalid save source. Please provide a valid source (postgres or s3)",
-      },
-      { status: 400 },
+  if (requestBody.metadata) {
+    return saveWithMetadata(
+      fhirBundle,
+      ecrId,
+      saveSource,
+      requestBody.metadata,
     );
+  } else {
+    return saveFhirData(fhirBundle, ecrId, saveSource);
   }
 }
diff --git a/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts b/containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts
@@ -7,6 +7,7 @@ import {
   PutObjectCommandOutput,
 } from "@aws-sdk/client-s3";
 import { Bundle } from "fhir/r4";
+import { S3_SOURCE, AZURE_SOURCE, POSTGRES_SOURCE } from "@/app/api/utils";
 
 const s3Client =
   process.env.APP_ENV === "dev"
@@ -20,7 +21,7 @@ const s3Client =
 /**
  * Saves a FHIR bundle to a postgres database.
  * @async
- * @function saveToS3
+ * @function saveToPostgres
  * @param fhirBundle - The FHIR bundle to be saved.
  * @param ecrId - The unique identifier for the Electronic Case Reporting (ECR) associated with the FHIR bundle.
  * @returns A promise that resolves when the FHIR bundle is successfully saved to postgres.
@@ -48,7 +49,7 @@ export const saveToPostgres = async (fhirBundle: Bundle, ecrId: string) => {
     console.error("Error inserting data to database:", error);
     return NextResponse.json(
       { message: "Failed to insert data to database. " + error.message },
-      { status: 400 },
+      { status: 500 },
     );
   }
 };
@@ -89,7 +90,7 @@ export const saveToS3 = async (fhirBundle: Bundle, ecrId: string) => {
   } catch (error: any) {
     return NextResponse.json(
       { message: "Failed to insert data to S3. " + error.message },
-      { status: 400 },
+      { status: 500 },
     );
   }
 };
@@ -139,7 +140,144 @@ export const saveToAzure = async (fhirBundle: Bundle, ecrId: string) => {
           "Failed to insert FHIR bundle to Azure Blob Storage. " +
           error.message,
       },
+      { status: 500 },
+    );
+  }
+};
+
+interface BundleMetadata {
+  last_name: string;
+  first_name: string;
+  birth_date: string;
+  data_source: string;
+  reportable_condition: string;
+  rule_summary: string;
+  report_date: string;
+}
+
+/**
+ * @async
+ * @function saveFhirData
+ * @param fhirBundle - The FHIR bundle to be saved.
+ * @param ecrId - The unique identifier for the Electronic Case Reporting (ECR) associated with the FHIR bundle.
+ * @param saveSource - The location to save the FHIR bundle. Valid values are "postgres", "s3", or "azure".
+ * @returns A `NextResponse` object with a JSON payload indicating the success message. The response content type is set to `application/json`.
+ */
+export const saveFhirData = async (
+  fhirBundle: Bundle,
+  ecrId: string,
+  saveSource: string,
+) => {
+  if (saveSource === S3_SOURCE) {
+    return saveToS3(fhirBundle, ecrId);
+  } else if (saveSource === AZURE_SOURCE) {
+    return saveToAzure(fhirBundle, ecrId);
+  } else if (saveSource === POSTGRES_SOURCE) {
+    return await saveToPostgres(fhirBundle, ecrId);
+  } else {
+    return NextResponse.json(
+      {
+        message:
+          'Invalid save source. Please provide a valid value for \'saveSource\' ("postgres", "s3", or "azure").',
+      },
       { status: 400 },
     );
   }
 };
+
+/**
+ * Saves a FHIR bundle metadata to a postgres database.
+ * @async
+ * @function saveToMetadataPostgres
+ * @param metadata - The FHIR bundle metadata to be saved.
+ * @param ecrId - The unique identifier for the Electronic Case Reporting (ECR) associated with the FHIR bundle.
+ * @returns A promise that resolves when the FHIR bundle metadata is successfully saved to postgres.
+ * @throws {Error} Throws an error if the FHIR bundle metadata cannot be saved to postgress.
+ */
+export const saveToMetadataPostgres = async (
+  metadata: BundleMetadata,
+  ecrId: string,
+) => {
+  const db_url = process.env.DATABASE_URL || "";
+  const db = pgPromise();
+  const database = db(db_url);
+
+  const { ParameterizedQuery: PQ } = pgPromise;
+  const addMetadata = new PQ({
+    text: "INSERT INTO fhir_metadata (ecr_id,patient_name_last,patient_name_first,patient_birth_date,data_source,reportable_condition,rule_summary,report_date) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING ecr_id",
+    values: [
+      ecrId,
+      metadata.first_name,
+      metadata.last_name,
+      metadata.birth_date,
+      "DB",
+      metadata.reportable_condition,
+      metadata.rule_summary,
+      metadata.report_date,
+    ],
+  });
+
+  try {
+    const saveECR = await database.one(addMetadata);
+
+    return NextResponse.json(
+      { message: "Success. Saved metadata to database: " + saveECR.ecr_id },
+      { status: 200 },
+    );
+  } catch (error: any) {
+    console.error("Error inserting metadata to database:", error);
+    return NextResponse.json(
+      { message: "Failed to insert metadata to database. " + error.message },
+      { status: 500 },
+    );
+  }
+};
+
+/**
+ * @async
+ * @function saveWithMetadata
+ * @param fhirBundle - The FHIR bundle to be saved.
+ * @param ecrId - The unique identifier for the Electronic Case Reporting (ECR) associated with the FHIR bundle.
+ * @param saveSource - The location to save the FHIR bundle. Valid values are "postgres", "s3", or "azure".
+ * @param metadata - The metadata to be saved with the FHIR bundle.
+ * @returns A `NextResponse` object with a JSON payload indicating the success message. The response content type is set to `application/json`.
+ * @throws {Error} Throws an error if the FHIR bundle or metadata cannot be saved.
+ */
+export const saveWithMetadata = async (
+  fhirBundle: Bundle,
+  ecrId: string,
+  saveSource: string,
+  metadata: BundleMetadata,
+) => {
+  let fhirDataResult;
+  let metadataResult;
+  try {
+    fhirDataResult = await saveFhirData(fhirBundle, ecrId, saveSource);
+    metadataResult = await saveToMetadataPostgres(metadata, ecrId);
+  } catch (error: any) {
+    return NextResponse.json(
+      { message: "Failed to save FHIR data with metadata. " + error.message },
+      { status: 500 },
+    );
+  }
+
+  let responseMessage = "";
+  let responseStatus = 200;
+  if (fhirDataResult.status !== 200) {
+    responseMessage += "Failed to save FHIR data.\n";
+    responseStatus = 500;
+  } else {
+    responseMessage += "Saved FHIR data.\n";
+  }
+  if (metadataResult.status !== 200) {
+    responseMessage += "Failed to save metadata.";
+    responseStatus = 500;
+  } else {
+    responseMessage += "Saved metadata.";
+  }
+
+  return NextResponse.json(
+    { message: responseMessage },
+    { status: responseStatus },
+  );
+};
diff --git a/containers/ecr-viewer/src/app/tests/save-fhir-data.test.tsx b/containers/ecr-viewer/src/app/tests/save-fhir-data.test.tsx
@@ -129,7 +129,7 @@ describe("POST Save FHIR Data API Route", () => {
 
     const response = await POST(request);
     const responseJson = await response.json();
-    expect(response.status).toBe(400);
+    expect(response.status).toBe(500);
     expect(responseJson.message).toBe(
       "Failed to insert data to S3. HTTP Status Code: 403",
     );
@@ -183,7 +183,7 @@ describe("POST Save FHIR Data API Route - Azure", () => {
     const response = await POST(request);
     const responseJson = await response.json();
 
-    expect(response.status).toBe(400);
+    expect(response.status).toBe(500);
     expect(responseJson.message).toInclude(
       "Failed to insert FHIR bundle to Azure Blob Storage.",
     );

diff --git a/containers/orchestration/tests/integration/test_orchestration.py b/containers/orchestration/tests/integration/test_orchestration.py
@@ -150,7 +150,7 @@ def test_failed_save_to_ecr_viewer(setup, clean_up_db):
         orchestration_response = httpx.post(
             PROCESS_ZIP_ENDPOINT, data=form_data, files=files, timeout=60
         )
-        assert orchestration_response.status_code == 400
+        assert orchestration_response.status_code == 500
 
 
 @pytest.mark.integration