From 8f92f2426f9a92bd0a16bb8d8b12636dcfbcfcf9 Mon Sep 17 00:00:00 2001 From: Jacopo Margutti Date: Sat, 2 Nov 2024 13:37:00 +0100 Subject: [PATCH] new endpoint: update choice list of linked kobo form --- routes/routesKobo.py | 203 ++++++++++++++++++++++++++++++++++--------- utils/utilsKobo.py | 10 +++ 2 files changed, 174 insertions(+), 39 deletions(-) diff --git a/routes/routesKobo.py b/routes/routesKobo.py index 102ca05..09fce14 100644 --- a/routes/routesKobo.py +++ b/routes/routesKobo.py @@ -4,123 +4,158 @@ import base64 import csv import io +import os +import uuid import json from enum import Enum from utils.utils121 import login121 -from utils.utilsKobo import required_headers_121_kobo +from utils.utilsKobo import ( + add_submission, + clean_kobo_data, + get_attachment_dict, + get_kobo_attachment, + update_submission_status, + required_headers_121_kobo, + required_headers_linked_kobo, +) +from utils.logger import logger router = APIRouter() + @router.post("/update-kobo-csv") -async def prepare_kobo_validation(request: Request, programId: int, kobousername: str, dependencies=Depends(required_headers_121_kobo)): +async def prepare_kobo_validation( + request: Request, + programId: int, + kobousername: str, + dependencies=Depends(required_headers_121_kobo), +): """ Prepare Kobo validation by fetching data from 121 platform, converting it to CSV, and uploading to Kobo. """ - access_token = login121(request.headers["url121"], request.headers["username121"], request.headers["password121"]) - + access_token = login121( + request.headers["url121"], + request.headers["username121"], + request.headers["password121"], + ) + # Fetch data from 121 platform response = requests.get( - f"{request.headers['url121']}/api/programs/{programId}/metrics/export-list/all-people-affected", - headers={'Cookie': f"access_token_general={access_token}"} - ) + f"{request.headers['url121']}/api/programs/{programId}/metrics/export-list/all-people-affected", + headers={"Cookie": f"access_token_general={access_token}"}, + ) if response.status_code != 200: - raise HTTPException(status_code=response.status_code, detail="Failed to fetch data from 121 platform") - + raise HTTPException( + status_code=response.status_code, + detail="Failed to fetch data from 121 platform", + ) + data = response.json() # Convert JSON to CSV output = io.StringIO() writer = csv.writer(output) - + # Ensure we have data to process - if data and 'data' in data and len(data['data']) > 0: + if data and "data" in data and len(data["data"]) > 0: # Get the keys (column names) from the first row - fieldnames = list(data['data'][0].keys()) + fieldnames = list(data["data"][0].keys()) # Write header writer.writerow(fieldnames) # Write rows - for row in data['data']: + for row in data["data"]: # Create a list of values in the same order as fieldnames - row_data = [row.get(field, '') for field in fieldnames] + row_data = [row.get(field, "") for field in fieldnames] writer.writerow(row_data) - csv_content = output.getvalue().encode('utf-8') + csv_content = output.getvalue().encode("utf-8") # Prepare the payload for Kobo - base64_encoded_csv = base64.b64encode(csv_content).decode('utf-8') + base64_encoded_csv = base64.b64encode(csv_content).decode("utf-8") metadata = json.dumps({"filename": "ValidationDataFrom121.csv"}) - + payload = { "description": "default", "file_type": "form_media", "metadata": metadata, - "base64Encoded": f"data:text/csv;base64,{base64_encoded_csv}" + "base64Encoded": f"data:text/csv;base64,{base64_encoded_csv}", } # Kobo headers headers = { "Authorization": f"Token {request.headers['kobotoken']}", - "Content-Type": "application/x-www-form-urlencoded" + "Content-Type": "application/x-www-form-urlencoded", } - #If exists, remove existing ValidationDataFrom121.csv + # If exists, remove existing ValidationDataFrom121.csv media_response = requests.get( f"https://kobo.ifrc.org/api/v2/assets/{request.headers['koboasset']}/files/", - headers=headers - ) + headers=headers, + ) if media_response.status_code != 200: - raise HTTPException(status_code=response.status_code, detail="Failed to fetch media from kobo") - + raise HTTPException( + status_code=response.status_code, detail="Failed to fetch media from kobo" + ) + media = media_response.json() # Check if ValidationDataFrom121.csv exists and get its uid existing_file_uid = None - for file in media.get('results', []): - if file.get('metadata', {}).get('filename') == "ValidationDataFrom121.csv": - existing_file_uid = file.get('uid') + for file in media.get("results", []): + if file.get("metadata", {}).get("filename") == "ValidationDataFrom121.csv": + existing_file_uid = file.get("uid") break # If the file exists, delete it if existing_file_uid: delete_response = requests.delete( f"https://kobo.ifrc.org/api/v2/assets/{request.headers['koboasset']}/files/{existing_file_uid}/", - headers={"Authorization": f"Token {request.headers['kobotoken']}"} + headers={"Authorization": f"Token {request.headers['kobotoken']}"}, ) if delete_response.status_code != 204: - raise HTTPException(status_code=delete_response.status_code, detail="Failed to delete existing file from Kobo") + raise HTTPException( + status_code=delete_response.status_code, + detail="Failed to delete existing file from Kobo", + ) - upload_response = requests.post( f"https://kobo.ifrc.org/api/v2/assets/{request.headers['koboasset']}/files/", headers=headers, - data=payload + data=payload, ) if upload_response.status_code != 201: - raise HTTPException(status_code=upload_response.status_code, detail="Failed to upload file to Kobo") + raise HTTPException( + status_code=upload_response.status_code, + detail="Failed to upload file to Kobo", + ) # Redeploy the Kobo form redeploy_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['koboasset']}/deployment/" redeploy_payload = {"active": True} - + redeploy_response = requests.patch( - redeploy_url, - headers=headers, - json=redeploy_payload + redeploy_url, headers=headers, json=redeploy_payload ) if redeploy_response.status_code != 200: - raise HTTPException(status_code=redeploy_response.status_code, detail="Failed to redeploy Kobo form") - + raise HTTPException( + status_code=redeploy_response.status_code, + detail="Failed to redeploy Kobo form", + ) - return {"message": "Validation data prepared and uploaded successfully", "kobo_response": upload_response.json()} + return { + "message": "Validation data prepared and uploaded successfully", + "kobo_response": upload_response.json(), + } ############### + class system(str, Enum): system_generic = "generic" system_espo = "espocrm" @@ -193,3 +228,93 @@ def remove_keys(data, keys_to_remove): content={"message": "Failed to post data to the target endpoint"}, status_code=response.status_code, ) + + +@router.post("/kobo-to-linked-kobo") +async def kobo_to_linked_kobo( + request: Request, dependencies=Depends(required_headers_linked_kobo) +): + """Update a linked Kobo form based on this submission.""" + + kobo_data = await request.json() + extra_logs = {"environment": os.getenv("ENV")} + try: + extra_logs["kobo_form_id"] = str(kobo_data["_xform_id_string"]) + extra_logs["kobo_form_version"] = str(kobo_data["__version__"]) + extra_logs["kobo_submission_id"] = str(kobo_data["_id"]) + except KeyError: + return JSONResponse( + status_code=422, + content={"detail": "Not a valid Kobo submission"}, + ) + + # store the submission uuid and status, to avoid duplicate submissions + submission = add_submission(kobo_data) + if submission["status"] == "success": + logger.info( + "Submission has already been successfully processed", extra=extra_logs + ) + return JSONResponse( + status_code=200, + content={"detail": "Submission has already been successfully processed"}, + ) + + # get submissions of parent form + target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['parentasset']}/data/?format=json" + koboheaders = {"Authorization": f"Token {request.headers['kobotoken']}"} + response = requests.get(target_url, headers=koboheaders) + submissions = json.loads(response.content) + + # create new choice list based on parent form submissions + new_choices_form, kuids = [], [] + for submission in submissions.items(): + if request.headers["parentquestion"] not in submission.keys(): + continue + kuid = str(uuid.uuid4())[:10].replace("-", "") + while kuid in kuids: + kuid = str(uuid.uuid4())[:10].replace("-", "") + kuids.append(kuid) + + new_choices_form.append( + { + "name": submission[request.headers["parentquestion"]], + "$kuid": kuid, + "label": [submission[request.headers["parentquestion"]]], + "list_name": request.headers["childlist"], + "$autovalue": submission[request.headers["parentquestion"]], + } + ) + + # get child form + target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" + response = requests.get(target_url, headers=koboheaders) + assetdata = json.loads(response.content) + + # update form + assetdata["content"]["choices"] = [ + choice + for choice in assetdata["content"]["choices"] + if choice["list_name"] != request.headers["childlist"] + ] + assetdata["content"]["choices"].extend(new_choices_form) + requests.patch(target_url, headers=koboheaders, json=assetdata) + + # get latest version id + target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" + response = requests.get(target_url, headers=koboheaders) + newassetdata = json.loads(response.content) + newversionid = newassetdata["version_id"] + + # deploy latest version id + target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/deployment/" + payload = {"version_id": newversionid, "active": True} + response = requests.patch(target_url, headers=koboheaders, data=payload) + + if response.status_code == 200: + logger.info("Success", extra=extra_logs) + update_submission_status(submission, "success") + return JSONResponse(status_code=200, content=response.content) + else: + logger.error("Failed", extra=extra_logs) + update_submission_status(submission, "failed", response.content) + return JSONResponse(status_code=500, content=response.content) diff --git a/utils/utilsKobo.py b/utils/utilsKobo.py index c03e339..bdb5866 100644 --- a/utils/utilsKobo.py +++ b/utils/utilsKobo.py @@ -102,3 +102,13 @@ def clean_kobo_data(kobo_data): new_key = key.split("/")[-1] kobo_data_clean[new_key] = kobo_data_clean.pop(key) return kobo_data_clean + + +def required_headers_linked_kobo( + kobotoken: str = Header(), + childasset: str = Header(), + childlist: str = Header(), + parentasset: str = Header(), + parentquestion: str = Header(), +): + return kobotoken, childasset, childlist, parentasset, parentquestion