Skip to content

Commit

Permalink
new endpoint: update choice list of linked kobo form
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmargutt
jmargutt committed Nov 2, 2024
1 parent 34669bb commit 8f92f24
Show file tree
Hide file tree
Showing 2 changed files with 174 additions and 39 deletions.
203 changes: 164 additions & 39 deletions routes/routesKobo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,123 +4,158 @@
import base64
import csv
import io
import os
import uuid
import json
from enum import Enum
from utils.utils121 import login121
from utils.utilsKobo import required_headers_121_kobo
from utils.utilsKobo import (
add_submission,
clean_kobo_data,
get_attachment_dict,
get_kobo_attachment,
update_submission_status,
required_headers_121_kobo,
required_headers_linked_kobo,
)
from utils.logger import logger

router = APIRouter()


@router.post("/update-kobo-csv")
async def prepare_kobo_validation(request: Request, programId: int, kobousername: str, dependencies=Depends(required_headers_121_kobo)):
async def prepare_kobo_validation(
request: Request,
programId: int,
kobousername: str,
dependencies=Depends(required_headers_121_kobo),
):
"""
Prepare Kobo validation by fetching data from 121 platform,
converting it to CSV, and uploading to Kobo.
"""

access_token = login121(request.headers["url121"], request.headers["username121"], request.headers["password121"])

access_token = login121(
request.headers["url121"],
request.headers["username121"],
request.headers["password121"],
)

# Fetch data from 121 platform
response = requests.get(
f"{request.headers['url121']}/api/programs/{programId}/metrics/export-list/all-people-affected",
headers={'Cookie': f"access_token_general={access_token}"}
)
f"{request.headers['url121']}/api/programs/{programId}/metrics/export-list/all-people-affected",
headers={"Cookie": f"access_token_general={access_token}"},
)
if response.status_code != 200:
raise HTTPException(status_code=response.status_code, detail="Failed to fetch data from 121 platform")

raise HTTPException(
status_code=response.status_code,
detail="Failed to fetch data from 121 platform",
)

data = response.json()

# Convert JSON to CSV
output = io.StringIO()
writer = csv.writer(output)

# Ensure we have data to process
if data and 'data' in data and len(data['data']) > 0:
if data and "data" in data and len(data["data"]) > 0:
# Get the keys (column names) from the first row
fieldnames = list(data['data'][0].keys())
fieldnames = list(data["data"][0].keys())

# Write header
writer.writerow(fieldnames)

# Write rows
for row in data['data']:
for row in data["data"]:
# Create a list of values in the same order as fieldnames
row_data = [row.get(field, '') for field in fieldnames]
row_data = [row.get(field, "") for field in fieldnames]
writer.writerow(row_data)

csv_content = output.getvalue().encode('utf-8')
csv_content = output.getvalue().encode("utf-8")

# Prepare the payload for Kobo
base64_encoded_csv = base64.b64encode(csv_content).decode('utf-8')
base64_encoded_csv = base64.b64encode(csv_content).decode("utf-8")
metadata = json.dumps({"filename": "ValidationDataFrom121.csv"})

payload = {
"description": "default",
"file_type": "form_media",
"metadata": metadata,
"base64Encoded": f"data:text/csv;base64,{base64_encoded_csv}"
"base64Encoded": f"data:text/csv;base64,{base64_encoded_csv}",
}

# Kobo headers
headers = {
"Authorization": f"Token {request.headers['kobotoken']}",
"Content-Type": "application/x-www-form-urlencoded"
"Content-Type": "application/x-www-form-urlencoded",
}
#If exists, remove existing ValidationDataFrom121.csv
# If exists, remove existing ValidationDataFrom121.csv
media_response = requests.get(
f"https://kobo.ifrc.org/api/v2/assets/{request.headers['koboasset']}/files/",
headers=headers
)
headers=headers,
)
if media_response.status_code != 200:
raise HTTPException(status_code=response.status_code, detail="Failed to fetch media from kobo")

raise HTTPException(
status_code=response.status_code, detail="Failed to fetch media from kobo"
)

media = media_response.json()

# Check if ValidationDataFrom121.csv exists and get its uid
existing_file_uid = None
for file in media.get('results', []):
if file.get('metadata', {}).get('filename') == "ValidationDataFrom121.csv":
existing_file_uid = file.get('uid')
for file in media.get("results", []):
if file.get("metadata", {}).get("filename") == "ValidationDataFrom121.csv":
existing_file_uid = file.get("uid")
break

# If the file exists, delete it
if existing_file_uid:
delete_response = requests.delete(
f"https://kobo.ifrc.org/api/v2/assets/{request.headers['koboasset']}/files/{existing_file_uid}/",
headers={"Authorization": f"Token {request.headers['kobotoken']}"}
headers={"Authorization": f"Token {request.headers['kobotoken']}"},
)
if delete_response.status_code != 204:
raise HTTPException(status_code=delete_response.status_code, detail="Failed to delete existing file from Kobo")
raise HTTPException(
status_code=delete_response.status_code,
detail="Failed to delete existing file from Kobo",
)


upload_response = requests.post(
f"https://kobo.ifrc.org/api/v2/assets/{request.headers['koboasset']}/files/",
headers=headers,
data=payload
data=payload,
)

if upload_response.status_code != 201:
raise HTTPException(status_code=upload_response.status_code, detail="Failed to upload file to Kobo")
raise HTTPException(
status_code=upload_response.status_code,
detail="Failed to upload file to Kobo",
)

# Redeploy the Kobo form
redeploy_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['koboasset']}/deployment/"
redeploy_payload = {"active": True}

redeploy_response = requests.patch(
redeploy_url,
headers=headers,
json=redeploy_payload
redeploy_url, headers=headers, json=redeploy_payload
)

if redeploy_response.status_code != 200:
raise HTTPException(status_code=redeploy_response.status_code, detail="Failed to redeploy Kobo form")

raise HTTPException(
status_code=redeploy_response.status_code,
detail="Failed to redeploy Kobo form",
)

return {"message": "Validation data prepared and uploaded successfully", "kobo_response": upload_response.json()}
return {
"message": "Validation data prepared and uploaded successfully",
"kobo_response": upload_response.json(),
}


###############


class system(str, Enum):
system_generic = "generic"
system_espo = "espocrm"
Expand Down Expand Up @@ -193,3 +228,93 @@ def remove_keys(data, keys_to_remove):
content={"message": "Failed to post data to the target endpoint"},
status_code=response.status_code,
)


@router.post("/kobo-to-linked-kobo")
async def kobo_to_linked_kobo(
request: Request, dependencies=Depends(required_headers_linked_kobo)
):
"""Update a linked Kobo form based on this submission."""

kobo_data = await request.json()
extra_logs = {"environment": os.getenv("ENV")}
try:
extra_logs["kobo_form_id"] = str(kobo_data["_xform_id_string"])
extra_logs["kobo_form_version"] = str(kobo_data["__version__"])
extra_logs["kobo_submission_id"] = str(kobo_data["_id"])
except KeyError:
return JSONResponse(
status_code=422,
content={"detail": "Not a valid Kobo submission"},
)

# store the submission uuid and status, to avoid duplicate submissions
submission = add_submission(kobo_data)
if submission["status"] == "success":
logger.info(
"Submission has already been successfully processed", extra=extra_logs
)
return JSONResponse(
status_code=200,
content={"detail": "Submission has already been successfully processed"},
)

# get submissions of parent form
target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['parentasset']}/data/?format=json"
koboheaders = {"Authorization": f"Token {request.headers['kobotoken']}"}
response = requests.get(target_url, headers=koboheaders)
submissions = json.loads(response.content)

# create new choice list based on parent form submissions
new_choices_form, kuids = [], []
for submission in submissions.items():
if request.headers["parentquestion"] not in submission.keys():
continue
kuid = str(uuid.uuid4())[:10].replace("-", "")
while kuid in kuids:
kuid = str(uuid.uuid4())[:10].replace("-", "")
kuids.append(kuid)

new_choices_form.append(
{
"name": submission[request.headers["parentquestion"]],
"$kuid": kuid,
"label": [submission[request.headers["parentquestion"]]],
"list_name": request.headers["childlist"],
"$autovalue": submission[request.headers["parentquestion"]],
}
)

# get child form
target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json"
response = requests.get(target_url, headers=koboheaders)
assetdata = json.loads(response.content)

# update form
assetdata["content"]["choices"] = [
choice
for choice in assetdata["content"]["choices"]
if choice["list_name"] != request.headers["childlist"]
]
assetdata["content"]["choices"].extend(new_choices_form)
requests.patch(target_url, headers=koboheaders, json=assetdata)

# get latest version id
target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json"
response = requests.get(target_url, headers=koboheaders)
newassetdata = json.loads(response.content)
newversionid = newassetdata["version_id"]

# deploy latest version id
target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/deployment/"
payload = {"version_id": newversionid, "active": True}
response = requests.patch(target_url, headers=koboheaders, data=payload)

if response.status_code == 200:
logger.info("Success", extra=extra_logs)
update_submission_status(submission, "success")
return JSONResponse(status_code=200, content=response.content)
else:
logger.error("Failed", extra=extra_logs)
update_submission_status(submission, "failed", response.content)
return JSONResponse(status_code=500, content=response.content)
10 changes: 10 additions & 0 deletions utils/utilsKobo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,3 +102,13 @@ def clean_kobo_data(kobo_data):
new_key = key.split("/")[-1]
kobo_data_clean[new_key] = kobo_data_clean.pop(key)
return kobo_data_clean


def required_headers_linked_kobo(
kobotoken: str = Header(),
childasset: str = Header(),
childlist: str = Header(),
parentasset: str = Header(),
parentquestion: str = Header(),
):
return kobotoken, childasset, childlist, parentasset, parentquestion

0 comments on commit 8f92f24

Please sign in to comment.