Merge pull request #26 from rodekruis/feat.create121fromkobo

Create 121 Programme from Kobo form (incl. REST service)

Dockerfile

@@ -3,6 +3,7 @@ FROM python:3.9-slim
 
 # copy files to the /app folder in the container
 ADD clients /app/clients
+ADD mappings /app/mappings
 COPY ./main.py /app/main.py
 COPY ./requirements.txt /app/requirements.txt
 

main.py

@@ -7,6 +7,9 @@
 from enum import Enum
 from clients.espo_api_client import EspoAPI
 import requests
+import csv
+import pandas as pd
+from datetime import datetime
 import os
 from azure.cosmos.exceptions import CosmosResourceExistsError
 import azure.cosmos.cosmos_client as cosmos_client
@@ -435,6 +438,236 @@ def remove_keys(data, keys_to_remove):
 ########################################################################################################################
 
 
+def required_headers_121_kobo(
+        url121: str = Header(),
+        username121: str = Header(),
+        password121: str = Header(),
+        kobotoken: str = Header(),
+        koboasset: str = Header()):
+    return url121, username121, password121, kobotoken, koboasset
+
+
+@app.post("/create-121-program-from-kobo")
+async def create_121_program_from_kobo(request: Request, dependencies=Depends(required_headers_121_kobo)):
+    """Utility endpoint to automatically create a 121 Program in 121 from a koboform, including REST Service \n
+    Does only support the IFRC server kobo.ifrc.org \n
+    ***NB: if you want to duplicate an endpoint, please also use the Hook ID query param***"""
+
+    koboUrl = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['koboasset']}"
+    koboheaders = {"Authorization": f"Token {request.headers['kobotoken']}"}
+    data_request = requests.get(f'{koboUrl}/?format=json', headers=koboheaders)
+    if data_request.status_code >= 400:
+        raise HTTPException(
+            status_code=data_request.status_code,
+            detail=data_request.content.decode("utf-8")
+        )
+    data = data_request.json()
+
+    survey = pd.DataFrame(data['content']['survey'])
+    choices = pd.DataFrame(data['content']['choices'])
+
+    type_mapping = {}
+    with open('mappings/kobo121fieldtypes.csv', newline='') as csvfile:
+        reader = csv.reader(csvfile, delimiter='\t')
+        for row in reader:
+            if len(row) == 2:
+                type_mapping[row[0]] = row[1]
+
+    mappingdf = pd.read_csv('mappings/kobo121fieldtypes.csv', delimiter='\t')
+
+    CHECKFIELDS = ['validation', 'phase', 'location', 'ngo', 'language', 'titlePortal', 'description',
+                'startDate', 'endDate', 'currency', 'distributionFrequency', 'distributionDuration', 'fixedTransferValue',
+                'financialServiceProviders', 'targetNrRegistrations', 'tryWhatsAppFirst', 'phoneNumberPlaceholder', 'aboutProgram',
+                'fullnameNamingConvention', 'enableMaxPayments', 'phoneNumber','preferredLanguage','maxPayments','fspName']
+
+    # First check if all setup fields are in the xlsform
+    FIELDNAMES = survey["name"].to_list()
+    MISSINGFIELDS = []
+    for checkfield in CHECKFIELDS:
+        if checkfield not in FIELDNAMES:
+            MISSINGFIELDS.append(checkfield)
+
+    if len(MISSINGFIELDS) != 0:
+        print('Missing hidden fields in the template: ', MISSINGFIELDS)
+
+    lookupdict = dict(zip(survey['name'], survey['default']))
+
+    if 'tags'in survey.columns:
+        dedupedict = dict(zip(survey['name'], survey['tags']))
+
+        for key, value in dedupedict.items():
+            if isinstance(value, list) and any('dedupe' in item for item in value):
+                dedupedict[key] = True
+            else:
+                dedupedict[key] = False
+    else:
+        survey['tags'] = False
+        dedupedict = dict(zip(survey['name'], survey['tags']))
+
+    # Create the JSON structure
+    data = {
+        "published": True,
+        "validation": lookupdict['validation'].upper() == 'TRUE',
+        "phase": lookupdict['phase'],
+        "location": lookupdict['location'],
+        "ngo": lookupdict['ngo'],
+        "titlePortal": {
+            lookupdict['language']: lookupdict['titlePortal']
+        },
+        "titlePaApp": {
+            lookupdict['language']: lookupdict['titlePortal']
+        },
+        "description": {
+            "en": ""
+        },
+        "startDate": datetime.strptime(lookupdict['startDate'], '%d/%m/%Y').isoformat(),
+        "endDate": datetime.strptime(lookupdict['endDate'], '%d/%m/%Y').isoformat(),
+        "currency": lookupdict['currency'],
+        "distributionFrequency": lookupdict['distributionFrequency'],
+        "distributionDuration": int(lookupdict['distributionDuration']),
+        "fixedTransferValue": int(lookupdict['fixedTransferValue']),
+        "paymentAmountMultiplierFormula": "",
+        "financialServiceProviders": [
+            {
+            "fsp": lookupdict['financialServiceProviders']
+            }
+        ],
+        "targetNrRegistrations": int(lookupdict['targetNrRegistrations']),
+        "tryWhatsAppFirst": lookupdict['tryWhatsAppFirst'].upper() == 'TRUE',
+        "phoneNumberPlaceholder": lookupdict['phoneNumberPlaceholder'],
+        "programCustomAttributes": [],
+        "programQuestions": [],
+        "aboutProgram": {
+            lookupdict['language']: lookupdict['aboutProgram']
+        },
+        "fullnameNamingConvention": [
+            lookupdict['fullnameNamingConvention']
+        ],
+        "languages": [
+            lookupdict['language']
+        ],
+        "enableMaxPayments": lookupdict['enableMaxPayments'].upper() == 'TRUE',
+        "allowEmptyPhoneNumber": False,
+        "enableScope": False
+    }
+
+    koboConnectHeader = ['fspName', 'preferredLanguage', 'maxPayments']
+
+    for index, row in survey.iterrows():
+        if row['type'].split()[0] in mappingdf['kobotype'].tolist() and row['name'] not in CHECKFIELDS:
+            koboConnectHeader.append(row['name'])
+            question = {
+                "name": row['name'],
+                "label": {
+                    "en": str(row['label'][0])
+                },
+                "answerType": type_mapping[row['type'].split()[0]],
+                "questionType": "standard",
+                "options": [],
+                "scoring": {},
+                "persistence": True,
+                "pattern": "",
+                "phases": [],
+                "editableInPortal": True,
+                "export": [
+                    "all-people-affected",
+                    "included"
+                ],
+                "shortLabel": {
+                    "en": row['name'],
+                },
+                "duplicateCheck": dedupedict[row['name']],
+                "placeholder": ""
+            }
+            if type_mapping[row['type'].split()[0]] == 'dropdown':
+                filtered_df = choices[choices['list_name'] == row['select_from_list_name']]
+                for index, row in filtered_df.iterrows():
+                    option = {
+                        "option": row['name'],
+                        "label": {
+                            "en": str(row['label'][0])
+                        }
+                    }
+                    question["options"].append(option)
+            data["programQuestions"].append(question)
+        if row['name'] == 'phoneNumber':
+            koboConnectHeader.append('phoneNumber')
+            question = {
+                "name": 'phoneNumber',
+                "label": {
+                    "en": 'Phone Number'
+                },
+                "answerType": "tel",
+                "questionType": "standard",
+                "options": [],
+                "scoring": {},
+                "persistence": True,
+                "pattern": "",
+                "phases": [],
+                "editableInPortal": True,
+                "export": [
+                    "all-people-affected",
+                    "included"
+                ],
+                "shortLabel": {
+                    "en": row['name'],
+                },
+                "duplicateCheck": dedupedict[row['name']],
+                "placeholder": ""
+            }
+            data["programQuestions"].append(question)
+
+    # Create program in 121
+    body = {'username': {request.headers['username121']}, 'password': {request.headers['password121']}}
+    url = f"{request.headers['url121']}/api/users/login"
+    login = requests.post(url, data=body)
+    if login.status_code >= 400:
+        raise HTTPException(
+            status_code=login.status_code,
+            detail=login.content.decode("utf-8")
+        )
+    access_token = login.json()['access_token_general']
+
+    # POST to target API
+    response = requests.post(
+        f"{request.headers['url121']}/api/programs",
+        headers={'Cookie': f"access_token_general={access_token}"},
+        json=data
+    )
+    if response.status_code >= 400:
+        raise HTTPException(
+            status_code=response.status_code,
+            detail=response.content.decode("utf-8")
+        )
+
+    # Create kobo-connect rest service
+    restServicePayload = {
+        "name": 'Kobo Connect',
+        "endpoint": 'https://kobo-connect.azurewebsites.net/kobo-to-121',
+        "active": True,
+        "email_notification": True,
+        "export_type": 'json',
+        "settings": {
+            "custom_headers": {
+            }
+        }
+    }
+    customHeaders = dict(zip(koboConnectHeader, koboConnectHeader))
+    restServicePayload['settings']['custom_headers'] = customHeaders
+
+    kobo_response = requests.post(
+        f'{koboUrl}/hooks/',
+        headers=koboheaders,
+        json=restServicePayload
+    )
+
+    if kobo_response.status_code == 200 or 201:
+        return JSONResponse(content={"message": "Sucess"})
+    else:
+        return JSONResponse(content={"message": "Failed"}, status_code=response.status_code)
+
+########################################################################################################################
+
 @app.post("/kobo-to-generic")
 async def kobo_to_generic(request: Request):
     """Send a Kobo submission to a generic API.

mappings/kobo121fieldtypes.csv

@@ -0,0 +1,17 @@
+kobotype	121type
+integer	numeric
+decimal	numeric
+range	numeric
+text	text
+select_one	dropdown
+select_multiple	text
+geopoint	text
+geotrace	text
+geoshape	text
+date	text
+time	text
+dateTime	text
+image	text
+calculate	text
+hidden	text
+tel	tel

requirements.txt

@@ -11,6 +11,7 @@ idna==3.4
 lxml==4.9.3
 multidict==6.0.4
 openai==0.27.8
+pandas==2.2.1
 pypdf==3.12.2
 python-docx==0.8.11
 python-dotenv==1.0.0