From 48de3fea92989a4451146b18ce383962ceeda349 Mon Sep 17 00:00:00 2001 From: tijsziere <39266480+tijsziere@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:14:21 +0100 Subject: [PATCH 01/12] fix empty label bug --- routes/routes121.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/routes121.py b/routes/routes121.py index 238cffe..c46392d 100644 --- a/routes/routes121.py +++ b/routes/routes121.py @@ -359,7 +359,7 @@ async def create_121_program_from_kobo( koboConnectHeader.append(row["name"]) question = { "name": row["name"], - "label": {"en": str(row["label"][0])}, + "label": {"en": str(row["label"][0]) if not isinstance(row["label"], float) else row["name"]}, "answerType": type_mapping[row["type"].split()[0]], "questionType": "standard", "options": [], From 9068c59c8454c17c5002b9fc6568784a0da58931 Mon Sep 17 00:00:00 2001 From: tijsziere <39266480+tijsziere@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:30:07 +0100 Subject: [PATCH 02/12] improve clarity --- routes/routes121.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/routes121.py b/routes/routes121.py index c46392d..2a1d2be 100644 --- a/routes/routes121.py +++ b/routes/routes121.py @@ -359,7 +359,7 @@ async def create_121_program_from_kobo( koboConnectHeader.append(row["name"]) question = { "name": row["name"], - "label": {"en": str(row["label"][0]) if not isinstance(row["label"], float) else row["name"]}, + "label": {"en": str(row["label"][0]) if not row["label"] else row["name"]}, "answerType": type_mapping[row["type"].split()[0]], "questionType": "standard", "options": [], From 04c52d1c284ad393c9cb1d674df23e48ddb28fab Mon Sep 17 00:00:00 2001 From: tijsziere <39266480+tijsziere@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:43:08 +0100 Subject: [PATCH 03/12] fix error --- routes/routes121.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/routes121.py b/routes/routes121.py index 2a1d2be..f6fbd27 100644 --- a/routes/routes121.py +++ b/routes/routes121.py @@ -359,7 +359,8 @@ async def create_121_program_from_kobo( koboConnectHeader.append(row["name"]) question = { "name": row["name"], - "label": {"en": str(row["label"][0]) if not row["label"] else row["name"]}, + # check if label exists, otherwise use name: + "label": {"en": str(row["label"][0]) if not isinstance(row["label"], float) else row["name"]}, "answerType": type_mapping[row["type"].split()[0]], "questionType": "standard", "options": [], From 4f30c11de504bc7065f71690ed43abb42d544162 Mon Sep 17 00:00:00 2001 From: tijsziere <39266480+tijsziere@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:28:29 +0100 Subject: [PATCH 04/12] add support for kobo barcode questiontype --- mappings/kobo121fieldtypes.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mappings/kobo121fieldtypes.csv b/mappings/kobo121fieldtypes.csv index 4ecc70b..c037059 100644 --- a/mappings/kobo121fieldtypes.csv +++ b/mappings/kobo121fieldtypes.csv @@ -14,4 +14,5 @@ dateTime text image text calculate text hidden text -tel tel \ No newline at end of file +tel tel +barcode text \ No newline at end of file From 265dc69ac5ac49f3600a78643f4a8cb025b0d7f5 Mon Sep 17 00:00:00 2001 From: Jacopo Margutti Date: Tue, 19 Nov 2024 15:42:21 +0100 Subject: [PATCH 05/12] add info to debug --- routes/routesKobo.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/routes/routesKobo.py b/routes/routesKobo.py index 7c34e9d..b445903 100644 --- a/routes/routesKobo.py +++ b/routes/routesKobo.py @@ -264,6 +264,8 @@ async def kobo_to_linked_kobo( koboheaders = {"Authorization": f"Token {request.headers['kobotoken']}"} response = requests.get(target_url, headers=koboheaders) parent_submissions = json.loads(response.content) + logging.info("parent_submissions") + logging.info(parent_submissions["results"][0]) # create new choice list based on parent form submissions new_choices_form, kuids, names = [], [], [] @@ -290,11 +292,15 @@ async def kobo_to_linked_kobo( "$autovalue": name, } ) + logging.info("new_choices_form") + logging.info(new_choices_form) # get child form target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" response = requests.get(target_url, headers=koboheaders) assetdata = json.loads(response.content) + logging.info("get child form") + logging.info(assetdata) # update child form with new choice list assetdata["content"]["choices"] = [ @@ -303,13 +309,19 @@ async def kobo_to_linked_kobo( if choice["list_name"] != request.headers["childlist"] ] assetdata["content"]["choices"].extend(new_choices_form) - requests.patch(target_url, headers=koboheaders, json=assetdata) + logging.info("update child form with new choice list") + logging.info(assetdata) + response = requests.patch(target_url, headers=koboheaders, json=assetdata) + logging.info(response.status_code) + logging.info(response.content) # get latest form version id target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" response = requests.get(target_url, headers=koboheaders) newassetdata = json.loads(response.content) newversionid = newassetdata["version_id"] + logging.info("get child form") + logging.info(assetdata) # deploy latest form version id target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/deployment/" From dfc19b630895320e07e4c3c7e3dbfb26f5bfff83 Mon Sep 17 00:00:00 2001 From: Jacopo Margutti Date: Tue, 19 Nov 2024 15:50:53 +0100 Subject: [PATCH 06/12] add info to debug --- routes/routesKobo.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/routes/routesKobo.py b/routes/routesKobo.py index b445903..8e32a21 100644 --- a/routes/routesKobo.py +++ b/routes/routesKobo.py @@ -264,8 +264,8 @@ async def kobo_to_linked_kobo( koboheaders = {"Authorization": f"Token {request.headers['kobotoken']}"} response = requests.get(target_url, headers=koboheaders) parent_submissions = json.loads(response.content) - logging.info("parent_submissions") - logging.info(parent_submissions["results"][0]) + logger.info("parent_submissions") + logger.info(parent_submissions["results"][0]) # create new choice list based on parent form submissions new_choices_form, kuids, names = [], [], [] @@ -292,15 +292,15 @@ async def kobo_to_linked_kobo( "$autovalue": name, } ) - logging.info("new_choices_form") - logging.info(new_choices_form) + logger.info("new_choices_form") + logger.info(new_choices_form) # get child form target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" response = requests.get(target_url, headers=koboheaders) assetdata = json.loads(response.content) - logging.info("get child form") - logging.info(assetdata) + logger.info("get child form") + logger.info(assetdata) # update child form with new choice list assetdata["content"]["choices"] = [ @@ -309,19 +309,19 @@ async def kobo_to_linked_kobo( if choice["list_name"] != request.headers["childlist"] ] assetdata["content"]["choices"].extend(new_choices_form) - logging.info("update child form with new choice list") - logging.info(assetdata) + logger.info("update child form with new choice list") + logger.info(assetdata) response = requests.patch(target_url, headers=koboheaders, json=assetdata) - logging.info(response.status_code) - logging.info(response.content) + logger.info(response.status_code) + logger.info(response.content) # get latest form version id target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" response = requests.get(target_url, headers=koboheaders) newassetdata = json.loads(response.content) newversionid = newassetdata["version_id"] - logging.info("get child form") - logging.info(assetdata) + logger.info("get child form") + logger.info(assetdata) # deploy latest form version id target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/deployment/" From 368bec5ee13a90580003f4ca792844ab91d099c8 Mon Sep 17 00:00:00 2001 From: Jacopo Margutti Date: Tue, 19 Nov 2024 16:00:52 +0100 Subject: [PATCH 07/12] add info to debug --- routes/routesKobo.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/routes/routesKobo.py b/routes/routesKobo.py index 8e32a21..db413bd 100644 --- a/routes/routesKobo.py +++ b/routes/routesKobo.py @@ -258,14 +258,16 @@ async def kobo_to_linked_kobo( status_code=200, content={"detail": "Submission has already been successfully processed"}, ) + print("kobo_data") + print(kobo_data) # get submissions of parent form target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['parentasset']}/data/?format=json" koboheaders = {"Authorization": f"Token {request.headers['kobotoken']}"} response = requests.get(target_url, headers=koboheaders) parent_submissions = json.loads(response.content) - logger.info("parent_submissions") - logger.info(parent_submissions["results"][0]) + print("parent_submissions") + print(parent_submissions["results"][0]) # create new choice list based on parent form submissions new_choices_form, kuids, names = [], [], [] @@ -292,15 +294,15 @@ async def kobo_to_linked_kobo( "$autovalue": name, } ) - logger.info("new_choices_form") - logger.info(new_choices_form) + print("new_choices_form") + print(new_choices_form) # get child form target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" response = requests.get(target_url, headers=koboheaders) assetdata = json.loads(response.content) - logger.info("get child form") - logger.info(assetdata) + print("get child form") + print(assetdata) # update child form with new choice list assetdata["content"]["choices"] = [ @@ -312,16 +314,16 @@ async def kobo_to_linked_kobo( logger.info("update child form with new choice list") logger.info(assetdata) response = requests.patch(target_url, headers=koboheaders, json=assetdata) - logger.info(response.status_code) - logger.info(response.content) + print(response.status_code) + print(response.content) # get latest form version id target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" response = requests.get(target_url, headers=koboheaders) newassetdata = json.loads(response.content) newversionid = newassetdata["version_id"] - logger.info("get child form") - logger.info(assetdata) + print("get child form") + print(assetdata) # deploy latest form version id target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/deployment/" From 17453fc4f6ac146b236daa7672a0fafa1b700caf Mon Sep 17 00:00:00 2001 From: Jacopo Margutti Date: Tue, 19 Nov 2024 20:37:28 +0100 Subject: [PATCH 08/12] add info to debug --- routes/routesKobo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/routes/routesKobo.py b/routes/routesKobo.py index db413bd..66c2c64 100644 --- a/routes/routesKobo.py +++ b/routes/routesKobo.py @@ -275,7 +275,9 @@ async def kobo_to_linked_kobo( if request.headers["parentquestion"] not in parent_submission.keys(): continue - name = parent_submission[request.headers["parentquestion"]] + parent_data = clean_kobo_data(parent_submission) + + name = parent_data[request.headers["parentquestion"].lower()] if name in names: continue # avoid duplicate names names.append(name) From 4b81c6eb66ab45aa2dbdc86ccc2ab99346511a4f Mon Sep 17 00:00:00 2001 From: Jacopo Margutti Date: Tue, 19 Nov 2024 20:44:12 +0100 Subject: [PATCH 09/12] add info to debug --- routes/routesKobo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/routes/routesKobo.py b/routes/routesKobo.py index 66c2c64..f352697 100644 --- a/routes/routesKobo.py +++ b/routes/routesKobo.py @@ -272,10 +272,12 @@ async def kobo_to_linked_kobo( # create new choice list based on parent form submissions new_choices_form, kuids, names = [], [], [] for parent_submission in parent_submissions["results"]: - if request.headers["parentquestion"] not in parent_submission.keys(): - continue parent_data = clean_kobo_data(parent_submission) + parent_question = request.headers["parentquestion"].lower() + + if parent_question not in parent_data.keys(): + continue name = parent_data[request.headers["parentquestion"].lower()] if name in names: From 3f5edf54717326c735cefa611ec41c733877ad8f Mon Sep 17 00:00:00 2001 From: Jacopo Margutti Date: Tue, 19 Nov 2024 20:55:43 +0100 Subject: [PATCH 10/12] add info to debug --- routes/routesKobo.py | 59 +++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/routes/routesKobo.py b/routes/routesKobo.py index f352697..9139acf 100644 --- a/routes/routesKobo.py +++ b/routes/routesKobo.py @@ -258,8 +258,6 @@ async def kobo_to_linked_kobo( status_code=200, content={"detail": "Submission has already been successfully processed"}, ) - print("kobo_data") - print(kobo_data) # get submissions of parent form target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['parentasset']}/data/?format=json" @@ -272,32 +270,29 @@ async def kobo_to_linked_kobo( # create new choice list based on parent form submissions new_choices_form, kuids, names = [], [], [] for parent_submission in parent_submissions["results"]: - - parent_data = clean_kobo_data(parent_submission) - parent_question = request.headers["parentquestion"].lower() - - if parent_question not in parent_data.keys(): - continue - - name = parent_data[request.headers["parentquestion"].lower()] - if name in names: - continue # avoid duplicate names - names.append(name) - - kuid = str(uuid.uuid4())[:10].replace("-", "") - while kuid in kuids: - kuid = str(uuid.uuid4())[:10].replace("-", "") # avoid duplicate kuids - kuids.append(kuid) - - new_choices_form.append( - { - "name": name, - "$kuid": kuid, - "label": [name], - "list_name": request.headers["childlist"], - "$autovalue": name, - } - ) + for key in parent_submission.keys(): + if key.split("/")[-1] == request.headers["parentquestion"]: + name = parent_submission[key] + if name in names: + continue # avoid duplicate names + names.append(name) + + kuid = str(uuid.uuid4())[:10].replace("-", "") + while kuid in kuids: + kuid = str(uuid.uuid4())[:10].replace( + "-", "" + ) # avoid duplicate kuids + kuids.append(kuid) + + new_choices_form.append( + { + "name": name, + "$kuid": kuid, + "label": [name], + "list_name": request.headers["childlist"], + "$autovalue": name, + } + ) print("new_choices_form") print(new_choices_form) @@ -305,8 +300,8 @@ async def kobo_to_linked_kobo( target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" response = requests.get(target_url, headers=koboheaders) assetdata = json.loads(response.content) - print("get child form") - print(assetdata) + print("child_choices") + print(assetdata["content"]["choices"]) # update child form with new choice list assetdata["content"]["choices"] = [ @@ -318,16 +313,12 @@ async def kobo_to_linked_kobo( logger.info("update child form with new choice list") logger.info(assetdata) response = requests.patch(target_url, headers=koboheaders, json=assetdata) - print(response.status_code) - print(response.content) # get latest form version id target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" response = requests.get(target_url, headers=koboheaders) newassetdata = json.loads(response.content) newversionid = newassetdata["version_id"] - print("get child form") - print(assetdata) # deploy latest form version id target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/deployment/" From a25b25bf74ece8578cc61199ca3b9efa1e4c32c7 Mon Sep 17 00:00:00 2001 From: Jacopo Margutti Date: Tue, 19 Nov 2024 21:16:23 +0100 Subject: [PATCH 11/12] add info to debug --- routes/routesKobo.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/routes/routesKobo.py b/routes/routesKobo.py index 9139acf..71f9a90 100644 --- a/routes/routesKobo.py +++ b/routes/routesKobo.py @@ -267,6 +267,18 @@ async def kobo_to_linked_kobo( print("parent_submissions") print(parent_submissions["results"][0]) + # get child form + target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" + response = requests.get(target_url, headers=koboheaders) + assetdata = json.loads(response.content) + print("child_choices") + print(assetdata["content"]["choices"]) + len_choices = [] + for choice in assetdata["content"]["choices"]: + if choice["list_name"] == request.headers["childlist"]: + len_choices.append(len(choice["label"])) + len_choices = max(len_choices) + # create new choice list based on parent form submissions new_choices_form, kuids, names = [], [], [] for parent_submission in parent_submissions["results"]: @@ -288,20 +300,11 @@ async def kobo_to_linked_kobo( { "name": name, "$kuid": kuid, - "label": [name], + "label": [name for i in range(len_choices)], "list_name": request.headers["childlist"], "$autovalue": name, } ) - print("new_choices_form") - print(new_choices_form) - - # get child form - target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" - response = requests.get(target_url, headers=koboheaders) - assetdata = json.loads(response.content) - print("child_choices") - print(assetdata["content"]["choices"]) # update child form with new choice list assetdata["content"]["choices"] = [ @@ -310,6 +313,8 @@ async def kobo_to_linked_kobo( if choice["list_name"] != request.headers["childlist"] ] assetdata["content"]["choices"].extend(new_choices_form) + print("new_child_choices") + print(assetdata["content"]["choices"]) logger.info("update child form with new choice list") logger.info(assetdata) response = requests.patch(target_url, headers=koboheaders, json=assetdata) From d3f9306ac61bffb604c3de9846b0d129fd225173 Mon Sep 17 00:00:00 2001 From: Jacopo Margutti Date: Tue, 19 Nov 2024 21:31:54 +0100 Subject: [PATCH 12/12] final commit debug groups and logo --- routes/routesKobo.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/routes/routesKobo.py b/routes/routesKobo.py index 71f9a90..71db937 100644 --- a/routes/routesKobo.py +++ b/routes/routesKobo.py @@ -264,15 +264,11 @@ async def kobo_to_linked_kobo( koboheaders = {"Authorization": f"Token {request.headers['kobotoken']}"} response = requests.get(target_url, headers=koboheaders) parent_submissions = json.loads(response.content) - print("parent_submissions") - print(parent_submissions["results"][0]) # get child form target_url = f"https://kobo.ifrc.org/api/v2/assets/{request.headers['childasset']}/?format=json" response = requests.get(target_url, headers=koboheaders) assetdata = json.loads(response.content) - print("child_choices") - print(assetdata["content"]["choices"]) len_choices = [] for choice in assetdata["content"]["choices"]: if choice["list_name"] == request.headers["childlist"]: @@ -313,8 +309,6 @@ async def kobo_to_linked_kobo( if choice["list_name"] != request.headers["childlist"] ] assetdata["content"]["choices"].extend(new_choices_form) - print("new_child_choices") - print(assetdata["content"]["choices"]) logger.info("update child form with new choice list") logger.info(assetdata) response = requests.patch(target_url, headers=koboheaders, json=assetdata)