From 5c766cb34ed4a3d1ba6189a69219e81c1aaccb15 Mon Sep 17 00:00:00 2001 From: James B Date: Fri, 1 Dec 2023 16:02:06 +0000 Subject: [PATCH 1/6] Links to codelists - rewrite to OCDS Standard https://github.com/OpenDataServices/lib-cove-web/issues/74 --- cove_ocds/views.py | 8 ++++++++ requirements.txt | 4 +--- requirements_dev.txt | 6 +----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cove_ocds/views.py b/cove_ocds/views.py index 11b5ea74..344664ce 100644 --- a/cove_ocds/views.py +++ b/cove_ocds/views.py @@ -231,6 +231,14 @@ def explore_ocds(request, pk): context = common_checks_ocds(context, upload_dir, json_data, schema_ocds) + # Construct the Open Contracting specific urls for codelists in the docs + for key in ['additional_closed_codelist_values', 'additional_open_codelist_values']: + for path_string, codelist_info in context[key].items(): + codelist_info['codelist_url'] = ( + 'https://standard.open-contracting.org/{}/en/schema/codelists/#'.format(db_data.data_schema_version) + + re.sub(r'([A-Z])', r'-\1', codelist_info['codelist'].split('.')[0]).lower() + ) + if schema_ocds.json_deref_error: exceptions.raise_json_deref_error(schema_ocds.json_deref_error) diff --git a/requirements.txt b/requirements.txt index e5c26198..e469b096 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,8 +44,6 @@ django-bootstrap3==15.0.0 # via # -r requirements.in # libcoveweb -django-environ==0.4.5 - # via libcoveweb et-xmlfile==1.1.0 # via openpyxl exceptiongroup==1.1.3 @@ -79,7 +77,7 @@ libcove[flatten] @ https://github.com/jpmckinney/lib-cove/archive/hotfix2.zip # libcoveweb libcoveocds[perf,web]==0.12.7 # via -r requirements.in -libcoveweb==0.28.0 +libcoveweb==0.30.0 # via -r requirements.in lxml==4.9.1 # via flattentool diff --git a/requirements_dev.txt b/requirements_dev.txt index b08f816f..0eae023c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -81,10 +81,6 @@ django-bootstrap3==15.0.0 # via # -r requirements.txt # libcoveweb -django-environ==0.4.5 - # via - # -r requirements.txt - # libcoveweb docopt==0.6.2 # via coveralls et-xmlfile==1.1.0 @@ -151,7 +147,7 @@ libcoveocds[perf,web]==0.12.7 # via # -r requirements.txt # libcoveocds -libcoveweb==0.28.0 +libcoveweb==0.30.0 # via -r requirements.txt libsass==0.20.1 # via -r requirements_dev.in From bec9835587960b076ff3cf76fc0927adaeef12c6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 1 Dec 2023 16:11:55 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cove_ocds/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cove_ocds/views.py b/cove_ocds/views.py index 344664ce..79072c71 100644 --- a/cove_ocds/views.py +++ b/cove_ocds/views.py @@ -232,11 +232,11 @@ def explore_ocds(request, pk): context = common_checks_ocds(context, upload_dir, json_data, schema_ocds) # Construct the Open Contracting specific urls for codelists in the docs - for key in ['additional_closed_codelist_values', 'additional_open_codelist_values']: + for key in ["additional_closed_codelist_values", "additional_open_codelist_values"]: for path_string, codelist_info in context[key].items(): - codelist_info['codelist_url'] = ( - 'https://standard.open-contracting.org/{}/en/schema/codelists/#'.format(db_data.data_schema_version) + - re.sub(r'([A-Z])', r'-\1', codelist_info['codelist'].split('.')[0]).lower() + codelist_info["codelist_url"] = ( + "https://standard.open-contracting.org/{}/en/schema/codelists/#".format(db_data.data_schema_version) + + re.sub(r"([A-Z])", r"-\1", codelist_info["codelist"].split(".")[0]).lower() ) if schema_ocds.json_deref_error: From 9eaf96731041a6e57aafca2fcdd80b2a516830f6 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 4 Dec 2023 15:30:28 -0500 Subject: [PATCH 3/6] chore: Modify context if not raising exception --- cove_ocds/views.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cove_ocds/views.py b/cove_ocds/views.py index 79072c71..fb3bb471 100644 --- a/cove_ocds/views.py +++ b/cove_ocds/views.py @@ -231,14 +231,6 @@ def explore_ocds(request, pk): context = common_checks_ocds(context, upload_dir, json_data, schema_ocds) - # Construct the Open Contracting specific urls for codelists in the docs - for key in ["additional_closed_codelist_values", "additional_open_codelist_values"]: - for path_string, codelist_info in context[key].items(): - codelist_info["codelist_url"] = ( - "https://standard.open-contracting.org/{}/en/schema/codelists/#".format(db_data.data_schema_version) - + re.sub(r"([A-Z])", r"-\1", codelist_info["codelist"].split(".")[0]).lower() - ) - if schema_ocds.json_deref_error: exceptions.raise_json_deref_error(schema_ocds.json_deref_error) @@ -250,6 +242,13 @@ def explore_ocds(request, pk): } ) + for key in ("additional_closed_codelist_values", "additional_open_codelist_values"): + for path_string, codelist_info in context[key].items(): + codelist_info["codelist_url"] = ( + "https://standard.open-contracting.org/{}/en/schema/codelists/#".format(db_data.data_schema_version) + + re.sub(r"([A-Z])", r"-\1", codelist_info["codelist"].split(".")[0]).lower() + ) + schema_version = getattr(schema_ocds, "version", None) if schema_version: db_data.schema_version = schema_version From 3f940a937c3ff6893856d9f0416586e444b33435 Mon Sep 17 00:00:00 2001 From: James B Date: Tue, 5 Dec 2023 14:32:28 +0000 Subject: [PATCH 4/6] Links to codelists - rewrite to OCDS Standard - check URL first https://github.com/OpenDataServices/lib-cove-web/issues/74 https://github.com/open-contracting/cove-ocds/pull/192#pullrequestreview-1763357587 --- cove_ocds/views.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cove_ocds/views.py b/cove_ocds/views.py index fb3bb471..10a4f47c 100644 --- a/cove_ocds/views.py +++ b/cove_ocds/views.py @@ -244,10 +244,13 @@ def explore_ocds(request, pk): for key in ("additional_closed_codelist_values", "additional_open_codelist_values"): for path_string, codelist_info in context[key].items(): - codelist_info["codelist_url"] = ( - "https://standard.open-contracting.org/{}/en/schema/codelists/#".format(db_data.data_schema_version) - + re.sub(r"([A-Z])", r"-\1", codelist_info["codelist"].split(".")[0]).lower() - ) + if codelist_info["codelist_url"].startswith(schema_ocds.codelists): + codelist_info["codelist_url"] = ( + "https://standard.open-contracting.org/{}/en/schema/codelists/#".format( + db_data.data_schema_version + ) + + re.sub(r"([A-Z])", r"-\1", codelist_info["codelist"].split(".")[0]).lower() + ) schema_version = getattr(schema_ocds, "version", None) if schema_version: From e87fcf3c7798cb6611f5da4b65858b9d6a966a4c Mon Sep 17 00:00:00 2001 From: James B Date: Tue, 5 Dec 2023 14:33:23 +0000 Subject: [PATCH 5/6] Links to codelists - rewrite to OCDS Standard - tests https://github.com/OpenDataServices/lib-cove-web/issues/74 https://github.com/open-contracting/cove-ocds/pull/192#pullrequestreview-1763357587 --- tests/test_general.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_general.py b/tests/test_general.py index 8d076ccf..8b2d0a27 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -619,3 +619,31 @@ def test_get_json_data_missing_ids(): missin_ids_paths = cove_common.get_json_data_missing_ids(user_data_paths, schema_obj) assert missin_ids_paths == results + + +@pytest.mark.django_db +def test_codelist_url_rewriting(client): + file_name = os.path.join( + "tests", + "fixtures", + "tenders_releases_2_releases_codelists.json", + ) + with open(os.path.join(file_name)) as fp: + user_data = fp.read() + data = SuppliedData.objects.create() + data.original_file.save("test.json", ContentFile(user_data)) + data.current_app = "cove_ocds" + resp = client.get(data.get_absolute_url()) + assert resp.status_code == 200 + assert len(resp.context["additional_closed_codelist_values"]) == 1 + assert ( + resp.context["additional_closed_codelist_values"]["releases/tag"].get("codelist_url") + == "https://standard.open-contracting.org/1.1/en/schema/codelists/#release-tag" + ) + assert len(resp.context["additional_open_codelist_values"]) == 1 + assert ( + resp.context["additional_open_codelist_values"]["releases/tender/items/classification/scheme"].get( + "codelist_url" + ) + == "https://standard.open-contracting.org/1.1/en/schema/codelists/#item-classification-scheme" + ) From c481de19f64243ae98f42e35fb68067bb73031ca Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Tue, 5 Dec 2023 10:49:39 -0500 Subject: [PATCH 6/6] test: Add test for extension codelists --- cove_ocds/views.py | 4 +--- tests/test_general.py | 37 ++++++++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cove_ocds/views.py b/cove_ocds/views.py index 10a4f47c..9d5e0488 100644 --- a/cove_ocds/views.py +++ b/cove_ocds/views.py @@ -246,9 +246,7 @@ def explore_ocds(request, pk): for path_string, codelist_info in context[key].items(): if codelist_info["codelist_url"].startswith(schema_ocds.codelists): codelist_info["codelist_url"] = ( - "https://standard.open-contracting.org/{}/en/schema/codelists/#".format( - db_data.data_schema_version - ) + f"https://standard.open-contracting.org/{db_data.data_schema_version}/en/schema/codelists/#" + re.sub(r"([A-Z])", r"-\1", codelist_info["codelist"].split(".")[0]).lower() ) diff --git a/tests/test_general.py b/tests/test_general.py index 8b2d0a27..131eb537 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -622,7 +622,7 @@ def test_get_json_data_missing_ids(): @pytest.mark.django_db -def test_codelist_url_rewriting(client): +def test_codelist_url_ocds_codelists(client): file_name = os.path.join( "tests", "fixtures", @@ -634,16 +634,43 @@ def test_codelist_url_rewriting(client): data.original_file.save("test.json", ContentFile(user_data)) data.current_app = "cove_ocds" resp = client.get(data.get_absolute_url()) + assert resp.status_code == 200 assert len(resp.context["additional_closed_codelist_values"]) == 1 assert ( - resp.context["additional_closed_codelist_values"]["releases/tag"].get("codelist_url") + resp.context["additional_closed_codelist_values"]["releases/tag"]["codelist_url"] == "https://standard.open-contracting.org/1.1/en/schema/codelists/#release-tag" ) assert len(resp.context["additional_open_codelist_values"]) == 1 assert ( - resp.context["additional_open_codelist_values"]["releases/tender/items/classification/scheme"].get( - "codelist_url" - ) + resp.context["additional_open_codelist_values"]["releases/tender/items/classification/scheme"]["codelist_url"] == "https://standard.open-contracting.org/1.1/en/schema/codelists/#item-classification-scheme" ) + + +@pytest.mark.django_db +def test_codelist_url_extension_codelists(client): + file_name = os.path.join( + "tests", + "fixtures", + "tenders_releases_1_release_with_various_codelists.json", + ) + with open(os.path.join(file_name)) as fp: + user_data = fp.read() + data = SuppliedData.objects.create() + data.original_file.save("test.json", ContentFile(user_data)) + data.current_app = "cove_ocds" + resp = client.get(data.get_absolute_url()) + + assert resp.status_code == 200 + assert len(resp.context["additional_closed_codelist_values"]) == 1 + assert ( + resp.context["additional_closed_codelist_values"]["releases/contracts/tariffs/paidBy"]["codelist_url"] + == "http://standard.open-contracting.org/profiles/ppp/latest/es/codelists/chargePaidBy.csv" + ) + # lib-cove-web displays codelist_amend_urls separately. + assert len(resp.context["additional_open_codelist_values"]) == 1 + assert ( + resp.context["additional_open_codelist_values"]["releases/tender/documents/documentType"]["codelist_url"] + == "https://standard.open-contracting.org/1.1/en/schema/codelists/#document-type" + )