Skip to content

Commit

Permalink
geojsontojson: Add inconsistent information to meta
Browse files Browse the repository at this point in the history
  • Loading branch information
odscjames committed Nov 9, 2022
1 parent 05a1c8e commit 7e7386a
Show file tree
Hide file tree
Showing 18 changed files with 887 additions and 9 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- JSONSchema validate:
- Add format checkers
- Add more fields to output object
- GeoJSON to JSON:
- meta information includes inconsistent ids on organisations and phases

## [0.3.0] - 2022-11-08

Expand Down
33 changes: 29 additions & 4 deletions libcoveofds/geojson.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import copy
from collections import defaultdict

from json_merge_patch import create_patch as json_diff_function

from libcove2.common import fields_present_generator

Expand Down Expand Up @@ -169,6 +172,10 @@ def _convert_span_to_feature(
class GeoJSONToJSONConverter:
def __init__(self):
self._networks: dict = {}
self._inconsistent_phase_ids_by_network_id: defaultdict = defaultdict(set)
self._inconsistent_organisation_ids_by_network_id: defaultdict = defaultdict(
set
)

def process_data(self, nodes_data: dict, spans_data: dict) -> None:
# Network
Expand Down Expand Up @@ -285,8 +292,11 @@ def _process_phase(self, network_id: str, phase: dict) -> str:
phase_id = phase.get("id")
if phase_id:
if phase_id in self._networks[network_id]["phases"]:
# TODO check value is same, add error if not
pass
# Is it inconsistent with what we have seen before?
if json_diff_function(
self._networks[network_id]["phases"][phase_id], phase
):
self._inconsistent_phase_ids_by_network_id[network_id].add(phase_id)
else:
self._networks[network_id]["phases"][phase_id] = phase
return phase_id
Expand All @@ -296,8 +306,14 @@ def _process_organisation(self, network_id: str, organisation: dict) -> str:
organisation_id = organisation.get("id")
if organisation_id:
if organisation_id in self._networks[network_id]["organisations"]:
# TODO check value is same, add error if not
pass
# Is it inconsistent with what we have seen before?
if json_diff_function(
self._networks[network_id]["organisations"][organisation_id],
organisation,
):
self._inconsistent_organisation_ids_by_network_id[network_id].add(
organisation_id
)
else:
self._networks[network_id]["organisations"][
organisation_id
Expand Down Expand Up @@ -329,5 +345,14 @@ def get_meta_json(self) -> dict:
out["output_field_coverage"][key] = {"count": 1}
else:
out["output_field_coverage"][key]["count"] += 1
# inconsistent
out["inconsistent_phases_by_network_id"] = {
k: {"phase_ids": sorted(list(v))}
for k, v in self._inconsistent_phase_ids_by_network_id.items()
}
out["inconsistent_organisations_by_network_id"] = {
k: {"organisation_ids": sorted(list(v))}
for k, v in self._inconsistent_organisation_ids_by_network_id.items()
}
# return
return out
2 changes: 2 additions & 0 deletions make_expected_test_data.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ libcoveofds gjtoj --outputmetafilename tests/fixtures/geojson_to_json/basic_1.me
libcoveofds gjtoj --outputmetafilename tests/fixtures/geojson_to_json/no_geometry_1.meta.expected.json tests/fixtures/geojson_to_json/no_geometry_1.nodes.geo.json tests/fixtures/geojson_to_json/no_geometry_1.spans.geo.json tests/fixtures/geojson_to_json/no_geometry_1.expected.json
libcoveofds gjtoj --outputmetafilename tests/fixtures/geojson_to_json/phases_1.meta.expected.json tests/fixtures/geojson_to_json/phases_1.nodes.geo.json tests/fixtures/geojson_to_json/phases_1.spans.geo.json tests/fixtures/geojson_to_json/phases_1.expected.json
libcoveofds gjtoj --outputmetafilename tests/fixtures/geojson_to_json/organisations_1.meta.expected.json tests/fixtures/geojson_to_json/organisations_1.nodes.geo.json tests/fixtures/geojson_to_json/organisations_1.spans.geo.json tests/fixtures/geojson_to_json/organisations_1.expected.json
libcoveofds gjtoj --outputmetafilename tests/fixtures/geojson_to_json/phases_inconsistent_1.meta.expected.json tests/fixtures/geojson_to_json/phases_inconsistent_1.nodes.geo.json tests/fixtures/geojson_to_json/phases_inconsistent_1.spans.geo.json tests/fixtures/geojson_to_json/phases_inconsistent_1.expected.json
libcoveofds gjtoj --outputmetafilename tests/fixtures/geojson_to_json/organisations_inconsistent_1.meta.expected.json tests/fixtures/geojson_to_json/organisations_inconsistent_1.nodes.geo.json tests/fixtures/geojson_to_json/organisations_inconsistent_1.spans.geo.json tests/fixtures/geojson_to_json/organisations_inconsistent_1.expected.json

# JSON to GeoJSON
libcoveofds jtogj --outputmetafilename tests/fixtures/json_to_geojson/basic_1.expected.meta.json tests/fixtures/json_to_geojson/basic_1.json tests/fixtures/json_to_geojson/basic_1.expected.nodes.geo.json tests/fixtures/json_to_geojson/basic_1.expected.spans.geo.json
Expand Down
3 changes: 3 additions & 0 deletions mypy.ini
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,6 @@ ignore_missing_imports = True

[mypy-jsonschema.*]
ignore_missing_imports = True

[mypy-json_merge_patch.*]
ignore_missing_imports = True
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
packages=find_packages(),
long_description="A data review library",
python_requires=">=3.8",
install_requires=["jsonschema", "requests", "jsonref"],
install_requires=["jsonschema", "requests", "jsonref", "json-merge-patch"],
extras_require={
"dev": ["pytest", "flake8", "black==22.3.0", "isort", "mypy", "Sphinx"]
},
Expand Down
4 changes: 3 additions & 1 deletion tests/fixtures/geojson_to_json/basic_1.meta.expected.json
Original file line number Diff line number Diff line change
Expand Up @@ -54,5 +54,7 @@
"/networks/spans/route/coordinates": {
"count": 1
}
}
},
"inconsistent_phases_by_network_id": {},
"inconsistent_organisations_by_network_id": {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,5 +36,7 @@
"/networks/spans/end": {
"count": 1
}
}
},
"inconsistent_phases_by_network_id": {},
"inconsistent_organisations_by_network_id": {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -87,5 +87,7 @@
"/networks/organisations/name": {
"count": 2
}
}
},
"inconsistent_phases_by_network_id": {},
"inconsistent_organisations_by_network_id": {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
{
"networks": [
{
"id": "a096d627-72e1-4f9b-b129-951b1737bff4",
"name": "Ghana Fibre Network",
"nodes": [
{
"id": "1",
"name": "Accra",
"physicalInfrastructureProvider": {
"id": "GH-RGD-CS111111111"
},
"networkProvider": {
"id": "GH-RGD-CS222222222"
},
"location": {
"type": "Point",
"coordinates": [
-0.174,
5.625
]
}
},
{
"id": "2",
"name": "Kumasi",
"status": "operational",
"physicalInfrastructureProvider": {
"id": "GH-RGD-CS111111111"
},
"networkProvider": {
"id": "GH-RGD-CS222222222"
},
"location": {
"type": "Point",
"coordinates": [
-1.628,
6.711
]
}
}
],
"spans": [
{
"id": "1",
"name": "Accra to Kumasi",
"start": "1",
"end": "2",
"physicalInfrastructureProvider": {
"id": "GH-RGD-CS111111111"
},
"networkProvider": {
"id": "GH-RGD-CS222222222"
},
"route": {
"type": "LineString",
"coordinates": [
[
-0.173,
5.626
],
[
-0.178,
5.807
],
[
-0.112,
5.971
],
[
-0.211,
5.963
],
[
-0.321,
6.17
],
[
-0.488,
6.29
],
[
-0.56,
6.421
],
[
-0.752,
6.533
],
[
-0.867,
6.607
],
[
-1.101,
6.585
],
[
-1.304,
6.623
],
[
-1.461,
6.727
],
[
-1.628,
6.713
]
]
}
}
],
"organisations": [
{
"id": "GH-RGD-CS111111111",
"name": "FibreCo"
},
{
"id": "GH-RGD-CS222222222",
"name": "FastWeb"
}
]
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
{
"output_field_coverage": {
"/networks": {
"count": 1
},
"/networks/id": {
"count": 1
},
"/networks/name": {
"count": 1
},
"/networks/nodes": {
"count": 1
},
"/networks/nodes/id": {
"count": 2
},
"/networks/nodes/name": {
"count": 2
},
"/networks/nodes/physicalInfrastructureProvider": {
"count": 2
},
"/networks/nodes/physicalInfrastructureProvider/id": {
"count": 2
},
"/networks/nodes/networkProvider": {
"count": 2
},
"/networks/nodes/networkProvider/id": {
"count": 2
},
"/networks/nodes/location": {
"count": 2
},
"/networks/nodes/location/type": {
"count": 2
},
"/networks/nodes/location/coordinates": {
"count": 2
},
"/networks/nodes/status": {
"count": 1
},
"/networks/spans": {
"count": 1
},
"/networks/spans/id": {
"count": 1
},
"/networks/spans/name": {
"count": 1
},
"/networks/spans/start": {
"count": 1
},
"/networks/spans/end": {
"count": 1
},
"/networks/spans/physicalInfrastructureProvider": {
"count": 1
},
"/networks/spans/physicalInfrastructureProvider/id": {
"count": 1
},
"/networks/spans/networkProvider": {
"count": 1
},
"/networks/spans/networkProvider/id": {
"count": 1
},
"/networks/spans/route": {
"count": 1
},
"/networks/spans/route/type": {
"count": 1
},
"/networks/spans/route/coordinates": {
"count": 1
},
"/networks/organisations": {
"count": 1
},
"/networks/organisations/id": {
"count": 2
},
"/networks/organisations/name": {
"count": 2
}
},
"inconsistent_phases_by_network_id": {},
"inconsistent_organisations_by_network_id": {
"a096d627-72e1-4f9b-b129-951b1737bff4": {
"organisation_ids": [
"GH-RGD-CS111111111",
"GH-RGD-CS222222222"
]
}
}
}
Loading

0 comments on commit 7e7386a

Please sign in to comment.