From cbfe2c1389181e42038ad56fc356c91cd7e1c36d Mon Sep 17 00:00:00 2001 From: Keyur Shah Date: Fri, 11 Aug 2023 09:24:52 -0700 Subject: [PATCH] Add support for coordinates resolution. (#209) --- .../main/java/org/datacommons/tool/Args.java | 3 + .../java/org/datacommons/tool/GenMcf.java | 1 + .../main/java/org/datacommons/tool/Lint.java | 1 + .../main/java/org/datacommons/tool/Main.java | 8 + .../java/org/datacommons/tool/Processor.java | 4 +- .../java/org/datacommons/tool/GenMcfTest.java | 23 +- .../tool/genmcf/fataltmcf/output/report.json | 3 +- .../genmcf/latlngresolution/input/LatLng.csv | 6 + .../genmcf/latlngresolution/input/LatLng.tmcf | 13 + .../genmcf/latlngresolution/input/args.txt | 1 + .../latlngresolution/output/report.json | 110 ++ .../output/summary_report.html | 622 +++++++++ .../output/table_mcf_nodes_LatLng.mcf | 85 ++ .../localidresolution/output/report.json | 3 +- .../manyinconsistent/output/report.json | 3 +- .../measurementresult/output/report.json | 3 +- .../tool/genmcf/resolution/output/report.json | 3 +- .../tool/genmcf/statchecks/output/report.json | 3 +- .../genmcf/successtmcf/output/report.json | 3 +- .../tool/lint/allfiletypes/output/report.json | 3 +- .../tool/lint/mcfonly/output/report.json | 3 +- .../tool/lint/nocsv/output/report.json | 3 +- .../tool/lint/statchecks/output/report.json | 3 +- util/pom.xml | 6 + .../java/org/datacommons/proto/Debug.java | 146 ++- .../java/org/datacommons/proto/Recon.java | 1164 ++++++++++++++++- .../datacommons/util/CoordinatesResolver.java | 102 ++ .../datacommons/util/ExternalIdResolver.java | 26 +- .../org/datacommons/util/ReconClient.java | 114 +- util/src/main/proto/Debug.proto | 3 +- util/src/main/proto/recon.proto | 5 + .../util/CoordinatesResolverTest.java | 66 + .../util/ExternalIdResolverTest.java | 79 +- .../org/datacommons/util/ReconClientTest.java | 77 +- 34 files changed, 2615 insertions(+), 83 deletions(-) create mode 100644 tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.csv create mode 100644 tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.tmcf create mode 100644 tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/args.txt create mode 100644 tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/report.json create mode 100644 tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/summary_report.html create mode 100644 tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/table_mcf_nodes_LatLng.mcf create mode 100644 util/src/main/java/org/datacommons/util/CoordinatesResolver.java create mode 100644 util/src/test/java/org/datacommons/util/CoordinatesResolverTest.java diff --git a/tool/src/main/java/org/datacommons/tool/Args.java b/tool/src/main/java/org/datacommons/tool/Args.java index e2724f2a..5d7fc07c 100644 --- a/tool/src/main/java/org/datacommons/tool/Args.java +++ b/tool/src/main/java/org/datacommons/tool/Args.java @@ -11,6 +11,7 @@ class Args { public boolean doExistenceChecks = false; public ResolutionMode resolutionMode = ResolutionMode.NONE; + public boolean doCoordinatesResolution = false; public boolean doStatChecks = false; public List samplePlaces = null; public boolean verbose = false; @@ -32,6 +33,7 @@ public String toString() { } argStr.append(" existence-checks=" + doExistenceChecks); argStr.append(", resolution=" + resolutionMode.name()); + argStr.append(", coordinates-resolution=" + doCoordinatesResolution); argStr.append(", num-threads=" + numThreads); argStr.append(", stat-checks=" + doStatChecks); if (samplePlaces != null) { @@ -55,6 +57,7 @@ public Debug.CommandArgs toProto() { } else if (resolutionMode == ResolutionMode.FULL) { argsBuilder.setResolution(Debug.CommandArgs.ResolutionMode.RESOLUTION_MODE_FULL); } + argsBuilder.setCoordinatesResolution(doCoordinatesResolution); argsBuilder.setStatChecks(doStatChecks); if (samplePlaces != null) argsBuilder.addAllSamplePlaces(samplePlaces); argsBuilder.setObservationAbout(checkObservationAbout); diff --git a/tool/src/main/java/org/datacommons/tool/GenMcf.java b/tool/src/main/java/org/datacommons/tool/GenMcf.java index 8a847948..0b064eec 100644 --- a/tool/src/main/java/org/datacommons/tool/GenMcf.java +++ b/tool/src/main/java/org/datacommons/tool/GenMcf.java @@ -58,6 +58,7 @@ public Integer call() throws IOException, TemplateException { Args args = new Args(); args.doExistenceChecks = parent.doExistenceChecks; args.resolutionMode = parent.resolutionMode; + args.doCoordinatesResolution = parent.doCoordinatesResolution; args.doStatChecks = parent.doStatChecks; args.samplePlaces = parent.samplePlaces; args.numThreads = parent.numThreads; diff --git a/tool/src/main/java/org/datacommons/tool/Lint.java b/tool/src/main/java/org/datacommons/tool/Lint.java index 3e4da98f..69e1a107 100644 --- a/tool/src/main/java/org/datacommons/tool/Lint.java +++ b/tool/src/main/java/org/datacommons/tool/Lint.java @@ -56,6 +56,7 @@ public Integer call() throws IOException, TemplateException { Args args = new Args(); args.doExistenceChecks = parent.doExistenceChecks; args.resolutionMode = parent.resolutionMode; + args.doCoordinatesResolution = parent.doCoordinatesResolution; args.doStatChecks = parent.doStatChecks; args.samplePlaces = parent.samplePlaces; args.numThreads = parent.numThreads; diff --git a/tool/src/main/java/org/datacommons/tool/Main.java b/tool/src/main/java/org/datacommons/tool/Main.java index c0fad292..0b2c6325 100644 --- a/tool/src/main/java/org/datacommons/tool/Main.java +++ b/tool/src/main/java/org/datacommons/tool/Main.java @@ -67,6 +67,14 @@ class Main { + "You rarely want to use this mode.") public Args.ResolutionMode resolutionMode = Args.ResolutionMode.NONE; + @CommandLine.Option( + names = {"-cr", "--coordinates-resolution"}, + defaultValue = "false", + scope = CommandLine.ScopeType.INHERIT, + description = + "If set, resolves lat-lng coordinates by making DC Recon API calls. This flag is only application in full resolution mode.") + public boolean doCoordinatesResolution; + @CommandLine.Option( names = {"-s", "--stat-checks"}, defaultValue = "true", diff --git a/tool/src/main/java/org/datacommons/tool/Processor.java b/tool/src/main/java/org/datacommons/tool/Processor.java index a3793c7f..05db2b26 100644 --- a/tool/src/main/java/org/datacommons/tool/Processor.java +++ b/tool/src/main/java/org/datacommons/tool/Processor.java @@ -131,7 +131,9 @@ private Processor(Args args) { existenceChecker = new ExistenceChecker(this.httpClient, args.verbose, logCtx); } if (args.resolutionMode == Args.ResolutionMode.FULL) { - idResolver = new ExternalIdResolver(this.httpClient, args.verbose, logCtx); + idResolver = + new ExternalIdResolver( + this.httpClient, args.doCoordinatesResolution, args.verbose, logCtx); } statVarState = new StatVarState(this.httpClient, logCtx); if (args.doStatChecks) { diff --git a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java index bca825d2..a17e773c 100644 --- a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java +++ b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java @@ -54,13 +54,22 @@ public class GenMcfTest { // count here. private static Map EXPECTED_FILES_TO_CHECK = Map.of( - "fataltmcf", 2, - "resolution", 5, - "statchecks", 3, - "successtmcf", 3, - "measurementresult", 4, - "localidresolution", 5, - "manyinconsistent", 4); + "fataltmcf", + 2, + "resolution", + 5, + "latlngresolution", + 3, + "statchecks", + 3, + "successtmcf", + 3, + "measurementresult", + 4, + "localidresolution", + 5, + "manyinconsistent", + 4); // Skip testing the following files. If this List is non-empty, the flaky files should be fixed // and removed from this list. diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json index 79bb69ec..535d2b3f 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json @@ -196,6 +196,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": false + "allowNanSvobs": false, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.csv b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.csv new file mode 100644 index 00000000..a29645f1 --- /dev/null +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.csv @@ -0,0 +1,6 @@ +Year,Place_Name,Lat,Lng,SV1 +2019,British–Columbia,53,-127,1 +2019,Saskatchewan,43,-116,5 +2019,San Francisco,37.77493,-122.41942,1 +2019,California,36,-119.4,1 +2019,Alberta,53.9,-116,6 \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.tmcf b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.tmcf new file mode 100644 index 00000000..4df55187 --- /dev/null +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.tmcf @@ -0,0 +1,13 @@ +Node: E:SVTest->E0 +typeOf: dcs:StatVarObservation +variableMeasured: dcs:SV1 +measurementMethod: dcs:SVTest +observationAbout: E:SVTest->E3 +observationDate: C:SVTest->Year +value: C:SVTest->SV1 + +Node: E:SVTest->E3 +typeOf: dcid:Place +name: C:SVTest->Place_Name +latitude: C:SVTest->Lat +longitude: C:SVTest->Lng \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/args.txt b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/args.txt new file mode 100644 index 00000000..fb05fafa --- /dev/null +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/args.txt @@ -0,0 +1 @@ +--coordinates-resolution=true diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/report.json new file mode 100644 index 00000000..76866681 --- /dev/null +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/report.json @@ -0,0 +1,110 @@ +{ + "levelSummary": { + "LEVEL_INFO": { + "counters": { + "NumRowSuccesses": "5", + "NumPVSuccesses": "65", + "Existence_NumChecks": "65", + "NumNodeSuccesses": "10", + "ReconClient_NumApiCalls": "1", + "Existence_NumDcCalls": "1" + } + }, + "LEVEL_WARNING": { + "counters": { + "Existence_MissingReference_measurementMethod": "5", + "Existence_MissingReference_variableMeasured": "5" + } + } + }, + "entries": [{ + "level": "LEVEL_WARNING", + "location": { + "file": "LatLng.csv", + "lineNumber": "2" + }, + "userMessage": "Failed reference existence check :: value-ref: 'SVTest', property: 'measurementMethod', node: 'E:SVTest->E0'", + "counterKey": "Existence_MissingReference_measurementMethod" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "LatLng.csv", + "lineNumber": "3" + }, + "userMessage": "Failed reference existence check :: value-ref: 'SVTest', property: 'measurementMethod', node: 'E:SVTest->E0'", + "counterKey": "Existence_MissingReference_measurementMethod" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "LatLng.csv", + "lineNumber": "4" + }, + "userMessage": "Failed reference existence check :: value-ref: 'SVTest', property: 'measurementMethod', node: 'E:SVTest->E0'", + "counterKey": "Existence_MissingReference_measurementMethod" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "LatLng.csv", + "lineNumber": "5" + }, + "userMessage": "Failed reference existence check :: value-ref: 'SVTest', property: 'measurementMethod', node: 'E:SVTest->E0'", + "counterKey": "Existence_MissingReference_measurementMethod" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "LatLng.csv", + "lineNumber": "6" + }, + "userMessage": "Failed reference existence check :: value-ref: 'SVTest', property: 'measurementMethod', node: 'E:SVTest->E0'", + "counterKey": "Existence_MissingReference_measurementMethod" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "LatLng.csv", + "lineNumber": "2" + }, + "userMessage": "Failed reference existence check :: value-ref: 'SV1', property: 'variableMeasured', node: 'E:SVTest->E0'", + "counterKey": "Existence_MissingReference_variableMeasured" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "LatLng.csv", + "lineNumber": "3" + }, + "userMessage": "Failed reference existence check :: value-ref: 'SV1', property: 'variableMeasured', node: 'E:SVTest->E0'", + "counterKey": "Existence_MissingReference_variableMeasured" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "LatLng.csv", + "lineNumber": "4" + }, + "userMessage": "Failed reference existence check :: value-ref: 'SV1', property: 'variableMeasured', node: 'E:SVTest->E0'", + "counterKey": "Existence_MissingReference_variableMeasured" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "LatLng.csv", + "lineNumber": "5" + }, + "userMessage": "Failed reference existence check :: value-ref: 'SV1', property: 'variableMeasured', node: 'E:SVTest->E0'", + "counterKey": "Existence_MissingReference_variableMeasured" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "LatLng.csv", + "lineNumber": "6" + }, + "userMessage": "Failed reference existence check :: value-ref: 'SV1', property: 'variableMeasured', node: 'E:SVTest->E0'", + "counterKey": "Existence_MissingReference_variableMeasured" + }], + "commandArgs": { + "existenceChecks": true, + "resolution": "RESOLUTION_MODE_FULL", + "numThreads": 1, + "statChecks": true, + "observationAbout": false, + "allowNanSvobs": false, + "coordinatesResolution": true + } +} \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/summary_report.html b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/summary_report.html new file mode 100644 index 00000000..8a6ecce5 --- /dev/null +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/summary_report.html @@ -0,0 +1,622 @@ + + + Summary Report + + + + + + + + + + +
+ Go to Top +
+

Summary Report

+

Table of Contents

+ + + + +
+

+ Import Run Details +

+ + + + + + + + + + + + + + + + + + + + + +
Existence Checks Enabledyes
Resolution ModeRESOLUTION_MODE_FULL
Num Threads1
Stat Checks Enabledyes
Sample Places Entered
+
+ + +
+

+ Counters +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Counter NameNum Occurences
LEVEL_INFO
NumRowSuccesses5
NumPVSuccesses65
Existence_NumChecks65
NumNodeSuccesses10
ReconClient_NumApiCalls1
Existence_NumDcCalls1
LEVEL_WARNING
Existence_MissingReference_measurementMethod5
Existence_MissingReference_variableMeasured5
+
+ +
+

+ StatVarObservations by StatVar +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
StatVarNum PlacesNum ObservationsNum Observation DatesMin DateMax DateMeasurement MethodsUnitsScaling FactorsObservation Periods
SV155120192019 +
SVTest
+ 		+
+ 		+
+ 		+
+
+
+
+

+ Series Summaries for Sample Places +

+ +
+ Canada (country/CAN) + Open this place (country/CAN) in Data Commons browser. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
StatVarNum ObservationsDatesCorresponding ValuesMeasurement MethodUnitScaling FactorObservation PeriodTime Series Chart
SV1120191 +
SVTest
+ 		+
+ 		+
+ 		+
+ 		+ + + +2018-12-312019-01-010.00.20.40.60.81.01.21.41.61.82.0
+
+
+ 53.75,-115.75 CAN (ipcc_50/53.75_-115.75_CAN) + Open this place (ipcc_50/53.75_-115.75_CAN) in Data Commons browser. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
StatVarNum ObservationsDatesCorresponding ValuesMeasurement MethodUnitScaling FactorObservation PeriodTime Series Chart
SV1120196 +
SVTest
+ 		+
+ 		+
+ 		+
+ 		+ + + +2018-12-312019-01-015.05.25.45.65.86.06.26.46.66.87.0
+
+
+ 83624 (zip/83624) + Open this place (zip/83624) in Data Commons browser. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
StatVarNum ObservationsDatesCorresponding ValuesMeasurement MethodUnitScaling FactorObservation PeriodTime Series Chart
SV1120195 +
SVTest
+ 		+
+ 		+
+ 		+
+ 		+ + + +2018-12-312019-01-014.04.24.44.64.85.05.25.45.65.86.0
+
+
+ 93212 (zip/93212) + Open this place (zip/93212) in Data Commons browser. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
StatVarNum ObservationsDatesCorresponding ValuesMeasurement MethodUnitScaling FactorObservation PeriodTime Series Chart
SV1120191 +
SVTest
+ 		+
+ 		+
+ 		+
+ 		+ + + +2018-12-312019-01-010.00.20.40.60.81.01.21.41.61.82.0
+
+
+ 94103 (zip/94103) + Open this place (zip/94103) in Data Commons browser. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
StatVarNum ObservationsDatesCorresponding ValuesMeasurement MethodUnitScaling FactorObservation PeriodTime Series Chart
SV1120191 +
SVTest
+ 		+
+ 		+
+ 		+
+ 		+ + + +2018-12-312019-01-010.00.20.40.60.81.01.21.41.61.82.0
+
+
+ + + + \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/table_mcf_nodes_LatLng.mcf b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/table_mcf_nodes_LatLng.mcf new file mode 100644 index 00000000..86da4ecc --- /dev/null +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/table_mcf_nodes_LatLng.mcf @@ -0,0 +1,85 @@ +Node: SVTest/E3/1 +latitude: 53 +name: "British–Columbia" +typeOf: dcid:Place +longitude: -127 +dcid: "country/CAN" + +Node: SVTest/E0/1 +observationDate: 2019 +observationAbout: dcid:country/CAN +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: 1 +typeOf: dcid:StatVarObservation +keyString: "observationAbout=country/CANvariableMeasured=SV1observationDate=2019value=1measurementMethod=SVTest" +dcid: "dc/o/6v44zdhbcq8" + +Node: SVTest/E3/2 +latitude: 43 +name: "Saskatchewan" +typeOf: dcid:Place +longitude: -116 +dcid: "zip/83624" + +Node: SVTest/E0/2 +observationDate: 2019 +observationAbout: dcid:zip/83624 +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: 5 +typeOf: dcid:StatVarObservation +keyString: "observationAbout=zip/83624variableMeasured=SV1observationDate=2019value=5measurementMethod=SVTest" +dcid: "dc/o/tjm0x6y8jv26f" + +Node: SVTest/E3/3 +latitude: 37.77493 +name: "San Francisco" +typeOf: dcid:Place +longitude: -122.41942 +dcid: "zip/94103" + +Node: SVTest/E0/3 +observationDate: 2019 +observationAbout: dcid:zip/94103 +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: 1 +typeOf: dcid:StatVarObservation +keyString: "observationAbout=zip/94103variableMeasured=SV1observationDate=2019value=1measurementMethod=SVTest" +dcid: "dc/o/1jm50b24f2mg" + +Node: SVTest/E3/4 +latitude: 36 +name: "California" +typeOf: dcid:Place +longitude: -119.4 +dcid: "zip/93212" + +Node: SVTest/E0/4 +observationDate: 2019 +observationAbout: dcid:zip/93212 +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: 1 +typeOf: dcid:StatVarObservation +keyString: "observationAbout=zip/93212variableMeasured=SV1observationDate=2019value=1measurementMethod=SVTest" +dcid: "dc/o/0r93tc1txjj1h" + +Node: SVTest/E3/5 +latitude: 53.9 +name: "Alberta" +typeOf: dcid:Place +longitude: -116 +dcid: "ipcc_50/53.75_-115.75_CAN" + +Node: SVTest/E0/5 +observationDate: 2019 +observationAbout: dcid:ipcc_50/53.75_-115.75_CAN +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: 6 +typeOf: dcid:StatVarObservation +keyString: "observationAbout=ipcc_50/53.75_-115.75_CANvariableMeasured=SV1observationDate=2019value=6measurementMethod=SVTest" +dcid: "dc/o/bf7wjkynq5sl" + diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/localidresolution/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/localidresolution/output/report.json index c01e4cbe..3008f8d4 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/localidresolution/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/localidresolution/output/report.json @@ -57,6 +57,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": false + "allowNanSvobs": false, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/manyinconsistent/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/manyinconsistent/output/report.json index 5894c9b1..1d587360 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/manyinconsistent/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/manyinconsistent/output/report.json @@ -106,6 +106,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": false + "allowNanSvobs": false, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/measurementresult/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/measurementresult/output/report.json index 2facdcfe..2d4fc8e4 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/measurementresult/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/measurementresult/output/report.json @@ -132,6 +132,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": true + "allowNanSvobs": true, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/resolution/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/resolution/output/report.json index 730f28c3..03adcd47 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/resolution/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/resolution/output/report.json @@ -196,6 +196,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": false + "allowNanSvobs": false, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/statchecks/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/statchecks/output/report.json index a3ea4b71..8e6711b9 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/statchecks/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/statchecks/output/report.json @@ -237,6 +237,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": false + "allowNanSvobs": false, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json index fb0a9306..96008a4d 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json @@ -720,6 +720,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": true + "allowNanSvobs": true, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json index 6f48ec7c..dcca0ca2 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json @@ -935,6 +935,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": false + "allowNanSvobs": false, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json index ffdb0808..719af430 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json @@ -563,6 +563,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": false + "allowNanSvobs": false, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json index 99dbd7e1..f7aecbcd 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json @@ -704,6 +704,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": false + "allowNanSvobs": false, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/lint/statchecks/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/statchecks/output/report.json index d63e3723..d9f9e054 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/statchecks/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/statchecks/output/report.json @@ -130,6 +130,7 @@ "numThreads": 1, "statChecks": true, "observationAbout": false, - "allowNanSvobs": false + "allowNanSvobs": false, + "coordinatesResolution": false } } \ No newline at end of file diff --git a/util/pom.xml b/util/pom.xml index 56b06e2f..07cd8397 100644 --- a/util/pom.xml +++ b/util/pom.xml @@ -73,6 +73,12 @@ 0.46 test + + com.google.truth.extensions + truth-java8-extension + 0.46 + test + com.google.truth.extensions truth-proto-extension diff --git a/util/src/main/java/org/datacommons/proto/Debug.java b/util/src/main/java/org/datacommons/proto/Debug.java index 96cf6e46..9a863267 100644 --- a/util/src/main/java/org/datacommons/proto/Debug.java +++ b/util/src/main/java/org/datacommons/proto/Debug.java @@ -4476,6 +4476,19 @@ public interface CommandArgsOrBuilder * @return The checkMeasurementResult. */ boolean getCheckMeasurementResult(); + + /** + * optional bool coordinates_resolution = 9; + * + * @return Whether the coordinatesResolution field is set. + */ + boolean hasCoordinatesResolution(); + /** + * optional bool coordinates_resolution = 9; + * + * @return The coordinatesResolution. + */ + boolean getCoordinatesResolution(); } /** Protobuf type {@code org.datacommons.proto.CommandArgs} */ public static final class CommandArgs extends com.google.protobuf.GeneratedMessageV3 @@ -4583,6 +4596,12 @@ private CommandArgs( checkMeasurementResult_ = input.readBool(); break; } + case 72: + { + bitField0_ |= 0x00000080; + coordinatesResolution_ = input.readBool(); + break; + } default: { if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { @@ -4967,6 +4986,25 @@ public boolean getCheckMeasurementResult() { return checkMeasurementResult_; } + public static final int COORDINATES_RESOLUTION_FIELD_NUMBER = 9; + private boolean coordinatesResolution_; + /** + * optional bool coordinates_resolution = 9; + * + * @return Whether the coordinatesResolution field is set. + */ + public boolean hasCoordinatesResolution() { + return ((bitField0_ & 0x00000080) != 0); + } + /** + * optional bool coordinates_resolution = 9; + * + * @return The coordinatesResolution. + */ + public boolean getCoordinatesResolution() { + return coordinatesResolution_; + } + private byte memoizedIsInitialized = -1; @java.lang.Override @@ -5005,6 +5043,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io if (((bitField0_ & 0x00000040) != 0)) { output.writeBool(8, checkMeasurementResult_); } + if (((bitField0_ & 0x00000080) != 0)) { + output.writeBool(9, coordinatesResolution_); + } unknownFields.writeTo(output); } @@ -5043,6 +5084,9 @@ public int getSerializedSize() { if (((bitField0_ & 0x00000040) != 0)) { size += com.google.protobuf.CodedOutputStream.computeBoolSize(8, checkMeasurementResult_); } + if (((bitField0_ & 0x00000080) != 0)) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(9, coordinatesResolution_); + } size += unknownFields.getSerializedSize(); memoizedSize = size; return size; @@ -5087,6 +5131,10 @@ public boolean equals(final java.lang.Object obj) { if (hasCheckMeasurementResult()) { if (getCheckMeasurementResult() != other.getCheckMeasurementResult()) return false; } + if (hasCoordinatesResolution() != other.hasCoordinatesResolution()) return false; + if (hasCoordinatesResolution()) { + if (getCoordinatesResolution() != other.getCoordinatesResolution()) return false; + } if (!unknownFields.equals(other.unknownFields)) return false; return true; } @@ -5130,6 +5178,10 @@ public int hashCode() { hash = (37 * hash) + CHECK_MEASUREMENT_RESULT_FIELD_NUMBER; hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getCheckMeasurementResult()); } + if (hasCoordinatesResolution()) { + hash = (37 * hash) + COORDINATES_RESOLUTION_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getCoordinatesResolution()); + } hash = (29 * hash) + unknownFields.hashCode(); memoizedHashCode = hash; return hash; @@ -5285,6 +5337,8 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00000040); checkMeasurementResult_ = false; bitField0_ = (bitField0_ & ~0x00000080); + coordinatesResolution_ = false; + bitField0_ = (bitField0_ & ~0x00000100); return this; } @@ -5347,6 +5401,10 @@ public org.datacommons.proto.Debug.CommandArgs buildPartial() { result.checkMeasurementResult_ = checkMeasurementResult_; to_bitField0_ |= 0x00000040; } + if (((from_bitField0_ & 0x00000100) != 0)) { + result.coordinatesResolution_ = coordinatesResolution_; + to_bitField0_ |= 0x00000080; + } result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -5430,6 +5488,9 @@ public Builder mergeFrom(org.datacommons.proto.Debug.CommandArgs other) { if (other.hasCheckMeasurementResult()) { setCheckMeasurementResult(other.getCheckMeasurementResult()); } + if (other.hasCoordinatesResolution()) { + setCoordinatesResolution(other.getCoordinatesResolution()); + } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; @@ -5917,6 +5978,47 @@ public Builder clearCheckMeasurementResult() { return this; } + private boolean coordinatesResolution_; + /** + * optional bool coordinates_resolution = 9; + * + * @return Whether the coordinatesResolution field is set. + */ + public boolean hasCoordinatesResolution() { + return ((bitField0_ & 0x00000100) != 0); + } + /** + * optional bool coordinates_resolution = 9; + * + * @return The coordinatesResolution. + */ + public boolean getCoordinatesResolution() { + return coordinatesResolution_; + } + /** + * optional bool coordinates_resolution = 9; + * + * @param value The coordinatesResolution to set. + * @return This builder for chaining. + */ + public Builder setCoordinatesResolution(boolean value) { + bitField0_ |= 0x00000100; + coordinatesResolution_ = value; + onChanged(); + return this; + } + /** + * optional bool coordinates_resolution = 9; + * + * @return This builder for chaining. + */ + public Builder clearCoordinatesResolution() { + bitField0_ = (bitField0_ & ~0x00000100); + coordinatesResolution_ = false; + onChanged(); + return this; + } + @java.lang.Override public final Builder setUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { @@ -12295,33 +12397,34 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + " \001(\0132%.org.datacommons.proto.Log.Counter" + "Set:\0028\001\"c\n\005Level\022\025\n\021LEVEL_UNSPECIFIED\020\000\022" + "\016\n\nLEVEL_INFO\020\001\022\021\n\rLEVEL_WARNING\020\002\022\017\n\013LE" - + "VEL_ERROR\020\003\022\017\n\013LEVEL_FATAL\020\004J\004\010\002\020\003\"\210\003\n\013C" + + "VEL_ERROR\020\003\022\017\n\013LEVEL_FATAL\020\004J\004\010\002\020\003\"\250\003\n\013C" + "ommandArgs\022\030\n\020existence_checks\030\001 \001(\010\022E\n\n" + "resolution\030\002 \001(\01621.org.datacommons.proto" + ".CommandArgs.ResolutionMode\022\023\n\013num_threa" + "ds\030\003 \001(\005\022\023\n\013stat_checks\030\004 \001(\010\022\025\n\rsample_" + "places\030\005 \003(\t\022\031\n\021observation_about\030\006 \001(\010\022" + "\027\n\017allow_nan_svobs\030\007 \001(\010\022 \n\030check_measur" - + "ement_result\030\010 \001(\010\"\200\001\n\016ResolutionMode\022\037\n" - + "\033RESOLUTION_MODE_UNSPECIFIED\020\000\022\030\n\024RESOLU" - + "TION_MODE_NONE\020\001\022\031\n\025RESOLUTION_MODE_LOCA" - + "L\020\002\022\030\n\024RESOLUTION_MODE_FULL\020\003\"\321\001\n\tDataPo" - + "int\022\014\n\004date\030\001 \001(\t\022:\n\006values\030\002 \003(\0132*.org." - + "datacommons.proto.DataPoint.DataValue\032z\n" - + "\tDataValue\0229\n\005value\030\001 \001(\0132*.org.datacomm" - + "ons.proto.McfGraph.TypedValue\0222\n\tlocatio" - + "ns\030\002 \003(\0132\037.org.datacommons.proto.Locatio" - + "n\"\234\003\n\024StatValidationResult\022\022\n\nplace_dcid" - + "\030\001 \001(\t\022\025\n\rstat_var_dcid\030\002 \001(\t\022\032\n\022measure" - + "ment_method\030\003 \001(\t\022\032\n\022observation_period\030" - + "\004 \001(\t\022\026\n\016scaling_factor\030\005 \001(\t\022\014\n\004unit\030\006 " - + "\001(\t\022\\\n\023validation_counters\030\007 \003(\0132?.org.d" - + "atacommons.proto.StatValidationResult.St" - + "atValidationEntry\032\234\001\n\023StatValidationEntr" - + "y\022\023\n\013counter_key\030\001 \001(\t\0228\n\016problem_points" - + "\030\002 \003(\0132 .org.datacommons.proto.DataPoint" - + "\022\032\n\022additional_details\030\003 \001(\t\022\032\n\022percent_" - + "difference\030\004 \001(\001" + + "ement_result\030\010 \001(\010\022\036\n\026coordinates_resolu" + + "tion\030\t \001(\010\"\200\001\n\016ResolutionMode\022\037\n\033RESOLUT" + + "ION_MODE_UNSPECIFIED\020\000\022\030\n\024RESOLUTION_MOD" + + "E_NONE\020\001\022\031\n\025RESOLUTION_MODE_LOCAL\020\002\022\030\n\024R" + + "ESOLUTION_MODE_FULL\020\003\"\321\001\n\tDataPoint\022\014\n\004d" + + "ate\030\001 \001(\t\022:\n\006values\030\002 \003(\0132*.org.datacomm" + + "ons.proto.DataPoint.DataValue\032z\n\tDataVal" + + "ue\0229\n\005value\030\001 \001(\0132*.org.datacommons.prot" + + "o.McfGraph.TypedValue\0222\n\tlocations\030\002 \003(\013" + + "2\037.org.datacommons.proto.Location\"\234\003\n\024St" + + "atValidationResult\022\022\n\nplace_dcid\030\001 \001(\t\022\025" + + "\n\rstat_var_dcid\030\002 \001(\t\022\032\n\022measurement_met" + + "hod\030\003 \001(\t\022\032\n\022observation_period\030\004 \001(\t\022\026\n" + + "\016scaling_factor\030\005 \001(\t\022\014\n\004unit\030\006 \001(\t\022\\\n\023v" + + "alidation_counters\030\007 \003(\0132?.org.datacommo" + + "ns.proto.StatValidationResult.StatValida" + + "tionEntry\032\234\001\n\023StatValidationEntry\022\023\n\013cou" + + "nter_key\030\001 \001(\t\0228\n\016problem_points\030\002 \003(\0132 " + + ".org.datacommons.proto.DataPoint\022\032\n\022addi" + + "tional_details\030\003 \001(\t\022\032\n\022percent_differen" + + "ce\030\004 \001(\001" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom( @@ -12383,6 +12486,7 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { "ObservationAbout", "AllowNanSvobs", "CheckMeasurementResult", + "CoordinatesResolution", }); internal_static_org_datacommons_proto_DataPoint_descriptor = getDescriptor().getMessageTypes().get(2); diff --git a/util/src/main/java/org/datacommons/proto/Recon.java b/util/src/main/java/org/datacommons/proto/Recon.java index adf1a100..134f1ed0 100644 --- a/util/src/main/java/org/datacommons/proto/Recon.java +++ b/util/src/main/java/org/datacommons/proto/Recon.java @@ -11439,6 +11439,743 @@ public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { org.datacommons.proto.Recon.ResolveCoordinatesResponse.Builder.class); } + public interface PlaceOrBuilder + extends + // @@protoc_insertion_point(interface_extends:org.datacommons.proto.ResolveCoordinatesResponse.Place) + com.google.protobuf.MessageOrBuilder { + + /** + * string dcid = 1; + * + * @return The dcid. + */ + java.lang.String getDcid(); + /** + * string dcid = 1; + * + * @return The bytes for dcid. + */ + com.google.protobuf.ByteString getDcidBytes(); + + /** + * string dominant_type = 2; + * + * @return The dominantType. + */ + java.lang.String getDominantType(); + /** + * string dominant_type = 2; + * + * @return The bytes for dominantType. + */ + com.google.protobuf.ByteString getDominantTypeBytes(); + } + /** Protobuf type {@code org.datacommons.proto.ResolveCoordinatesResponse.Place} */ + public static final class Place extends com.google.protobuf.GeneratedMessageV3 + implements + // @@protoc_insertion_point(message_implements:org.datacommons.proto.ResolveCoordinatesResponse.Place) + PlaceOrBuilder { + private static final long serialVersionUID = 0L; + // Use Place.newBuilder() to construct. + private Place(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private Place() { + dcid_ = ""; + dominantType_ = ""; + } + + @java.lang.Override + @SuppressWarnings({"unused"}) + protected java.lang.Object newInstance(UnusedPrivateParameter unused) { + return new Place(); + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private Place( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + case 10: + { + java.lang.String s = input.readStringRequireUtf8(); + + dcid_ = s; + break; + } + case 18: + { + java.lang.String s = input.readStringRequireUtf8(); + + dominantType_ = s; + break; + } + default: + { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e) + .setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return org.datacommons.proto.Recon + .internal_static_org_datacommons_proto_ResolveCoordinatesResponse_Place_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.datacommons.proto.Recon + .internal_static_org_datacommons_proto_ResolveCoordinatesResponse_Place_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.class, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder.class); + } + + public static final int DCID_FIELD_NUMBER = 1; + private volatile java.lang.Object dcid_; + /** + * string dcid = 1; + * + * @return The dcid. + */ + public java.lang.String getDcid() { + java.lang.Object ref = dcid_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + dcid_ = s; + return s; + } + } + /** + * string dcid = 1; + * + * @return The bytes for dcid. + */ + public com.google.protobuf.ByteString getDcidBytes() { + java.lang.Object ref = dcid_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + dcid_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int DOMINANT_TYPE_FIELD_NUMBER = 2; + private volatile java.lang.Object dominantType_; + /** + * string dominant_type = 2; + * + * @return The dominantType. + */ + public java.lang.String getDominantType() { + java.lang.Object ref = dominantType_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + dominantType_ = s; + return s; + } + } + /** + * string dominant_type = 2; + * + * @return The bytes for dominantType. + */ + public com.google.protobuf.ByteString getDominantTypeBytes() { + java.lang.Object ref = dominantType_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + dominantType_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + @java.lang.Override + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) return true; + if (isInitialized == 0) return false; + + memoizedIsInitialized = 1; + return true; + } + + @java.lang.Override + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (!getDcidBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, dcid_); + } + if (!getDominantTypeBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, dominantType_); + } + unknownFields.writeTo(output); + } + + @java.lang.Override + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) return size; + + size = 0; + if (!getDcidBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, dcid_); + } + if (!getDominantTypeBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, dominantType_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place)) { + return super.equals(obj); + } + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place other = + (org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place) obj; + + if (!getDcid().equals(other.getDcid())) return false; + if (!getDominantType().equals(other.getDominantType())) return false; + if (!unknownFields.equals(other.unknownFields)) return false; + return true; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + hash = (37 * hash) + DCID_FIELD_NUMBER; + hash = (53 * hash) + getDcid().hashCode(); + hash = (37 * hash) + DOMINANT_TYPE_FIELD_NUMBER; + hash = (53 * hash) + getDominantType().hashCode(); + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseFrom( + java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseFrom( + java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseFrom( + byte[] data) throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseFrom( + byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseFrom( + java.io.InputStream input) throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseFrom( + java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException( + PARSER, input, extensionRegistry); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseDelimitedFrom( + java.io.InputStream input) throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseDelimitedFrom( + java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException( + PARSER, input, extensionRegistry); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseFrom( + com.google.protobuf.CodedInputStream input) throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException( + PARSER, input, extensionRegistry); + } + + @java.lang.Override + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder( + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + @java.lang.Override + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** Protobuf type {@code org.datacommons.proto.ResolveCoordinatesResponse.Place} */ + public static final class Builder + extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:org.datacommons.proto.ResolveCoordinatesResponse.Place) + org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return org.datacommons.proto.Recon + .internal_static_org_datacommons_proto_ResolveCoordinatesResponse_Place_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.datacommons.proto.Recon + .internal_static_org_datacommons_proto_ResolveCoordinatesResponse_Place_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.class, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder.class); + } + + // Construct using org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) {} + } + + @java.lang.Override + public Builder clear() { + super.clear(); + dcid_ = ""; + + dominantType_ = ""; + + return this; + } + + @java.lang.Override + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return org.datacommons.proto.Recon + .internal_static_org_datacommons_proto_ResolveCoordinatesResponse_Place_descriptor; + } + + @java.lang.Override + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place + getDefaultInstanceForType() { + return org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.getDefaultInstance(); + } + + @java.lang.Override + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place build() { + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + @java.lang.Override + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place buildPartial() { + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place result = + new org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place(this); + result.dcid_ = dcid_; + result.dominantType_ = dominantType_; + onBuilt(); + return result; + } + + @java.lang.Override + public Builder clone() { + return super.clone(); + } + + @java.lang.Override + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return super.setField(field, value); + } + + @java.lang.Override + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return super.clearField(field); + } + + @java.lang.Override + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return super.clearOneof(oneof); + } + + @java.lang.Override + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, + java.lang.Object value) { + return super.setRepeatedField(field, index, value); + } + + @java.lang.Override + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return super.addRepeatedField(field, value); + } + + @java.lang.Override + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place) { + return mergeFrom((org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom( + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place other) { + if (other + == org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.getDefaultInstance()) + return this; + if (!other.getDcid().isEmpty()) { + dcid_ = other.dcid_; + onChanged(); + } + if (!other.getDominantType().isEmpty()) { + dominantType_ = other.dominantType_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + @java.lang.Override + public final boolean isInitialized() { + return true; + } + + @java.lang.Override + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = + (org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place) + e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private java.lang.Object dcid_ = ""; + /** + * string dcid = 1; + * + * @return The dcid. + */ + public java.lang.String getDcid() { + java.lang.Object ref = dcid_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + dcid_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * string dcid = 1; + * + * @return The bytes for dcid. + */ + public com.google.protobuf.ByteString getDcidBytes() { + java.lang.Object ref = dcid_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + dcid_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * string dcid = 1; + * + * @param value The dcid to set. + * @return This builder for chaining. + */ + public Builder setDcid(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + + dcid_ = value; + onChanged(); + return this; + } + /** + * string dcid = 1; + * + * @return This builder for chaining. + */ + public Builder clearDcid() { + + dcid_ = getDefaultInstance().getDcid(); + onChanged(); + return this; + } + /** + * string dcid = 1; + * + * @param value The bytes for dcid to set. + * @return This builder for chaining. + */ + public Builder setDcidBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + dcid_ = value; + onChanged(); + return this; + } + + private java.lang.Object dominantType_ = ""; + /** + * string dominant_type = 2; + * + * @return The dominantType. + */ + public java.lang.String getDominantType() { + java.lang.Object ref = dominantType_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + dominantType_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * string dominant_type = 2; + * + * @return The bytes for dominantType. + */ + public com.google.protobuf.ByteString getDominantTypeBytes() { + java.lang.Object ref = dominantType_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + dominantType_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * string dominant_type = 2; + * + * @param value The dominantType to set. + * @return This builder for chaining. + */ + public Builder setDominantType(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + + dominantType_ = value; + onChanged(); + return this; + } + /** + * string dominant_type = 2; + * + * @return This builder for chaining. + */ + public Builder clearDominantType() { + + dominantType_ = getDefaultInstance().getDominantType(); + onChanged(); + return this; + } + /** + * string dominant_type = 2; + * + * @param value The bytes for dominantType to set. + * @return This builder for chaining. + */ + public Builder setDominantTypeBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + dominantType_ = value; + onChanged(); + return this; + } + + @java.lang.Override + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + @java.lang.Override + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:org.datacommons.proto.ResolveCoordinatesResponse.Place) + } + + // @@protoc_insertion_point(class_scope:org.datacommons.proto.ResolveCoordinatesResponse.Place) + private static final org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place + DEFAULT_INSTANCE; + + static { + DEFAULT_INSTANCE = new org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place(); + } + + public static org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place + getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + @java.lang.Override + public Place parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new Place(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + @java.lang.Override + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place + getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + } + public interface PlaceCoordinateOrBuilder extends // @@protoc_insertion_point(interface_extends:org.datacommons.proto.ResolveCoordinatesResponse.PlaceCoordinate) @@ -11484,6 +12221,30 @@ public interface PlaceCoordinateOrBuilder * @return The bytes of the placeDcids at the given index. */ com.google.protobuf.ByteString getPlaceDcidsBytes(int index); + + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + java.util.List getPlacesList(); + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place getPlaces(int index); + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + int getPlacesCount(); + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + java.util.List< + ? extends org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceOrBuilder> + getPlacesOrBuilderList(); + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceOrBuilder getPlacesOrBuilder( + int index); } /** Protobuf type {@code org.datacommons.proto.ResolveCoordinatesResponse.PlaceCoordinate} */ public static final class PlaceCoordinate extends com.google.protobuf.GeneratedMessageV3 @@ -11498,6 +12259,7 @@ private PlaceCoordinate(com.google.protobuf.GeneratedMessageV3.Builder builde private PlaceCoordinate() { placeDcids_ = com.google.protobuf.LazyStringArrayList.EMPTY; + places_ = java.util.Collections.emptyList(); } @java.lang.Override @@ -11550,6 +12312,20 @@ private PlaceCoordinate( placeDcids_.add(s); break; } + case 34: + { + if (!((mutable_bitField0_ & 0x00000002) != 0)) { + places_ = + new java.util.ArrayList< + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place>(); + mutable_bitField0_ |= 0x00000002; + } + places_.add( + input.readMessage( + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.parser(), + extensionRegistry)); + break; + } default: { if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { @@ -11568,6 +12344,9 @@ private PlaceCoordinate( if (((mutable_bitField0_ & 0x00000001) != 0)) { placeDcids_ = placeDcids_.getUnmodifiableView(); } + if (((mutable_bitField0_ & 0x00000002) != 0)) { + places_ = java.util.Collections.unmodifiableList(places_); + } this.unknownFields = unknownFields.build(); makeExtensionsImmutable(); } @@ -11648,6 +12427,43 @@ public com.google.protobuf.ByteString getPlaceDcidsBytes(int index) { return placeDcids_.getByteString(index); } + public static final int PLACES_FIELD_NUMBER = 4; + private java.util.List places_; + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public java.util.List + getPlacesList() { + return places_; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public java.util.List< + ? extends org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceOrBuilder> + getPlacesOrBuilderList() { + return places_; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public int getPlacesCount() { + return places_.size(); + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place getPlaces(int index) { + return places_.get(index); + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceOrBuilder + getPlacesOrBuilder(int index) { + return places_.get(index); + } + private byte memoizedIsInitialized = -1; @java.lang.Override @@ -11671,6 +12487,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io for (int i = 0; i < placeDcids_.size(); i++) { com.google.protobuf.GeneratedMessageV3.writeString(output, 3, placeDcids_.getRaw(i)); } + for (int i = 0; i < places_.size(); i++) { + output.writeMessage(4, places_.get(i)); + } unknownFields.writeTo(output); } @@ -11694,6 +12513,9 @@ public int getSerializedSize() { size += dataSize; size += 1 * getPlaceDcidsList().size(); } + for (int i = 0; i < places_.size(); i++) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(4, places_.get(i)); + } size += unknownFields.getSerializedSize(); memoizedSize = size; return size; @@ -11716,6 +12538,7 @@ public boolean equals(final java.lang.Object obj) { if (java.lang.Double.doubleToLongBits(getLongitude()) != java.lang.Double.doubleToLongBits(other.getLongitude())) return false; if (!getPlaceDcidsList().equals(other.getPlaceDcidsList())) return false; + if (!getPlacesList().equals(other.getPlacesList())) return false; if (!unknownFields.equals(other.unknownFields)) return false; return true; } @@ -11741,6 +12564,10 @@ public int hashCode() { hash = (37 * hash) + PLACE_DCIDS_FIELD_NUMBER; hash = (53 * hash) + getPlaceDcidsList().hashCode(); } + if (getPlacesCount() > 0) { + hash = (37 * hash) + PLACES_FIELD_NUMBER; + hash = (53 * hash) + getPlacesList().hashCode(); + } hash = (29 * hash) + unknownFields.hashCode(); memoizedHashCode = hash; return hash; @@ -11885,7 +12712,9 @@ private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { } private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) {} + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getPlacesFieldBuilder(); + } } @java.lang.Override @@ -11897,6 +12726,12 @@ public Builder clear() { placeDcids_ = com.google.protobuf.LazyStringArrayList.EMPTY; bitField0_ = (bitField0_ & ~0x00000001); + if (placesBuilder_ == null) { + places_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + } else { + placesBuilder_.clear(); + } return this; } @@ -11936,6 +12771,15 @@ public org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceCoordinate bu bitField0_ = (bitField0_ & ~0x00000001); } result.placeDcids_ = placeDcids_; + if (placesBuilder_ == null) { + if (((bitField0_ & 0x00000002) != 0)) { + places_ = java.util.Collections.unmodifiableList(places_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.places_ = places_; + } else { + result.places_ = placesBuilder_.build(); + } onBuilt(); return result; } @@ -12008,6 +12852,33 @@ public Builder mergeFrom( } onChanged(); } + if (placesBuilder_ == null) { + if (!other.places_.isEmpty()) { + if (places_.isEmpty()) { + places_ = other.places_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensurePlacesIsMutable(); + places_.addAll(other.places_); + } + onChanged(); + } + } else { + if (!other.places_.isEmpty()) { + if (placesBuilder_.isEmpty()) { + placesBuilder_.dispose(); + placesBuilder_ = null; + places_ = other.places_; + bitField0_ = (bitField0_ & ~0x00000002); + placesBuilder_ = + com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders + ? getPlacesFieldBuilder() + : null; + } else { + placesBuilder_.addAllMessages(other.places_); + } + } + } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; @@ -12222,6 +13093,268 @@ public Builder addPlaceDcidsBytes(com.google.protobuf.ByteString value) { return this; } + private java.util.List + places_ = java.util.Collections.emptyList(); + + private void ensurePlacesIsMutable() { + if (!((bitField0_ & 0x00000002) != 0)) { + places_ = + new java.util.ArrayList< + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place>(places_); + bitField0_ |= 0x00000002; + } + } + + private com.google.protobuf.RepeatedFieldBuilderV3< + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceOrBuilder> + placesBuilder_; + + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public java.util.List + getPlacesList() { + if (placesBuilder_ == null) { + return java.util.Collections.unmodifiableList(places_); + } else { + return placesBuilder_.getMessageList(); + } + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public int getPlacesCount() { + if (placesBuilder_ == null) { + return places_.size(); + } else { + return placesBuilder_.getCount(); + } + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place getPlaces(int index) { + if (placesBuilder_ == null) { + return places_.get(index); + } else { + return placesBuilder_.getMessage(index); + } + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public Builder setPlaces( + int index, org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place value) { + if (placesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePlacesIsMutable(); + places_.set(index, value); + onChanged(); + } else { + placesBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public Builder setPlaces( + int index, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder builderForValue) { + if (placesBuilder_ == null) { + ensurePlacesIsMutable(); + places_.set(index, builderForValue.build()); + onChanged(); + } else { + placesBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public Builder addPlaces( + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place value) { + if (placesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePlacesIsMutable(); + places_.add(value); + onChanged(); + } else { + placesBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public Builder addPlaces( + int index, org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place value) { + if (placesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePlacesIsMutable(); + places_.add(index, value); + onChanged(); + } else { + placesBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public Builder addPlaces( + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder builderForValue) { + if (placesBuilder_ == null) { + ensurePlacesIsMutable(); + places_.add(builderForValue.build()); + onChanged(); + } else { + placesBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public Builder addPlaces( + int index, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder builderForValue) { + if (placesBuilder_ == null) { + ensurePlacesIsMutable(); + places_.add(index, builderForValue.build()); + onChanged(); + } else { + placesBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public Builder addAllPlaces( + java.lang.Iterable< + ? extends org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place> + values) { + if (placesBuilder_ == null) { + ensurePlacesIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, places_); + onChanged(); + } else { + placesBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public Builder clearPlaces() { + if (placesBuilder_ == null) { + places_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + } else { + placesBuilder_.clear(); + } + return this; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public Builder removePlaces(int index) { + if (placesBuilder_ == null) { + ensurePlacesIsMutable(); + places_.remove(index); + onChanged(); + } else { + placesBuilder_.remove(index); + } + return this; + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder + getPlacesBuilder(int index) { + return getPlacesFieldBuilder().getBuilder(index); + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceOrBuilder + getPlacesOrBuilder(int index) { + if (placesBuilder_ == null) { + return places_.get(index); + } else { + return placesBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public java.util.List< + ? extends org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceOrBuilder> + getPlacesOrBuilderList() { + if (placesBuilder_ != null) { + return placesBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(places_); + } + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder + addPlacesBuilder() { + return getPlacesFieldBuilder() + .addBuilder( + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place + .getDefaultInstance()); + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder + addPlacesBuilder(int index) { + return getPlacesFieldBuilder() + .addBuilder( + index, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place + .getDefaultInstance()); + } + /** + * repeated .org.datacommons.proto.ResolveCoordinatesResponse.Place places = 4; + */ + public java.util.List + getPlacesBuilderList() { + return getPlacesFieldBuilder().getBuilderList(); + } + + private com.google.protobuf.RepeatedFieldBuilderV3< + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceOrBuilder> + getPlacesFieldBuilder() { + if (placesBuilder_ == null) { + placesBuilder_ = + new com.google.protobuf.RepeatedFieldBuilderV3< + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place.Builder, + org.datacommons.proto.Recon.ResolveCoordinatesResponse.PlaceOrBuilder>( + places_, ((bitField0_ & 0x00000002) != 0), getParentForChildren(), isClean()); + places_ = null; + } + return placesBuilder_; + } + @java.lang.Override public final Builder setUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { @@ -13102,6 +14235,10 @@ public org.datacommons.proto.Recon.ResolveCoordinatesResponse getDefaultInstance internal_static_org_datacommons_proto_ResolveCoordinatesResponse_descriptor; private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_org_datacommons_proto_ResolveCoordinatesResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_org_datacommons_proto_ResolveCoordinatesResponse_Place_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_org_datacommons_proto_ResolveCoordinatesResponse_Place_fieldAccessorTable; private static final com.google.protobuf.Descriptors.Descriptor internal_static_org_datacommons_proto_ResolveCoordinatesResponse_PlaceCoordinate_descriptor; private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable @@ -13147,12 +14284,15 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + "dinatesRequest\022P\n\013coordinates\030\001 \003(\0132;.or" + "g.datacommons.proto.ResolveCoordinatesRe" + "quest.Coordinate\0321\n\nCoordinate\022\020\n\010latitu" - + "de\030\001 \001(\001\022\021\n\tlongitude\030\002 \001(\001\"\307\001\n\032ResolveC" + + "de\030\001 \001(\001\022\021\n\tlongitude\030\002 \001(\001\"\277\002\n\032ResolveC" + "oordinatesResponse\022\\\n\021place_coordinates\030" + "\001 \003(\0132A.org.datacommons.proto.ResolveCoo" - + "rdinatesResponse.PlaceCoordinate\032K\n\017Plac" - + "eCoordinate\022\020\n\010latitude\030\001 \001(\001\022\021\n\tlongitu" - + "de\030\002 \001(\001\022\023\n\013place_dcids\030\003 \003(\tb\006proto3" + + "rdinatesResponse.PlaceCoordinate\032,\n\005Plac" + + "e\022\014\n\004dcid\030\001 \001(\t\022\025\n\rdominant_type\030\002 \001(\t\032\224" + + "\001\n\017PlaceCoordinate\022\020\n\010latitude\030\001 \001(\001\022\021\n\t" + + "longitude\030\002 \001(\001\022\023\n\013place_dcids\030\003 \003(\t\022G\n\006" + + "places\030\004 \003(\01327.org.datacommons.proto.Res" + + "olveCoordinatesResponse.Placeb\006proto3" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom( @@ -13280,15 +14420,25 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { new java.lang.String[] { "PlaceCoordinates", }); - internal_static_org_datacommons_proto_ResolveCoordinatesResponse_PlaceCoordinate_descriptor = + internal_static_org_datacommons_proto_ResolveCoordinatesResponse_Place_descriptor = internal_static_org_datacommons_proto_ResolveCoordinatesResponse_descriptor .getNestedTypes() .get(0); + internal_static_org_datacommons_proto_ResolveCoordinatesResponse_Place_fieldAccessorTable = + new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_org_datacommons_proto_ResolveCoordinatesResponse_Place_descriptor, + new java.lang.String[] { + "Dcid", "DominantType", + }); + internal_static_org_datacommons_proto_ResolveCoordinatesResponse_PlaceCoordinate_descriptor = + internal_static_org_datacommons_proto_ResolveCoordinatesResponse_descriptor + .getNestedTypes() + .get(1); internal_static_org_datacommons_proto_ResolveCoordinatesResponse_PlaceCoordinate_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_org_datacommons_proto_ResolveCoordinatesResponse_PlaceCoordinate_descriptor, new java.lang.String[] { - "Latitude", "Longitude", "PlaceDcids", + "Latitude", "Longitude", "PlaceDcids", "Places", }); org.datacommons.proto.Mcf.getDescriptor(); } diff --git a/util/src/main/java/org/datacommons/util/CoordinatesResolver.java b/util/src/main/java/org/datacommons/util/CoordinatesResolver.java new file mode 100644 index 00000000..3c99e3b3 --- /dev/null +++ b/util/src/main/java/org/datacommons/util/CoordinatesResolver.java @@ -0,0 +1,102 @@ +package org.datacommons.util; + +import static java.util.stream.Collectors.toList; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import org.datacommons.proto.Mcf.McfGraph.PropertyValues; +import org.datacommons.proto.Mcf.McfGraph.Values; +import org.datacommons.proto.Mcf.ValueType; +import org.datacommons.proto.Recon.ResolveCoordinatesRequest; +import org.datacommons.proto.Recon.ResolveCoordinatesRequest.Coordinate; +import org.datacommons.proto.Recon.ResolveCoordinatesResponse; +import org.datacommons.proto.Recon.ResolveCoordinatesResponse.Place; + +/** Resolves nodes with lat-lngs by calling the DC coordinates resolution API. */ +// TODO: Add counters for errors. +final class CoordinatesResolver { + private final Set resolveCoordinates = ConcurrentHashMap.newKeySet(); + + private final ConcurrentHashMap> resolvedCoordinates = + new ConcurrentHashMap<>(); + + private final ReconClient client; + + CoordinatesResolver(ReconClient client) { + this.client = client; + } + + boolean submit(PropertyValues node) { + Optional optionalCoordinate = getCoordinate(node); + if (optionalCoordinate.isPresent()) { + resolveCoordinates.add(optionalCoordinate.get()); + return true; + } + return false; + } + + void drain() { + if (!resolveCoordinates.isEmpty()) { + populateResolvedCandidates( + client.resolveCoordinates( + ResolveCoordinatesRequest.newBuilder() + .addAllCoordinates(resolveCoordinates) + .build())); + } + } + + Optional resolve(PropertyValues node) { + return getCoordinate(node) + .filter(resolvedCoordinates::containsKey) + .flatMap(coordinate -> resolvedCoordinates.get(coordinate).stream().findFirst()); + } + + private void populateResolvedCandidates(ResolveCoordinatesResponse response) { + response + .getPlaceCoordinatesList() + .forEach( + placeCoordinate -> { + if (placeCoordinate.getPlacesCount() > 0) { + resolvedCoordinates.put( + Coordinate.newBuilder() + .setLatitude(placeCoordinate.getLatitude()) + .setLongitude(placeCoordinate.getLongitude()) + .build(), + new LinkedHashSet<>( + placeCoordinate.getPlacesList().stream() + .map(Place::getDcid) + .collect(toList()))); + } + }); + } + + private static Optional getCoordinate(PropertyValues node) { + if (node.containsPvs(Vocabulary.LATITUDE) && node.containsPvs(Vocabulary.LONGITUDE)) { + + Optional optLat = getDoubleValue(node.getPvsMap().get(Vocabulary.LATITUDE)); + Optional optLng = getDoubleValue(node.getPvsMap().get(Vocabulary.LONGITUDE)); + + if (optLat.isPresent() && optLng.isPresent()) { + double lat = optLat.get(); + double lng = optLng.get(); + + if (!Double.isNaN(lat) && !Double.isNaN(lng)) { + return Optional.of(Coordinate.newBuilder().setLatitude(lat).setLongitude(lng).build()); + } + } + } + + return Optional.empty(); + } + + // TODO: Add support for other formats of values (e.g. 12.34N, 45.56W, etc.) + private static Optional getDoubleValue(Values prop) { + return prop.getTypedValuesList().stream() + .filter( + typedValue -> + ValueType.NUMBER.equals(typedValue.getType()) + || ValueType.TEXT.equals(typedValue.getType())) + .findFirst() + .map(typedValue -> Double.parseDouble(typedValue.getValue())); + } +} diff --git a/util/src/main/java/org/datacommons/util/ExternalIdResolver.java b/util/src/main/java/org/datacommons/util/ExternalIdResolver.java index 6bf68516..925f6f62 100644 --- a/util/src/main/java/org/datacommons/util/ExternalIdResolver.java +++ b/util/src/main/java/org/datacommons/util/ExternalIdResolver.java @@ -1,5 +1,7 @@ package org.datacommons.util; +import static org.apache.commons.lang3.StringUtils.isEmpty; + import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import java.io.IOException; @@ -40,6 +42,7 @@ public class ExternalIdResolver { private final boolean verbose; private final LogWrapper logCtx; private final HttpClient httpClient; + private final CoordinatesResolver coordinatesResolver; // IDs waiting to be mapped. // Key: ID property, Value: set of external IDs @@ -54,10 +57,17 @@ public class ExternalIdResolver { // phase), so we use RW locks. private final ReadWriteLock rwlock = new ReentrantReadWriteLock(); - public ExternalIdResolver(HttpClient httpClient, boolean verbose, LogWrapper logCtx) { + public ExternalIdResolver( + HttpClient httpClient, boolean doCoordinatesResolution, boolean verbose, LogWrapper logCtx) { this.httpClient = httpClient; this.verbose = verbose; this.logCtx = logCtx; + if (doCoordinatesResolution) { + this.coordinatesResolver = + new CoordinatesResolver(new ReconClient(httpClient, logCtx, MAX_RESOLUTION_BATCH_IDS)); + } else { + this.coordinatesResolver = null; + } } public void submitNode(Mcf.McfGraph.PropertyValues node) @@ -89,6 +99,10 @@ public void submitNode(Mcf.McfGraph.PropertyValues node) } } + if (coordinatesResolver != null) { + coordinatesResolver.submit(node); + } + if (numBatchedIds >= MAX_RESOLUTION_BATCH_IDS) { if (verbose) { logger.info("Processing batched external-IDs due to MAX_RESOLUTION_BATCH_IDS threshold"); @@ -107,6 +121,11 @@ public void drainRemoteCalls() throws IOException, InterruptedException { throw new UnexpectedException("drainRemoteCalls() can only be called once!"); } drainRemoteCallsInternal(); + + if (coordinatesResolver != null) { + coordinatesResolver.drain(); + } + drained = true; } finally { rwlock.writeLock().unlock(); @@ -184,7 +203,10 @@ public String resolveNode(String nodeId, Mcf.McfGraph.PropertyValues node) foundExternalId = id; } } - return foundDcid; + if (!isEmpty(foundDcid)) { + return foundDcid; + } + return coordinatesResolver != null ? coordinatesResolver.resolve(node).orElse("") : foundDcid; } finally { rwlock.readLock().unlock(); } diff --git a/util/src/main/java/org/datacommons/util/ReconClient.java b/util/src/main/java/org/datacommons/util/ReconClient.java index 7c7d3d72..9040b932 100644 --- a/util/src/main/java/org/datacommons/util/ReconClient.java +++ b/util/src/main/java/org/datacommons/util/ReconClient.java @@ -1,51 +1,133 @@ package org.datacommons.util; +import static com.google.common.collect.Lists.partition; import static java.net.http.HttpClient.Version.HTTP_1_1; +import static java.util.stream.Collectors.toList; +import static org.datacommons.util.StringUtil.msgToJson; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import com.google.protobuf.util.JsonFormat; -import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse.BodyHandlers; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; import org.datacommons.proto.Recon.ResolveCoordinatesRequest; import org.datacommons.proto.Recon.ResolveCoordinatesResponse; /** * Client to the DC resolution APIs. * - *

Currently it only resolves coordinates. + *

Currently it only resolves coordinates. If the number of coordinates to resolve are greater + * than {@code chunkSize}, the API calls will be partitioned into max {@code chunkSize}d batches. */ public class ReconClient { private static final String RESOLVE_COORDINATES_API_URL = "https://api.datacommons.org/v1/recon/resolve/coordinate"; + static final String NUM_API_CALLS_COUNTER = "ReconClient_NumApiCalls"; + + private static final int DEFAULT_CHUNK_SIZE = 500; + + private final int chunkSize; + private final HttpClient httpClient; - public ReconClient(HttpClient httpClient) { + private final LogWrapper logWrapper; + + public ReconClient(HttpClient httpClient, LogWrapper logWrapper) { + this(httpClient, logWrapper, DEFAULT_CHUNK_SIZE); + } + + public ReconClient(HttpClient httpClient, LogWrapper logWrapper, int chunkSize) { this.httpClient = httpClient; + this.logWrapper = logWrapper; + this.chunkSize = chunkSize; } - public ResolveCoordinatesResponse resolveCoordinates(ResolveCoordinatesRequest request) - throws IOException, InterruptedException { - return callApi( - RESOLVE_COORDINATES_API_URL, request, ResolveCoordinatesResponse.getDefaultInstance()); + public ResolveCoordinatesResponse resolveCoordinates(ResolveCoordinatesRequest request) { + try { + return resolveCoordinatesAsync(request).get(); + } catch (Exception e) { + throw new RuntimeException("Error resolving candidates", e); + } } - private T callApi( - String apiUrl, Message requestMessage, T responseDefaultInstance) - throws IOException, InterruptedException { - var request = + public CompletableFuture resolveCoordinatesAsync( + ResolveCoordinatesRequest request) { + ResolveCoordinatesResponse defaultResponse = ResolveCoordinatesResponse.getDefaultInstance(); + if (request.getCoordinatesCount() < 1) { + return CompletableFuture.completedFuture(defaultResponse); + } + + return toFutureOfList( + // Partition request into chunkSize batches. + // e.g. if chunkSize = 3 then Request(C1, C2, C3) will be chunked into [Request(C1, C2), + // Request(C3)] + partition(request.getCoordinatesList(), chunkSize).stream() + .map( + chunk -> + request.toBuilder().clearCoordinates().addAllCoordinates(chunk).build()) + .map( + // Call API for each chunked request. + chunkedRequest -> + callApi(RESOLVE_COORDINATES_API_URL, chunkedRequest, defaultResponse)) + .collect(toList())) + .thenApply( + // Aggregate chunked responses. + // e.g. [Response(P1, P2), Response(P3)] will be aggregated into Response(P1, P2, P3) + chunkedResponses -> + ResolveCoordinatesResponse.newBuilder() + .addAllPlaceCoordinates( + chunkedResponses.stream() + .flatMap( + chunkedResponse -> + chunkedResponse.getPlaceCoordinatesList().stream()) + .collect(toList())) + .build()); + } + + private CompletableFuture callApi( + String apiUrl, Message requestMessage, T responseDefaultInstance) { + logWrapper.incrementInfoCounterBy(NUM_API_CALLS_COUNTER, 1); + HttpRequest request = HttpRequest.newBuilder(URI.create(apiUrl)) .version(HTTP_1_1) .header("accept", "application/json") - .POST(BodyPublishers.ofString(StringUtil.msgToJson(requestMessage))) + .POST(BodyPublishers.ofString(toJson(requestMessage))) .build(); - var response = httpClient.send(request, BodyHandlers.ofString()); - var responseMessageBuilder = responseDefaultInstance.newBuilderForType(); - JsonFormat.parser().merge(response.body().trim(), responseMessageBuilder); - return (T) responseMessageBuilder.build(); + return httpClient + .sendAsync(request, BodyHandlers.ofString()) + .thenApply( + response -> { + Message.Builder responseMessageBuilder = responseDefaultInstance.newBuilderForType(); + fromJson(response.body().trim(), responseMessageBuilder); + return (T) responseMessageBuilder.build(); + }); + } + + private static CompletableFuture> toFutureOfList(List> futures) { + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply(v -> futures.stream().map(CompletableFuture::join).collect(Collectors.toList())); + } + + private static String toJson(Message message) { + try { + return msgToJson(message); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } + + private static void fromJson(String json, Message.Builder builder) { + try { + JsonFormat.parser().merge(json, builder); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } } } diff --git a/util/src/main/proto/Debug.proto b/util/src/main/proto/Debug.proto index b6073b33..a5c7db95 100644 --- a/util/src/main/proto/Debug.proto +++ b/util/src/main/proto/Debug.proto @@ -81,7 +81,8 @@ message CommandArgs { optional bool observation_about = 6; optional bool allow_nan_svobs = 7; // allow not-a-number svobs values // check the existence of SVObs value references if the SV they are measuring has statType: measurementResult - optional bool check_measurement_result = 8; + optional bool check_measurement_result = 8; + optional bool coordinates_resolution = 9; } message DataPoint { diff --git a/util/src/main/proto/recon.proto b/util/src/main/proto/recon.proto index 613cbe56..1562839e 100644 --- a/util/src/main/proto/recon.proto +++ b/util/src/main/proto/recon.proto @@ -86,10 +86,15 @@ message ResolveCoordinatesRequest { } message ResolveCoordinatesResponse { + message Place { + string dcid = 1; + string dominant_type = 2; + } message PlaceCoordinate { double latitude = 1; double longitude = 2; repeated string place_dcids = 3; + repeated Place places = 4; } repeated PlaceCoordinate place_coordinates = 1; } diff --git a/util/src/test/java/org/datacommons/util/CoordinatesResolverTest.java b/util/src/test/java/org/datacommons/util/CoordinatesResolverTest.java new file mode 100644 index 00000000..5cb0a6c2 --- /dev/null +++ b/util/src/test/java/org/datacommons/util/CoordinatesResolverTest.java @@ -0,0 +1,66 @@ +package org.datacommons.util; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static java.net.http.HttpClient.newHttpClient; +import static org.datacommons.util.TestUtil.newLogCtx; +import static org.datacommons.util.Vocabulary.LATITUDE; +import static org.datacommons.util.Vocabulary.LONGITUDE; + +import java.util.List; +import java.util.Map; +import org.datacommons.proto.Mcf.McfGraph.PropertyValues; +import org.datacommons.proto.Mcf.ValueType; +import org.junit.Test; + +public class CoordinatesResolverTest { + private static final PropertyValues SF = + newNode("City", Map.of(LATITUDE, "37.77493", LONGITUDE, "-122.41942")); + private static final String SF_ZIP_DCID = "zip/94103"; + + private static final PropertyValues BIG_BEN = + newNode("Place", Map.of(LATITUDE, "51.510357", LONGITUDE, "-0.116773")); + private static final String BIG_BEN_NUTS_DCID = "nuts/UKI32"; + + private static final PropertyValues NON_LAT_LNG_NODE = newNode("Place", Map.of("isoCode", "IN")); + + private static final List TEST_NODES = List.of(SF, BIG_BEN, NON_LAT_LNG_NODE); + + private static final PropertyValues UNSUBMITTED_NODE = + newNode("City", Map.of(LATITUDE, "12.34", LONGITUDE, "56.78")); + + @Test + public void endToEnd() { + CoordinatesResolver resolver = + new CoordinatesResolver(new ReconClient(newHttpClient(), newLogCtx())); + + for (PropertyValues node : TEST_NODES) { + resolver.submit(node); + } + + resolver.drain(); + + assertThat(resolver.resolve(SF)).hasValue(SF_ZIP_DCID); + assertThat(resolver.resolve(BIG_BEN)).hasValue(BIG_BEN_NUTS_DCID); + assertThat(resolver.resolve(UNSUBMITTED_NODE)).isEmpty(); + } + + @Test + public void submitNode() { + CoordinatesResolver resolver = + new CoordinatesResolver(new ReconClient(newHttpClient(), newLogCtx())); + + assertThat(resolver.submit(SF)).isTrue(); + assertThat(resolver.submit(BIG_BEN)).isTrue(); + assertThat(resolver.submit(NON_LAT_LNG_NODE)).isFalse(); + } + + private static PropertyValues newNode(String typeOf, Map props) { + PropertyValues.Builder node = PropertyValues.newBuilder(); + node.putPvs(Vocabulary.TYPE_OF, McfUtil.newValues(ValueType.RESOLVED_REF, typeOf)); + for (var pv : props.entrySet()) { + node.putPvs(pv.getKey(), McfUtil.newValues(ValueType.TEXT, pv.getValue())); + } + return node.build(); + } +} diff --git a/util/src/test/java/org/datacommons/util/ExternalIdResolverTest.java b/util/src/test/java/org/datacommons/util/ExternalIdResolverTest.java index 07c9ce6d..63f87acc 100644 --- a/util/src/test/java/org/datacommons/util/ExternalIdResolverTest.java +++ b/util/src/test/java/org/datacommons/util/ExternalIdResolverTest.java @@ -1,8 +1,12 @@ package org.datacommons.util; +import static com.google.common.truth.Truth.assertThat; +import static org.datacommons.util.Vocabulary.LATITUDE; +import static org.datacommons.util.Vocabulary.LONGITUDE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import com.google.common.collect.ImmutableList; import java.io.IOException; import java.net.http.HttpClient; import java.nio.file.Path; @@ -35,13 +39,23 @@ public class ExternalIdResolverTest { List testPlaceNodes = List.of(in, ca, sf, vz, unk, tn); + McfGraph.PropertyValues bigBenWithLatLng = + buildNode("Place", Map.of(LATITUDE, "51.510357", LONGITUDE, "-0.116773")); + String bigBenDcid = "nuts/UKI32"; + + List testPlaceNodesPlusLatLngNodes = + ImmutableList.builder() + .addAll(testPlaceNodes) + .add(bigBenWithLatLng) + .build(); + @Test public void endToEndWithApiCalls() throws IOException, InterruptedException { Debug.Log.Builder lb = Debug.Log.newBuilder(); LogWrapper lw = new LogWrapper(lb, Path.of("InMemory")); ExternalIdResolver.MAX_RESOLUTION_BATCH_IDS = 4; - var resolver = new ExternalIdResolver(HttpClient.newHttpClient(), true, lw); + var resolver = new ExternalIdResolver(HttpClient.newHttpClient(), false, true, lw); for (var node : testPlaceNodes) { resolver.submitNode(node); } @@ -63,7 +77,7 @@ public void endToEndWithLocalSideMcf() throws IOException, InterruptedException LogWrapper lw = new LogWrapper(lb, Path.of("InMemory")); ExternalIdResolver.MAX_RESOLUTION_BATCH_IDS = 1; // This allows us to count the number of DC calls exactly - var resolver = new ExternalIdResolver(HttpClient.newHttpClient(), true, lw); + var resolver = new ExternalIdResolver(HttpClient.newHttpClient(), false, true, lw); // Construct input side MCF where we also provide the DCIDs of the nodes var inWithDcid = addDcidToNode(in, inDcid); @@ -100,6 +114,67 @@ public void endToEndWithLocalSideMcf() throws IOException, InterruptedException assertTrue(TestUtil.checkCounter(lw.getLog(), "Resolution_NumDcCalls", 1)); } + @Test + public void endToEndWithApiCalls_withLatLngNodes_withCoordinatesResolutionDisabled() + throws IOException, InterruptedException { + Debug.Log.Builder lb = Debug.Log.newBuilder(); + LogWrapper lw = new LogWrapper(lb, Path.of("InMemory")); + ExternalIdResolver.MAX_RESOLUTION_BATCH_IDS = 4; + + var resolver = new ExternalIdResolver(HttpClient.newHttpClient(), false, true, lw); + for (var node : testPlaceNodesPlusLatLngNodes) { + resolver.submitNode(node); + } + // Issue 20 more SF calls, which should all be batched. + for (int i = 0; i < 20; i++) { + resolver.submitNode(sf); + } + resolver.drainRemoteCalls(); + + testAssertionSuiteOnResolverInstance(resolver, lw); + + // There are 7 IDs, and batch-size if 4, so we must have done 2 calls. + assertTrue(TestUtil.checkCounter(lw.getLog(), "Resolution_NumDcCalls", 2)); + + // Coordinates assertions. + + // There is 1 lat-lng node but coordinates resolution is disabled so there should be no counters + // (represented as -1). + assertThat(TestUtil.getCounter(lw.getLog(), ReconClient.NUM_API_CALLS_COUNTER)).isEqualTo(-1); + // Since coordinates resolution is disabled, this won't resolve. + assertEquals("", resolver.resolveNode("bigben", bigBenWithLatLng)); + } + + @Test + public void endToEndWithApiCalls_withLatLngNodes_withCoordinatesResolutionEnabled() + throws IOException, InterruptedException { + Debug.Log.Builder lb = Debug.Log.newBuilder(); + LogWrapper lw = new LogWrapper(lb, Path.of("InMemory")); + ExternalIdResolver.MAX_RESOLUTION_BATCH_IDS = 4; + + var resolver = new ExternalIdResolver(HttpClient.newHttpClient(), true, true, lw); + for (var node : testPlaceNodesPlusLatLngNodes) { + resolver.submitNode(node); + } + // Issue 20 more SF calls, which should all be batched. + for (int i = 0; i < 20; i++) { + resolver.submitNode(sf); + } + resolver.drainRemoteCalls(); + + testAssertionSuiteOnResolverInstance(resolver, lw); + + // There are 7 IDs, and batch-size if 4, so we must have done 2 calls. + assertTrue(TestUtil.checkCounter(lw.getLog(), "Resolution_NumDcCalls", 2)); + + // Coordinates assertions. + + // There is 1 lat-lng node so there should be 1 DC call. + assertThat(TestUtil.getCounter(lw.getLog(), ReconClient.NUM_API_CALLS_COUNTER)).isEqualTo(1); + // big ben with lat lng should be resolved. + assertEquals(bigBenDcid, resolver.resolveNode("bigben", bigBenWithLatLng)); + } + // Runs assertions on the place constants as defined in the class constants. // These assertions are factored out of individual tests to allow testing different // input methods (API, addLocalGraph) have the same local behavior with the same input diff --git a/util/src/test/java/org/datacommons/util/ReconClientTest.java b/util/src/test/java/org/datacommons/util/ReconClientTest.java index 879c03aa..357367f8 100644 --- a/util/src/test/java/org/datacommons/util/ReconClientTest.java +++ b/util/src/test/java/org/datacommons/util/ReconClientTest.java @@ -1,29 +1,76 @@ package org.datacommons.util; import static com.google.common.truth.Truth.assertThat; +import static java.util.stream.Collectors.toList; +import static org.datacommons.util.ReconClient.NUM_API_CALLS_COUNTER; +import static org.datacommons.util.TestUtil.getCounter; +import static org.datacommons.util.TestUtil.newLogCtx; import java.net.http.HttpClient; -import org.datacommons.proto.Recon; +import org.datacommons.proto.Recon.ResolveCoordinatesRequest; +import org.datacommons.proto.Recon.ResolveCoordinatesRequest.Coordinate; +import org.datacommons.proto.Recon.ResolveCoordinatesResponse; import org.junit.Test; public class ReconClientTest { + private static final String USA_DCID = "country/USA"; + private static final String GBR_DCID = "country/GBR"; + private static final Coordinate SF_COORDINATES = + Coordinate.newBuilder().setLatitude(37.77493).setLongitude(-122.41942).build(); + private static final Coordinate BIG_BEN_COORDINATES = + Coordinate.newBuilder().setLatitude(51.510357).setLongitude(-0.116773).build(); + @Test - public void resolveCoordinates_endToEndApiCall() throws Exception { - var client = new ReconClient(HttpClient.newHttpClient()); - - // San Francisco coordinates - var request = - Recon.ResolveCoordinatesRequest.newBuilder() - .addCoordinates( - Recon.ResolveCoordinatesRequest.Coordinate.newBuilder() - .setLatitude(37.77493) - .setLongitude(-122.41942) - .build()) + public void resolveCoordinates() { + LogWrapper logWrapper = newLogCtx(); + ReconClient client = new ReconClient(HttpClient.newHttpClient(), logWrapper); + + ResolveCoordinatesRequest request = + ResolveCoordinatesRequest.newBuilder() + .addCoordinates(SF_COORDINATES) + .addCoordinates(BIG_BEN_COORDINATES) + .build(); + + ResolveCoordinatesResponse result = client.resolveCoordinates(request); + + assertThat(result.getPlaceCoordinatesCount()).isEqualTo(2); + assertThat( + result.getPlaceCoordinates(0).getPlacesList().stream() + .map(ResolveCoordinatesResponse.Place::getDcid) + .collect(toList())) + .contains(USA_DCID); + assertThat( + result.getPlaceCoordinates(1).getPlacesList().stream() + .map(ResolveCoordinatesResponse.Place::getDcid) + .collect(toList())) + .contains(GBR_DCID); + assertThat(getCounter(logWrapper.getLog(), NUM_API_CALLS_COUNTER)).isEqualTo(1); + } + + @Test + public void resolveCoordinates_chunked() { + LogWrapper logWrapper = newLogCtx(); + ReconClient client = new ReconClient(HttpClient.newHttpClient(), logWrapper, 1); + + ResolveCoordinatesRequest request = + ResolveCoordinatesRequest.newBuilder() + .addCoordinates(SF_COORDINATES) + .addCoordinates(BIG_BEN_COORDINATES) .build(); - var result = client.resolveCoordinates(request); + ResolveCoordinatesResponse result = client.resolveCoordinates(request); - assertThat(result.getPlaceCoordinatesCount()).isEqualTo(1); - assertThat(result.getPlaceCoordinates(0).getPlaceDcidsList()).contains("country/USA"); + assertThat(result.getPlaceCoordinatesCount()).isEqualTo(2); + assertThat( + result.getPlaceCoordinates(0).getPlacesList().stream() + .map(ResolveCoordinatesResponse.Place::getDcid) + .collect(toList())) + .contains(USA_DCID); + assertThat( + result.getPlaceCoordinates(1).getPlacesList().stream() + .map(ResolveCoordinatesResponse.Place::getDcid) + .collect(toList())) + .contains(GBR_DCID); + assertThat(getCounter(logWrapper.getLog(), NUM_API_CALLS_COUNTER)).isEqualTo(2); } }