Add support for coordinates resolution. (#209)

datacommonsorg · Aug 11, 2023 · cbfe2c1 · cbfe2c1
1 parent ec43cb2
commit cbfe2c1
Show file tree

Hide file tree

Showing 34 changed files with 2,615 additions and 83 deletions.
diff --git a/tool/src/main/java/org/datacommons/tool/Args.java b/tool/src/main/java/org/datacommons/tool/Args.java
@@ -11,6 +11,7 @@
 class Args {
   public boolean doExistenceChecks = false;
   public ResolutionMode resolutionMode = ResolutionMode.NONE;
+  public boolean doCoordinatesResolution = false;
   public boolean doStatChecks = false;
   public List<String> samplePlaces = null;
   public boolean verbose = false;
@@ -32,6 +33,7 @@ public String toString() {
     }
     argStr.append(" existence-checks=" + doExistenceChecks);
     argStr.append(", resolution=" + resolutionMode.name());
+    argStr.append(", coordinates-resolution=" + doCoordinatesResolution);
     argStr.append(", num-threads=" + numThreads);
     argStr.append(", stat-checks=" + doStatChecks);
     if (samplePlaces != null) {
@@ -55,6 +57,7 @@ public Debug.CommandArgs toProto() {
     } else if (resolutionMode == ResolutionMode.FULL) {
       argsBuilder.setResolution(Debug.CommandArgs.ResolutionMode.RESOLUTION_MODE_FULL);
     }
+    argsBuilder.setCoordinatesResolution(doCoordinatesResolution);
     argsBuilder.setStatChecks(doStatChecks);
     if (samplePlaces != null) argsBuilder.addAllSamplePlaces(samplePlaces);
     argsBuilder.setObservationAbout(checkObservationAbout);

diff --git a/tool/src/main/java/org/datacommons/tool/GenMcf.java b/tool/src/main/java/org/datacommons/tool/GenMcf.java
@@ -58,6 +58,7 @@ public Integer call() throws IOException, TemplateException {
     Args args = new Args();
     args.doExistenceChecks = parent.doExistenceChecks;
     args.resolutionMode = parent.resolutionMode;
+    args.doCoordinatesResolution = parent.doCoordinatesResolution;
     args.doStatChecks = parent.doStatChecks;
     args.samplePlaces = parent.samplePlaces;
     args.numThreads = parent.numThreads;

diff --git a/tool/src/main/java/org/datacommons/tool/Lint.java b/tool/src/main/java/org/datacommons/tool/Lint.java
@@ -56,6 +56,7 @@ public Integer call() throws IOException, TemplateException {
     Args args = new Args();
     args.doExistenceChecks = parent.doExistenceChecks;
     args.resolutionMode = parent.resolutionMode;
+    args.doCoordinatesResolution = parent.doCoordinatesResolution;
     args.doStatChecks = parent.doStatChecks;
     args.samplePlaces = parent.samplePlaces;
     args.numThreads = parent.numThreads;

diff --git a/tool/src/main/java/org/datacommons/tool/Main.java b/tool/src/main/java/org/datacommons/tool/Main.java
@@ -67,6 +67,14 @@ class Main {
               + "You rarely want to use this mode.")
   public Args.ResolutionMode resolutionMode = Args.ResolutionMode.NONE;
 
+  @CommandLine.Option(
+      names = {"-cr", "--coordinates-resolution"},
+      defaultValue = "false",
+      scope = CommandLine.ScopeType.INHERIT,
+      description =
+          "If set, resolves lat-lng coordinates by making DC Recon API calls. This flag is only application in full resolution mode.")
+  public boolean doCoordinatesResolution;
+
   @CommandLine.Option(
       names = {"-s", "--stat-checks"},
       defaultValue = "true",

diff --git a/tool/src/main/java/org/datacommons/tool/Processor.java b/tool/src/main/java/org/datacommons/tool/Processor.java
@@ -131,7 +131,9 @@ private Processor(Args args) {
       existenceChecker = new ExistenceChecker(this.httpClient, args.verbose, logCtx);
     }
     if (args.resolutionMode == Args.ResolutionMode.FULL) {
-      idResolver = new ExternalIdResolver(this.httpClient, args.verbose, logCtx);
+      idResolver =
+          new ExternalIdResolver(
+              this.httpClient, args.doCoordinatesResolution, args.verbose, logCtx);
     }
     statVarState = new StatVarState(this.httpClient, logCtx);
     if (args.doStatChecks) {

diff --git a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java
@@ -54,13 +54,22 @@ public class GenMcfTest {
   // count here.
   private static Map<String, Integer> EXPECTED_FILES_TO_CHECK =
       Map.of(
-          "fataltmcf", 2,
-          "resolution", 5,
-          "statchecks", 3,
-          "successtmcf", 3,
-          "measurementresult", 4,
-          "localidresolution", 5,
-          "manyinconsistent", 4);
+          "fataltmcf",
+          2,
+          "resolution",
+          5,
+          "latlngresolution",
+          3,
+          "statchecks",
+          3,
+          "successtmcf",
+          3,
+          "measurementresult",
+          4,
+          "localidresolution",
+          5,
+          "manyinconsistent",
+          4);
 
   // Skip testing the following files. If this List is non-empty, the flaky files should be fixed
   // and removed from this list.

diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json
@@ -196,6 +196,7 @@
     "numThreads": 1,
     "statChecks": true,
     "observationAbout": false,
-    "allowNanSvobs": false
+    "allowNanSvobs": false,
+    "coordinatesResolution": false
   }
 }
diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.csv b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.csv
@@ -0,0 +1,6 @@
+Year,Place_Name,Lat,Lng,SV1
+2019,British–Columbia,53,-127,1
+2019,Saskatchewan,43,-116,5
+2019,San Francisco,37.77493,-122.41942,1
+2019,California,36,-119.4,1
+2019,Alberta,53.9,-116,6
diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.tmcf b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/LatLng.tmcf
@@ -0,0 +1,13 @@
+Node: E:SVTest->E0
+typeOf: dcs:StatVarObservation
+variableMeasured: dcs:SV1
+measurementMethod: dcs:SVTest
+observationAbout: E:SVTest->E3
+observationDate: C:SVTest->Year
+value: C:SVTest->SV1
+
+Node: E:SVTest->E3
+typeOf: dcid:Place
+name: C:SVTest->Place_Name
+latitude: C:SVTest->Lat
+longitude: C:SVTest->Lng
diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/args.txt b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/input/args.txt
@@ -0,0 +1 @@
+--coordinates-resolution=true
diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/latlngresolution/output/report.json
@@ -0,0 +1,110 @@
+{
+  "levelSummary": {
+    "LEVEL_INFO": {
+      "counters": {
+        "NumRowSuccesses": "5",
+        "NumPVSuccesses": "65",
+        "Existence_NumChecks": "65",
+        "NumNodeSuccesses": "10",
+        "ReconClient_NumApiCalls": "1",
+        "Existence_NumDcCalls": "1"
+      }
+    },
+    "LEVEL_WARNING": {
+      "counters": {
+        "Existence_MissingReference_measurementMethod": "5",
+        "Existence_MissingReference_variableMeasured": "5"
+      }
+    }
+  },
+  "entries": [{
+    "level": "LEVEL_WARNING",
+    "location": {
+      "file": "LatLng.csv",
+      "lineNumber": "2"
+    },
+    "userMessage": "Failed reference existence check :: value-ref: 'SVTest', property: 'measurementMethod', node: 'E:SVTest->E0'",
+    "counterKey": "Existence_MissingReference_measurementMethod"
+  }, {
+    "level": "LEVEL_WARNING",
+    "location": {
+      "file": "LatLng.csv",
+      "lineNumber": "3"
+    },
+    "userMessage": "Failed reference existence check :: value-ref: 'SVTest', property: 'measurementMethod', node: 'E:SVTest->E0'",
+    "counterKey": "Existence_MissingReference_measurementMethod"
+  }, {
+    "level": "LEVEL_WARNING",
+    "location": {
+      "file": "LatLng.csv",
+      "lineNumber": "4"
+    },
+    "userMessage": "Failed reference existence check :: value-ref: 'SVTest', property: 'measurementMethod', node: 'E:SVTest->E0'",
+    "counterKey": "Existence_MissingReference_measurementMethod"
+  }, {
+    "level": "LEVEL_WARNING",
+    "location": {
+      "file": "LatLng.csv",
+      "lineNumber": "5"
+    },
+    "userMessage": "Failed reference existence check :: value-ref: 'SVTest', property: 'measurementMethod', node: 'E:SVTest->E0'",
+    "counterKey": "Existence_MissingReference_measurementMethod"
+  }, {
+    "level": "LEVEL_WARNING",
+    "location": {
+      "file": "LatLng.csv",
+      "lineNumber": "6"
+    },
+    "userMessage": "Failed reference existence check :: value-ref: 'SVTest', property: 'measurementMethod', node: 'E:SVTest->E0'",
+    "counterKey": "Existence_MissingReference_measurementMethod"
+  }, {
+    "level": "LEVEL_WARNING",
+    "location": {
+      "file": "LatLng.csv",
+      "lineNumber": "2"
+    },
+    "userMessage": "Failed reference existence check :: value-ref: 'SV1', property: 'variableMeasured', node: 'E:SVTest->E0'",
+    "counterKey": "Existence_MissingReference_variableMeasured"
+  }, {
+    "level": "LEVEL_WARNING",
+    "location": {
+      "file": "LatLng.csv",
+      "lineNumber": "3"
+    },
+    "userMessage": "Failed reference existence check :: value-ref: 'SV1', property: 'variableMeasured', node: 'E:SVTest->E0'",
+    "counterKey": "Existence_MissingReference_variableMeasured"
+  }, {
+    "level": "LEVEL_WARNING",
+    "location": {
+      "file": "LatLng.csv",
+      "lineNumber": "4"
+    },
+    "userMessage": "Failed reference existence check :: value-ref: 'SV1', property: 'variableMeasured', node: 'E:SVTest->E0'",
+    "counterKey": "Existence_MissingReference_variableMeasured"
+  }, {
+    "level": "LEVEL_WARNING",
+    "location": {
+      "file": "LatLng.csv",
+      "lineNumber": "5"
+    },
+    "userMessage": "Failed reference existence check :: value-ref: 'SV1', property: 'variableMeasured', node: 'E:SVTest->E0'",
+    "counterKey": "Existence_MissingReference_variableMeasured"
+  }, {
+    "level": "LEVEL_WARNING",
+    "location": {
+      "file": "LatLng.csv",
+      "lineNumber": "6"
+    },
+    "userMessage": "Failed reference existence check :: value-ref: 'SV1', property: 'variableMeasured', node: 'E:SVTest->E0'",
+    "counterKey": "Existence_MissingReference_variableMeasured"
+  }],
+  "commandArgs": {
+    "existenceChecks": true,
+    "resolution": "RESOLUTION_MODE_FULL",
+    "numThreads": 1,
+    "statChecks": true,
+    "observationAbout": false,
+    "allowNanSvobs": false,
+    "coordinatesResolution": true
+  }
+}