Add existence checks for observationAbout references (#131)

* add existence checks for `observationAbout` references - checks that the ID referenced in the `observationAbout` field of SVObs nodes exist against the API - this check is behind a new flag (`-ep` / `--existence-checks-place`) that defaults to `false` Co-authored-by: Prashanth R <[email protected]>
datacommonsorg · Jun 30, 2022 · 59736fa · 59736fa
1 parent 088c09b
commit 59736fa
Show file tree

Hide file tree

Showing 18 changed files with 208 additions and 43 deletions.
diff --git a/docs/usage.md b/docs/usage.md
@@ -121,6 +121,12 @@ Specifies the directory to write output files.
 
 Default is `dc_generated/` within current working directory.
 
+### `-ep`, `--existence-checks-place`
+
+Specifies whether to perform existence checks for places found in the `observationAbout` property in StatVarObservation nodes.
+
+Defaults to `false`.
+
 ### `-s`, `--stat-checks`
 
 Checks integrity of time series by checking for holes, variance in values, etc.

diff --git a/tool/src/main/java/org/datacommons/tool/Args.java b/tool/src/main/java/org/datacommons/tool/Args.java
@@ -19,6 +19,7 @@ class Args {
   public int numThreads = 1;
   public Path outputDir = null;
   public boolean generateSummaryReport = true;
+  public boolean checkObservationAbout = false;
 
   public String toString() {
     StringBuilder argStr = new StringBuilder();
@@ -34,6 +35,7 @@ public String toString() {
     if (samplePlaces != null) {
       argStr.append(", sample-places=" + Strings.join(samplePlaces, ':'));
     }
+    argStr.append(", observation-about=" + checkObservationAbout);
     return argStr.toString();
   }
 
@@ -50,6 +52,7 @@ public Debug.CommandArgs toProto() {
     }
     argsBuilder.setStatChecks(doStatChecks);
     if (samplePlaces != null) argsBuilder.addAllSamplePlaces(samplePlaces);
+    argsBuilder.setObservationAbout(checkObservationAbout);
     return argsBuilder.build();
   }
 

diff --git a/tool/src/main/java/org/datacommons/tool/GenMcf.java b/tool/src/main/java/org/datacommons/tool/GenMcf.java
@@ -74,6 +74,7 @@ public Integer call() throws IOException, TemplateException {
       args.outputFiles.put(type, Paths.get(parent.outputDir.getPath(), fName));
     }
     args.generateSummaryReport = parent.generateSummaryReport;
+    args.checkObservationAbout = parent.checkObservationAbout;
 
     // Process all the things.
     return Processor.process(args);

diff --git a/tool/src/main/java/org/datacommons/tool/Lint.java b/tool/src/main/java/org/datacommons/tool/Lint.java
@@ -67,6 +67,7 @@ public Integer call() throws IOException, TemplateException {
     args.fileGroup = FileGroup.build(files, spec, delimiter, logger);
     args.outputDir = parent.outputDir.toPath();
     args.generateSummaryReport = parent.generateSummaryReport;
+    args.checkObservationAbout = parent.checkObservationAbout;
     return Processor.process(args);
   }
 }
diff --git a/tool/src/main/java/org/datacommons/tool/Main.java b/tool/src/main/java/org/datacommons/tool/Main.java
@@ -106,6 +106,16 @@ class Main {
       description = "Generates a summary report in html format. Defaults to true.")
   public boolean generateSummaryReport;
 
+  @CommandLine.Option(
+      names = {"-ep", "--existence-checks-place"},
+      defaultValue = "false",
+      scope = CommandLine.ScopeType.INHERIT,
+      description =
+          "Specifies whether to perform existence checks for places found in "
+              + "the `observationAbout` property in StatVarObservation nodes."
+              + "Defaults to true.")
+  public boolean checkObservationAbout;
+
   public static void main(String... args) {
     System.exit(
         new CommandLine(new Main()).setCaseInsensitiveEnumValuesAllowed(true).execute(args));

diff --git a/tool/src/main/java/org/datacommons/tool/Processor.java b/tool/src/main/java/org/datacommons/tool/Processor.java
@@ -237,7 +237,8 @@ private void processTable(File csvFile)
       g = McfMutator.mutate(g.toBuilder(), logCtx);
 
       // This will set counters/messages in logCtx.
-      boolean success = McfChecker.check(g, existenceChecker, statVarState, logCtx);
+      boolean success =
+          McfChecker.check(g, existenceChecker, statVarState, args.checkObservationAbout, logCtx);
 
       if (args.resolutionMode != Args.ResolutionMode.NONE) {
         g = resolveCommon(g, writerPair);

diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json
@@ -116,24 +116,24 @@
       "file": "FatalTmcf.tmcf",
       "lineNumber": "19"
     },
-    "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'",
+    "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'",
     "counterKey": "Sanity_TmcfMissingEntityDef"
   }, {
     "level": "LEVEL_ERROR",
     "location": {
       "file": "FatalTmcf.tmcf",
       "lineNumber": "19"
     },
-    "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'",
-    "counterKey": "Sanity_TmcfMissingEntityDef"
+    "userMessage": "Column referred to in TMCF is missing from CSV header :: column: 'dcid1', node: 'E:SVTest->E3'",
+    "counterKey": "Sanity_TmcfMissingColumn"
   }, {
     "level": "LEVEL_ERROR",
     "location": {
       "file": "FatalTmcf.tmcf",
       "lineNumber": "19"
     },
-    "userMessage": "Column referred to in TMCF is missing from CSV header :: column: 'dcid1', node: 'E:SVTest->E3'",
-    "counterKey": "Sanity_TmcfMissingColumn"
+    "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'",
+    "counterKey": "Sanity_TmcfMissingEntityDef"
   }, {
     "level": "LEVEL_ERROR",
     "location": {
@@ -195,6 +195,7 @@
     "existenceChecks": true,
     "resolution": "RESOLUTION_MODE_FULL",
     "numThreads": 1,
-    "statChecks": true
+    "statChecks": true,
+    "observationAbout": false
   }
 }
diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/localidresolution/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/localidresolution/output/report.json
@@ -55,6 +55,7 @@
     "existenceChecks": true,
     "resolution": "RESOLUTION_MODE_FULL",
     "numThreads": 1,
-    "statChecks": true
+    "statChecks": true,
+    "observationAbout": false
   }
 }
diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/resolution/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/resolution/output/report.json
@@ -194,6 +194,7 @@
     "existenceChecks": true,
     "resolution": "RESOLUTION_MODE_FULL",
     "numThreads": 1,
-    "statChecks": true
+    "statChecks": true,
+    "observationAbout": false
   }
 }
diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/statchecks/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/statchecks/output/report.json
@@ -215,6 +215,7 @@
     "existenceChecks": true,
     "resolution": "RESOLUTION_MODE_FULL",
     "numThreads": 1,
-    "statChecks": true
+    "statChecks": true,
+    "observationAbout": false
   }
 }
diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json
@@ -710,6 +710,7 @@
     "existenceChecks": true,
     "resolution": "RESOLUTION_MODE_FULL",
     "numThreads": 1,
-    "statChecks": true
+    "statChecks": true,
+    "observationAbout": false
   }
 }
diff --git a/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json
@@ -933,6 +933,7 @@
     "existenceChecks": true,
     "resolution": "RESOLUTION_MODE_LOCAL",
     "numThreads": 1,
-    "statChecks": true
+    "statChecks": true,
+    "observationAbout": false
   }
 }
diff --git a/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json
@@ -561,6 +561,7 @@
     "existenceChecks": true,
     "resolution": "RESOLUTION_MODE_LOCAL",
     "numThreads": 1,
-    "statChecks": true
+    "statChecks": true,
+    "observationAbout": false
   }
 }
diff --git a/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json
@@ -583,15 +583,15 @@
       "file": "NoCsv.tmcf",
       "lineNumber": "3"
     },
-    "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'",
+    "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E3', property: 'observationAbout' node: 'E:SVTest->E0'",
     "counterKey": "Sanity_TmcfMissingEntityDef"
   }, {
     "level": "LEVEL_ERROR",
     "location": {
       "file": "NoCsv.tmcf",
       "lineNumber": "3"
     },
-    "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E3', property: 'observationAbout' node: 'E:SVTest->E0'",
+    "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'",
     "counterKey": "Sanity_TmcfMissingEntityDef"
   }, {
     "level": "LEVEL_ERROR",
@@ -702,6 +702,7 @@
     "existenceChecks": true,
     "resolution": "RESOLUTION_MODE_LOCAL",
     "numThreads": 1,
-    "statChecks": true
+    "statChecks": true,
+    "observationAbout": false
   }
 }
diff --git a/tool/src/test/resources/org/datacommons/tool/lint/statchecks/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/statchecks/output/report.json
@@ -113,6 +113,7 @@
     "existenceChecks": true,
     "resolution": "RESOLUTION_MODE_LOCAL",
     "numThreads": 1,
-    "statChecks": true
+    "statChecks": true,
+    "observationAbout": false
   }
 }