From 8452b6dc952c000bdf5b731c18bb3b2808461c9e Mon Sep 17 00:00:00 2001 From: Prashanth Radhakrishnan Date: Thu, 26 Aug 2021 19:20:52 -0700 Subject: [PATCH 1/7] Implement additional checks --- .../java/org/datacommons/util/McfChecker.java | 172 ++++++++++++------ .../java/org/datacommons/util/McfUtil.java | 13 ++ .../java/org/datacommons/util/Vocabulary.java | 3 +- .../org/datacommons/util/McfCheckerTest.java | 51 +++--- 4 files changed, 156 insertions(+), 83 deletions(-) diff --git a/util/src/main/java/org/datacommons/util/McfChecker.java b/util/src/main/java/org/datacommons/util/McfChecker.java index 92d3f8ac..921513c2 100644 --- a/util/src/main/java/org/datacommons/util/McfChecker.java +++ b/util/src/main/java/org/datacommons/util/McfChecker.java @@ -16,9 +16,11 @@ import com.google.common.base.Charsets; import java.io.IOException; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import org.datacommons.proto.Debug; import org.datacommons.proto.Mcf; import org.datacommons.util.McfUtil.LogCb; @@ -32,12 +34,18 @@ public class McfChecker { private final List PROPS_ONLY_IN_PROP = List.of(Vocabulary.DOMAIN_INCLUDES, Vocabulary.RANGE_INCLUDES, Vocabulary.SUB_PROPERTY_OF); private final List PROPS_ONLY_IN_CLASS = List.of(Vocabulary.SUB_CLASS_OF); - private final Set CLASS_REFS_IN_CLASS = - Set.of(Vocabulary.NAME, Vocabulary.LABEL, Vocabulary.DCID, Vocabulary.SUB_CLASS_OF); + private final Set CLASS_REFS_IN_CLASS = Set.of(Vocabulary.DCID, Vocabulary.SUB_CLASS_OF); private final Set CLASS_REFS_IN_PROP = Set.of(Vocabulary.DOMAIN_INCLUDES, Vocabulary.RANGE_INCLUDES); - private final Set PROP_REFS_IN_PROP = - Set.of(Vocabulary.NAME, Vocabulary.LABEL, Vocabulary.DCID, Vocabulary.SUB_PROPERTY_OF); + private final Set PROP_REFS_IN_PROP = Set.of(Vocabulary.DCID, Vocabulary.SUB_PROPERTY_OF); + + // Includes: a-z A-Z 0-9 _ & + - % / . )( + private final Pattern VALID_DCID_PATTERN = Pattern.compile("^[\\w&/%\\)\\(+\\-\\.]+$"); + // Everything in VALID_DCID_PATTERN, and then: ' * >< ][ | : ; + // TODO: Drop this after Bio DCIDs are fixed + private final Pattern VALID_BIO_DCID_PATTERN = + Pattern.compile("^[\\w&/%\\)\\(+\\-\\.'\\*><\\]\\[|:; ]+$"); + private Mcf.McfGraph graph; private LogWrapper logCtx; private Set columns; // Relevant only when graph.type() == TEMPLATE_MCF @@ -211,22 +219,8 @@ private void checkStatVar(String nodeId, Mcf.McfGraph.PropertyValues node) private void checkSVObs(String nodeId, Mcf.McfGraph.PropertyValues node) throws IOException, InterruptedException { - var statVar = - checkRequiredSingleValueProp( - nodeId, node, Vocabulary.STAT_VAR_OBSERVATION_TYPE, Vocabulary.VARIABLE_MEASURED); - // For SVObs, the only ref check we do is the existence of StatVar, and its an error if missing. - if (existenceChecker != null && !existenceChecker.checkNode(statVar)) { - addLog( - "Existence_MissingValueRef_variableMeasured", - "Failed StatVar existence check :: reference: '" - + statVar - + "', property: '" - + Vocabulary.VARIABLE_MEASURED - + "', node: '" - + nodeId - + "'", - node); - } + checkRequiredSingleValueProp( + nodeId, node, Vocabulary.STAT_VAR_OBSERVATION_TYPE, Vocabulary.VARIABLE_MEASURED); checkRequiredSingleValueProp( nodeId, node, Vocabulary.STAT_VAR_OBSERVATION_TYPE, Vocabulary.OBSERVATION_ABOUT); String obsDate = @@ -329,24 +323,26 @@ private void checkLegacyObs(String nodeId, Mcf.McfGraph.PropertyValues node) { } } - private boolean performExistenceChecks(List types) { - // For types that aren't SVObs, legacy StatPop and legacy Observation we perform existence - // check. - if (existenceChecker == null) return false; - for (String t : types) { - if (Vocabulary.isStatVarObs(t) - || Vocabulary.isPopulation(t) - || Vocabulary.isLegacyObservation(t)) { - return false; - } + // Returns true if we want to check existence for the given prop and types. + private boolean shouldCheckExistence(String prop, Set types) { + if (types.contains(Vocabulary.LEGACY_POPULATION_TYPE_SUFFIX) + || types.contains(Vocabulary.LEGACY_OBSERVATION_TYPE_SUFFIX)) { + return false; + } + // For StatVarObs, we check variableMeasured, measurementMethod and unit only. + if (types.contains(Vocabulary.STAT_VAR_OBSERVATION_TYPE) + && !prop.equals(Vocabulary.VARIABLE_MEASURED) + && !prop.equals(Vocabulary.MEASUREMENT_METHOD) + && !prop.equals(Vocabulary.UNIT)) { + return false; } return true; } - private List checkCommon(String nodeId, Mcf.McfGraph.PropertyValues node) + private Set checkCommon(String nodeId, Mcf.McfGraph.PropertyValues node) throws IOException, InterruptedException { - var types = checkRequiredValueProp(nodeId, node, "Thing", Vocabulary.TYPE_OF); - boolean doExistence = performExistenceChecks(types); + Set types = + new HashSet<>(checkRequiredValueProp(nodeId, node, "Thing", Vocabulary.TYPE_OF)); for (Map.Entry pv : node.getPvsMap().entrySet()) { String prop = pv.getKey(); if (prop.isEmpty()) { @@ -379,19 +375,19 @@ private List checkCommon(String nodeId, Mcf.McfGraph.PropertyValues node node); continue; } - String dcid = vals.getTypedValues(0).getValue(); + var dcid = vals.getTypedValues(0); if (vals.getTypedValues(0).getType() == Mcf.ValueType.TABLE_ENTITY) { addLog( "Sanity_DcidTableEntity", "Value of dcid property must not be an 'E:' reference :: value: '" - + dcid + + dcid.getValue() + "', node: '" + nodeId + "'", node); continue; } - if (dcid.length() > MAX_DCID_LENGTH) { + if (dcid.getValue().length() > MAX_DCID_LENGTH) { addLog( "Sanity_VeryLongDcid", "Found a very long dcid value; must be less than " @@ -402,17 +398,14 @@ private List checkCommon(String nodeId, Mcf.McfGraph.PropertyValues node node); continue; } - // TODO: Expand this to include other special characters. - if (!dcid.startsWith("bio/") && dcid.contains(" ")) { - addLog( - "Sanity_SpaceInDcid", - "Found a whitespace in dcid value :: value: '" + dcid + "', node: '" + nodeId + "'", - node); + // dcid value should typically be TEXT, but some MCFs mistakenly have refs too. + if ((dcid.getType() == Mcf.ValueType.TEXT || dcid.getType() == Mcf.ValueType.RESOLVED_REF) + && !checkDcid(dcid.getValue(), Vocabulary.DCID, nodeId, node)) { continue; } } - if (doExistence && !existenceChecker.checkNode(prop)) { + if (existenceChecker != null && !existenceChecker.checkNode(prop)) { // Mark reference check misses as warnings to flag the user to add schema. addLog( Debug.Log.Level.LEVEL_WARNING, @@ -452,27 +445,69 @@ private List checkCommon(String nodeId, Mcf.McfGraph.PropertyValues node + "'", node); } - if (doExistence - && tv.getType() == Mcf.ValueType.RESOLVED_REF - && !existenceChecker.checkNode(tv.getValue())) { - // Mark reference check misses as warnings to flag the user to add schema. - addLog( - Debug.Log.Level.LEVEL_WARNING, - "Existence_MissingValueRef_" + prop, - "Failed existence check :: reference: '" - + tv.getValue() - + "', property: '" - + prop - + "', node: '" - + nodeId - + "'", - node); + if (tv.getType() == Mcf.ValueType.RESOLVED_REF) { + if (!checkDcid(tv.getValue(), prop, nodeId, node)) { + System.err.println("FAiled for " + tv); + // Failed. checkDcid would have updated logCtx, pass through... + } else if (shouldCheckExistence(prop, types) + && existenceChecker != null + && !existenceChecker.checkNode(tv.getValue())) { + // Mark reference check misses as warnings to flag the user to add schema. + addLog( + Debug.Log.Level.LEVEL_WARNING, + "Existence_MissingValueRef_" + prop, + "Failed existence check :: reference: '" + + tv.getValue() + + "', property: '" + + prop + + "', node: '" + + nodeId + + "'", + node); + } } } } return types; } + private String getInvalidChars(String ref, Pattern p) { + StringBuilder result = new StringBuilder(); + for (int i = 0; i < ref.length(); i++) { + var c = ref.charAt(i); + if (!p.matcher(Character.toString(c)).matches()) { + result.append(c); + } + } + return result.toString(); + } + + private boolean checkDcid( + String ref, String prop, String nodeId, Mcf.McfGraph.PropertyValues node) { + Pattern p; + if (ref.startsWith("bio/")) { + p = VALID_BIO_DCID_PATTERN; + } else { + p = VALID_DCID_PATTERN; + } + if (!p.matcher(ref).matches()) { + addLog( + "Sanity_InvalidChars_" + prop, + "Found invalid chars in dcid value :: value: '" + + ref + + "', invalid-chars: '" + + getInvalidChars(ref, p) + + "', property: '" + + prop + + "', node: '" + + nodeId + + "'", + node); + return false; + } + return true; + } + private void checkClassOrProp(String typeOf, String nodeId, Mcf.McfGraph.PropertyValues node) { List unexpectedProps = typeOf.equals(Vocabulary.CLASS_TYPE) ? PROPS_ONLY_IN_PROP : PROPS_ONLY_IN_CLASS; @@ -526,8 +561,25 @@ private void checkClassOrProp(String typeOf, String nodeId, Mcf.McfGraph.Propert } } } - if (typeOf.equals(Vocabulary.CLASS_TYPE) - && !McfUtil.getPropVal(node, Vocabulary.DCID).equals(Vocabulary.THING_TYPE)) { + // Assert that the DCID matches name/label. + var dcid = McfUtil.getPropVal(node, Vocabulary.DCID); + var name = McfUtil.getPropVal(node, Vocabulary.NAME); + if (name.isEmpty()) name = McfUtil.getPropVal(node, Vocabulary.LABEL); + if (!dcid.isEmpty() && !name.isEmpty() && !dcid.equals(name)) { + addLog( + "Sanity_DcidNameMismatchInSchema", + "Schema node with dcid/name mismatch :: name: '" + + name + + "', dcid: '" + + dcid + + "'," + + " node: '" + + nodeId + + "'", + node); + } + + if (typeOf.equals(Vocabulary.CLASS_TYPE) && !dcid.equals(Vocabulary.THING_TYPE)) { checkRequiredValueProp(nodeId, node, Vocabulary.CLASS_TYPE, Vocabulary.SUB_CLASS_OF); } } diff --git a/util/src/main/java/org/datacommons/util/McfUtil.java b/util/src/main/java/org/datacommons/util/McfUtil.java index de99f244..f39ee6f6 100644 --- a/util/src/main/java/org/datacommons/util/McfUtil.java +++ b/util/src/main/java/org/datacommons/util/McfUtil.java @@ -94,6 +94,19 @@ public static List getPropTvs( return null; } + public static Mcf.McfGraph.TypedValue getPropTv( + Mcf.McfGraph.PropertyValues node, String property) { + try { + var tvs = node.getPvsOrThrow(property).getTypedValuesList(); + if (!tvs.isEmpty()) { + return tvs.get(0); + } + } catch (IllegalArgumentException ex) { + // Not having a value is not an error. + } + return null; + } + public static Mcf.McfGraph.Values newValues(Mcf.ValueType type, String value) { Mcf.McfGraph.Values.Builder vals = Mcf.McfGraph.Values.newBuilder(); Mcf.McfGraph.TypedValue.Builder tv = vals.addTypedValuesBuilder(); diff --git a/util/src/main/java/org/datacommons/util/Vocabulary.java b/util/src/main/java/org/datacommons/util/Vocabulary.java index e722487e..846aad08 100644 --- a/util/src/main/java/org/datacommons/util/Vocabulary.java +++ b/util/src/main/java/org/datacommons/util/Vocabulary.java @@ -165,7 +165,8 @@ public static boolean isReferenceProperty(String prop) { || prop.equals(MEASUREMENT_METHOD) || prop.equals(MEASUREMENT_DENOMINATOR) || prop.equals(MEASUREMENT_QUALIFIER) - || prop.equals(STAT_TYPE); + || prop.equals(STAT_TYPE) + || prop.equals(UNIT); } public static boolean isGlobalReference(String val) { diff --git a/util/src/test/java/org/datacommons/util/McfCheckerTest.java b/util/src/test/java/org/datacommons/util/McfCheckerTest.java index d830b54c..6d27559e 100644 --- a/util/src/test/java/org/datacommons/util/McfCheckerTest.java +++ b/util/src/test/java/org/datacommons/util/McfCheckerTest.java @@ -14,15 +14,16 @@ package org.datacommons.util; -import static org.junit.Assert.assertTrue; +import org.datacommons.proto.Debug; +import org.datacommons.proto.Mcf; +import org.junit.Test; import java.io.IOException; import java.net.http.HttpClient; import java.nio.file.Path; import java.util.Set; -import org.datacommons.proto.Debug; -import org.datacommons.proto.Mcf; -import org.junit.Test; + +import static org.junit.Assert.assertTrue; public class McfCheckerTest { @@ -73,7 +74,7 @@ public void checkDcid() throws IOException, InterruptedException { // DCID must not have space. mcf = "Node: USState\n" + "typeOf: schema:State\n" + "dcid: \"dc/Not Allowed\"\n"; - assertTrue(failure(mcf, "Sanity_SpaceInDcid", "whitespace in dcid")); + assertTrue(failure(mcf, "Sanity_InvalidChars_dcid", "invalid chars in dcid")); // Temporarily, bio/ DCIDs can have space. mcf = "Node: USState\n" + "typeOf: schema:State\n" + "dcid: \"bio/For Now Allowed\"\n"; @@ -324,14 +325,24 @@ public void checkSchema() throws IOException, InterruptedException { assertTrue(failure(mcf, "Sanity_NonAsciiValueInSchema", "“Place”")); // Property types must have lower-case names/IDs. + mcf = + "Node: dcid:Age\n" + + "typeOf: schema:Property\n" + + "name: \"age\"\n" + + "domainIncludes: schema:Person\n" + + "rangeIncludes: schema:Number\n" + + "description: \"Person Age.\"\n"; + assertTrue(failure(mcf, "Sanity_NotInitLower_dcidInProperty", "Age")); + + // Property types must have names and IDs matching. mcf = "Node: dcid:age\n" + "typeOf: schema:Property\n" - + "name: \"Age\"\n" + + "name: \"aGe\"\n" + "domainIncludes: schema:Person\n" + "rangeIncludes: schema:Number\n" + "description: \"Person Age.\"\n"; - assertTrue(failure(mcf, "Sanity_NotInitLower_nameInProperty", "Age")); + assertTrue(failure(mcf, "Sanity_DcidNameMismatchInSchema", "aGe")); // Property types must not have subClassOf. mcf = @@ -426,8 +437,8 @@ public void checkTemplate() throws IOException, InterruptedException { + "typeOf: City\n" + "name: C:CityStats->City\n" + "containedIn: E:CityStats->E1\n"; - assertTrue(success(mcf, Set.of("StateId", "StateName", "City"), false)); - assertTrue(success(mcf)); + assertTrue(success(mcf, Set.of("StateId", "StateName", "City"), false, true)); + assertTrue(success(mcf, null, false, true)); } // NOTE: This test actually makes RPCs to staging mixer. @@ -441,7 +452,7 @@ public void checkExistence() throws IOException, InterruptedException { + "observationAbout: dcid:country/IdontKnow\n" + "observationDate: \"2019\"\n" + "value: 10000\n"; - assertTrue(success(mcf, null, true)); + assertTrue(success(mcf, null, true, false)); // SVObs with a bogus StatVar. mcf = @@ -500,7 +511,7 @@ public void checkExistence() throws IOException, InterruptedException { null, true)); - // StatVar where constrainrProp's value doesn't exist. + // StatVar where constraintProp's value doesn't exist. mcf = "Node: SV4\n" + "typeOf: dcs:StatisticalVariable\n" @@ -516,20 +527,15 @@ public void checkExistence() throws IOException, InterruptedException { null, true)); - // Space in a value should still be handled right. + // Space in a value should fail local sanity check. mcf = "Node: SV5\n" + "typeOf: dcs:RaceCodeEnum\n" + "measuredProperty: dcs:count\n" + "race: dcs:Jupiterian Saturnarian\n" + "statType: dcs:measuredValue\n"; - assertTrue( - failure( - mcf, - "Existence_MissingValueRef_race", - "reference: 'Jupiterian Saturnarian', property: 'race'", - null, - true)); + assertTrue(failure(mcf, "Sanity_InvalidChars_race", "invalid-chars: ' '", + null, true)); } private static boolean failure( @@ -563,12 +569,13 @@ private static boolean failure(String mcfString, String counter, String message) return failure(mcfString, counter, message, null, false); } - private static boolean success(String mcfString, Set columns, boolean doExistenceCheck) + private static boolean success( + String mcfString, Set columns, boolean doExistenceCheck, boolean isTemplate) throws IOException, InterruptedException { Debug.Log.Builder log = Debug.Log.newBuilder(); LogWrapper lw = new LogWrapper(log, Path.of("InMemory")); Mcf.McfGraph graph; - if (columns != null) { + if (isTemplate) { graph = McfParser.parseTemplateMcfString(mcfString, lw); } else { graph = McfParser.parseInstanceMcfString(mcfString, false, lw); @@ -585,6 +592,6 @@ private static boolean success(String mcfString, Set columns, boolean do } private static boolean success(String mcfString) throws IOException, InterruptedException { - return success(mcfString, null, false); + return success(mcfString, null, false, false); } } From b4068c7f36f20beab7f5e80ce4e7ef8c06e2f0da Mon Sep 17 00:00:00 2001 From: Prashanth Radhakrishnan Date: Thu, 26 Aug 2021 21:12:44 -0700 Subject: [PATCH 2/7] Improve tests --- tool/pom.xml | 17 ++ .../java/org/datacommons/tool/GenMcfTest.java | 41 +--- .../java/org/datacommons/tool/LintTest.java | 4 +- .../java/org/datacommons/tool/TestUtil.java | 82 +++++-- .../genmcf/fataltmcf/output/generated.mcf | 0 .../genmcf/successtmcf/output/generated.mcf | 221 ++++++++---------- .../genmcf/successtmcf/output/report.json | 34 ++- .../tool/lint/allfiletypes/output/report.json | 83 ++++++- .../tool/lint/mcfonly/output/report.json | 52 ++++- .../tool/lint/nocsv/output/report.json | 62 ++++- .../java/org/datacommons/util/McfChecker.java | 7 +- .../java/org/datacommons/util/McfUtil.java | 15 +- .../org/datacommons/util/McfCheckerTest.java | 12 +- 13 files changed, 412 insertions(+), 218 deletions(-) create mode 100644 tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/generated.mcf diff --git a/tool/pom.xml b/tool/pom.xml index 0d428974..9dd34e65 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -63,6 +63,23 @@ 4.11 test + + com.google.truth + truth + 0.46 + test + + + com.google.truth.extensions + truth-proto-extension + 0.46 + test + + + com.google.guava + guava + 25.1-jre + diff --git a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java index dc943d7d..3aa791c6 100644 --- a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java +++ b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java @@ -14,16 +14,15 @@ package org.datacommons.tool; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; +import org.datacommons.util.TmcfCsvParser; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -38,6 +37,7 @@ public class GenMcfTest { @Test public void GenMcfTest() throws IOException { + TmcfCsvParser.TEST_mode = true; Main app = new Main(); CommandLine cmd = new CommandLine(app); File[] testDirectories = new File(resourceFile("genmcf")).listFiles(File::isDirectory); @@ -53,36 +53,19 @@ public void GenMcfTest() throws IOException { cmd.execute(args); String actualReportString = TestUtil.getStringFromTestFile(testFolder, "report.json"); String expectedReportString = TestUtil.getStringFromOutputReport(directory.getPath()); - TestUtil.assertReportFilesAreSimilar(expectedReportString, actualReportString); - Path actualGeneratedFilePath = Paths.get(testFolder.getRoot().getPath(), "generated.mcf"); - Path expectedGeneratedFilePath = Path.of(directory.getPath(), "output", "generated.mcf"); - assertTrue(areSimilarGeneratedMcf(expectedGeneratedFilePath, actualGeneratedFilePath)); + TestUtil.assertReportFilesAreSimilar(directory, expectedReportString, actualReportString); + + String actualGeneratedFilePath = + Paths.get(testFolder.getRoot().getPath(), "generated.mcf").toString(); + String expectedGeneratedFilePath = + Path.of(directory.getPath(), "output", "generated.mcf").toString(); + assertEquals( + org.datacommons.util.TestUtil.mcfFromFile(expectedGeneratedFilePath), + org.datacommons.util.TestUtil.mcfFromFile(actualGeneratedFilePath)); } } private String resourceFile(String resource) { return this.getClass().getResource(resource).getPath(); } - - // When testing GeneratedMcf, can't just check against an expected file because When generating - // SVO MCF from csv and tmcf, Nodes will be assigned an ID that may not always be the same - private boolean areSimilarGeneratedMcf(Path expectedFilePath, Path actualFilePath) - throws IOException { - Iterator actualFileLines = Files.lines(actualFilePath).iterator(); - if (!new File(expectedFilePath.toString()).isFile()) { - return !actualFileLines.hasNext(); - } - Iterator expectedFileLines = Files.lines(expectedFilePath).iterator(); - while (expectedFileLines.hasNext() && actualFileLines.hasNext()) { - String expectedLine = expectedFileLines.next(); - String actualLine = actualFileLines.next(); - if (expectedLine.contains("Node") && !expectedLine.contains("dcid")) { - continue; - } - if (!actualLine.trim().equals(expectedLine.trim())) { - return false; - } - } - return true; - } } diff --git a/tool/src/test/java/org/datacommons/tool/LintTest.java b/tool/src/test/java/org/datacommons/tool/LintTest.java index 4f13fbbf..e449a969 100644 --- a/tool/src/test/java/org/datacommons/tool/LintTest.java +++ b/tool/src/test/java/org/datacommons/tool/LintTest.java @@ -5,6 +5,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import org.datacommons.util.TmcfCsvParser; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -20,6 +21,7 @@ public class LintTest { @Test public void LintTest() throws IOException { + TmcfCsvParser.TEST_mode = true; Main app = new Main(); CommandLine cmd = new CommandLine(app); File[] testDirectories = new File(resourceFile("lint")).listFiles(File::isDirectory); @@ -35,7 +37,7 @@ public void LintTest() throws IOException { cmd.execute(args); String actualReportString = TestUtil.getStringFromTestFile(testFolder, "report.json"); String expectedReportString = TestUtil.getStringFromOutputReport(directory.getPath()); - TestUtil.assertReportFilesAreSimilar(expectedReportString, actualReportString); + TestUtil.assertReportFilesAreSimilar(directory, expectedReportString, actualReportString); } } diff --git a/tool/src/test/java/org/datacommons/tool/TestUtil.java b/tool/src/test/java/org/datacommons/tool/TestUtil.java index ba4dda21..520145f3 100644 --- a/tool/src/test/java/org/datacommons/tool/TestUtil.java +++ b/tool/src/test/java/org/datacommons/tool/TestUtil.java @@ -14,7 +14,7 @@ package org.datacommons.tool; -import static org.junit.Assert.assertEquals; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; import static org.junit.Assert.assertTrue; import com.google.protobuf.util.JsonFormat; @@ -23,44 +23,84 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Collection; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.datacommons.proto.Debug; import org.junit.rules.TemporaryFolder; // Common set of utils used in e2e tests. public class TestUtil { - public static void assertReportFilesAreSimilar(String expected, String actual) + public static void assertReportFilesAreSimilar(File directory, String expected, String actual) throws IOException { + String testCase = directory.getName(); Debug.Log.Builder expectedLogBuilder = Debug.Log.newBuilder(); Debug.Log.Builder actualLogBuilder = Debug.Log.newBuilder(); JsonFormat.parser().merge(expected, expectedLogBuilder); JsonFormat.parser().merge(actual, actualLogBuilder); Debug.Log expectedLog = expectedLogBuilder.build(); Debug.Log actualLog = actualLogBuilder.build(); - assertMapsAreEqual( - "Counter Set", - expectedLog.getCounterSet().getCountersMap(), - actualLog.getCounterSet().getCountersMap()); - assertMapsAreEqual( - "Level Summary", expectedLog.getLevelSummaryMap(), actualLog.getLevelSummaryMap()); - assertTrue(actualLog.getEntriesList().containsAll(expectedLog.getEntriesList())); - assertTrue(expectedLog.getEntriesList().containsAll(actualLog.getEntriesList())); + boolean pass = true; + pass &= + areMapsEqual( + testCase, + "Counter Set", + expectedLog.getCounterSet().getCountersMap(), + actualLog.getCounterSet().getCountersMap()); + pass &= + areMapsEqual( + testCase, + "Level Summary", + expectedLog.getLevelSummaryMap(), + actualLog.getLevelSummaryMap()); + pass &= actualLog.getEntriesList().containsAll(expectedLog.getEntriesList()); + pass &= expectedLog.getEntriesList().containsAll(actualLog.getEntriesList()); + if (!pass) { + System.err.println("ACTUAL REPORT for " + testCase + " :: \n\n" + actual); + } + assertThat(actualLog).ignoringRepeatedFieldOrder().isEqualTo(expectedLog); + assertTrue(pass); } - public static void assertMapsAreEqual( - String mapType, Map expected, Map actual) { - assertEquals( - mapType + " has different size between actual and expected", - expected.keySet().size(), - actual.keySet().size()); + private static boolean areMapsEqual( + String testCase, String mapType, Map expected, Map actual) { + boolean equal = true; + if (expected.keySet().size() > actual.keySet().size()) { + equal = false; + Set diff = new HashSet<>((Collection) expected.keySet()); + diff.removeAll((Collection) actual.keySet()); + System.err.println( + testCase + " :: " + mapType + " has some missing keys: " + String.join(", ", diff)); + } else if (expected.keySet().size() < actual.keySet().size()) { + equal = false; + Set diff = new HashSet<>((Collection) actual.keySet()); + diff.removeAll((Collection) expected.keySet()); + System.err.println( + testCase + " :: " + mapType + " has some extra keys: " + String.join(", ", diff)); + } for (String key : expected.keySet()) { - assertTrue(mapType + " actual report is missing the key: " + key, actual.containsKey(key)); - assertEquals( - mapType + " has different values for the key: " + key, - expected.get(key), - actual.get(key)); + if (!actual.containsKey(key)) { + equal = false; + System.err.println(mapType + " actual report is missing the key: " + key); + } + if (expected.get(key) != actual.get(key)) { + equal = false; + System.err.println( + testCase + + " :: " + + mapType + + " has different values for the key " + + key + + " : expected (" + + expected.get(key) + + ") vs. actual (" + + actual.get(key) + + ")"); + } } + return equal; } public static String getStringFromTestFile(TemporaryFolder testFolder, String fileName) diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/generated.mcf b/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/generated.mcf new file mode 100644 index 00000000..e69de29b diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/generated.mcf b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/generated.mcf index 533e7047..ac7d8bef 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/generated.mcf +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/generated.mcf @@ -1,178 +1,161 @@ -Node: SVTest/E0/80541484-a9f6-4950-9acf-9c5e47958a80 -observationDate: 2019 -observationAbout: dcid:CA-BC -variableMeasured: dcid:SV1 +Node: 65 +dcid: "65" +name: "6 5" +typeOf: dcid:Quantity +unit: dcid:6 +value: 5 + +Node: SVTest/E0/10 measurementMethod: dcid:SVTest -value: 1 +observationAbout: dcid:veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylongdcid +observationDate: 2019 typeOf: dcid:StatVarObservation +value: 1 +variableMeasured: dcid:SV1 -Node: SVTest/E1/80541484-a9f6-4950-9acf-9c5e47958a80 -observationDate: 2019 -observationAbout: dcid:CA-BC -variableMeasured: dcid:SV2 +Node: SVTest/E0/11 measurementMethod: dcid:SVTest -value: 2 +observationAbout: dcid:CA-MN +observationDate: 2019 typeOf: dcid:StatVarObservation +value: 1 +variableMeasured: dcid:SV1 -Node: SVTest/E0/b1e3ba3f-866f-42e3-9940-4ac05373826f +Node: SVTest/E0/12 +measurementMethod: dcid:SVTest +observationAbout: dcid:CA-SK, dcid:CA-YU observationDate: 2019 -observationAbout: dcid:CA-AB +typeOf: dcid:StatVarObservation +value: "[5]", dcid:65 variableMeasured: dcid:SV1 + +Node: SVTest/E0/2 measurementMethod: dcid:SVTest -value: 1 +observationAbout: dcid:CA-BC +observationDate: 2019 typeOf: dcid:StatVarObservation +value: 1 +variableMeasured: dcid:SV1 -Node: SVTest/E1/b1e3ba3f-866f-42e3-9940-4ac05373826f -observationDate: 2019 -observationAbout: dcid:CA-AB -variableMeasured: dcid:SV2 +Node: SVTest/E0/3 measurementMethod: dcid:SVTest -value: 2 +observationAbout: dcid:CA-AB +observationDate: 2019 typeOf: dcid:StatVarObservation +value: 1 +variableMeasured: dcid:SV1 -Node: SVTest/E0/5076a445-c80e-4587-834f-ffb83e69c119 -observationDate: 2019 +Node: SVTest/E0/4 +measurementMethod: dcid:SVTest observationAbout: dcid:US-SF +observationDate: 2019 +typeOf: dcid:StatVarObservation +value: 1 variableMeasured: dcid:SV1 + +Node: SVTest/E0/8 measurementMethod: dcid:SVTest -value: 1 +observationAbout: dcid:CH-SH +observationDate: 2019 typeOf: dcid:StatVarObservation +value: 1 +variableMeasured: dcid:SV1 -Node: SVTest/E1/5076a445-c80e-4587-834f-ffb83e69c119 -observationDate: 2019 -observationAbout: dcid:US-SF -variableMeasured: dcid:SV2 +Node: SVTest/E1/10 measurementMethod: dcid:SVTest -value: 2 +observationAbout: dcid:veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylongdcid +observationDate: 2019 typeOf: dcid:StatVarObservation +value: 2 +variableMeasured: dcid:SV2 -Node: dcid:CA-VN -dcid: "CA-VN" -name: "Vancouver" -location: [] -typeOf: dcid:City - -Node: dcid:CH-SH -dcid: "CH-SH" -name: "Shanghai" -location: -typeOf: dcid:City - -Node: SVTest/E0/69bb3fad-75b6-493e-aea6-d49b626a601a -observationDate: 2019 -observationAbout: dcid:CH-SH -variableMeasured: dcid:SV1 +Node: SVTest/E1/11 measurementMethod: dcid:SVTest -value: 1 +observationAbout: dcid:CA-MN +observationDate: 2019 typeOf: dcid:StatVarObservation +value: 2 +variableMeasured: dcid:SV2 -Node: SVTest/E1/69bb3fad-75b6-493e-aea6-d49b626a601a +Node: SVTest/E1/12 +measurementMethod: dcid:SVTest +observationAbout: dcid:CA-SK, dcid:CA-YU observationDate: 2019 -observationAbout: dcid:CH-SH +typeOf: dcid:StatVarObservation +value: 2 variableMeasured: dcid:SV2 + +Node: SVTest/E1/2 measurementMethod: dcid:SVTest -value: 2 +observationAbout: dcid:CA-BC +observationDate: 2019 typeOf: dcid:StatVarObservation +value: 2 +variableMeasured: dcid:SV2 -Node: SVTest/E0/a547a571-70d1-47bc-b71d-b7ad5bb8dbb2 -observationDate: 2019 -observationAbout: dcid:veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylongdcid -variableMeasured: dcid:SV1 +Node: SVTest/E1/3 measurementMethod: dcid:SVTest -value: 1 +observationAbout: dcid:CA-AB +observationDate: 2019 typeOf: dcid:StatVarObservation +value: 2 +variableMeasured: dcid:SV2 -Node: SVTest/E1/a547a571-70d1-47bc-b71d-b7ad5bb8dbb2 +Node: SVTest/E1/4 +measurementMethod: dcid:SVTest +observationAbout: dcid:US-SF observationDate: 2019 -observationAbout: dcid:veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylongdcid +typeOf: dcid:StatVarObservation +value: 2 variableMeasured: dcid:SV2 + +Node: SVTest/E1/8 measurementMethod: dcid:SVTest -value: 2 +observationAbout: dcid:CH-SH +observationDate: 2019 typeOf: dcid:StatVarObservation +value: 2 +variableMeasured: dcid:SV2 Node: dcid:CA-MN dcid: "CA-MN" -name: "Manitoba" location: [Lat Long] +name: "Manitoba" typeOf: dcid:Province -Node: SVTest/E0/34b6f7f2-a791-4f0f-a39d-f865f10a5e0e -observationDate: 2019 -observationAbout: dcid:CA-MN -variableMeasured: dcid:SV1 -measurementMethod: dcid:SVTest -value: 1 -typeOf: dcid:StatVarObservation - -Node: SVTest/E1/34b6f7f2-a791-4f0f-a39d-f865f10a5e0e -observationDate: 2019 -observationAbout: dcid:CA-MN -variableMeasured: dcid:SV2 -measurementMethod: dcid:SVTest -value: 2 -typeOf: dcid:StatVarObservation - Node: dcid:CA-SK dcid: "CA-SK" -name: "Saskatchewan" location: dcid:latLong/4300000_-11600000 +name: "Saskatchewan" typeOf: dcid:Province -Node: SVTest/E0/83e82312-f727-4fee-852c-80aab6927acd -observationDate: 2019 -observationAbout: dcid:CA-SK -variableMeasured: dcid:SV1 -measurementMethod: dcid:SVTest -value: dcid:65 -typeOf: dcid:StatVarObservation - -Node: SVTest/E1/83e82312-f727-4fee-852c-80aab6927acd -observationDate: 2019 -observationAbout: dcid:CA-SK -variableMeasured: dcid:SV2 -measurementMethod: dcid:SVTest -value: 2 -typeOf: dcid:StatVarObservation - -Node: latLong/4300000_-11600000 -dcid: "latLong/4300000_-11600000" -name: "43.00000,-116.00000" -typeOf: dcid:GeoCoordinates -latitude: "43N" -longitude: "116W" - -Node: 65 -dcid: "65" -name: "6 5" -typeOf: dcid:Quantity -value: 5 -unit: dcid:6 +Node: dcid:CA-VN +dcid: "CA-VN" +location: [] +name: "Vancouver" +typeOf: dcid:City Node: dcid:CA-YU dcid: "CA-YU" -name: "Yukon" location: dcid:latLong/-4300000_11600000 +name: "Yukon" typeOf: dcid:Territory -Node: SVTest/E0/f63ab200-f511-4326-a38a-cf7e354ef307 -observationDate: 2019 -observationAbout: dcid:CA-YU -variableMeasured: dcid:SV1 -measurementMethod: dcid:SVTest -value: "[5]" -typeOf: dcid:StatVarObservation - -Node: SVTest/E1/f63ab200-f511-4326-a38a-cf7e354ef307 -observationDate: 2019 -observationAbout: dcid:CA-YU -variableMeasured: dcid:SV2 -measurementMethod: dcid:SVTest -value: 2 -typeOf: dcid:StatVarObservation +Node: dcid:CH-SH +dcid: "CH-SH" +name: "Shanghai" +typeOf: dcid:City Node: latLong/-4300000_11600000 dcid: "latLong/-4300000_11600000" -name: "-43.00000,116.00000" -typeOf: dcid:GeoCoordinates latitude: "43S" longitude: "116E" +name: "-43.00000,116.00000" +typeOf: dcid:GeoCoordinates + +Node: latLong/4300000_-11600000 +dcid: "latLong/4300000_-11600000" +latitude: "43N" +longitude: "116W" +name: "43.00000,-116.00000" +typeOf: dcid:GeoCoordinates diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json index 9df1c953..fa248374 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json @@ -1,17 +1,19 @@ { "levelSummary": { - "LEVEL_ERROR": "17", + "LEVEL_ERROR": "20", "LEVEL_WARNING": "4" }, "counterSet": { "counters": { - "Sanity_SpaceInDcid": "1", + "Sanity_InvalidChars_dcid": "1", + "Sanity_InvalidChars_observationAbout": "2", "Sanity_InvalidObsDate": "2", "NumRowSuccesses": "13", "Sanity_MultipleDcidValues": "1", "NumNodeSuccesses": "21", "NumPVSuccesses": "116", "Sanity_NonAsciiValueInNonText": "1", + "Sanity_InvalidChars_typeOf": "1", "StrSplit_EmptyToken_typeOf": "1", "Sanity_MissingOrEmpty_typeOf": "1", "CSV_InconsistentRows": "2", @@ -33,8 +35,16 @@ "file": "SuccessTmcf.csv", "lineNumber": "2" }, - "userMessage": "Found a whitespace in dcid value :: value: 'US- CA', node: 'E:SVTest->E3'", - "counterKey": "Sanity_SpaceInDcid" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'dcid', node: 'E:SVTest->E3'", + "counterKey": "Sanity_InvalidChars_dcid" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "SuccessTmcf.csv", + "lineNumber": "2" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { @@ -43,6 +53,14 @@ }, "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E0'", "counterKey": "Sanity_InvalidObsDate" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "SuccessTmcf.csv", + "lineNumber": "2" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E1'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { @@ -67,6 +85,14 @@ }, "userMessage": "Found non-ascii characters in a value that is not text :: value: 'Administrative–Area1', type: 'RESOLVED_REF', property: 'typeOf', node: 'E:SVTest->E3'", "counterKey": "Sanity_NonAsciiValueInNonText" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "SuccessTmcf.csv", + "lineNumber": "4" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'Administrative–Area1', invalid-chars: '–', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_WARNING", "location": { diff --git a/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json index 6bceba30..514b8de0 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json @@ -1,6 +1,6 @@ { "levelSummary": { - "LEVEL_ERROR": "48", + "LEVEL_ERROR": "56", "LEVEL_WARNING": "5" }, "counterSet": { @@ -9,12 +9,14 @@ "MCF_MalformedColonLessLine": "1", "Mutator_MissingTypeOf": "1", "Sanity_MissingOrEmpty_typeOf": "2", + "Sanity_InvalidChars_typeOf": "2", "Sanity_InvalidObsDate": "3", "Sanity_NonDoubleObsValue": "1", "Sanity_ObsMissingValueProp": "1", "MCF_MalformedNodeName": "1", - "Sanity_SpaceInDcid": "2", + "Sanity_InvalidChars_dcid": "2", "Sanity_NonAsciiValueInNonText": "2", + "Sanity_InvalidChars_location": "1", "MCF_InvalidLatitude": "1", "Sanity_MultipleDcidValues": "2", "MCF_InvalidLongitude": "1", @@ -27,13 +29,16 @@ "MCF_QuantityMalformedValue": "2", "MCF_QuantityRangeMalformedValues": "3", "Sanity_NotInitLowerPropName": "1", + "Sanity_InvalidChars_domainIncludes": "1", "Sanity_UnexpectedPropInProperty": "1", "Sanity_NonAsciiValueInSchema": "1", "Sanity_NotInitLower_labelInProperty": "1", "Sanity_EmptySchemaValue": "1", + "Sanity_DcidNameMismatchInSchema": "2", "Sanity_UnexpectedPropInClass": "1", "Sanity_NotInitUpper_nameInClass": "1", "Sanity_MissingOrEmpty_subClassOf": "1", + "Sanity_InvalidChars_observationAbout": "2", "NumRowSuccesses": "11", "NumNodeSuccesses": "15", "NumPVSuccesses": "84", @@ -80,6 +85,14 @@ }, "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", "counterKey": "Sanity_MissingOrEmpty_typeOf" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.mcf", + "lineNumber": "15" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'Statistical\nPopulation', invalid-chars: '\', property: 'typeOf', node: 'CityStats/E3/2'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_ERROR", "location": { @@ -118,8 +131,8 @@ "file": "AllFileTypes.mcf", "lineNumber": "49" }, - "userMessage": "Found a whitespace in dcid value :: value: 'dc/ 2sffw13', node: 'CityStats/E5/2'", - "counterKey": "Sanity_SpaceInDcid" + "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { @@ -128,6 +141,14 @@ }, "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", "counterKey": "Sanity_NonAsciiValueInNonText" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.mcf", + "lineNumber": "49" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_location" }, { "level": "LEVEL_ERROR", "location": { @@ -248,6 +269,14 @@ }, "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", "counterKey": "Sanity_MissingOrEmpty_populationType" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.mcf", + "lineNumber": "116" + }, + "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", + "counterKey": "Sanity_InvalidChars_domainIncludes" }, { "level": "LEVEL_ERROR", "location": { @@ -280,6 +309,14 @@ }, "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", "counterKey": "Sanity_EmptySchemaValue" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.mcf", + "lineNumber": "116" + }, + "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { @@ -296,6 +333,14 @@ }, "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", "counterKey": "Sanity_NotInitUpper_nameInClass" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.mcf", + "lineNumber": "123" + }, + "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { @@ -310,8 +355,16 @@ "file": "AllFileTypes.csv", "lineNumber": "2" }, - "userMessage": "Found a whitespace in dcid value :: value: 'US- CA', node: 'E:SVTest->E3'", - "counterKey": "Sanity_SpaceInDcid" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'dcid', node: 'E:SVTest->E3'", + "counterKey": "Sanity_InvalidChars_dcid" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.csv", + "lineNumber": "2" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { @@ -320,6 +373,14 @@ }, "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E0'", "counterKey": "Sanity_InvalidObsDate" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.csv", + "lineNumber": "2" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E1'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { @@ -344,6 +405,14 @@ }, "userMessage": "Found non-ascii characters in a value that is not text :: value: 'Administrative–Area1', type: 'RESOLVED_REF', property: 'typeOf', node: 'E:SVTest->E3'", "counterKey": "Sanity_NonAsciiValueInNonText" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.csv", + "lineNumber": "4" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'Administrative–Area1', invalid-chars: '–', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_WARNING", "location": { @@ -473,4 +542,4 @@ "userMessage": "Quantity value must be a number :: value: '[Lat Long]', property: 'location', node: 'dcid:CA-MN'", "counterKey": "MCF_QuantityMalformedValue" }] -} \ No newline at end of file +} diff --git a/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json index 00440bf0..9044b967 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json @@ -1,6 +1,6 @@ { "levelSummary": { - "LEVEL_ERROR": "31", + "LEVEL_ERROR": "36", "LEVEL_WARNING": "1" }, "counterSet": { @@ -9,12 +9,14 @@ "MCF_MalformedColonLessLine": "1", "Mutator_MissingTypeOf": "1", "Sanity_MissingOrEmpty_typeOf": "1", + "Sanity_InvalidChars_typeOf": "1", "Sanity_InvalidObsDate": "1", "Sanity_NonDoubleObsValue": "1", "Sanity_ObsMissingValueProp": "1", "MCF_MalformedNodeName": "1", - "Sanity_SpaceInDcid": "1", + "Sanity_InvalidChars_dcid": "1", "Sanity_NonAsciiValueInNonText": "1", + "Sanity_InvalidChars_location": "1", "MCF_InvalidLatitude": "1", "Sanity_MultipleDcidValues": "1", "MCF_InvalidLongitude": "1", @@ -27,10 +29,12 @@ "MCF_QuantityMalformedValue": "1", "MCF_QuantityRangeMalformedValues": "3", "Sanity_NotInitLowerPropName": "1", + "Sanity_InvalidChars_domainIncludes": "1", "Sanity_UnexpectedPropInProperty": "1", "Sanity_NonAsciiValueInSchema": "1", "Sanity_NotInitLower_labelInProperty": "1", "Sanity_EmptySchemaValue": "1", + "Sanity_DcidNameMismatchInSchema": "2", "Sanity_UnexpectedPropInClass": "1", "Sanity_NotInitUpper_nameInClass": "1", "Sanity_MissingOrEmpty_subClassOf": "1" @@ -68,6 +72,14 @@ }, "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", "counterKey": "Sanity_MissingOrEmpty_typeOf" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "McfOnly.mcf", + "lineNumber": "15" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'Statistical\nPopulation', invalid-chars: '\', property: 'typeOf', node: 'CityStats/E3/2'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_ERROR", "location": { @@ -106,8 +118,8 @@ "file": "McfOnly.mcf", "lineNumber": "49" }, - "userMessage": "Found a whitespace in dcid value :: value: 'dc/ 2sffw13', node: 'CityStats/E5/2'", - "counterKey": "Sanity_SpaceInDcid" + "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { @@ -116,6 +128,14 @@ }, "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", "counterKey": "Sanity_NonAsciiValueInNonText" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "McfOnly.mcf", + "lineNumber": "49" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_location" }, { "level": "LEVEL_ERROR", "location": { @@ -236,6 +256,14 @@ }, "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", "counterKey": "Sanity_MissingOrEmpty_populationType" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "McfOnly.mcf", + "lineNumber": "116" + }, + "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", + "counterKey": "Sanity_InvalidChars_domainIncludes" }, { "level": "LEVEL_ERROR", "location": { @@ -268,6 +296,14 @@ }, "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", "counterKey": "Sanity_EmptySchemaValue" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "McfOnly.mcf", + "lineNumber": "116" + }, + "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { @@ -284,6 +320,14 @@ }, "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", "counterKey": "Sanity_NotInitUpper_nameInClass" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "McfOnly.mcf", + "lineNumber": "123" + }, + "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { diff --git a/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json index 82e6be93..dca2e096 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json @@ -1,6 +1,6 @@ { "levelSummary": { - "LEVEL_ERROR": "51", + "LEVEL_ERROR": "56", "LEVEL_WARNING": "2" }, "counterSet": { @@ -9,12 +9,14 @@ "MCF_MalformedColonLessLine": "2", "Mutator_MissingTypeOf": "2", "Sanity_MissingOrEmpty_typeOf": "2", + "Sanity_InvalidChars_typeOf": "1", "Sanity_InvalidObsDate": "1", "Sanity_NonDoubleObsValue": "1", "Sanity_ObsMissingValueProp": "1", "MCF_MalformedNodeName": "2", - "Sanity_SpaceInDcid": "1", + "Sanity_InvalidChars_dcid": "1", "Sanity_NonAsciiValueInNonText": "1", + "Sanity_InvalidChars_location": "1", "MCF_InvalidLatitude": "1", "Sanity_MultipleDcidValues": "2", "MCF_InvalidLongitude": "1", @@ -27,10 +29,12 @@ "MCF_QuantityMalformedValue": "1", "MCF_QuantityRangeMalformedValues": "3", "Sanity_NotInitLowerPropName": "3", + "Sanity_InvalidChars_domainIncludes": "1", "Sanity_UnexpectedPropInProperty": "1", "Sanity_NonAsciiValueInSchema": "1", "Sanity_NotInitLower_labelInProperty": "1", "Sanity_EmptySchemaValue": "1", + "Sanity_DcidNameMismatchInSchema": "2", "Sanity_UnexpectedPropInClass": "1", "Sanity_NotInitUpper_nameInClass": "1", "Sanity_MissingOrEmpty_subClassOf": "1", @@ -75,6 +79,14 @@ }, "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", "counterKey": "Sanity_MissingOrEmpty_typeOf" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "NoCsv.mcf", + "lineNumber": "15" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'Statistical\nPopulation', invalid-chars: '\', property: 'typeOf', node: 'CityStats/E3/2'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_ERROR", "location": { @@ -113,8 +125,8 @@ "file": "NoCsv.mcf", "lineNumber": "49" }, - "userMessage": "Found a whitespace in dcid value :: value: 'dc/ 2sffw13', node: 'CityStats/E5/2'", - "counterKey": "Sanity_SpaceInDcid" + "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { @@ -123,6 +135,14 @@ }, "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", "counterKey": "Sanity_NonAsciiValueInNonText" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "NoCsv.mcf", + "lineNumber": "49" + }, + "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_location" }, { "level": "LEVEL_ERROR", "location": { @@ -243,6 +263,14 @@ }, "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", "counterKey": "Sanity_MissingOrEmpty_populationType" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "NoCsv.mcf", + "lineNumber": "116" + }, + "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", + "counterKey": "Sanity_InvalidChars_domainIncludes" }, { "level": "LEVEL_ERROR", "location": { @@ -275,6 +303,14 @@ }, "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", "counterKey": "Sanity_EmptySchemaValue" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "NoCsv.mcf", + "lineNumber": "116" + }, + "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { @@ -291,6 +327,14 @@ }, "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", "counterKey": "Sanity_NotInitUpper_nameInClass" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "NoCsv.mcf", + "lineNumber": "123" + }, + "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { @@ -345,7 +389,7 @@ "file": "NoCsv.tmcf", "lineNumber": "3" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E3', property: 'observationAbout' node: 'E:SVTest->E0'", + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'", "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", @@ -353,7 +397,7 @@ "file": "NoCsv.tmcf", "lineNumber": "3" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'", + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E3', property: 'observationAbout' node: 'E:SVTest->E0'", "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", @@ -417,7 +461,7 @@ "file": "NoCsv.tmcf", "lineNumber": "19" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'", + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'", "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", @@ -425,7 +469,7 @@ "file": "NoCsv.tmcf", "lineNumber": "19" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'", + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'", "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", @@ -468,4 +512,4 @@ "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E1', property: 'dcid' node: 'E:SVTest->E4'", "counterKey": "Sanity_TmcfMissingEntityDef" }] -} \ No newline at end of file +} diff --git a/util/src/main/java/org/datacommons/util/McfChecker.java b/util/src/main/java/org/datacommons/util/McfChecker.java index 921513c2..edff6540 100644 --- a/util/src/main/java/org/datacommons/util/McfChecker.java +++ b/util/src/main/java/org/datacommons/util/McfChecker.java @@ -34,10 +34,12 @@ public class McfChecker { private final List PROPS_ONLY_IN_PROP = List.of(Vocabulary.DOMAIN_INCLUDES, Vocabulary.RANGE_INCLUDES, Vocabulary.SUB_PROPERTY_OF); private final List PROPS_ONLY_IN_CLASS = List.of(Vocabulary.SUB_CLASS_OF); - private final Set CLASS_REFS_IN_CLASS = Set.of(Vocabulary.DCID, Vocabulary.SUB_CLASS_OF); + private final Set CLASS_REFS_IN_CLASS = + Set.of(Vocabulary.NAME, Vocabulary.LABEL, Vocabulary.DCID, Vocabulary.SUB_CLASS_OF); private final Set CLASS_REFS_IN_PROP = Set.of(Vocabulary.DOMAIN_INCLUDES, Vocabulary.RANGE_INCLUDES); - private final Set PROP_REFS_IN_PROP = Set.of(Vocabulary.DCID, Vocabulary.SUB_PROPERTY_OF); + private final Set PROP_REFS_IN_PROP = + Set.of(Vocabulary.NAME, Vocabulary.LABEL, Vocabulary.DCID, Vocabulary.SUB_PROPERTY_OF); // Includes: a-z A-Z 0-9 _ & + - % / . )( private final Pattern VALID_DCID_PATTERN = Pattern.compile("^[\\w&/%\\)\\(+\\-\\.]+$"); @@ -447,7 +449,6 @@ private Set checkCommon(String nodeId, Mcf.McfGraph.PropertyValues node) } if (tv.getType() == Mcf.ValueType.RESOLVED_REF) { if (!checkDcid(tv.getValue(), prop, nodeId, node)) { - System.err.println("FAiled for " + tv); // Failed. checkDcid would have updated logCtx, pass through... } else if (shouldCheckExistence(prop, types) && existenceChecker != null diff --git a/util/src/main/java/org/datacommons/util/McfUtil.java b/util/src/main/java/org/datacommons/util/McfUtil.java index f39ee6f6..767d1ab5 100644 --- a/util/src/main/java/org/datacommons/util/McfUtil.java +++ b/util/src/main/java/org/datacommons/util/McfUtil.java @@ -94,19 +94,6 @@ public static List getPropTvs( return null; } - public static Mcf.McfGraph.TypedValue getPropTv( - Mcf.McfGraph.PropertyValues node, String property) { - try { - var tvs = node.getPvsOrThrow(property).getTypedValuesList(); - if (!tvs.isEmpty()) { - return tvs.get(0); - } - } catch (IllegalArgumentException ex) { - // Not having a value is not an error. - } - return null; - } - public static Mcf.McfGraph.Values newValues(Mcf.ValueType type, String value) { Mcf.McfGraph.Values.Builder vals = Mcf.McfGraph.Values.newBuilder(); Mcf.McfGraph.TypedValue.Builder tv = vals.addTypedValuesBuilder(); @@ -118,7 +105,7 @@ public static Mcf.McfGraph.Values newValues(Mcf.ValueType type, String value) { // Given a list of MCF graphs, merges common nodes and de-duplicates PVs. public static Mcf.McfGraph mergeGraphs(List graphs) throws AssertionError { if (graphs.isEmpty()) { - throw new AssertionError("mergeGraphs called with empty graphs!"); + return Mcf.McfGraph.newBuilder().build(); } // node-id -> {prop -> vals} diff --git a/util/src/test/java/org/datacommons/util/McfCheckerTest.java b/util/src/test/java/org/datacommons/util/McfCheckerTest.java index 6d27559e..0dffb31f 100644 --- a/util/src/test/java/org/datacommons/util/McfCheckerTest.java +++ b/util/src/test/java/org/datacommons/util/McfCheckerTest.java @@ -14,16 +14,15 @@ package org.datacommons.util; -import org.datacommons.proto.Debug; -import org.datacommons.proto.Mcf; -import org.junit.Test; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.net.http.HttpClient; import java.nio.file.Path; import java.util.Set; - -import static org.junit.Assert.assertTrue; +import org.datacommons.proto.Debug; +import org.datacommons.proto.Mcf; +import org.junit.Test; public class McfCheckerTest { @@ -534,8 +533,7 @@ public void checkExistence() throws IOException, InterruptedException { + "measuredProperty: dcs:count\n" + "race: dcs:Jupiterian Saturnarian\n" + "statType: dcs:measuredValue\n"; - assertTrue(failure(mcf, "Sanity_InvalidChars_race", "invalid-chars: ' '", - null, true)); + assertTrue(failure(mcf, "Sanity_InvalidChars_race", "invalid-chars: ' '", null, true)); } private static boolean failure( From e4af2b6239ab9ca6a1f81c44720e0ea3c118a01c Mon Sep 17 00:00:00 2001 From: Prashanth Radhakrishnan Date: Thu, 26 Aug 2021 22:22:33 -0700 Subject: [PATCH 3/7] Improve tests --- .../java/org/datacommons/tool/GenMcfTest.java | 2 ++ .../java/org/datacommons/tool/LintTest.java | 2 ++ .../java/org/datacommons/tool/TestUtil.java | 2 ++ .../java/org/datacommons/util/McfParser.java | 6 ++++- .../org/datacommons/util/McfCheckerTest.java | 18 +++++++++++++++ .../org/datacommons/util/McfParserTest.java | 23 +++++++++++++++++++ 6 files changed, 52 insertions(+), 1 deletion(-) diff --git a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java index 3aa791c6..65ccc09d 100644 --- a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java +++ b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java @@ -37,7 +37,9 @@ public class GenMcfTest { @Test public void GenMcfTest() throws IOException { + // Set this so that the generated node IDs are deterministic TmcfCsvParser.TEST_mode = true; + Main app = new Main(); CommandLine cmd = new CommandLine(app); File[] testDirectories = new File(resourceFile("genmcf")).listFiles(File::isDirectory); diff --git a/tool/src/test/java/org/datacommons/tool/LintTest.java b/tool/src/test/java/org/datacommons/tool/LintTest.java index e449a969..ec64c41e 100644 --- a/tool/src/test/java/org/datacommons/tool/LintTest.java +++ b/tool/src/test/java/org/datacommons/tool/LintTest.java @@ -21,7 +21,9 @@ public class LintTest { @Test public void LintTest() throws IOException { + // Set this so that the generated node IDs are deterministic TmcfCsvParser.TEST_mode = true; + Main app = new Main(); CommandLine cmd = new CommandLine(app); File[] testDirectories = new File(resourceFile("lint")).listFiles(File::isDirectory); diff --git a/tool/src/test/java/org/datacommons/tool/TestUtil.java b/tool/src/test/java/org/datacommons/tool/TestUtil.java index 520145f3..07d5e45e 100644 --- a/tool/src/test/java/org/datacommons/tool/TestUtil.java +++ b/tool/src/test/java/org/datacommons/tool/TestUtil.java @@ -42,6 +42,8 @@ public static void assertReportFilesAreSimilar(File directory, String expected, JsonFormat.parser().merge(actual, actualLogBuilder); Debug.Log expectedLog = expectedLogBuilder.build(); Debug.Log actualLog = actualLogBuilder.build(); + // Compare the maps, printing log messages along the way and assert only at the very end. On + // failure, dump the whole report out for easy copy/paste. boolean pass = true; pass &= areMapsEqual( diff --git a/util/src/main/java/org/datacommons/util/McfParser.java b/util/src/main/java/org/datacommons/util/McfParser.java index f2ccc04d..28c8588b 100644 --- a/util/src/main/java/org/datacommons/util/McfParser.java +++ b/util/src/main/java/org/datacommons/util/McfParser.java @@ -493,7 +493,11 @@ public static List splitAndStripWithQuoteEscape( } List parts = new ArrayList<>(); if (!StringUtil.SplitStructuredLineWithEscapes(orig, arg.delimiter, '"', parts)) { - throw new AssertionError("Failed to split and strip string: " + orig); + if (logCb != null) { + logCb.logError( + "StrSplit_BadQuotesInToken", "Found token with incorrectly double-quoted value"); + } + return results; } for (String s : parts) { String ss = arg.stripEnclosingQuotes ? stripEnclosingQuotePair(s.trim()) : s.trim(); diff --git a/util/src/test/java/org/datacommons/util/McfCheckerTest.java b/util/src/test/java/org/datacommons/util/McfCheckerTest.java index 0dffb31f..db1f7c14 100644 --- a/util/src/test/java/org/datacommons/util/McfCheckerTest.java +++ b/util/src/test/java/org/datacommons/util/McfCheckerTest.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.net.http.HttpClient; import java.nio.file.Path; +import java.util.List; import java.util.Set; import org.datacommons.proto.Debug; import org.datacommons.proto.Mcf; @@ -78,6 +79,23 @@ public void checkDcid() throws IOException, InterruptedException { // Temporarily, bio/ DCIDs can have space. mcf = "Node: USState\n" + "typeOf: schema:State\n" + "dcid: \"bio/For Now Allowed\"\n"; assertTrue(success(mcf)); + + String okCharsId = "A_B&C/D.F-G%H)I(J+K"; + mcf = "Node: ID\n" + "typeOf: schema:State\n" + "dcid: \"" + okCharsId + "\"\n"; + assertTrue(success(mcf)); + + String bioOnlyBadCharsId = "A*BD]E[F|G:H;I J'K"; + mcf = "Node: ID\n" + "typeOf: schema:State\n" + "dcid: \"bio/" + bioOnlyBadCharsId + "\"\n"; + assertTrue(success(mcf)); + // Without bio/ prefix, it should fail + mcf = "Node: ID\n" + "typeOf: schema:State\n" + "dcid: \"" + bioOnlyBadCharsId + "\"\n"; + assertTrue(failure(mcf, "Sanity_InvalidChars_dcid", "invalid-chars: '*<>][|:; ''")); + + String badCharsId = "A^B#C~D\\E`F"; + for (var id : List.of(badCharsId, "bio/" + badCharsId)) { + mcf = "Node: ID\n" + "typeOf: schema:State\n" + "dcid: \"" + id + "\"\n"; + assertTrue(failure(mcf, "Sanity_InvalidChars_dcid", "invalid-chars: '^#~\\`'")); + } } @Test diff --git a/util/src/test/java/org/datacommons/util/McfParserTest.java b/util/src/test/java/org/datacommons/util/McfParserTest.java index a5e2e31d..27984412 100644 --- a/util/src/test/java/org/datacommons/util/McfParserTest.java +++ b/util/src/test/java/org/datacommons/util/McfParserTest.java @@ -18,12 +18,15 @@ import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; import static org.datacommons.util.McfParser.SplitAndStripArg; import static org.datacommons.util.McfParser.splitAndStripWithQuoteEscape; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.Map; import org.apache.commons.io.IOUtils; +import org.datacommons.proto.Debug; import org.datacommons.proto.Mcf.McfGraph; import org.datacommons.proto.Mcf.McfType; import org.junit.Test; @@ -75,6 +78,26 @@ public void funcSplitAndStripWithQuoteEscape() { .containsExactly("{ \"type\": \"feature\" }"); } + @Test + public void testQuoting() throws IOException, URISyntaxException { + Debug.Log.Builder logCtx = Debug.Log.newBuilder(); + LogWrapper lw = new LogWrapper(logCtx, Path.of("InMemory")); + + // Parsing weird double quotes fails. + String mcf = + "Node: US1\n" + + "typeOf: schema:State\n" + + "description: \"Odd\"double\"quotes" + + "\"fails\"\n"; + var graph = McfParser.parseInstanceMcfString(mcf, true, lw); + assertTrue(TestUtil.checkLog(logCtx.build(), "StrSplit_BadQuotesInToken_description", "US1")); + + // New-line in value. + mcf = "Node: US2\n" + "typeOf: schema:State\n" + "description: \"Line with\nnewline\"\n"; + graph = McfParser.parseInstanceMcfString(mcf, true, lw); + assertTrue(TestUtil.checkLog(logCtx.build(), "StrSplit_BadQuotesInToken_description", "US2")); + } + @Test public void funcParseUnresolvedGraph() throws IOException, URISyntaxException { McfGraph act = actual("McfParserTest_UnresolvedGraph.mcf", false); From f973b6e132d965fea26a19f7f214e44d5e10060d Mon Sep 17 00:00:00 2001 From: Prashanth Radhakrishnan Date: Fri, 27 Aug 2021 01:41:41 -0700 Subject: [PATCH 4/7] Improve tests --- .../java/org/datacommons/tool/GenMcfTest.java | 43 +- .../java/org/datacommons/tool/LintTest.java | 21 +- .../java/org/datacommons/tool/TestUtil.java | 61 ++- .../tool/genmcf/fataltmcf/output/report.json | 112 ++--- .../genmcf/successtmcf/output/generated.mcf | 218 ++++----- .../genmcf/successtmcf/output/report.json | 118 ++--- .../tool/lint/allfiletypes/output/report.json | 425 +++++++++--------- .../tool/lint/mcfonly/output/report.json | 221 ++++----- .../tool/lint/nocsv/output/report.json | 323 ++++++------- 9 files changed, 809 insertions(+), 733 deletions(-) diff --git a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java index 65ccc09d..96674ff8 100644 --- a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java +++ b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java @@ -14,12 +14,13 @@ package org.datacommons.tool; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static org.junit.Assert.assertEquals; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import org.datacommons.util.TmcfCsvParser; @@ -32,6 +33,10 @@ // directory, add an input directory and an output directory. In the input directory, put the test // files you want to run the lint tool against. In the output directory, put a report.json file with // the expected report output. +// +// These tests can be run in a mode to produce golden files, as below: +// mvn -DgoldenFilesPrefix=$PWD/tool/src/test/resources/org/datacommons/tool test +// public class GenMcfTest { @Rule public TemporaryFolder testFolder = new TemporaryFolder(); @@ -40,10 +45,14 @@ public void GenMcfTest() throws IOException { // Set this so that the generated node IDs are deterministic TmcfCsvParser.TEST_mode = true; + String goldenFilesPrefix = System.getProperty("goldenFilesPrefix"); + goldenFilesPrefix = + "/Users/shanth/work/git/import/tool/src/test/resources/org/datacommons" + "/tool"; Main app = new Main(); CommandLine cmd = new CommandLine(app); File[] testDirectories = new File(resourceFile("genmcf")).listFiles(File::isDirectory); for (File directory : testDirectories) { + System.err.println("Processing " + directory.getName()); List argsList = new ArrayList<>(); argsList.add("genmcf"); File[] inputFiles = new File(Path.of(directory.getPath(), "input").toString()).listFiles(); @@ -53,17 +62,29 @@ public void GenMcfTest() throws IOException { argsList.add("--output-dir=" + testFolder.getRoot().getPath()); String[] args = argsList.toArray(new String[argsList.size()]); cmd.execute(args); - String actualReportString = TestUtil.getStringFromTestFile(testFolder, "report.json"); - String expectedReportString = TestUtil.getStringFromOutputReport(directory.getPath()); - TestUtil.assertReportFilesAreSimilar(directory, expectedReportString, actualReportString); - String actualGeneratedFilePath = - Paths.get(testFolder.getRoot().getPath(), "generated.mcf").toString(); - String expectedGeneratedFilePath = - Path.of(directory.getPath(), "output", "generated.mcf").toString(); - assertEquals( - org.datacommons.util.TestUtil.mcfFromFile(expectedGeneratedFilePath), - org.datacommons.util.TestUtil.mcfFromFile(actualGeneratedFilePath)); + Path actualGeneratedFilePath = TestUtil.getTestFilePath(testFolder, "generated.mcf"); + Path actualReportPath = TestUtil.getTestFilePath(testFolder, "report.json"); + + if (goldenFilesPrefix != null && !goldenFilesPrefix.isEmpty()) { + Path goldenGeneratedPath = + Path.of(goldenFilesPrefix, "genmcf", directory.getName(), "output", "generated.mcf"); + Files.copy(actualGeneratedFilePath, goldenGeneratedPath, REPLACE_EXISTING); + Path goldenReportPath = + Path.of(goldenFilesPrefix, "genmcf", directory.getName(), "output", "report.json"); + TestUtil.writeSortedReport(actualReportPath, goldenReportPath); + } else { + Path expectedGeneratedFilePath = + TestUtil.getOutputFilePath(directory.getPath(), "generated.mcf"); + Path expectedReportPath = TestUtil.getOutputFilePath(directory.getPath(), "report.json"); + TestUtil.assertReportFilesAreSimilar( + directory, + TestUtil.readStringFromPath(expectedReportPath), + TestUtil.readStringFromPath(actualReportPath)); + assertEquals( + org.datacommons.util.TestUtil.mcfFromFile(expectedGeneratedFilePath.toString()), + org.datacommons.util.TestUtil.mcfFromFile(actualGeneratedFilePath.toString())); + } } } diff --git a/tool/src/test/java/org/datacommons/tool/LintTest.java b/tool/src/test/java/org/datacommons/tool/LintTest.java index ec64c41e..025dae9f 100644 --- a/tool/src/test/java/org/datacommons/tool/LintTest.java +++ b/tool/src/test/java/org/datacommons/tool/LintTest.java @@ -15,6 +15,10 @@ // directory, add an input directory and an output directory. In the input directory, put the test // files you want to run the lint tool against. In the output directory, put a report.json file with // the expected report output. +// +// These tests can be run in a mode to produce golden files, as below: +// mvn -DgoldenFilesPrefix=$PWD/tool/src/test/resources/org/datacommons/tool test +// // TODO(shanth): Incorporate e2e test-cases for existence checks once this is generalized. public class LintTest { @Rule public TemporaryFolder testFolder = new TemporaryFolder(); @@ -24,10 +28,12 @@ public void LintTest() throws IOException { // Set this so that the generated node IDs are deterministic TmcfCsvParser.TEST_mode = true; + String goldenFilesPrefix = System.getProperty("goldenFilesPrefix"); Main app = new Main(); CommandLine cmd = new CommandLine(app); File[] testDirectories = new File(resourceFile("lint")).listFiles(File::isDirectory); for (File directory : testDirectories) { + System.err.println("Processing " + directory.getName()); List argsList = new ArrayList<>(); argsList.add("lint"); File[] inputFiles = new File(Path.of(directory.getPath(), "input").toString()).listFiles(); @@ -37,9 +43,18 @@ public void LintTest() throws IOException { argsList.add("--output-dir=" + testFolder.getRoot().getPath()); String[] args = argsList.toArray(new String[argsList.size()]); cmd.execute(args); - String actualReportString = TestUtil.getStringFromTestFile(testFolder, "report.json"); - String expectedReportString = TestUtil.getStringFromOutputReport(directory.getPath()); - TestUtil.assertReportFilesAreSimilar(directory, expectedReportString, actualReportString); + Path actualReportPath = TestUtil.getTestFilePath(testFolder, "report.json"); + if (goldenFilesPrefix != null && !goldenFilesPrefix.isEmpty()) { + Path goldenPath = + Path.of(goldenFilesPrefix, "lint", directory.getName(), "output", "report.json"); + TestUtil.writeSortedReport(actualReportPath, goldenPath); + } else { + Path expectedReportPath = TestUtil.getOutputFilePath(directory.getPath(), "report.json"); + TestUtil.assertReportFilesAreSimilar( + directory, + TestUtil.readStringFromPath(expectedReportPath), + TestUtil.readStringFromPath(actualReportPath)); + } } } diff --git a/tool/src/test/java/org/datacommons/tool/TestUtil.java b/tool/src/test/java/org/datacommons/tool/TestUtil.java index 07d5e45e..703b5aef 100644 --- a/tool/src/test/java/org/datacommons/tool/TestUtil.java +++ b/tool/src/test/java/org/datacommons/tool/TestUtil.java @@ -17,17 +17,16 @@ import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; import static org.junit.Assert.assertTrue; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.apache.commons.io.FileUtils; +import org.apache.commons.text.StringEscapeUtils; import org.datacommons.proto.Debug; import org.junit.rules.TemporaryFolder; @@ -36,14 +35,9 @@ public class TestUtil { public static void assertReportFilesAreSimilar(File directory, String expected, String actual) throws IOException { String testCase = directory.getName(); - Debug.Log.Builder expectedLogBuilder = Debug.Log.newBuilder(); - Debug.Log.Builder actualLogBuilder = Debug.Log.newBuilder(); - JsonFormat.parser().merge(expected, expectedLogBuilder); - JsonFormat.parser().merge(actual, actualLogBuilder); - Debug.Log expectedLog = expectedLogBuilder.build(); - Debug.Log actualLog = actualLogBuilder.build(); - // Compare the maps, printing log messages along the way and assert only at the very end. On - // failure, dump the whole report out for easy copy/paste. + Debug.Log expectedLog = reportToProto(expected).build(); + Debug.Log actualLog = reportToProto(actual).build(); + // Compare the maps, printing log messages along the way and assert only at the very end. boolean pass = true; pass &= areMapsEqual( @@ -59,9 +53,6 @@ public static void assertReportFilesAreSimilar(File directory, String expected, actualLog.getLevelSummaryMap()); pass &= actualLog.getEntriesList().containsAll(expectedLog.getEntriesList()); pass &= expectedLog.getEntriesList().containsAll(actualLog.getEntriesList()); - if (!pass) { - System.err.println("ACTUAL REPORT for " + testCase + " :: \n\n" + actual); - } assertThat(actualLog).ignoringRepeatedFieldOrder().isEqualTo(expectedLog); assertTrue(pass); } @@ -87,7 +78,7 @@ private static boolean areMapsEqual( equal = false; System.err.println(mapType + " actual report is missing the key: " + key); } - if (expected.get(key) != actual.get(key)) { + if (!expected.get(key).equals(actual.get(key))) { equal = false; System.err.println( testCase @@ -105,14 +96,42 @@ private static boolean areMapsEqual( return equal; } - public static String getStringFromTestFile(TemporaryFolder testFolder, String fileName) + private static Debug.Log.Builder reportToProto(String report) + throws InvalidProtocolBufferException { + Debug.Log.Builder logBuilder = Debug.Log.newBuilder(); + JsonFormat.parser().merge(report, logBuilder); + return logBuilder; + } + + public static void writeSortedReport(Path inputPath, Path outputPath) throws IOException { + Debug.Log.Builder logBuilder = reportToProto(readStringFromPath(inputPath)); + List entries = new ArrayList<>(logBuilder.getEntriesList()); + Collections.sort( + entries, + new Comparator() { + @Override + public int compare(Debug.Log.Entry o1, Debug.Log.Entry o2) { + return o1.toString().compareTo(o2.toString()); + } + }); + logBuilder.clearEntries(); + logBuilder.addAllEntries(entries); + String jsonStr = StringEscapeUtils.unescapeJson(JsonFormat.printer().print(logBuilder.build())); + FileUtils.writeStringToFile(new File(outputPath.toString()), jsonStr, StandardCharsets.UTF_8); + } + + public static Path getTestFilePath(TemporaryFolder testFolder, String fileName) throws IOException { - File file = new File(Paths.get(testFolder.getRoot().getPath(), fileName).toString()); - return FileUtils.readFileToString(file, StandardCharsets.UTF_8); + return Paths.get(testFolder.getRoot().getPath(), fileName); + } + + public static Path getOutputFilePath(String parentDirectoryPath, String fileName) + throws IOException { + return Path.of(parentDirectoryPath, "output", fileName); } - public static String getStringFromOutputReport(String parentDirectoryPath) throws IOException { - File file = new File(Path.of(parentDirectoryPath, "output", "report.json").toString()); + public static String readStringFromPath(Path filePath) throws IOException { + File file = new File(filePath.toString()); return FileUtils.readFileToString(file, StandardCharsets.UTF_8); } } diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json index 057b1564..f3f709c3 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json @@ -32,126 +32,118 @@ }, "userMessage": "Property found without a preceding line with 'Node' :: line: 'typeOf dcs:StatVarObservation'", "counterKey": "MCF_UnexpectedProperty" - }, { - "level": "LEVEL_ERROR", - "location": { - "file": "FatalTmcf.tmcf", - "lineNumber": "5" - }, - "userMessage": "Malformed line without a colon delimiter :: line: ': dcs:SV1'", - "counterKey": "MCF_MalformedColonLessLine" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", "lineNumber": "11" }, - "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: '\"E:SVTest->E1\"'", + "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: '"E:SVTest->E1"'", "counterKey": "TMCF_MalformedEntity" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "22" + "lineNumber": "19" }, - "userMessage": "Malformed column value; must have a '->' delimiter :: value: 'C:SVTest>Place_Name', property: 'name', node: 'E:SVTest->E3'", - "counterKey": "TMCF_MalformedSchemaTerm" + "userMessage": "Column referred to in TMCF is missing from CSV header :: column: 'dcid1', node: 'E:SVTest->E3'", + "counterKey": "Sanity_TmcfMissingColumn" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "25" + "lineNumber": "19" }, - "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: 'C:SVTest->NodeDcid'", - "counterKey": "TMCF_MalformedEntity" + "userMessage": "Found dcid with more than one value :: count: 3, node: 'E:SVTest->E3'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "27" + "lineNumber": "19" }, - "userMessage": "TMCF properties cannot refer to CSV columns yet :: value: 'SVTest->Property: C:SVTest->Property_Value', property: 'C', node: 'E:SVTest->E3'", - "counterKey": "TMCF_UnsupportedColumnNameInProperty" + "userMessage": "Found property name that does not start with a lower-case :: property: 'C', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "29" + "lineNumber": "19" }, - "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'E:SVTest->E5,E:SVTest->E6'", - "counterKey": "MCF_MalformedNodeName" + "userMessage": "Found property name that does not start with a lower-case :: property: 'Dcid', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "34" + "lineNumber": "19" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'age', node: 'E:SVTest->E4'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", "lineNumber": "19" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'C', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "22" }, - "userMessage": "Found dcid with more than one value :: count: 3, node: 'E:SVTest->E3'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Malformed column value; must have a '->' delimiter :: value: 'C:SVTest>Place_Name', property: 'name', node: 'E:SVTest->E3'", + "counterKey": "TMCF_MalformedSchemaTerm" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "25" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'Dcid', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: 'C:SVTest->NodeDcid'", + "counterKey": "TMCF_MalformedEntity" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "27" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "TMCF properties cannot refer to CSV columns yet :: value: 'SVTest->Property: C:SVTest->Property_Value', property: 'C', node: 'E:SVTest->E3'", + "counterKey": "TMCF_UnsupportedColumnNameInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "29" }, - "userMessage": "Column referred to in TMCF is missing from CSV header :: column: 'dcid1', node: 'E:SVTest->E3'", - "counterKey": "Sanity_TmcfMissingColumn" + "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'E:SVTest->E5,E:SVTest->E6'", + "counterKey": "MCF_MalformedNodeName" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "3" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Found multiple values for single-value property :: property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_observationAbout" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "32" + "lineNumber": "3" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'E:SVTest->E4', type: 'Thing'", - "counterKey": "Sanity_MissingOrEmpty_typeOf" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", "lineNumber": "32" }, - "userMessage": "Value of dcid property must not be an 'E:' reference :: value: 'E:SVTest->E1', node: 'E:SVTest->E4'", - "counterKey": "Sanity_DcidTableEntity" + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'E:SVTest->E4', type: 'Thing'", + "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { @@ -164,26 +156,26 @@ "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "3" + "lineNumber": "32" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationAbout', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_observationAbout" + "userMessage": "Value of dcid property must not be an 'E:' reference :: value: 'E:SVTest->E1', node: 'E:SVTest->E4'", + "counterKey": "Sanity_DcidTableEntity" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "3" + "lineNumber": "34" }, - "userMessage": "Found multiple values for single-value property :: property: 'value', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_value" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'age', node: 'E:SVTest->E4'", + "counterKey": "MCF_MalformedComplexValue" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "3" + "lineNumber": "5" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Malformed line without a colon delimiter :: line: ': dcs:SV1'", + "counterKey": "MCF_MalformedColonLessLine" }, { "level": "LEVEL_FATAL", "location": { @@ -192,5 +184,13 @@ }, "userMessage": "Found fatal sanity error in TMCF; check Sanity_ counter messages :: TMCF-file: FatalTmcf.tmcf", "counterKey": "CSV_TmcfCheckFailure" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "FatalTmcf.tmcf", + "lineNumber": "3" + }, + "userMessage": "Found multiple values for single-value property :: property: 'value', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_value" }] } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/generated.mcf b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/generated.mcf index ac7d8bef..c935d2bb 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/generated.mcf +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/generated.mcf @@ -1,161 +1,179 @@ -Node: 65 -dcid: "65" -name: "6 5" -typeOf: dcid:Quantity -unit: dcid:6 -value: 5 - -Node: SVTest/E0/10 -measurementMethod: dcid:SVTest -observationAbout: dcid:veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylongdcid +Node: SVTest/E0/2 observationDate: 2019 -typeOf: dcid:StatVarObservation -value: 1 +observationAbout: dcid:CA-BC variableMeasured: dcid:SV1 - -Node: SVTest/E0/11 measurementMethod: dcid:SVTest -observationAbout: dcid:CA-MN -observationDate: 2019 -typeOf: dcid:StatVarObservation value: 1 -variableMeasured: dcid:SV1 - -Node: SVTest/E0/12 -measurementMethod: dcid:SVTest -observationAbout: dcid:CA-SK, dcid:CA-YU -observationDate: 2019 typeOf: dcid:StatVarObservation -value: "[5]", dcid:65 -variableMeasured: dcid:SV1 -Node: SVTest/E0/2 -measurementMethod: dcid:SVTest -observationAbout: dcid:CA-BC +Node: SVTest/E1/2 observationDate: 2019 +observationAbout: dcid:CA-BC +variableMeasured: dcid:SV2 +measurementMethod: dcid:SVTest +value: 2 typeOf: dcid:StatVarObservation -value: 1 -variableMeasured: dcid:SV1 Node: SVTest/E0/3 -measurementMethod: dcid:SVTest +observationDate: 2019 observationAbout: dcid:CA-AB +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: 1 +typeOf: dcid:StatVarObservation + +Node: SVTest/E1/3 observationDate: 2019 +observationAbout: dcid:CA-AB +variableMeasured: dcid:SV2 +measurementMethod: dcid:SVTest +value: 2 typeOf: dcid:StatVarObservation -value: 1 -variableMeasured: dcid:SV1 Node: SVTest/E0/4 -measurementMethod: dcid:SVTest -observationAbout: dcid:US-SF observationDate: 2019 -typeOf: dcid:StatVarObservation -value: 1 +observationAbout: dcid:US-SF variableMeasured: dcid:SV1 - -Node: SVTest/E0/8 measurementMethod: dcid:SVTest -observationAbout: dcid:CH-SH -observationDate: 2019 -typeOf: dcid:StatVarObservation value: 1 -variableMeasured: dcid:SV1 +typeOf: dcid:StatVarObservation -Node: SVTest/E1/10 -measurementMethod: dcid:SVTest -observationAbout: dcid:veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylongdcid +Node: SVTest/E1/4 observationDate: 2019 -typeOf: dcid:StatVarObservation -value: 2 +observationAbout: dcid:US-SF variableMeasured: dcid:SV2 - -Node: SVTest/E1/11 measurementMethod: dcid:SVTest -observationAbout: dcid:CA-MN -observationDate: 2019 -typeOf: dcid:StatVarObservation value: 2 -variableMeasured: dcid:SV2 +typeOf: dcid:StatVarObservation -Node: SVTest/E1/12 -measurementMethod: dcid:SVTest -observationAbout: dcid:CA-SK, dcid:CA-YU +Node: dcid:CA-VN +dcid: "CA-VN" +name: "Vancouver" +location: [] +typeOf: dcid:City + +Node: dcid:CH-SH +dcid: "CH-SH" +name: "Shanghai" +location: +typeOf: dcid:City + +Node: SVTest/E0/8 observationDate: 2019 +observationAbout: dcid:CH-SH +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: 1 typeOf: dcid:StatVarObservation -value: 2 -variableMeasured: dcid:SV2 -Node: SVTest/E1/2 -measurementMethod: dcid:SVTest -observationAbout: dcid:CA-BC +Node: SVTest/E1/8 observationDate: 2019 -typeOf: dcid:StatVarObservation -value: 2 +observationAbout: dcid:CH-SH variableMeasured: dcid:SV2 - -Node: SVTest/E1/3 measurementMethod: dcid:SVTest -observationAbout: dcid:CA-AB -observationDate: 2019 -typeOf: dcid:StatVarObservation value: 2 -variableMeasured: dcid:SV2 +typeOf: dcid:StatVarObservation -Node: SVTest/E1/4 -measurementMethod: dcid:SVTest -observationAbout: dcid:US-SF +Node: SVTest/E0/10 observationDate: 2019 +observationAbout: dcid:veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylongdcid +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: 1 typeOf: dcid:StatVarObservation -value: 2 -variableMeasured: dcid:SV2 -Node: SVTest/E1/8 -measurementMethod: dcid:SVTest -observationAbout: dcid:CH-SH +Node: SVTest/E1/10 observationDate: 2019 -typeOf: dcid:StatVarObservation -value: 2 +observationAbout: dcid:veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylongdcid variableMeasured: dcid:SV2 +measurementMethod: dcid:SVTest +value: 2 +typeOf: dcid:StatVarObservation Node: dcid:CA-MN dcid: "CA-MN" -location: [Lat Long] name: "Manitoba" +location: [Lat Long] typeOf: dcid:Province +Node: SVTest/E0/11 +observationDate: 2019 +observationAbout: dcid:CA-MN +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: 1 +typeOf: dcid:StatVarObservation + +Node: SVTest/E1/11 +observationDate: 2019 +observationAbout: dcid:CA-MN +variableMeasured: dcid:SV2 +measurementMethod: dcid:SVTest +value: 2 +typeOf: dcid:StatVarObservation + Node: dcid:CA-SK dcid: "CA-SK" -location: dcid:latLong/4300000_-11600000 name: "Saskatchewan" +location: dcid:latLong/4300000_-11600000 typeOf: dcid:Province -Node: dcid:CA-VN -dcid: "CA-VN" -location: [] -name: "Vancouver" -typeOf: dcid:City +Node: SVTest/E0/12 +observationDate: 2019 +observationAbout: dcid:CA-SK +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: dcid:65 +typeOf: dcid:StatVarObservation + +Node: SVTest/E1/12 +observationDate: 2019 +observationAbout: dcid:CA-SK +variableMeasured: dcid:SV2 +measurementMethod: dcid:SVTest +value: 2 +typeOf: dcid:StatVarObservation + +Node: latLong/4300000_-11600000 +dcid: "latLong/4300000_-11600000" +name: "43.00000,-116.00000" +typeOf: dcid:GeoCoordinates +latitude: "43N" +longitude: "116W" + +Node: 65 +dcid: "65" +name: "6 5" +typeOf: dcid:Quantity +value: 5 +unit: dcid:6 Node: dcid:CA-YU dcid: "CA-YU" -location: dcid:latLong/-4300000_11600000 name: "Yukon" +location: dcid:latLong/-4300000_11600000 typeOf: dcid:Territory -Node: dcid:CH-SH -dcid: "CH-SH" -name: "Shanghai" -typeOf: dcid:City +Node: SVTest/E0/12 +observationDate: 2019 +observationAbout: dcid:CA-YU +variableMeasured: dcid:SV1 +measurementMethod: dcid:SVTest +value: "[5]" +typeOf: dcid:StatVarObservation + +Node: SVTest/E1/12 +observationDate: 2019 +observationAbout: dcid:CA-YU +variableMeasured: dcid:SV2 +measurementMethod: dcid:SVTest +value: 2 +typeOf: dcid:StatVarObservation Node: latLong/-4300000_11600000 dcid: "latLong/-4300000_11600000" -latitude: "43S" -longitude: "116E" name: "-43.00000,116.00000" typeOf: dcid:GeoCoordinates +latitude: "43S" +longitude: "116E" -Node: latLong/4300000_-11600000 -dcid: "latLong/4300000_-11600000" -latitude: "43N" -longitude: "116W" -name: "43.00000,-116.00000" -typeOf: dcid:GeoCoordinates diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json index fa248374..d1acac5b 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json @@ -33,18 +33,34 @@ "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "2" + "lineNumber": "10" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'dcid', node: 'E:SVTest->E3'", - "counterKey": "Sanity_InvalidChars_dcid" + "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E0', type: 'StatVarObservation'", + "counterKey": "Sanity_MissingOrEmpty_observationAbout" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "2" + "lineNumber": "10" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E0'", - "counterKey": "Sanity_InvalidChars_observationAbout" + "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E1', type: 'StatVarObservation'", + "counterKey": "Sanity_MissingOrEmpty_observationAbout" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "SuccessTmcf.csv", + "lineNumber": "11" + }, + "userMessage": "Found a very long dcid value; must be less than 256 :: node: 'E:SVTest->E3'", + "counterKey": "Sanity_VeryLongDcid" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "SuccessTmcf.csv", + "lineNumber": "12" + }, + "userMessage": "Quantity value must be a number :: value: '[Lat Long]', property: 'location', node: 'dcid:CA-MN'", + "counterKey": "MCF_QuantityMalformedValue" }, { "level": "LEVEL_ERROR", "location": { @@ -59,32 +75,40 @@ "file": "SuccessTmcf.csv", "lineNumber": "2" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E1'", - "counterKey": "Sanity_InvalidChars_observationAbout" + "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E1'", + "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "2" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E1'", - "counterKey": "Sanity_InvalidObsDate" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'dcid', node: 'E:SVTest->E3'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "3" + "lineNumber": "2" }, - "userMessage": "Found dcid with more than one value :: count: 2, node: 'E:SVTest->E3'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "4" + "lineNumber": "2" }, - "userMessage": "Found non-ascii characters in a value that is not text :: value: 'Administrative–Area1', type: 'RESOLVED_REF', property: 'typeOf', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NonAsciiValueInNonText" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E1'", + "counterKey": "Sanity_InvalidChars_observationAbout" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "SuccessTmcf.csv", + "lineNumber": "3" + }, + "userMessage": "Found dcid with more than one value :: count: 2, node: 'E:SVTest->E3'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { @@ -94,13 +118,13 @@ "userMessage": "Found invalid chars in dcid value :: value: 'Administrative–Area1', invalid-chars: '–', property: 'typeOf', node: 'E:SVTest->E3'", "counterKey": "Sanity_InvalidChars_typeOf" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "5" + "lineNumber": "4" }, - "userMessage": "Empty value found :: value: '', column: 'Place_Type', property: 'typeOf', node: 'E:SVTest->E3'", - "counterKey": "StrSplit_EmptyToken_typeOf" + "userMessage": "Found non-ascii characters in a value that is not text :: value: 'Administrative–Area1', type: 'RESOLVED_REF', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NonAsciiValueInNonText" }, { "level": "LEVEL_ERROR", "location": { @@ -123,32 +147,32 @@ "file": "SuccessTmcf.csv", "lineNumber": "7" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_observationDate" + "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[]', components: 0, property: 'location', node: 'dcid:CA-VN'", + "counterKey": "MCF_MalformedComplexValueParts" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "7" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E1'", - "counterKey": "Sanity_MultipleVals_observationDate" + "userMessage": "Empty value found :: value: '[]', property: 'location', node: 'dcid:CA-VN'", + "counterKey": "StrSplit_EmptyToken_location" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "7" }, - "userMessage": "Empty value found :: value: '[]', property: 'location', node: 'dcid:CA-VN'", - "counterKey": "StrSplit_EmptyToken_location" + "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_observationDate" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "7" }, - "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[]', components: 0, property: 'location', node: 'dcid:CA-VN'", - "counterKey": "MCF_MalformedComplexValueParts" + "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E1'", + "counterKey": "Sanity_MultipleVals_observationDate" }, { "level": "LEVEL_ERROR", "location": { @@ -165,14 +189,6 @@ }, "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'location', node: 'E:SVTest->E3'", "counterKey": "MCF_MalformedComplexValue" - }, { - "level": "LEVEL_WARNING", - "location": { - "file": "SuccessTmcf.csv", - "lineNumber": "10" - }, - "userMessage": "Malformed CSV value for dcid property; must be a text or reference :: value: '123', node: 'E:SVTest->E3'", - "counterKey": "CSV_MalformedDCIDFailures" }, { "level": "LEVEL_WARNING", "location": { @@ -181,14 +197,6 @@ }, "userMessage": "In dcid:{entity} reference, found {entity} to be empty :: property: 'observationAbout', node: 'E:SVTest->E0'", "counterKey": "CSV_EmptyDcidReferences" - }, { - "level": "LEVEL_ERROR", - "location": { - "file": "SuccessTmcf.csv", - "lineNumber": "10" - }, - "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E0', type: 'StatVarObservation'", - "counterKey": "Sanity_MissingOrEmpty_observationAbout" }, { "level": "LEVEL_WARNING", "location": { @@ -198,28 +206,20 @@ "userMessage": "In dcid:{entity} reference, found {entity} to be empty :: property: 'observationAbout', node: 'E:SVTest->E1'", "counterKey": "CSV_EmptyDcidReferences" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { "file": "SuccessTmcf.csv", "lineNumber": "10" }, - "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E1', type: 'StatVarObservation'", - "counterKey": "Sanity_MissingOrEmpty_observationAbout" - }, { - "level": "LEVEL_ERROR", - "location": { - "file": "SuccessTmcf.csv", - "lineNumber": "11" - }, - "userMessage": "Found a very long dcid value; must be less than 256 :: node: 'E:SVTest->E3'", - "counterKey": "Sanity_VeryLongDcid" + "userMessage": "Malformed CSV value for dcid property; must be a text or reference :: value: '123', node: 'E:SVTest->E3'", + "counterKey": "CSV_MalformedDCIDFailures" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "12" + "lineNumber": "5" }, - "userMessage": "Quantity value must be a number :: value: '[Lat Long]', property: 'location', node: 'dcid:CA-MN'", - "counterKey": "MCF_QuantityMalformedValue" + "userMessage": "Empty value found :: value: '', column: 'Place_Type', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "StrSplit_EmptyToken_typeOf" }] } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json index 514b8de0..ac2eb0ff 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json @@ -56,187 +56,171 @@ "entries": [{ "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "1" - }, - "userMessage": "Property found without a preceding line with 'Node' :: line: 'typeOf: State'", - "counterKey": "MCF_UnexpectedProperty" - }, { - "level": "LEVEL_ERROR", - "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "8" + "file": "AllFileTypes.csv", + "lineNumber": "10" }, - "userMessage": "Malformed line without a colon delimiter :: line: ': State'", - "counterKey": "MCF_MalformedColonLessLine" + "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E0', type: 'StatVarObservation'", + "counterKey": "Sanity_MissingOrEmpty_observationAbout" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "7" + "file": "AllFileTypes.csv", + "lineNumber": "10" }, - "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", - "counterKey": "Mutator_MissingTypeOf" + "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E1', type: 'StatVarObservation'", + "counterKey": "Sanity_MissingOrEmpty_observationAbout" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "7" + "file": "AllFileTypes.csv", + "lineNumber": "11" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", - "counterKey": "Sanity_MissingOrEmpty_typeOf" + "userMessage": "Found a very long dcid value; must be less than 256 :: node: 'E:SVTest->E3'", + "counterKey": "Sanity_VeryLongDcid" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "15" + "file": "AllFileTypes.csv", + "lineNumber": "11" }, - "userMessage": "Found invalid chars in dcid value :: value: 'Statistical\nPopulation', invalid-chars: '\', property: 'typeOf', node: 'CityStats/E3/2'", - "counterKey": "Sanity_InvalidChars_typeOf" + "userMessage": "Quantity value must be a number :: value: '[Lat Long]', property: 'location', node: 'dcid:CA-MN'", + "counterKey": "MCF_QuantityMalformedValue" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "23" + "file": "AllFileTypes.csv", + "lineNumber": "2" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", + "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E0'", "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "32" - }, - "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", - "counterKey": "Sanity_NonDoubleObsValue" - }, { - "level": "LEVEL_WARNING", - "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "41" + "file": "AllFileTypes.csv", + "lineNumber": "2" }, - "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", - "counterKey": "Sanity_ObsMissingValueProp" + "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E1'", + "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "54" + "file": "AllFileTypes.csv", + "lineNumber": "2" }, - "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", - "counterKey": "MCF_MalformedNodeName" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'dcid', node: 'E:SVTest->E3'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "49" + "file": "AllFileTypes.csv", + "lineNumber": "2" }, - "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_dcid" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "49" + "file": "AllFileTypes.csv", + "lineNumber": "2" }, - "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_NonAsciiValueInNonText" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E1'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "49" + "file": "AllFileTypes.csv", + "lineNumber": "3" }, - "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_location" + "userMessage": "Found dcid with more than one value :: count: 2, node: 'E:SVTest->E3'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "67" + "file": "AllFileTypes.csv", + "lineNumber": "4" }, - "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '\"CANCountry\"'", - "counterKey": "MCF_InvalidLatitude" + "userMessage": "Found invalid chars in dcid value :: value: 'Administrative–Area1', invalid-chars: '–', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "67" + "file": "AllFileTypes.csv", + "lineNumber": "4" }, - "userMessage": "Found dcid with more than one value :: count: 2, node: '\"CANCountry\"'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Found non-ascii characters in a value that is not text :: value: 'Administrative–Area1', type: 'RESOLVED_REF', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NonAsciiValueInNonText" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "73" + "file": "AllFileTypes.csv", + "lineNumber": "5" }, - "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", - "counterKey": "MCF_InvalidLongitude" + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'E:SVTest->E3', type: 'Thing'", + "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "79" + "file": "AllFileTypes.csv", + "lineNumber": "6" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "Found CSV row with different number of columns", + "counterKey": "CSV_InconsistentRows" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "77" + "file": "AllFileTypes.csv", + "lineNumber": "7" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[]', components: 0, property: 'location', node: 'dcid:CA-VN'", + "counterKey": "MCF_MalformedComplexValueParts" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "77" + "file": "AllFileTypes.csv", + "lineNumber": "7" }, - "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_measuredProperty" + "userMessage": "Empty value found :: value: '[]', property: 'location', node: 'dcid:CA-VN'", + "counterKey": "StrSplit_EmptyToken_location" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "77" + "file": "AllFileTypes.csv", + "lineNumber": "7" }, - "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_statType" + "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_observationDate" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "77" + "file": "AllFileTypes.csv", + "lineNumber": "7" }, - "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", - "counterKey": "Sanity_UnknownStatType" + "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E1'", + "counterKey": "Sanity_MultipleVals_observationDate" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "81" + "file": "AllFileTypes.csv", + "lineNumber": "8" }, - "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", - "counterKey": "MCF_MalformedComplexValueParts" + "userMessage": "Found CSV row with different number of columns", + "counterKey": "CSV_InconsistentRows" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "88" + "file": "AllFileTypes.csv", + "lineNumber": "9" }, - "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", - "counterKey": "MCF_QuantityMalformedValue" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'location', node: 'E:SVTest->E3'", + "counterKey": "MCF_MalformedComplexValue" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "95" + "lineNumber": "1" }, - "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Property found without a preceding line with 'Node' :: line: 'typeOf: State'", + "counterKey": "MCF_UnexpectedProperty" }, { "level": "LEVEL_ERROR", "location": { @@ -251,8 +235,8 @@ "file": "AllFileTypes.mcf", "lineNumber": "109" }, - "userMessage": "Malformed start+end components in QuantityRange value; one of them must be a number :: startValue: '-', endValue: '-', property: 'age', node: 'dcid:Count_Death_15YearsPlus'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { @@ -267,72 +251,72 @@ "file": "AllFileTypes.mcf", "lineNumber": "109" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Malformed start+end components in QuantityRange value; one of them must be a number :: startValue: '-', endValue: '-', property: 'age', node: 'dcid:Count_Death_15YearsPlus'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", "lineNumber": "116" }, - "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", - "counterKey": "Sanity_InvalidChars_domainIncludes" + "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", + "counterKey": "Sanity_NotInitLower_labelInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", "lineNumber": "116" }, - "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", - "counterKey": "Sanity_UnexpectedPropInProperty" + "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", + "counterKey": "Sanity_EmptySchemaValue" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", "lineNumber": "116" }, - "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", - "counterKey": "Sanity_NonAsciiValueInSchema" + "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", + "counterKey": "Sanity_InvalidChars_domainIncludes" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", "lineNumber": "116" }, - "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", - "counterKey": "Sanity_NotInitLower_labelInProperty" + "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", + "counterKey": "Sanity_NonAsciiValueInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", "lineNumber": "116" }, - "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", - "counterKey": "Sanity_EmptySchemaValue" + "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", "lineNumber": "116" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", - "counterKey": "Sanity_DcidNameMismatchInSchema" + "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", + "counterKey": "Sanity_UnexpectedPropInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", "lineNumber": "123" }, - "userMessage": "Unexpected property in Class node :: property: 'domainIncludes', node: 'dcid:Pop'", - "counterKey": "Sanity_UnexpectedPropInClass" + "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", + "counterKey": "Sanity_NotInitUpper_nameInClass" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", "lineNumber": "123" }, - "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", - "counterKey": "Sanity_NotInitUpper_nameInClass" + "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", + "counterKey": "Sanity_MissingOrEmpty_subClassOf" }, { "level": "LEVEL_ERROR", "location": { @@ -347,168 +331,185 @@ "file": "AllFileTypes.mcf", "lineNumber": "123" }, - "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", - "counterKey": "Sanity_MissingOrEmpty_subClassOf" + "userMessage": "Unexpected property in Class node :: property: 'domainIncludes', node: 'dcid:Pop'", + "counterKey": "Sanity_UnexpectedPropInClass" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "2" + "file": "AllFileTypes.mcf", + "lineNumber": "15" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'dcid', node: 'E:SVTest->E3'", - "counterKey": "Sanity_InvalidChars_dcid" + "userMessage": "Found invalid chars in dcid value :: value: 'Statistical +Population', invalid-chars: '', property: 'typeOf', node: 'CityStats/E3/2'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "2" + "file": "AllFileTypes.mcf", + "lineNumber": "23" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E0'", - "counterKey": "Sanity_InvalidChars_observationAbout" + "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", + "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "2" + "file": "AllFileTypes.mcf", + "lineNumber": "32" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E0'", - "counterKey": "Sanity_InvalidObsDate" + "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", + "counterKey": "Sanity_NonDoubleObsValue" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "2" + "file": "AllFileTypes.mcf", + "lineNumber": "49" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E1'", - "counterKey": "Sanity_InvalidChars_observationAbout" + "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "2" + "file": "AllFileTypes.mcf", + "lineNumber": "49" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E1'", - "counterKey": "Sanity_InvalidObsDate" + "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_location" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "3" + "file": "AllFileTypes.mcf", + "lineNumber": "49" }, - "userMessage": "Found dcid with more than one value :: count: 2, node: 'E:SVTest->E3'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_NonAsciiValueInNonText" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "4" + "file": "AllFileTypes.mcf", + "lineNumber": "54" }, - "userMessage": "Found non-ascii characters in a value that is not text :: value: 'Administrative–Area1', type: 'RESOLVED_REF', property: 'typeOf', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NonAsciiValueInNonText" + "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", + "counterKey": "MCF_MalformedNodeName" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "4" + "file": "AllFileTypes.mcf", + "lineNumber": "67" }, - "userMessage": "Found invalid chars in dcid value :: value: 'Administrative–Area1', invalid-chars: '–', property: 'typeOf', node: 'E:SVTest->E3'", - "counterKey": "Sanity_InvalidChars_typeOf" + "userMessage": "Found dcid with more than one value :: count: 2, node: '"CANCountry"'", + "counterKey": "Sanity_MultipleDcidValues" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "5" + "file": "AllFileTypes.mcf", + "lineNumber": "67" }, - "userMessage": "Empty value found :: value: '', column: 'Place_Type', property: 'typeOf', node: 'E:SVTest->E3'", - "counterKey": "StrSplit_EmptyToken_typeOf" + "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '"CANCountry"'", + "counterKey": "MCF_InvalidLatitude" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "5" + "file": "AllFileTypes.mcf", + "lineNumber": "7" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'E:SVTest->E3', type: 'Thing'", + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "6" + "file": "AllFileTypes.mcf", + "lineNumber": "7" }, - "userMessage": "Found CSV row with different number of columns", - "counterKey": "CSV_InconsistentRows" + "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", + "counterKey": "Mutator_MissingTypeOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "7" + "file": "AllFileTypes.mcf", + "lineNumber": "73" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_observationDate" + "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", + "counterKey": "MCF_InvalidLongitude" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "7" + "file": "AllFileTypes.mcf", + "lineNumber": "77" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E1'", - "counterKey": "Sanity_MultipleVals_observationDate" + "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_measuredProperty" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "7" + "file": "AllFileTypes.mcf", + "lineNumber": "77" }, - "userMessage": "Empty value found :: value: '[]', property: 'location', node: 'dcid:CA-VN'", - "counterKey": "StrSplit_EmptyToken_location" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "7" + "file": "AllFileTypes.mcf", + "lineNumber": "77" }, - "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[]', components: 0, property: 'location', node: 'dcid:CA-VN'", - "counterKey": "MCF_MalformedComplexValueParts" + "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_statType" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "8" + "file": "AllFileTypes.mcf", + "lineNumber": "77" }, - "userMessage": "Found CSV row with different number of columns", - "counterKey": "CSV_InconsistentRows" + "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", + "counterKey": "Sanity_UnknownStatType" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "9" + "file": "AllFileTypes.mcf", + "lineNumber": "79" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'location', node: 'E:SVTest->E3'", + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", "counterKey": "MCF_MalformedComplexValue" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "10" + "file": "AllFileTypes.mcf", + "lineNumber": "8" }, - "userMessage": "Malformed CSV value for dcid property; must be a text or reference :: value: '123', node: 'E:SVTest->E3'", - "counterKey": "CSV_MalformedDCIDFailures" + "userMessage": "Malformed line without a colon delimiter :: line: ': State'", + "counterKey": "MCF_MalformedColonLessLine" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "10" + "file": "AllFileTypes.mcf", + "lineNumber": "81" }, - "userMessage": "In dcid:{entity} reference, found {entity} to be empty :: property: 'observationAbout', node: 'E:SVTest->E0'", - "counterKey": "CSV_EmptyDcidReferences" + "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", + "counterKey": "MCF_MalformedComplexValueParts" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.mcf", + "lineNumber": "88" + }, + "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", + "counterKey": "MCF_QuantityMalformedValue" }, { "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.mcf", + "lineNumber": "95" + }, + "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", + "counterKey": "MCF_QuantityRangeMalformedValues" + }, { + "level": "LEVEL_WARNING", "location": { "file": "AllFileTypes.csv", "lineNumber": "10" }, - "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E0', type: 'StatVarObservation'", - "counterKey": "Sanity_MissingOrEmpty_observationAbout" + "userMessage": "In dcid:{entity} reference, found {entity} to be empty :: property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "CSV_EmptyDcidReferences" }, { "level": "LEVEL_WARNING", "location": { @@ -518,28 +519,28 @@ "userMessage": "In dcid:{entity} reference, found {entity} to be empty :: property: 'observationAbout', node: 'E:SVTest->E1'", "counterKey": "CSV_EmptyDcidReferences" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { "file": "AllFileTypes.csv", "lineNumber": "10" }, - "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E1', type: 'StatVarObservation'", - "counterKey": "Sanity_MissingOrEmpty_observationAbout" + "userMessage": "Malformed CSV value for dcid property; must be a text or reference :: value: '123', node: 'E:SVTest->E3'", + "counterKey": "CSV_MalformedDCIDFailures" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { "file": "AllFileTypes.csv", - "lineNumber": "11" + "lineNumber": "5" }, - "userMessage": "Found a very long dcid value; must be less than 256 :: node: 'E:SVTest->E3'", - "counterKey": "Sanity_VeryLongDcid" + "userMessage": "Empty value found :: value: '', column: 'Place_Type', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "StrSplit_EmptyToken_typeOf" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "11" + "file": "AllFileTypes.mcf", + "lineNumber": "41" }, - "userMessage": "Quantity value must be a number :: value: '[Lat Long]', property: 'location', node: 'dcid:CA-MN'", - "counterKey": "MCF_QuantityMalformedValue" + "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", + "counterKey": "Sanity_ObsMissingValueProp" }] -} +} \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json index 9044b967..d31ed5d3 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json @@ -52,289 +52,290 @@ "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "8" + "lineNumber": "102" }, - "userMessage": "Malformed line without a colon delimiter :: line: ': State'", - "counterKey": "MCF_MalformedColonLessLine" + "userMessage": "Malformed end component in QuantityRange value; must be a number or '-' :: value: 'years', property: 'age', node: 'dcid:Count_Death_5YearsPlus'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "7" + "lineNumber": "109" }, - "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", - "counterKey": "Mutator_MissingTypeOf" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "7" + "lineNumber": "109" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", - "counterKey": "Sanity_MissingOrEmpty_typeOf" + "userMessage": "Found property name that does not start with a lower-case :: property: 'PopulationType', node: 'dcid:Count_Death_15YearsPlus'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "15" + "lineNumber": "109" }, - "userMessage": "Found invalid chars in dcid value :: value: 'Statistical\nPopulation', invalid-chars: '\', property: 'typeOf', node: 'CityStats/E3/2'", - "counterKey": "Sanity_InvalidChars_typeOf" + "userMessage": "Malformed start+end components in QuantityRange value; one of them must be a number :: startValue: '-', endValue: '-', property: 'age', node: 'dcid:Count_Death_15YearsPlus'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "23" + "lineNumber": "116" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", - "counterKey": "Sanity_InvalidObsDate" + "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", + "counterKey": "Sanity_NotInitLower_labelInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "32" + "lineNumber": "116" }, - "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", - "counterKey": "Sanity_NonDoubleObsValue" + "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", + "counterKey": "Sanity_EmptySchemaValue" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "41" + "lineNumber": "116" }, - "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", - "counterKey": "Sanity_ObsMissingValueProp" + "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", + "counterKey": "Sanity_InvalidChars_domainIncludes" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "54" + "lineNumber": "116" }, - "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", - "counterKey": "MCF_MalformedNodeName" + "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", + "counterKey": "Sanity_NonAsciiValueInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "49" + "lineNumber": "116" }, - "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_dcid" + "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "49" + "lineNumber": "116" }, - "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_NonAsciiValueInNonText" + "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", + "counterKey": "Sanity_UnexpectedPropInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "49" + "lineNumber": "123" }, - "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_location" + "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", + "counterKey": "Sanity_NotInitUpper_nameInClass" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "67" + "lineNumber": "123" }, - "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '\"CANCountry\"'", - "counterKey": "MCF_InvalidLatitude" + "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", + "counterKey": "Sanity_MissingOrEmpty_subClassOf" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "67" + "lineNumber": "123" }, - "userMessage": "Found dcid with more than one value :: count: 2, node: '\"CANCountry\"'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "73" + "lineNumber": "123" }, - "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", - "counterKey": "MCF_InvalidLongitude" + "userMessage": "Unexpected property in Class node :: property: 'domainIncludes', node: 'dcid:Pop'", + "counterKey": "Sanity_UnexpectedPropInClass" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "79" + "lineNumber": "15" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "Found invalid chars in dcid value :: value: 'Statistical +Population', invalid-chars: '', property: 'typeOf', node: 'CityStats/E3/2'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "77" + "lineNumber": "23" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", + "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "77" + "lineNumber": "32" }, - "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_measuredProperty" + "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", + "counterKey": "Sanity_NonDoubleObsValue" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "77" + "lineNumber": "49" }, - "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_statType" + "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "77" + "lineNumber": "49" }, - "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", - "counterKey": "Sanity_UnknownStatType" + "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_location" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "81" + "lineNumber": "49" }, - "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", - "counterKey": "MCF_MalformedComplexValueParts" + "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_NonAsciiValueInNonText" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "88" + "lineNumber": "54" }, - "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", - "counterKey": "MCF_QuantityMalformedValue" + "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", + "counterKey": "MCF_MalformedNodeName" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "95" + "lineNumber": "67" }, - "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Found dcid with more than one value :: count: 2, node: '"CANCountry"'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "102" + "lineNumber": "67" }, - "userMessage": "Malformed end component in QuantityRange value; must be a number or '-' :: value: 'years', property: 'age', node: 'dcid:Count_Death_5YearsPlus'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '"CANCountry"'", + "counterKey": "MCF_InvalidLatitude" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "109" + "lineNumber": "7" }, - "userMessage": "Malformed start+end components in QuantityRange value; one of them must be a number :: startValue: '-', endValue: '-', property: 'age', node: 'dcid:Count_Death_15YearsPlus'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", + "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "109" + "lineNumber": "7" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'PopulationType', node: 'dcid:Count_Death_15YearsPlus'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", + "counterKey": "Mutator_MissingTypeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "109" + "lineNumber": "73" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", + "counterKey": "MCF_InvalidLongitude" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "77" }, - "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", - "counterKey": "Sanity_InvalidChars_domainIncludes" + "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_measuredProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "77" }, - "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", - "counterKey": "Sanity_UnexpectedPropInProperty" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "77" }, - "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", - "counterKey": "Sanity_NonAsciiValueInSchema" + "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_statType" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "77" }, - "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", - "counterKey": "Sanity_NotInitLower_labelInProperty" + "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", + "counterKey": "Sanity_UnknownStatType" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "79" }, - "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", - "counterKey": "Sanity_EmptySchemaValue" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", + "counterKey": "MCF_MalformedComplexValue" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "8" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", - "counterKey": "Sanity_DcidNameMismatchInSchema" + "userMessage": "Malformed line without a colon delimiter :: line: ': State'", + "counterKey": "MCF_MalformedColonLessLine" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "123" + "lineNumber": "81" }, - "userMessage": "Unexpected property in Class node :: property: 'domainIncludes', node: 'dcid:Pop'", - "counterKey": "Sanity_UnexpectedPropInClass" + "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", + "counterKey": "MCF_MalformedComplexValueParts" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "123" + "lineNumber": "88" }, - "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", - "counterKey": "Sanity_NotInitUpper_nameInClass" + "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", + "counterKey": "MCF_QuantityMalformedValue" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "123" + "lineNumber": "95" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", - "counterKey": "Sanity_DcidNameMismatchInSchema" + "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { "file": "McfOnly.mcf", - "lineNumber": "123" + "lineNumber": "41" }, - "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", - "counterKey": "Sanity_MissingOrEmpty_subClassOf" + "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", + "counterKey": "Sanity_ObsMissingValueProp" }] } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json index dca2e096..1b90c99c 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json @@ -59,290 +59,283 @@ "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "8" + "lineNumber": "102" }, - "userMessage": "Malformed line without a colon delimiter :: line: ': State'", - "counterKey": "MCF_MalformedColonLessLine" + "userMessage": "Malformed end component in QuantityRange value; must be a number or '-' :: value: 'years', property: 'age', node: 'dcid:Count_Death_5YearsPlus'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "7" + "lineNumber": "109" }, - "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", - "counterKey": "Mutator_MissingTypeOf" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "7" + "lineNumber": "109" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", - "counterKey": "Sanity_MissingOrEmpty_typeOf" + "userMessage": "Found property name that does not start with a lower-case :: property: 'PopulationType', node: 'dcid:Count_Death_15YearsPlus'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "15" + "lineNumber": "109" }, - "userMessage": "Found invalid chars in dcid value :: value: 'Statistical\nPopulation', invalid-chars: '\', property: 'typeOf', node: 'CityStats/E3/2'", - "counterKey": "Sanity_InvalidChars_typeOf" + "userMessage": "Malformed start+end components in QuantityRange value; one of them must be a number :: startValue: '-', endValue: '-', property: 'age', node: 'dcid:Count_Death_15YearsPlus'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "23" + "lineNumber": "116" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", - "counterKey": "Sanity_InvalidObsDate" + "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", + "counterKey": "Sanity_NotInitLower_labelInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "32" - }, - "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", - "counterKey": "Sanity_NonDoubleObsValue" - }, { - "level": "LEVEL_WARNING", - "location": { - "file": "NoCsv.mcf", - "lineNumber": "41" + "lineNumber": "116" }, - "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", - "counterKey": "Sanity_ObsMissingValueProp" + "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", + "counterKey": "Sanity_EmptySchemaValue" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "54" + "lineNumber": "116" }, - "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", - "counterKey": "MCF_MalformedNodeName" + "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", + "counterKey": "Sanity_InvalidChars_domainIncludes" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "49" + "lineNumber": "116" }, - "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_dcid" + "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", + "counterKey": "Sanity_NonAsciiValueInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "49" + "lineNumber": "116" }, - "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_NonAsciiValueInNonText" + "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "49" + "lineNumber": "116" }, - "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_location" + "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", + "counterKey": "Sanity_UnexpectedPropInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "67" + "lineNumber": "123" }, - "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '\"CANCountry\"'", - "counterKey": "MCF_InvalidLatitude" + "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", + "counterKey": "Sanity_NotInitUpper_nameInClass" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "67" + "lineNumber": "123" }, - "userMessage": "Found dcid with more than one value :: count: 2, node: '\"CANCountry\"'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", + "counterKey": "Sanity_MissingOrEmpty_subClassOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "73" + "lineNumber": "123" }, - "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", - "counterKey": "MCF_InvalidLongitude" + "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "79" + "lineNumber": "123" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "Unexpected property in Class node :: property: 'domainIncludes', node: 'dcid:Pop'", + "counterKey": "Sanity_UnexpectedPropInClass" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "77" + "lineNumber": "15" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Found invalid chars in dcid value :: value: 'Statistical +Population', invalid-chars: '', property: 'typeOf', node: 'CityStats/E3/2'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "77" + "lineNumber": "23" }, - "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_measuredProperty" + "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", + "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "77" + "lineNumber": "32" }, - "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_statType" + "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", + "counterKey": "Sanity_NonDoubleObsValue" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "77" + "lineNumber": "49" }, - "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", - "counterKey": "Sanity_UnknownStatType" + "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "81" + "lineNumber": "49" }, - "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", - "counterKey": "MCF_MalformedComplexValueParts" + "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_location" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "88" + "lineNumber": "49" }, - "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", - "counterKey": "MCF_QuantityMalformedValue" + "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_NonAsciiValueInNonText" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "95" + "lineNumber": "54" }, - "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", + "counterKey": "MCF_MalformedNodeName" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "102" + "lineNumber": "67" }, - "userMessage": "Malformed end component in QuantityRange value; must be a number or '-' :: value: 'years', property: 'age', node: 'dcid:Count_Death_5YearsPlus'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Found dcid with more than one value :: count: 2, node: '"CANCountry"'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "109" + "lineNumber": "67" }, - "userMessage": "Malformed start+end components in QuantityRange value; one of them must be a number :: startValue: '-', endValue: '-', property: 'age', node: 'dcid:Count_Death_15YearsPlus'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '"CANCountry"'", + "counterKey": "MCF_InvalidLatitude" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "109" + "lineNumber": "7" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'PopulationType', node: 'dcid:Count_Death_15YearsPlus'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", + "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "109" + "lineNumber": "7" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", + "counterKey": "Mutator_MissingTypeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "73" }, - "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", - "counterKey": "Sanity_InvalidChars_domainIncludes" + "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", + "counterKey": "MCF_InvalidLongitude" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "77" }, - "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", - "counterKey": "Sanity_UnexpectedPropInProperty" + "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_measuredProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "77" }, - "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", - "counterKey": "Sanity_NonAsciiValueInSchema" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "77" }, - "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", - "counterKey": "Sanity_NotInitLower_labelInProperty" + "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_statType" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "77" }, - "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", - "counterKey": "Sanity_EmptySchemaValue" + "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", + "counterKey": "Sanity_UnknownStatType" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "79" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", - "counterKey": "Sanity_DcidNameMismatchInSchema" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", + "counterKey": "MCF_MalformedComplexValue" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "123" + "lineNumber": "8" }, - "userMessage": "Unexpected property in Class node :: property: 'domainIncludes', node: 'dcid:Pop'", - "counterKey": "Sanity_UnexpectedPropInClass" + "userMessage": "Malformed line without a colon delimiter :: line: ': State'", + "counterKey": "MCF_MalformedColonLessLine" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "123" + "lineNumber": "81" }, - "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", - "counterKey": "Sanity_NotInitUpper_nameInClass" + "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", + "counterKey": "MCF_MalformedComplexValueParts" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "123" + "lineNumber": "88" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", - "counterKey": "Sanity_DcidNameMismatchInSchema" + "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", + "counterKey": "MCF_QuantityMalformedValue" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "123" + "lineNumber": "95" }, - "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", - "counterKey": "Sanity_MissingOrEmpty_subClassOf" + "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { @@ -355,49 +348,49 @@ "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "5" + "lineNumber": "11" }, - "userMessage": "Malformed line without a colon delimiter :: line: ': dcs:SV1'", - "counterKey": "MCF_MalformedColonLessLine" + "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: '"E:SVTest->E1"'", + "counterKey": "TMCF_MalformedEntity" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "11" + "lineNumber": "19" }, - "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: '\"E:SVTest->E1\"'", - "counterKey": "TMCF_MalformedEntity" + "userMessage": "Found dcid with more than one value :: count: 3, node: 'E:SVTest->E3'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "3" + "lineNumber": "19" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationAbout', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_observationAbout" + "userMessage": "Found property name that does not start with a lower-case :: property: 'C', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NotInitLowerPropName" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "3" + "lineNumber": "19" }, - "userMessage": "Found multiple values for single-value property :: property: 'value', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_value" + "userMessage": "Found property name that does not start with a lower-case :: property: 'Dcid', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "3" + "lineNumber": "19" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'", + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'", "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "3" + "lineNumber": "19" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E3', property: 'observationAbout' node: 'E:SVTest->E0'", + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'", "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", @@ -435,81 +428,89 @@ "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "19" + "lineNumber": "3" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'C', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "Found multiple values for single-value property :: property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_observationAbout" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "19" + "lineNumber": "3" }, - "userMessage": "Found dcid with more than one value :: count: 3, node: 'E:SVTest->E3'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "19" + "lineNumber": "3" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'Dcid', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E3', property: 'observationAbout' node: 'E:SVTest->E0'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "19" + "lineNumber": "32" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'E:SVTest->E4', type: 'Thing'", + "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "19" + "lineNumber": "32" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Missing typeOf value for node :: node: 'E:SVTest->E4'", + "counterKey": "Mutator_MissingTypeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "34" + "lineNumber": "32" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'age', node: 'E:SVTest->E4'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E1', property: 'dcid' node: 'E:SVTest->E4'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", "lineNumber": "32" }, - "userMessage": "Missing typeOf value for node :: node: 'E:SVTest->E4'", - "counterKey": "Mutator_MissingTypeOf" + "userMessage": "Value of dcid property must not be an 'E:' reference :: value: 'E:SVTest->E1', node: 'E:SVTest->E4'", + "counterKey": "Sanity_DcidTableEntity" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "32" + "lineNumber": "34" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'E:SVTest->E4', type: 'Thing'", - "counterKey": "Sanity_MissingOrEmpty_typeOf" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'age', node: 'E:SVTest->E4'", + "counterKey": "MCF_MalformedComplexValue" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "32" + "lineNumber": "5" }, - "userMessage": "Value of dcid property must not be an 'E:' reference :: value: 'E:SVTest->E1', node: 'E:SVTest->E4'", - "counterKey": "Sanity_DcidTableEntity" + "userMessage": "Malformed line without a colon delimiter :: line: ': dcs:SV1'", + "counterKey": "MCF_MalformedColonLessLine" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", + "location": { + "file": "NoCsv.mcf", + "lineNumber": "41" + }, + "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", + "counterKey": "Sanity_ObsMissingValueProp" + }, { + "level": "LEVEL_WARNING", "location": { "file": "NoCsv.tmcf", - "lineNumber": "32" + "lineNumber": "3" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E1', property: 'dcid' node: 'E:SVTest->E4'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Found multiple values for single-value property :: property: 'value', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_value" }] -} +} \ No newline at end of file From dd6a997261f5bfac881ede128db77f5fc80ca80d Mon Sep 17 00:00:00 2001 From: Prashanth Radhakrishnan Date: Fri, 27 Aug 2021 01:45:02 -0700 Subject: [PATCH 5/7] Improve tests --- .../java/org/datacommons/tool/GenMcfTest.java | 2 +- .../java/org/datacommons/tool/LintTest.java | 5 +- .../java/org/datacommons/tool/TestUtil.java | 23 +- .../tool/genmcf/fataltmcf/output/report.json | 112 ++--- .../genmcf/successtmcf/output/report.json | 118 ++--- .../tool/lint/allfiletypes/output/report.json | 423 +++++++++--------- .../tool/lint/mcfonly/output/report.json | 221 +++++---- .../tool/lint/nocsv/output/report.json | 321 +++++++------ 8 files changed, 605 insertions(+), 620 deletions(-) diff --git a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java index 96674ff8..e5007e1b 100644 --- a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java +++ b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java @@ -72,7 +72,7 @@ public void GenMcfTest() throws IOException { Files.copy(actualGeneratedFilePath, goldenGeneratedPath, REPLACE_EXISTING); Path goldenReportPath = Path.of(goldenFilesPrefix, "genmcf", directory.getName(), "output", "report.json"); - TestUtil.writeSortedReport(actualReportPath, goldenReportPath); + Files.copy(actualReportPath, goldenReportPath, REPLACE_EXISTING); } else { Path expectedGeneratedFilePath = TestUtil.getOutputFilePath(directory.getPath(), "generated.mcf"); diff --git a/tool/src/test/java/org/datacommons/tool/LintTest.java b/tool/src/test/java/org/datacommons/tool/LintTest.java index 025dae9f..46f13e16 100644 --- a/tool/src/test/java/org/datacommons/tool/LintTest.java +++ b/tool/src/test/java/org/datacommons/tool/LintTest.java @@ -1,7 +1,10 @@ package org.datacommons.tool; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -47,7 +50,7 @@ public void LintTest() throws IOException { if (goldenFilesPrefix != null && !goldenFilesPrefix.isEmpty()) { Path goldenPath = Path.of(goldenFilesPrefix, "lint", directory.getName(), "output", "report.json"); - TestUtil.writeSortedReport(actualReportPath, goldenPath); + Files.copy(actualReportPath, goldenPath, REPLACE_EXISTING); } else { Path expectedReportPath = TestUtil.getOutputFilePath(directory.getPath(), "report.json"); TestUtil.assertReportFilesAreSimilar( diff --git a/tool/src/test/java/org/datacommons/tool/TestUtil.java b/tool/src/test/java/org/datacommons/tool/TestUtil.java index 703b5aef..25033311 100644 --- a/tool/src/test/java/org/datacommons/tool/TestUtil.java +++ b/tool/src/test/java/org/datacommons/tool/TestUtil.java @@ -24,9 +24,11 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import org.apache.commons.io.FileUtils; -import org.apache.commons.text.StringEscapeUtils; import org.datacommons.proto.Debug; import org.junit.rules.TemporaryFolder; @@ -103,23 +105,6 @@ private static Debug.Log.Builder reportToProto(String report) return logBuilder; } - public static void writeSortedReport(Path inputPath, Path outputPath) throws IOException { - Debug.Log.Builder logBuilder = reportToProto(readStringFromPath(inputPath)); - List entries = new ArrayList<>(logBuilder.getEntriesList()); - Collections.sort( - entries, - new Comparator() { - @Override - public int compare(Debug.Log.Entry o1, Debug.Log.Entry o2) { - return o1.toString().compareTo(o2.toString()); - } - }); - logBuilder.clearEntries(); - logBuilder.addAllEntries(entries); - String jsonStr = StringEscapeUtils.unescapeJson(JsonFormat.printer().print(logBuilder.build())); - FileUtils.writeStringToFile(new File(outputPath.toString()), jsonStr, StandardCharsets.UTF_8); - } - public static Path getTestFilePath(TemporaryFolder testFolder, String fileName) throws IOException { return Paths.get(testFolder.getRoot().getPath(), fileName); diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json index f3f709c3..057b1564 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/fataltmcf/output/report.json @@ -36,105 +36,105 @@ "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "11" + "lineNumber": "5" }, - "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: '"E:SVTest->E1"'", - "counterKey": "TMCF_MalformedEntity" + "userMessage": "Malformed line without a colon delimiter :: line: ': dcs:SV1'", + "counterKey": "MCF_MalformedColonLessLine" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "11" }, - "userMessage": "Column referred to in TMCF is missing from CSV header :: column: 'dcid1', node: 'E:SVTest->E3'", - "counterKey": "Sanity_TmcfMissingColumn" + "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: '\"E:SVTest->E1\"'", + "counterKey": "TMCF_MalformedEntity" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "22" }, - "userMessage": "Found dcid with more than one value :: count: 3, node: 'E:SVTest->E3'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Malformed column value; must have a '->' delimiter :: value: 'C:SVTest>Place_Name', property: 'name', node: 'E:SVTest->E3'", + "counterKey": "TMCF_MalformedSchemaTerm" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "25" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'C', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: 'C:SVTest->NodeDcid'", + "counterKey": "TMCF_MalformedEntity" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "27" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'Dcid', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "TMCF properties cannot refer to CSV columns yet :: value: 'SVTest->Property: C:SVTest->Property_Value', property: 'C', node: 'E:SVTest->E3'", + "counterKey": "TMCF_UnsupportedColumnNameInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "29" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'E:SVTest->E5,E:SVTest->E6'", + "counterKey": "MCF_MalformedNodeName" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "19" + "lineNumber": "34" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'age', node: 'E:SVTest->E4'", + "counterKey": "MCF_MalformedComplexValue" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "22" + "lineNumber": "19" }, - "userMessage": "Malformed column value; must have a '->' delimiter :: value: 'C:SVTest>Place_Name', property: 'name', node: 'E:SVTest->E3'", - "counterKey": "TMCF_MalformedSchemaTerm" + "userMessage": "Found property name that does not start with a lower-case :: property: 'C', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "25" + "lineNumber": "19" }, - "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: 'C:SVTest->NodeDcid'", - "counterKey": "TMCF_MalformedEntity" + "userMessage": "Found dcid with more than one value :: count: 3, node: 'E:SVTest->E3'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "27" + "lineNumber": "19" }, - "userMessage": "TMCF properties cannot refer to CSV columns yet :: value: 'SVTest->Property: C:SVTest->Property_Value', property: 'C', node: 'E:SVTest->E3'", - "counterKey": "TMCF_UnsupportedColumnNameInProperty" + "userMessage": "Found property name that does not start with a lower-case :: property: 'Dcid', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "29" + "lineNumber": "19" }, - "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'E:SVTest->E5,E:SVTest->E6'", - "counterKey": "MCF_MalformedNodeName" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "3" + "lineNumber": "19" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationAbout', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_observationAbout" + "userMessage": "Column referred to in TMCF is missing from CSV header :: column: 'dcid1', node: 'E:SVTest->E3'", + "counterKey": "Sanity_TmcfMissingColumn" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "3" + "lineNumber": "19" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'", + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'", "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", @@ -150,32 +150,40 @@ "file": "FatalTmcf.tmcf", "lineNumber": "32" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E1', property: 'dcid' node: 'E:SVTest->E4'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Value of dcid property must not be an 'E:' reference :: value: 'E:SVTest->E1', node: 'E:SVTest->E4'", + "counterKey": "Sanity_DcidTableEntity" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", "lineNumber": "32" }, - "userMessage": "Value of dcid property must not be an 'E:' reference :: value: 'E:SVTest->E1', node: 'E:SVTest->E4'", - "counterKey": "Sanity_DcidTableEntity" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E1', property: 'dcid' node: 'E:SVTest->E4'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "34" + "lineNumber": "3" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'age', node: 'E:SVTest->E4'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "Found multiple values for single-value property :: property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_observationAbout" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "FatalTmcf.tmcf", + "lineNumber": "3" + }, + "userMessage": "Found multiple values for single-value property :: property: 'value', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_value" }, { "level": "LEVEL_ERROR", "location": { "file": "FatalTmcf.tmcf", - "lineNumber": "5" + "lineNumber": "3" }, - "userMessage": "Malformed line without a colon delimiter :: line: ': dcs:SV1'", - "counterKey": "MCF_MalformedColonLessLine" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_FATAL", "location": { @@ -184,13 +192,5 @@ }, "userMessage": "Found fatal sanity error in TMCF; check Sanity_ counter messages :: TMCF-file: FatalTmcf.tmcf", "counterKey": "CSV_TmcfCheckFailure" - }, { - "level": "LEVEL_WARNING", - "location": { - "file": "FatalTmcf.tmcf", - "lineNumber": "3" - }, - "userMessage": "Found multiple values for single-value property :: property: 'value', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_value" }] } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json index d1acac5b..fa248374 100644 --- a/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/genmcf/successtmcf/output/report.json @@ -33,49 +33,25 @@ "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "10" - }, - "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E0', type: 'StatVarObservation'", - "counterKey": "Sanity_MissingOrEmpty_observationAbout" - }, { - "level": "LEVEL_ERROR", - "location": { - "file": "SuccessTmcf.csv", - "lineNumber": "10" - }, - "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E1', type: 'StatVarObservation'", - "counterKey": "Sanity_MissingOrEmpty_observationAbout" - }, { - "level": "LEVEL_ERROR", - "location": { - "file": "SuccessTmcf.csv", - "lineNumber": "11" - }, - "userMessage": "Found a very long dcid value; must be less than 256 :: node: 'E:SVTest->E3'", - "counterKey": "Sanity_VeryLongDcid" - }, { - "level": "LEVEL_ERROR", - "location": { - "file": "SuccessTmcf.csv", - "lineNumber": "12" + "lineNumber": "2" }, - "userMessage": "Quantity value must be a number :: value: '[Lat Long]', property: 'location', node: 'dcid:CA-MN'", - "counterKey": "MCF_QuantityMalformedValue" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'dcid', node: 'E:SVTest->E3'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "2" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E0'", - "counterKey": "Sanity_InvalidObsDate" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "2" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E1'", + "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E0'", "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", @@ -83,32 +59,32 @@ "file": "SuccessTmcf.csv", "lineNumber": "2" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'dcid', node: 'E:SVTest->E3'", - "counterKey": "Sanity_InvalidChars_dcid" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E1'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "2" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E0'", - "counterKey": "Sanity_InvalidChars_observationAbout" + "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E1'", + "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "2" + "lineNumber": "3" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E1'", - "counterKey": "Sanity_InvalidChars_observationAbout" + "userMessage": "Found dcid with more than one value :: count: 2, node: 'E:SVTest->E3'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "3" + "lineNumber": "4" }, - "userMessage": "Found dcid with more than one value :: count: 2, node: 'E:SVTest->E3'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Found non-ascii characters in a value that is not text :: value: 'Administrative–Area1', type: 'RESOLVED_REF', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NonAsciiValueInNonText" }, { "level": "LEVEL_ERROR", "location": { @@ -118,13 +94,13 @@ "userMessage": "Found invalid chars in dcid value :: value: 'Administrative–Area1', invalid-chars: '–', property: 'typeOf', node: 'E:SVTest->E3'", "counterKey": "Sanity_InvalidChars_typeOf" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "4" + "lineNumber": "5" }, - "userMessage": "Found non-ascii characters in a value that is not text :: value: 'Administrative–Area1', type: 'RESOLVED_REF', property: 'typeOf', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NonAsciiValueInNonText" + "userMessage": "Empty value found :: value: '', column: 'Place_Type', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "StrSplit_EmptyToken_typeOf" }, { "level": "LEVEL_ERROR", "location": { @@ -147,32 +123,32 @@ "file": "SuccessTmcf.csv", "lineNumber": "7" }, - "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[]', components: 0, property: 'location', node: 'dcid:CA-VN'", - "counterKey": "MCF_MalformedComplexValueParts" + "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_observationDate" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "7" }, - "userMessage": "Empty value found :: value: '[]', property: 'location', node: 'dcid:CA-VN'", - "counterKey": "StrSplit_EmptyToken_location" + "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E1'", + "counterKey": "Sanity_MultipleVals_observationDate" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "7" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_observationDate" + "userMessage": "Empty value found :: value: '[]', property: 'location', node: 'dcid:CA-VN'", + "counterKey": "StrSplit_EmptyToken_location" }, { "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "7" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E1'", - "counterKey": "Sanity_MultipleVals_observationDate" + "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[]', components: 0, property: 'location', node: 'dcid:CA-VN'", + "counterKey": "MCF_MalformedComplexValueParts" }, { "level": "LEVEL_ERROR", "location": { @@ -189,6 +165,14 @@ }, "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'location', node: 'E:SVTest->E3'", "counterKey": "MCF_MalformedComplexValue" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "SuccessTmcf.csv", + "lineNumber": "10" + }, + "userMessage": "Malformed CSV value for dcid property; must be a text or reference :: value: '123', node: 'E:SVTest->E3'", + "counterKey": "CSV_MalformedDCIDFailures" }, { "level": "LEVEL_WARNING", "location": { @@ -197,6 +181,14 @@ }, "userMessage": "In dcid:{entity} reference, found {entity} to be empty :: property: 'observationAbout', node: 'E:SVTest->E0'", "counterKey": "CSV_EmptyDcidReferences" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "SuccessTmcf.csv", + "lineNumber": "10" + }, + "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E0', type: 'StatVarObservation'", + "counterKey": "Sanity_MissingOrEmpty_observationAbout" }, { "level": "LEVEL_WARNING", "location": { @@ -206,20 +198,28 @@ "userMessage": "In dcid:{entity} reference, found {entity} to be empty :: property: 'observationAbout', node: 'E:SVTest->E1'", "counterKey": "CSV_EmptyDcidReferences" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", "lineNumber": "10" }, - "userMessage": "Malformed CSV value for dcid property; must be a text or reference :: value: '123', node: 'E:SVTest->E3'", - "counterKey": "CSV_MalformedDCIDFailures" + "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E1', type: 'StatVarObservation'", + "counterKey": "Sanity_MissingOrEmpty_observationAbout" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { "file": "SuccessTmcf.csv", - "lineNumber": "5" + "lineNumber": "11" }, - "userMessage": "Empty value found :: value: '', column: 'Place_Type', property: 'typeOf', node: 'E:SVTest->E3'", - "counterKey": "StrSplit_EmptyToken_typeOf" + "userMessage": "Found a very long dcid value; must be less than 256 :: node: 'E:SVTest->E3'", + "counterKey": "Sanity_VeryLongDcid" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "SuccessTmcf.csv", + "lineNumber": "12" + }, + "userMessage": "Quantity value must be a number :: value: '[Lat Long]', property: 'location', node: 'dcid:CA-MN'", + "counterKey": "MCF_QuantityMalformedValue" }] } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json index ac2eb0ff..c92ccf8b 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/allfiletypes/output/report.json @@ -56,195 +56,195 @@ "entries": [{ "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "10" + "file": "AllFileTypes.mcf", + "lineNumber": "1" }, - "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E0', type: 'StatVarObservation'", - "counterKey": "Sanity_MissingOrEmpty_observationAbout" + "userMessage": "Property found without a preceding line with 'Node' :: line: 'typeOf: State'", + "counterKey": "MCF_UnexpectedProperty" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "10" + "file": "AllFileTypes.mcf", + "lineNumber": "8" }, - "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E1', type: 'StatVarObservation'", - "counterKey": "Sanity_MissingOrEmpty_observationAbout" + "userMessage": "Malformed line without a colon delimiter :: line: ': State'", + "counterKey": "MCF_MalformedColonLessLine" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "11" + "file": "AllFileTypes.mcf", + "lineNumber": "7" }, - "userMessage": "Found a very long dcid value; must be less than 256 :: node: 'E:SVTest->E3'", - "counterKey": "Sanity_VeryLongDcid" + "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", + "counterKey": "Mutator_MissingTypeOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "11" + "file": "AllFileTypes.mcf", + "lineNumber": "7" }, - "userMessage": "Quantity value must be a number :: value: '[Lat Long]', property: 'location', node: 'dcid:CA-MN'", - "counterKey": "MCF_QuantityMalformedValue" + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", + "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "2" + "file": "AllFileTypes.mcf", + "lineNumber": "15" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E0'", - "counterKey": "Sanity_InvalidObsDate" + "userMessage": "Found invalid chars in dcid value :: value: 'Statistical\nPopulation', invalid-chars: '\', property: 'typeOf', node: 'CityStats/E3/2'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "2" + "file": "AllFileTypes.mcf", + "lineNumber": "23" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E1'", + "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "2" + "file": "AllFileTypes.mcf", + "lineNumber": "32" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'dcid', node: 'E:SVTest->E3'", - "counterKey": "Sanity_InvalidChars_dcid" + "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", + "counterKey": "Sanity_NonDoubleObsValue" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "2" + "file": "AllFileTypes.mcf", + "lineNumber": "41" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E0'", - "counterKey": "Sanity_InvalidChars_observationAbout" + "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", + "counterKey": "Sanity_ObsMissingValueProp" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "2" + "file": "AllFileTypes.mcf", + "lineNumber": "54" }, - "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E1'", - "counterKey": "Sanity_InvalidChars_observationAbout" + "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", + "counterKey": "MCF_MalformedNodeName" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "3" + "file": "AllFileTypes.mcf", + "lineNumber": "49" }, - "userMessage": "Found dcid with more than one value :: count: 2, node: 'E:SVTest->E3'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "4" + "file": "AllFileTypes.mcf", + "lineNumber": "49" }, - "userMessage": "Found invalid chars in dcid value :: value: 'Administrative–Area1', invalid-chars: '–', property: 'typeOf', node: 'E:SVTest->E3'", - "counterKey": "Sanity_InvalidChars_typeOf" + "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_NonAsciiValueInNonText" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "4" + "file": "AllFileTypes.mcf", + "lineNumber": "49" }, - "userMessage": "Found non-ascii characters in a value that is not text :: value: 'Administrative–Area1', type: 'RESOLVED_REF', property: 'typeOf', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NonAsciiValueInNonText" + "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_location" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "5" + "file": "AllFileTypes.mcf", + "lineNumber": "67" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'E:SVTest->E3', type: 'Thing'", - "counterKey": "Sanity_MissingOrEmpty_typeOf" + "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '\"CANCountry\"'", + "counterKey": "MCF_InvalidLatitude" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "6" + "file": "AllFileTypes.mcf", + "lineNumber": "67" }, - "userMessage": "Found CSV row with different number of columns", - "counterKey": "CSV_InconsistentRows" + "userMessage": "Found dcid with more than one value :: count: 2, node: '\"CANCountry\"'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "7" + "file": "AllFileTypes.mcf", + "lineNumber": "73" }, - "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[]', components: 0, property: 'location', node: 'dcid:CA-VN'", - "counterKey": "MCF_MalformedComplexValueParts" + "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", + "counterKey": "MCF_InvalidLongitude" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "7" + "file": "AllFileTypes.mcf", + "lineNumber": "79" }, - "userMessage": "Empty value found :: value: '[]', property: 'location', node: 'dcid:CA-VN'", - "counterKey": "StrSplit_EmptyToken_location" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", + "counterKey": "MCF_MalformedComplexValue" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "7" + "file": "AllFileTypes.mcf", + "lineNumber": "77" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_observationDate" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "7" + "file": "AllFileTypes.mcf", + "lineNumber": "77" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E1'", - "counterKey": "Sanity_MultipleVals_observationDate" + "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_measuredProperty" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "8" + "file": "AllFileTypes.mcf", + "lineNumber": "77" }, - "userMessage": "Found CSV row with different number of columns", - "counterKey": "CSV_InconsistentRows" + "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_statType" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.csv", - "lineNumber": "9" + "file": "AllFileTypes.mcf", + "lineNumber": "77" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'location', node: 'E:SVTest->E3'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", + "counterKey": "Sanity_UnknownStatType" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "1" + "lineNumber": "81" }, - "userMessage": "Property found without a preceding line with 'Node' :: line: 'typeOf: State'", - "counterKey": "MCF_UnexpectedProperty" + "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", + "counterKey": "MCF_MalformedComplexValueParts" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "102" + "lineNumber": "88" }, - "userMessage": "Malformed end component in QuantityRange value; must be a number or '-' :: value: 'years', property: 'age', node: 'dcid:Count_Death_5YearsPlus'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", + "counterKey": "MCF_QuantityMalformedValue" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "109" + "lineNumber": "95" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "109" + "lineNumber": "102" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'PopulationType', node: 'dcid:Count_Death_15YearsPlus'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "Malformed end component in QuantityRange value; must be a number or '-' :: value: 'years', property: 'age', node: 'dcid:Count_Death_5YearsPlus'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { @@ -257,18 +257,18 @@ "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "116" + "lineNumber": "109" }, - "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", - "counterKey": "Sanity_NotInitLower_labelInProperty" + "userMessage": "Found property name that does not start with a lower-case :: property: 'PopulationType', node: 'dcid:Count_Death_15YearsPlus'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "116" + "lineNumber": "109" }, - "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", - "counterKey": "Sanity_EmptySchemaValue" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { @@ -283,47 +283,39 @@ "file": "AllFileTypes.mcf", "lineNumber": "116" }, - "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", - "counterKey": "Sanity_NonAsciiValueInSchema" + "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", + "counterKey": "Sanity_UnexpectedPropInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", "lineNumber": "116" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", - "counterKey": "Sanity_DcidNameMismatchInSchema" + "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", + "counterKey": "Sanity_NonAsciiValueInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", "lineNumber": "116" }, - "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", - "counterKey": "Sanity_UnexpectedPropInProperty" - }, { - "level": "LEVEL_ERROR", - "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "123" - }, - "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", - "counterKey": "Sanity_NotInitUpper_nameInClass" + "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", + "counterKey": "Sanity_NotInitLower_labelInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "123" + "lineNumber": "116" }, - "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", - "counterKey": "Sanity_MissingOrEmpty_subClassOf" + "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", + "counterKey": "Sanity_EmptySchemaValue" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "123" + "lineNumber": "116" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", + "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", @@ -337,171 +329,170 @@ "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "15" + "lineNumber": "123" }, - "userMessage": "Found invalid chars in dcid value :: value: 'Statistical -Population', invalid-chars: '', property: 'typeOf', node: 'CityStats/E3/2'", - "counterKey": "Sanity_InvalidChars_typeOf" + "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", + "counterKey": "Sanity_NotInitUpper_nameInClass" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "23" + "lineNumber": "123" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", - "counterKey": "Sanity_InvalidObsDate" + "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.mcf", - "lineNumber": "32" + "lineNumber": "123" }, - "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", - "counterKey": "Sanity_NonDoubleObsValue" + "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", + "counterKey": "Sanity_MissingOrEmpty_subClassOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "49" + "file": "AllFileTypes.csv", + "lineNumber": "2" }, - "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'dcid', node: 'E:SVTest->E3'", "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "49" + "file": "AllFileTypes.csv", + "lineNumber": "2" }, - "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_location" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "49" + "file": "AllFileTypes.csv", + "lineNumber": "2" }, - "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_NonAsciiValueInNonText" + "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E0'", + "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "54" + "file": "AllFileTypes.csv", + "lineNumber": "2" }, - "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", - "counterKey": "MCF_MalformedNodeName" + "userMessage": "Found invalid chars in dcid value :: value: 'US- CA', invalid-chars: ' ', property: 'observationAbout', node: 'E:SVTest->E1'", + "counterKey": "Sanity_InvalidChars_observationAbout" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "67" + "file": "AllFileTypes.csv", + "lineNumber": "2" }, - "userMessage": "Found dcid with more than one value :: count: 2, node: '"CANCountry"'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Found a non-ISO8601 compliant date value :: value: '19', property: 'observationDate', node: 'E:SVTest->E1'", + "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "67" + "file": "AllFileTypes.csv", + "lineNumber": "3" }, - "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '"CANCountry"'", - "counterKey": "MCF_InvalidLatitude" + "userMessage": "Found dcid with more than one value :: count: 2, node: 'E:SVTest->E3'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "7" + "file": "AllFileTypes.csv", + "lineNumber": "4" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", - "counterKey": "Sanity_MissingOrEmpty_typeOf" + "userMessage": "Found non-ascii characters in a value that is not text :: value: 'Administrative–Area1', type: 'RESOLVED_REF', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NonAsciiValueInNonText" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "7" + "file": "AllFileTypes.csv", + "lineNumber": "4" }, - "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", - "counterKey": "Mutator_MissingTypeOf" + "userMessage": "Found invalid chars in dcid value :: value: 'Administrative–Area1', invalid-chars: '–', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "73" + "file": "AllFileTypes.csv", + "lineNumber": "5" }, - "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", - "counterKey": "MCF_InvalidLongitude" + "userMessage": "Empty value found :: value: '', column: 'Place_Type', property: 'typeOf', node: 'E:SVTest->E3'", + "counterKey": "StrSplit_EmptyToken_typeOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "77" + "file": "AllFileTypes.csv", + "lineNumber": "5" }, - "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_measuredProperty" + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'E:SVTest->E3', type: 'Thing'", + "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "77" + "file": "AllFileTypes.csv", + "lineNumber": "6" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Found CSV row with different number of columns", + "counterKey": "CSV_InconsistentRows" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "77" + "file": "AllFileTypes.csv", + "lineNumber": "7" }, - "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_statType" + "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_observationDate" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "77" + "file": "AllFileTypes.csv", + "lineNumber": "7" }, - "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", - "counterKey": "Sanity_UnknownStatType" + "userMessage": "Found multiple values for single-value property :: property: 'observationDate', column: 'Year', node: 'E:SVTest->E1'", + "counterKey": "Sanity_MultipleVals_observationDate" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "79" + "file": "AllFileTypes.csv", + "lineNumber": "7" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "Empty value found :: value: '[]', property: 'location', node: 'dcid:CA-VN'", + "counterKey": "StrSplit_EmptyToken_location" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "8" + "file": "AllFileTypes.csv", + "lineNumber": "7" }, - "userMessage": "Malformed line without a colon delimiter :: line: ': State'", - "counterKey": "MCF_MalformedColonLessLine" + "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[]', components: 0, property: 'location', node: 'dcid:CA-VN'", + "counterKey": "MCF_MalformedComplexValueParts" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "81" + "file": "AllFileTypes.csv", + "lineNumber": "8" }, - "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", - "counterKey": "MCF_MalformedComplexValueParts" + "userMessage": "Found CSV row with different number of columns", + "counterKey": "CSV_InconsistentRows" }, { "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "88" + "file": "AllFileTypes.csv", + "lineNumber": "9" }, - "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", - "counterKey": "MCF_QuantityMalformedValue" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'location', node: 'E:SVTest->E3'", + "counterKey": "MCF_MalformedComplexValue" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "95" + "file": "AllFileTypes.csv", + "lineNumber": "10" }, - "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Malformed CSV value for dcid property; must be a text or reference :: value: '123', node: 'E:SVTest->E3'", + "counterKey": "CSV_MalformedDCIDFailures" }, { "level": "LEVEL_WARNING", "location": { @@ -510,6 +501,14 @@ Population', invalid-chars: '', property: 'typeOf', node: 'CityStats/E3/2'", }, "userMessage": "In dcid:{entity} reference, found {entity} to be empty :: property: 'observationAbout', node: 'E:SVTest->E0'", "counterKey": "CSV_EmptyDcidReferences" + }, { + "level": "LEVEL_ERROR", + "location": { + "file": "AllFileTypes.csv", + "lineNumber": "10" + }, + "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E0', type: 'StatVarObservation'", + "counterKey": "Sanity_MissingOrEmpty_observationAbout" }, { "level": "LEVEL_WARNING", "location": { @@ -519,28 +518,28 @@ Population', invalid-chars: '', property: 'typeOf', node: 'CityStats/E3/2'", "userMessage": "In dcid:{entity} reference, found {entity} to be empty :: property: 'observationAbout', node: 'E:SVTest->E1'", "counterKey": "CSV_EmptyDcidReferences" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.csv", "lineNumber": "10" }, - "userMessage": "Malformed CSV value for dcid property; must be a text or reference :: value: '123', node: 'E:SVTest->E3'", - "counterKey": "CSV_MalformedDCIDFailures" + "userMessage": "Found a missing or empty property value :: property: 'observationAbout', node: 'E:SVTest->E1', type: 'StatVarObservation'", + "counterKey": "Sanity_MissingOrEmpty_observationAbout" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { "file": "AllFileTypes.csv", - "lineNumber": "5" + "lineNumber": "11" }, - "userMessage": "Empty value found :: value: '', column: 'Place_Type', property: 'typeOf', node: 'E:SVTest->E3'", - "counterKey": "StrSplit_EmptyToken_typeOf" + "userMessage": "Found a very long dcid value; must be less than 256 :: node: 'E:SVTest->E3'", + "counterKey": "Sanity_VeryLongDcid" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { - "file": "AllFileTypes.mcf", - "lineNumber": "41" + "file": "AllFileTypes.csv", + "lineNumber": "11" }, - "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", - "counterKey": "Sanity_ObsMissingValueProp" + "userMessage": "Quantity value must be a number :: value: '[Lat Long]', property: 'location', node: 'dcid:CA-MN'", + "counterKey": "MCF_QuantityMalformedValue" }] } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json index d31ed5d3..9044b967 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/mcfonly/output/report.json @@ -52,290 +52,289 @@ "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "102" + "lineNumber": "8" }, - "userMessage": "Malformed end component in QuantityRange value; must be a number or '-' :: value: 'years', property: 'age', node: 'dcid:Count_Death_5YearsPlus'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Malformed line without a colon delimiter :: line: ': State'", + "counterKey": "MCF_MalformedColonLessLine" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "109" + "lineNumber": "7" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", + "counterKey": "Mutator_MissingTypeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "109" + "lineNumber": "7" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'PopulationType', node: 'dcid:Count_Death_15YearsPlus'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", + "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "109" + "lineNumber": "15" }, - "userMessage": "Malformed start+end components in QuantityRange value; one of them must be a number :: startValue: '-', endValue: '-', property: 'age', node: 'dcid:Count_Death_15YearsPlus'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Found invalid chars in dcid value :: value: 'Statistical\nPopulation', invalid-chars: '\', property: 'typeOf', node: 'CityStats/E3/2'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "23" }, - "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", - "counterKey": "Sanity_NotInitLower_labelInProperty" + "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", + "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "32" }, - "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", - "counterKey": "Sanity_EmptySchemaValue" + "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", + "counterKey": "Sanity_NonDoubleObsValue" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "41" }, - "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", - "counterKey": "Sanity_InvalidChars_domainIncludes" + "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", + "counterKey": "Sanity_ObsMissingValueProp" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "54" }, - "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", - "counterKey": "Sanity_NonAsciiValueInSchema" + "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", + "counterKey": "MCF_MalformedNodeName" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "49" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", - "counterKey": "Sanity_DcidNameMismatchInSchema" + "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "116" + "lineNumber": "49" }, - "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", - "counterKey": "Sanity_UnexpectedPropInProperty" + "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_NonAsciiValueInNonText" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "123" + "lineNumber": "49" }, - "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", - "counterKey": "Sanity_NotInitUpper_nameInClass" + "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_location" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "123" + "lineNumber": "67" }, - "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", - "counterKey": "Sanity_MissingOrEmpty_subClassOf" + "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '\"CANCountry\"'", + "counterKey": "MCF_InvalidLatitude" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "123" + "lineNumber": "67" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", - "counterKey": "Sanity_DcidNameMismatchInSchema" + "userMessage": "Found dcid with more than one value :: count: 2, node: '\"CANCountry\"'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "123" + "lineNumber": "73" }, - "userMessage": "Unexpected property in Class node :: property: 'domainIncludes', node: 'dcid:Pop'", - "counterKey": "Sanity_UnexpectedPropInClass" + "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", + "counterKey": "MCF_InvalidLongitude" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "15" + "lineNumber": "79" }, - "userMessage": "Found invalid chars in dcid value :: value: 'Statistical -Population', invalid-chars: '', property: 'typeOf', node: 'CityStats/E3/2'", - "counterKey": "Sanity_InvalidChars_typeOf" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", + "counterKey": "MCF_MalformedComplexValue" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "23" + "lineNumber": "77" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", - "counterKey": "Sanity_InvalidObsDate" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "32" + "lineNumber": "77" }, - "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", - "counterKey": "Sanity_NonDoubleObsValue" + "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_measuredProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "49" + "lineNumber": "77" }, - "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_dcid" + "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_statType" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "49" + "lineNumber": "77" }, - "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_location" + "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", + "counterKey": "Sanity_UnknownStatType" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "49" + "lineNumber": "81" }, - "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_NonAsciiValueInNonText" + "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", + "counterKey": "MCF_MalformedComplexValueParts" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "54" + "lineNumber": "88" }, - "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", - "counterKey": "MCF_MalformedNodeName" + "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", + "counterKey": "MCF_QuantityMalformedValue" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "67" + "lineNumber": "95" }, - "userMessage": "Found dcid with more than one value :: count: 2, node: '"CANCountry"'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "67" + "lineNumber": "102" }, - "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '"CANCountry"'", - "counterKey": "MCF_InvalidLatitude" + "userMessage": "Malformed end component in QuantityRange value; must be a number or '-' :: value: 'years', property: 'age', node: 'dcid:Count_Death_5YearsPlus'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "7" + "lineNumber": "109" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", - "counterKey": "Sanity_MissingOrEmpty_typeOf" + "userMessage": "Malformed start+end components in QuantityRange value; one of them must be a number :: startValue: '-', endValue: '-', property: 'age', node: 'dcid:Count_Death_15YearsPlus'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "7" + "lineNumber": "109" }, - "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", - "counterKey": "Mutator_MissingTypeOf" + "userMessage": "Found property name that does not start with a lower-case :: property: 'PopulationType', node: 'dcid:Count_Death_15YearsPlus'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "73" + "lineNumber": "109" }, - "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", - "counterKey": "MCF_InvalidLongitude" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "77" + "lineNumber": "116" }, - "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_measuredProperty" + "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", + "counterKey": "Sanity_InvalidChars_domainIncludes" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "77" + "lineNumber": "116" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", + "counterKey": "Sanity_UnexpectedPropInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "77" + "lineNumber": "116" }, - "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_statType" + "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", + "counterKey": "Sanity_NonAsciiValueInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "77" + "lineNumber": "116" }, - "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", - "counterKey": "Sanity_UnknownStatType" + "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", + "counterKey": "Sanity_NotInitLower_labelInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "79" + "lineNumber": "116" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", + "counterKey": "Sanity_EmptySchemaValue" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "8" + "lineNumber": "116" }, - "userMessage": "Malformed line without a colon delimiter :: line: ': State'", - "counterKey": "MCF_MalformedColonLessLine" + "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "81" + "lineNumber": "123" }, - "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", - "counterKey": "MCF_MalformedComplexValueParts" + "userMessage": "Unexpected property in Class node :: property: 'domainIncludes', node: 'dcid:Pop'", + "counterKey": "Sanity_UnexpectedPropInClass" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "88" + "lineNumber": "123" }, - "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", - "counterKey": "MCF_QuantityMalformedValue" + "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", + "counterKey": "Sanity_NotInitUpper_nameInClass" }, { "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "95" + "lineNumber": "123" }, - "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { "file": "McfOnly.mcf", - "lineNumber": "41" + "lineNumber": "123" }, - "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", - "counterKey": "Sanity_ObsMissingValueProp" + "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", + "counterKey": "Sanity_MissingOrEmpty_subClassOf" }] } \ No newline at end of file diff --git a/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json b/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json index 1b90c99c..2ea8589a 100644 --- a/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json +++ b/tool/src/test/resources/org/datacommons/tool/lint/nocsv/output/report.json @@ -59,283 +59,290 @@ "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "102" + "lineNumber": "8" }, - "userMessage": "Malformed end component in QuantityRange value; must be a number or '-' :: value: 'years', property: 'age', node: 'dcid:Count_Death_5YearsPlus'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Malformed line without a colon delimiter :: line: ': State'", + "counterKey": "MCF_MalformedColonLessLine" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "109" + "lineNumber": "7" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", + "counterKey": "Mutator_MissingTypeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "109" + "lineNumber": "7" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'PopulationType', node: 'dcid:Count_Death_15YearsPlus'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", + "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "109" + "lineNumber": "15" }, - "userMessage": "Malformed start+end components in QuantityRange value; one of them must be a number :: startValue: '-', endValue: '-', property: 'age', node: 'dcid:Count_Death_15YearsPlus'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Found invalid chars in dcid value :: value: 'Statistical\nPopulation', invalid-chars: '\', property: 'typeOf', node: 'CityStats/E3/2'", + "counterKey": "Sanity_InvalidChars_typeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "23" }, - "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", - "counterKey": "Sanity_NotInitLower_labelInProperty" + "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", + "counterKey": "Sanity_InvalidObsDate" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "32" }, - "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", - "counterKey": "Sanity_EmptySchemaValue" + "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", + "counterKey": "Sanity_NonDoubleObsValue" + }, { + "level": "LEVEL_WARNING", + "location": { + "file": "NoCsv.mcf", + "lineNumber": "41" + }, + "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", + "counterKey": "Sanity_ObsMissingValueProp" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "54" }, - "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", - "counterKey": "Sanity_InvalidChars_domainIncludes" + "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", + "counterKey": "MCF_MalformedNodeName" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "49" }, - "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", - "counterKey": "Sanity_NonAsciiValueInSchema" + "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_dcid" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "49" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", - "counterKey": "Sanity_DcidNameMismatchInSchema" + "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_NonAsciiValueInNonText" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "116" + "lineNumber": "49" }, - "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", - "counterKey": "Sanity_UnexpectedPropInProperty" + "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", + "counterKey": "Sanity_InvalidChars_location" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "123" + "lineNumber": "67" }, - "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", - "counterKey": "Sanity_NotInitUpper_nameInClass" + "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '\"CANCountry\"'", + "counterKey": "MCF_InvalidLatitude" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "123" + "lineNumber": "67" }, - "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", - "counterKey": "Sanity_MissingOrEmpty_subClassOf" + "userMessage": "Found dcid with more than one value :: count: 2, node: '\"CANCountry\"'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "123" + "lineNumber": "73" }, - "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", - "counterKey": "Sanity_DcidNameMismatchInSchema" + "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", + "counterKey": "MCF_InvalidLongitude" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "123" + "lineNumber": "79" }, - "userMessage": "Unexpected property in Class node :: property: 'domainIncludes', node: 'dcid:Pop'", - "counterKey": "Sanity_UnexpectedPropInClass" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", + "counterKey": "MCF_MalformedComplexValue" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "15" + "lineNumber": "77" }, - "userMessage": "Found invalid chars in dcid value :: value: 'Statistical -Population', invalid-chars: '', property: 'typeOf', node: 'CityStats/E3/2'", - "counterKey": "Sanity_InvalidChars_typeOf" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "23" + "lineNumber": "77" }, - "userMessage": "Found a non-ISO8601 compliant date value :: value: '7', property: 'observationDate', node: 'CityStats/E4/2'", - "counterKey": "Sanity_InvalidObsDate" + "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_measuredProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "32" + "lineNumber": "77" }, - "userMessage": "Found a non-double Observation value :: value: 'onethousand', property: 'measuredValue', node: 'CityStats/E4/3'", - "counterKey": "Sanity_NonDoubleObsValue" + "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_statType" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "49" + "lineNumber": "77" }, - "userMessage": "Found invalid chars in dcid value :: value: 'dc/ 2sffw13', invalid-chars: ' ', property: 'dcid', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_dcid" + "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", + "counterKey": "Sanity_UnknownStatType" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "49" + "lineNumber": "81" }, - "userMessage": "Found invalid chars in dcid value :: value: 'geoId–SFCounty', invalid-chars: '–', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_InvalidChars_location" + "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", + "counterKey": "MCF_MalformedComplexValueParts" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "49" + "lineNumber": "88" }, - "userMessage": "Found non-ascii characters in a value that is not text :: value: 'geoId–SFCounty', type: 'RESOLVED_REF', property: 'location', node: 'CityStats/E5/2'", - "counterKey": "Sanity_NonAsciiValueInNonText" + "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", + "counterKey": "MCF_QuantityMalformedValue" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "54" + "lineNumber": "95" }, - "userMessage": "Found malformed Node value with a comma; must be a unary value :: node: 'USA,Country'", - "counterKey": "MCF_MalformedNodeName" + "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "67" + "lineNumber": "102" }, - "userMessage": "Found dcid with more than one value :: count: 2, node: '"CANCountry"'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Malformed end component in QuantityRange value; must be a number or '-' :: value: 'years', property: 'age', node: 'dcid:Count_Death_5YearsPlus'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "67" + "lineNumber": "109" }, - "userMessage": "Invalid latitude value; must be decimal degrees with an optional N/S suffix :: value: '-100', property: 'location', node: '"CANCountry"'", - "counterKey": "MCF_InvalidLatitude" + "userMessage": "Malformed start+end components in QuantityRange value; one of them must be a number :: startValue: '-', endValue: '-', property: 'age', node: 'dcid:Count_Death_15YearsPlus'", + "counterKey": "MCF_QuantityRangeMalformedValues" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "7" + "lineNumber": "109" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'dcid:geoId/CA', type: 'Thing'", - "counterKey": "Sanity_MissingOrEmpty_typeOf" + "userMessage": "Found property name that does not start with a lower-case :: property: 'PopulationType', node: 'dcid:Count_Death_15YearsPlus'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "7" + "lineNumber": "109" }, - "userMessage": "Missing typeOf value for node :: node: 'dcid:geoId/CA'", - "counterKey": "Mutator_MissingTypeOf" + "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_15YearsPlus', type: 'StatisticalVariable'", + "counterKey": "Sanity_MissingOrEmpty_populationType" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "73" + "lineNumber": "116" }, - "userMessage": "Invalid longitude value; must be decimal degrees with an optional E/W suffix :: value: '181', property: 'location', node: 'dcid:geoId/sf'", - "counterKey": "MCF_InvalidLongitude" + "userMessage": "Found invalid chars in dcid value :: value: '', invalid-chars: '', property: 'domainIncludes', node: 'dcid:beverageType'", + "counterKey": "Sanity_InvalidChars_domainIncludes" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "77" + "lineNumber": "116" }, - "userMessage": "Found a missing or empty property value :: property: 'measuredProperty', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_measuredProperty" + "userMessage": "Unexpected property in Property node :: property: 'subClassOf', node: 'dcid:beverageType'", + "counterKey": "Sanity_UnexpectedPropInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "77" + "lineNumber": "116" }, - "userMessage": "Found a missing or empty property value :: property: 'populationType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_populationType" + "userMessage": "Schema node has property values with non-ascii characters :: value: 'Beverage–type', property: 'name', node: 'dcid:beverageType'", + "counterKey": "Sanity_NonAsciiValueInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "77" + "lineNumber": "116" }, - "userMessage": "Found a missing or empty property value :: property: 'statType', node: 'dcid:Count_Death_10To12', type: 'StatisticalVariable'", - "counterKey": "Sanity_MissingOrEmpty_statType" + "userMessage": "Found a property reference that does not start with a lower-case :: reference: 'BeverageType', property: 'label, node: 'dcid:beverageType'", + "counterKey": "Sanity_NotInitLower_labelInProperty" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "77" + "lineNumber": "116" }, - "userMessage": "Found an unknown statType value :: value: '', node: 'dcid:Count_Death_10To12'", - "counterKey": "Sanity_UnknownStatType" + "userMessage": "Found empty property value :: property: 'domainIncludes', node 'dcid:beverageType'", + "counterKey": "Sanity_EmptySchemaValue" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "79" + "lineNumber": "116" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[10 12', property: 'age', node: 'dcid:Count_Death_10To12'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "Schema node with dcid/name mismatch :: name: 'Beverage–type', dcid: 'beverageType', node: 'dcid:beverageType'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "8" + "lineNumber": "123" }, - "userMessage": "Malformed line without a colon delimiter :: line: ': State'", - "counterKey": "MCF_MalformedColonLessLine" + "userMessage": "Unexpected property in Class node :: property: 'domainIncludes', node: 'dcid:Pop'", + "counterKey": "Sanity_UnexpectedPropInClass" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "81" + "lineNumber": "123" }, - "userMessage": "Complex value must have 2 (e.g., [Years 10]) or 3 (e.g., [Years 10 20]) components :: value: '[10]', components: 1, property: 'age', node: 'dcid:Count_Death_10YearsPlus'", - "counterKey": "MCF_MalformedComplexValueParts" + "userMessage": "Found a class reference that does not start with an upper-case :: reference: 'pop', property: 'name, node: 'dcid:Pop'", + "counterKey": "Sanity_NotInitUpper_nameInClass" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "88" + "lineNumber": "123" }, - "userMessage": "Quantity value must be a number :: value: '[5Years 10Years]', property: 'age', node: 'dcid:Count_Death_5To10'", - "counterKey": "MCF_QuantityMalformedValue" + "userMessage": "Schema node with dcid/name mismatch :: name: 'pop', dcid: 'Pop', node: 'dcid:Pop'", + "counterKey": "Sanity_DcidNameMismatchInSchema" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.mcf", - "lineNumber": "95" + "lineNumber": "123" }, - "userMessage": "Malformed start component in QuantityRange value; must be a number or '-' :: value: 'less', property: 'age', node: 'dcid:Count_Death_YearsLessThan5'", - "counterKey": "MCF_QuantityRangeMalformedValues" + "userMessage": "Found a missing or empty property value :: property: 'subClassOf', node: 'dcid:Pop', type: 'Class'", + "counterKey": "Sanity_MissingOrEmpty_subClassOf" }, { "level": "LEVEL_ERROR", "location": { @@ -348,49 +355,49 @@ Population', invalid-chars: '', property: 'typeOf', node: 'CityStats/E3/2'", "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "11" + "lineNumber": "5" }, - "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: '"E:SVTest->E1"'", - "counterKey": "TMCF_MalformedEntity" + "userMessage": "Malformed line without a colon delimiter :: line: ': dcs:SV1'", + "counterKey": "MCF_MalformedColonLessLine" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "19" + "lineNumber": "11" }, - "userMessage": "Found dcid with more than one value :: count: 3, node: 'E:SVTest->E3'", - "counterKey": "Sanity_MultipleDcidValues" + "userMessage": "Found malformed entity name that is not an entity prefix (E:) :: name: '\"E:SVTest->E1\"'", + "counterKey": "TMCF_MalformedEntity" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "19" + "lineNumber": "3" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'C', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "Found multiple values for single-value property :: property: 'observationAbout', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_observationAbout" }, { - "level": "LEVEL_ERROR", + "level": "LEVEL_WARNING", "location": { "file": "NoCsv.tmcf", - "lineNumber": "19" + "lineNumber": "3" }, - "userMessage": "Found property name that does not start with a lower-case :: property: 'Dcid', node: 'E:SVTest->E3'", - "counterKey": "Sanity_NotInitLowerPropName" + "userMessage": "Found multiple values for single-value property :: property: 'value', node: 'E:SVTest->E0'", + "counterKey": "Sanity_MultipleVals_value" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "19" + "lineNumber": "3" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'", + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'", "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "19" + "lineNumber": "3" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'", + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E3', property: 'observationAbout' node: 'E:SVTest->E0'", "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", @@ -428,89 +435,81 @@ Population', invalid-chars: '', property: 'typeOf', node: 'CityStats/E3/2'", "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "3" + "lineNumber": "19" }, - "userMessage": "Found multiple values for single-value property :: property: 'observationAbout', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_observationAbout" + "userMessage": "Found property name that does not start with a lower-case :: property: 'C', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "3" + "lineNumber": "19" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E30', property: 'observationAbout' node: 'E:SVTest->E0'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Found dcid with more than one value :: count: 3, node: 'E:SVTest->E3'", + "counterKey": "Sanity_MultipleDcidValues" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "3" + "lineNumber": "19" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E3', property: 'observationAbout' node: 'E:SVTest->E0'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Found property name that does not start with a lower-case :: property: 'Dcid', node: 'E:SVTest->E3'", + "counterKey": "Sanity_NotInitLowerPropName" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "32" + "lineNumber": "19" }, - "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'E:SVTest->E4', type: 'Thing'", - "counterKey": "Sanity_MissingOrEmpty_typeOf" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E10', property: 'dcid' node: 'E:SVTest->E3'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "32" + "lineNumber": "19" }, - "userMessage": "Missing typeOf value for node :: node: 'E:SVTest->E4'", - "counterKey": "Mutator_MissingTypeOf" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E13', property: 'dcid' node: 'E:SVTest->E3'", + "counterKey": "Sanity_TmcfMissingEntityDef" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "32" + "lineNumber": "34" }, - "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E1', property: 'dcid' node: 'E:SVTest->E4'", - "counterKey": "Sanity_TmcfMissingEntityDef" + "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'age', node: 'E:SVTest->E4'", + "counterKey": "MCF_MalformedComplexValue" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", "lineNumber": "32" }, - "userMessage": "Value of dcid property must not be an 'E:' reference :: value: 'E:SVTest->E1', node: 'E:SVTest->E4'", - "counterKey": "Sanity_DcidTableEntity" + "userMessage": "Missing typeOf value for node :: node: 'E:SVTest->E4'", + "counterKey": "Mutator_MissingTypeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "34" + "lineNumber": "32" }, - "userMessage": "Found malformed Complex value without a closing ] bracket :: value: '[', property: 'age', node: 'E:SVTest->E4'", - "counterKey": "MCF_MalformedComplexValue" + "userMessage": "Found a missing or empty property value :: property: 'typeOf', node: 'E:SVTest->E4', type: 'Thing'", + "counterKey": "Sanity_MissingOrEmpty_typeOf" }, { "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "5" - }, - "userMessage": "Malformed line without a colon delimiter :: line: ': dcs:SV1'", - "counterKey": "MCF_MalformedColonLessLine" - }, { - "level": "LEVEL_WARNING", - "location": { - "file": "NoCsv.mcf", - "lineNumber": "41" + "lineNumber": "32" }, - "userMessage": "Observation node missing value property :: node: 'CityStats/E4/4'", - "counterKey": "Sanity_ObsMissingValueProp" + "userMessage": "Value of dcid property must not be an 'E:' reference :: value: 'E:SVTest->E1', node: 'E:SVTest->E4'", + "counterKey": "Sanity_DcidTableEntity" }, { - "level": "LEVEL_WARNING", + "level": "LEVEL_ERROR", "location": { "file": "NoCsv.tmcf", - "lineNumber": "3" + "lineNumber": "32" }, - "userMessage": "Found multiple values for single-value property :: property: 'value', node: 'E:SVTest->E0'", - "counterKey": "Sanity_MultipleVals_value" + "userMessage": "No definition found for a referenced 'E:' value :: reference: 'E:SVTest->E1', property: 'dcid' node: 'E:SVTest->E4'", + "counterKey": "Sanity_TmcfMissingEntityDef" }] } \ No newline at end of file From 18eea446db434470e0d1d8a413dc7eb1ccb290e1 Mon Sep 17 00:00:00 2001 From: Prashanth Radhakrishnan Date: Fri, 27 Aug 2021 01:48:49 -0700 Subject: [PATCH 6/7] Improve tests --- tool/src/test/java/org/datacommons/tool/GenMcfTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java index e5007e1b..b3c35379 100644 --- a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java +++ b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java @@ -46,8 +46,6 @@ public void GenMcfTest() throws IOException { TmcfCsvParser.TEST_mode = true; String goldenFilesPrefix = System.getProperty("goldenFilesPrefix"); - goldenFilesPrefix = - "/Users/shanth/work/git/import/tool/src/test/resources/org/datacommons" + "/tool"; Main app = new Main(); CommandLine cmd = new CommandLine(app); File[] testDirectories = new File(resourceFile("genmcf")).listFiles(File::isDirectory); From aab8a398966593acb6cab20363a158891f28d0b9 Mon Sep 17 00:00:00 2001 From: Prashanth Radhakrishnan Date: Fri, 27 Aug 2021 10:42:04 -0700 Subject: [PATCH 7/7] Address PR comments --- .../java/org/datacommons/tool/GenMcfTest.java | 3 + .../java/org/datacommons/tool/LintTest.java | 3 + .../java/org/datacommons/tool/TestUtil.java | 79 +++---------------- 3 files changed, 19 insertions(+), 66 deletions(-) diff --git a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java index b3c35379..770ff2bd 100644 --- a/tool/src/test/java/org/datacommons/tool/GenMcfTest.java +++ b/tool/src/test/java/org/datacommons/tool/GenMcfTest.java @@ -17,6 +17,7 @@ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static org.junit.Assert.assertEquals; +import com.google.common.truth.Expect; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -39,6 +40,7 @@ // public class GenMcfTest { @Rule public TemporaryFolder testFolder = new TemporaryFolder(); + @Rule public final Expect expect = Expect.create(); @Test public void GenMcfTest() throws IOException { @@ -76,6 +78,7 @@ public void GenMcfTest() throws IOException { TestUtil.getOutputFilePath(directory.getPath(), "generated.mcf"); Path expectedReportPath = TestUtil.getOutputFilePath(directory.getPath(), "report.json"); TestUtil.assertReportFilesAreSimilar( + expect, directory, TestUtil.readStringFromPath(expectedReportPath), TestUtil.readStringFromPath(actualReportPath)); diff --git a/tool/src/test/java/org/datacommons/tool/LintTest.java b/tool/src/test/java/org/datacommons/tool/LintTest.java index 46f13e16..6b3a6b1e 100644 --- a/tool/src/test/java/org/datacommons/tool/LintTest.java +++ b/tool/src/test/java/org/datacommons/tool/LintTest.java @@ -2,6 +2,7 @@ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import com.google.common.truth.Expect; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -25,6 +26,7 @@ // TODO(shanth): Incorporate e2e test-cases for existence checks once this is generalized. public class LintTest { @Rule public TemporaryFolder testFolder = new TemporaryFolder(); + @Rule public final Expect expect = Expect.create(); @Test public void LintTest() throws IOException { @@ -54,6 +56,7 @@ public void LintTest() throws IOException { } else { Path expectedReportPath = TestUtil.getOutputFilePath(directory.getPath(), "report.json"); TestUtil.assertReportFilesAreSimilar( + expect, directory, TestUtil.readStringFromPath(expectedReportPath), TestUtil.readStringFromPath(actualReportPath)); diff --git a/tool/src/test/java/org/datacommons/tool/TestUtil.java b/tool/src/test/java/org/datacommons/tool/TestUtil.java index 25033311..ea6dcc35 100644 --- a/tool/src/test/java/org/datacommons/tool/TestUtil.java +++ b/tool/src/test/java/org/datacommons/tool/TestUtil.java @@ -14,9 +14,8 @@ package org.datacommons.tool; -import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; -import static org.junit.Assert.assertTrue; - +import com.google.common.truth.Expect; +import com.google.common.truth.extensions.proto.ProtoTruth; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import java.io.File; @@ -24,78 +23,26 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; import org.apache.commons.io.FileUtils; import org.datacommons.proto.Debug; import org.junit.rules.TemporaryFolder; // Common set of utils used in e2e tests. public class TestUtil { - public static void assertReportFilesAreSimilar(File directory, String expected, String actual) - throws IOException { + public static void assertReportFilesAreSimilar( + Expect expect, File directory, String expected, String actual) throws IOException { String testCase = directory.getName(); Debug.Log expectedLog = reportToProto(expected).build(); Debug.Log actualLog = reportToProto(actual).build(); - // Compare the maps, printing log messages along the way and assert only at the very end. - boolean pass = true; - pass &= - areMapsEqual( - testCase, - "Counter Set", - expectedLog.getCounterSet().getCountersMap(), - actualLog.getCounterSet().getCountersMap()); - pass &= - areMapsEqual( - testCase, - "Level Summary", - expectedLog.getLevelSummaryMap(), - actualLog.getLevelSummaryMap()); - pass &= actualLog.getEntriesList().containsAll(expectedLog.getEntriesList()); - pass &= expectedLog.getEntriesList().containsAll(actualLog.getEntriesList()); - assertThat(actualLog).ignoringRepeatedFieldOrder().isEqualTo(expectedLog); - assertTrue(pass); - } - - private static boolean areMapsEqual( - String testCase, String mapType, Map expected, Map actual) { - boolean equal = true; - if (expected.keySet().size() > actual.keySet().size()) { - equal = false; - Set diff = new HashSet<>((Collection) expected.keySet()); - diff.removeAll((Collection) actual.keySet()); - System.err.println( - testCase + " :: " + mapType + " has some missing keys: " + String.join(", ", diff)); - } else if (expected.keySet().size() < actual.keySet().size()) { - equal = false; - Set diff = new HashSet<>((Collection) actual.keySet()); - diff.removeAll((Collection) expected.keySet()); - System.err.println( - testCase + " :: " + mapType + " has some extra keys: " + String.join(", ", diff)); - } - for (String key : expected.keySet()) { - if (!actual.containsKey(key)) { - equal = false; - System.err.println(mapType + " actual report is missing the key: " + key); - } - if (!expected.get(key).equals(actual.get(key))) { - equal = false; - System.err.println( - testCase - + " :: " - + mapType - + " has different values for the key " - + key - + " : expected (" - + expected.get(key) - + ") vs. actual (" - + actual.get(key) - + ")"); - } - } - return equal; + expect + .that(expectedLog.getCounterSet().getCountersMap()) + .isEqualTo(actualLog.getCounterSet().getCountersMap()); + expect.that(expectedLog.getLevelSummaryMap()).isEqualTo(actualLog.getLevelSummaryMap()); + expect + .about(ProtoTruth.protos()) + .that(expectedLog.getEntriesList()) + .ignoringRepeatedFieldOrder() + .containsExactlyElementsIn(expectedLog.getEntriesList()); } private static Debug.Log.Builder reportToProto(String report)