Introduce some tests of non-determinism

README.md

@@ -20,3 +20,7 @@ Many editors support that file natively. Others (such as VS code) require a plug
 To add or modify a test suite, edit the corresponding file in the `tests` directory.
 To generate `cts.json`, run the `build.sh` located in the root folder. Do not modify `cts.json` directly.
 More details are available in the [Contributor Guide](./CONTRIBUTING.md).
+
+### Non-determinism
+
+Where the spec allows non-deterministic results for a given testcase, the testcase should specify an array of all the valid results (each of which is itself an array representing the resultant nodelist from the query) in the "results" member (and should not specify a "result" member).

cts.json

@@ -93,9 +93,15 @@
         "a": "A",
         "b": "B"
       },
-      "result": [
-        "A",
-        "B"
+      "results": [
+        [
+          "A",
+          "B"
+        ],
+        [
+          "B",
+          "A"
+        ]
       ]
     },
     {
@@ -135,9 +141,15 @@
           "b": "By"
         }
       },
-      "result": [
-        "Ax",
-        "Ay"
+      "results": [
+        [
+          "Ax",
+          "Ay"
+        ],
+        [
+          "Ay",
+          "Ax"
+        ]
       ]
     },
     {
@@ -464,6 +476,34 @@
         "f"
       ]
     },
+    {
+      "name": "basic, descendant segment, object traversal, multiple selectors",
+      "selector": "$..['a','d']",
+      "document": {
+        "x": {
+          "a": "b",
+          "d": "e"
+        },
+        "y": {
+          "a": "c",
+          "d": "f"
+        }
+      },
+      "results": [
+        [
+          "b",
+          "e",
+          "c",
+          "f"
+        ],
+        [
+          "c",
+          "f",
+          "b",
+          "e"
+        ]
+      ]
+    },
     {
       "name": "basic, bald descendant segment",
       "selector": "$..",

cts.schema.json

@@ -32,6 +32,9 @@
         "result": {
           "$ref": "#/$defs/test_case_result"
         },
+        "results": {
+          "$ref": "#/$defs/test_case_results"
+        },
         "invalid_selector": {
           "$ref": "#/$defs/invalid_selector"
         }
@@ -47,6 +50,12 @@
             "result"
           ]
         },
+        {
+          "required": [
+            "document",
+            "results"
+          ]
+        },
         {
           "required": [
             "invalid_selector"
@@ -63,6 +72,12 @@
       "type": "array",
       "description": "The expected result of applying the selector to the document, contains all the matched values"
     },
+    "test_case_results": {
+      "type": "array",
+      "items": {"$ref": "#/$defs/test_case_result"},
+      "minItems": 2,
+      "description": "An array of possible expected results of applying the selector to the document, each element of which contains all the matched values"
+    },
     "selector": {
       "description": "The JSONPath selector",
       "type": "string"

tests/basic.json

@@ -92,10 +92,13 @@
         "a": "A",
         "b": "B"
       },
-      "result": [
+      "results": [[
         "A",
         "B"
-      ]
+      ],[
+        "B",
+        "A"
+      ]]
     },
     {
       "name": "wildcard shorthand, array data",
@@ -134,10 +137,13 @@
           "b": "By"
         }
       },
-      "result": [
+      "results": [[
         "Ax",
         "Ay"
-      ]
+      ],[
+        "Ay",
+        "Ax"
+      ]]
     },
     {
       "name": "multiple selectors",
@@ -424,6 +430,22 @@
         "f"
       ]
     },
+    {
+      "name": "descendant segment, object traversal, multiple selectors",
+      "selector" : "$..['a','d']",
+      "document" : {"x": {"a": "b", "d": "e"}, "y": {"a":"c", "d": "f"}},
+      "results": [[
+        "b",
+        "e",
+        "c",
+        "f"
+      ],[
+        "c",
+        "f",
+        "b",
+        "e"
+      ]]
+    },
     {
       "name": "bald descendant segment",
       "selector": "$..",