Add support for lazy JSONPath queries.

.eslintignore

@@ -5,3 +5,4 @@ benchmark
 tests/browser
 docs
 tests/path/cts
+performance/index.js

.gitignore

@@ -13,5 +13,9 @@ coverage/
 benchmark/*.log
 benchmark/*.txt
 
+# vscode profiler logs
+*.heapprofile
+*.cpuprofile
+
 # dev
 tests/dev.test.ts

CHANGELOG.md

@@ -1,5 +1,16 @@
 # JSON P3 Change Log
 
+## Version 0.3.0 (unreleased)
+
+**Fixes**
+
+- Fixed call stack size issues when querying large datasets with the recursive descent selector. This was mostly due to extending arrays using the spread operator. We now iterate and use `Array.push()`.
+
+**Features**
+
+- Added `jsonpath.lazyQuery()`, a lazy alternative to `jsonpath.query()`. `lazyQuery()` can be faster and more memory efficient if querying large datasets, especially when using recursive descent selectors. Conversely, `query()` is usually the better choice when working with small datasets.
+- `jsonpath.match()` now uses `lazyQuery()` internally, potentially avoiding a lot of unnecessary work.
+
 # Version 0.2.1
 
 **Fixes**

docs/docs/quick-start.md

@@ -97,6 +97,33 @@ Sally @ $['users'][2]['name']
 Jane @ $['users'][3]['name']
 ```
 
+### Lazy queries
+
+[`lazyQuery()`](./api/namespaces/jsonpath.md#lazyquery) is an alternative to `query()`. `lazyQuery()` can be faster and more memory efficient if querying large datasets, especially when using recursive descent selectors. Conversely, `query()` is usually the better choice when working with small datasets.
+
+`lazyQuery()` returns an iterable sequence of [`JSONPathNode`](./api/classes/jsonpath.JSONPathNode.md) objects which is not a `JSONPathNodeList`.
+
+```javascript
+import { lazyQuery } from "json-p3";
+
+const data = {
+  users: [
+    { name: "Sue", score: 100 },
+    { name: "John", score: 86 },
+    { name: "Sally", score: 84 },
+    { name: "Jane", score: 55 },
+  ],
+};
+
+for (const node of lazyQuery("$.users[[email protected] < 100].name", data)) {
+  console.log(node.value);
+}
+
+// John
+// Sally
+// Jane
+```
+
 ### Compilation
 
 `query()` is a convenience function equivalent to `new JSONPathEnvironment().compile(path).query(data)`. Use `jsonpath.compile()` to construct a [`JSONPath`](./api/classes/jsonpath.JSONPath.md) object that can be applied to different data repeatedly.

docs/package.json

@@ -26,7 +26,6 @@
     "allotment": "^1.19.3",
     "clsx": "^1.2.1",
     "docusaurus-plugin-typedoc": "^0.20.1",
-    "json-p3": "^0.2.0",
     "monaco-editor": "^0.43.0",
     "monaco-themes": "^0.4.4",
     "prism-react-renderer": "^1.3.5",

docs/src/components/JSONPathPlayground/index.js

@@ -12,7 +12,7 @@ import "allotment/dist/style.css";
 
 import styles from "./styles.module.css";
 
-import { jsonpath, version as p3version } from "json-p3/dist/json-p3.esm";
+import { jsonpath, version as p3version } from "@site/../dist/json-p3.esm";
 
 const commonEditorOptions = {
   codeLens: false,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "json-p3",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "author": "James Prior",
   "license": "MIT",
   "description": "JSONPath, JSON Pointer and JSON Patch",

performance/index.js

@@ -0,0 +1,33 @@
+/**
+ * This is incomplete and, thus far, has been used in an adhoc manner.
+ */
+const { performance } = require("perf_hooks");
+const { jsonpath } = require("../dist/json-p3.cjs");
+const cts = require("../tests/path/cts/cts.json");
+
+function validQueries() {
+  return cts.tests
+    .filter((testCase) => testCase.invalid_selector !== true)
+    .map((testCase) => {
+      return [testCase.selector, testCase.document];
+    });
+}
+
+function perf(repeat) {
+  const env = new jsonpath.JSONPathEnvironment();
+  const queries = validQueries();
+  console.log(
+    `repeating ${queries.length} queries on small datasets ${repeat} times`,
+  );
+  const start = performance.now();
+  for (let i = 0; i < repeat; i++) {
+    for (const [query, data] of queries) {
+      env.query(query, data);
+      // Array.from(env.lazyQuery(query, data));
+    }
+  }
+  const stop = performance.now();
+  return (stop - start) / 1e3;
+}
+
+console.log(perf(1000));

src/index.ts

@@ -17,6 +17,7 @@ export {
   Token,
   TokenKind,
   Nothing,
+  lazyQuery,
   query,
   compile,
 } from "./path";

src/path/environment.ts

@@ -130,6 +130,22 @@ export class JSONPathEnvironment {
     return this.compile(path).query(value);
   }
 
+  /**
+   * A lazy version of {@link query} which is faster and more memory
+   * efficient when querying some large datasets.
+   *
+   * @param path - A JSONPath query to parse and evaluate against _value_.
+   * @param value - Data to which _path_ will be applied.
+   * @returns A sequence of {@link JSONPathNode} objects resulting from
+   * applying _path_ to _value_.
+   */
+  public lazyQuery(
+    path: string,
+    value: JSONValue,
+  ): IterableIterator<JSONPathNode> {
+    return this.compile(path).lazyQuery(value);
+  }
+
   /**
    * Return a {@link JSONPathNode} instance for the first object found in
    * _value_ matching _path_.

src/path/expression.ts

@@ -190,7 +190,7 @@ export abstract class JSONPathQuery extends FilterExpression {
 
 export class RelativeQuery extends JSONPathQuery {
   public evaluate(context: FilterContext): JSONPathNodeList {
-    return this.path.query(context.currentValue);
+    return this.path.query(context.currentValue); // TODO: lazy query?
   }
 
   public toString(): string {
@@ -200,7 +200,7 @@ export class RelativeQuery extends JSONPathQuery {
 
 export class RootQuery extends JSONPathQuery {
   public evaluate(context: FilterContext): JSONPathNodeList {
-    return this.path.query(context.rootValue);
+    return this.path.query(context.rootValue); // TODO: lazy query?
   }
 
   public toString(): string {

src/path/index.ts

@@ -49,6 +49,30 @@ export function query(path: string, value: JSONValue): JSONPathNodeList {
   return DEFAULT_ENVIRONMENT.query(path, value);
 }
 
+/**
+ * Lazily query JSON value _value_ with JSONPath expression _path_.
+ * Lazy queries can be faster and more memory efficient when querying
+ * large datasets, especially when using recursive decent selectors.
+ *
+ * @param path - A JSONPath expression/query.
+ * @param value - The JSON-like value the JSONPath query is applied to.
+ * @returns A sequence of {@link JSONPathNode} objects resulting from
+ * applying _path_ to _value_.
+ *
+ * @throws {@link JSONPathSyntaxError}
+ * If the path does not conform to standard syntax.
+ *
+ * @throws {@link JSONPathTypeError}
+ * If filter function arguments are invalid, or filter expression are
+ * used in an invalid way.
+ */
+export function lazyQuery(
+  path: string,
+  value: JSONValue,
+): IterableIterator<JSONPathNode> {
+  return DEFAULT_ENVIRONMENT.lazyQuery(path, value);
+}
+
 /**
  * Compile JSONPath _path_ for later use.
  * @param path - A JSONPath expression/query.

src/path/parse.ts

@@ -99,42 +99,55 @@ export class Parser {
     inFilter: boolean = false,
   ): JSONPathSelector[] {
     const selectors: JSONPathSelector[] = [];
-    loop: for (;;) {
-      switch (stream.current.kind) {
-        case TokenKind.NAME:
-          selectors.push(
-            new NameSelector(
-              this.environment,
-              stream.current,
-              stream.current.value,
-              true,
-            ),
-          );
-          break;
-        case TokenKind.WILD:
-          selectors.push(
-            new WildcardSelector(this.environment, stream.current, true),
-          );
-          break;
-        case TokenKind.DDOT:
-          selectors.push(
-            new RecursiveDescentSegment(this.environment, stream.current),
-          );
-          break;
-        case TokenKind.LBRACKET:
-          selectors.push(this.parseBracketedSelection(stream));
-          break;
-        default:
-          if (inFilter) {
-            stream.backup();
-          }
-          break loop;
+    for (;;) {
+      const selector = this.parseSegment(stream);
+      if (!selector) {
+        if (inFilter) {
+          stream.backup();
+        }
+        break;
       }
+
+      selectors.push(selector);
       stream.next();
     }
     return selectors;
   }
 
+  protected parseSegment(stream: TokenStream): JSONPathSelector | null {
+    switch (stream.current.kind) {
+      case TokenKind.NAME:
+        return new NameSelector(
+          this.environment,
+          stream.current,
+          stream.current.value,
+          true,
+        );
+      case TokenKind.WILD:
+        return new WildcardSelector(this.environment, stream.current, true);
+      case TokenKind.DDOT: {
+        const segmentToken = stream.current;
+        stream.next();
+        const selector = this.parseSegment(stream);
+        if (!selector) {
+          throw new JSONPathSyntaxError(
+            "bald descendant segment",
+            stream.current,
+          );
+        }
+        return new RecursiveDescentSegment(
+          this.environment,
+          segmentToken,
+          selector,
+        );
+      }
+      case TokenKind.LBRACKET:
+        return this.parseBracketedSelection(stream);
+      default:
+        return null;
+    }
+  }
+
   protected parseIndex(stream: TokenStream): IndexSelector {
     if (
       (stream.current.value.length > 1 &&

src/path/path.ts

@@ -35,6 +35,21 @@ export class JSONPath {
     return nodes;
   }
 
+  /**
+   *
+   * @param value -
+   * @returns
+   */
+  public lazyQuery(value: JSONValue): IterableIterator<JSONPathNode> {
+    let nodes: IterableIterator<JSONPathNode> = [
+      new JSONPathNode(value, [], value),
+    ][Symbol.iterator]();
+    for (const selector of this.selectors) {
+      nodes = selector.lazyResolve(nodes);
+    }
+    return nodes;
+  }
+
   /**
    * Return a {@link JSONPathNode} instance for the first object found in
    * _value_ matching this query.
@@ -44,7 +59,10 @@ export class JSONPath {
    * there are no matches.
    */
   public match(value: JSONValue): JSONPathNode | undefined {
-    return this.query(value).nodes.at(0);
+    const it = this.lazyQuery(value);
+    const rv = it.next();
+    if (rv.done) return undefined;
+    return rv.value;
   }
 
   /**