feat: control UTF-16 decoding in JSONPath strings, closes #25

CHANGELOG.md

@@ -20,6 +20,7 @@
 - Added `JSONPointer.exists()`, a method that returns `True` if a the pointer can be resolved against some data, or `False` otherwise.
 - Added the `RelativeJSONPointer` class for building new `JSONPointer` instances from Relative JSON Pointer syntax.
 - Added support for a non-standard index/property pointer using `#<property or index>`. This is to support Relative JSON Pointer's use of hash (`#`) when building `JSONPointer` instances from relative JSON Pointers.
+- Added the `unicode_escape` argument to `JSONPathEnvironment`. When `True` (the default), UTF-16 escaped sequences found in JSONPath string literals will be decoded.
 
 ## Version 0.8.1
 

jsonpath/cli.py

@@ -239,7 +239,9 @@ def handle_path_command(args: argparse.Namespace) -> None: # noqa: PLR0912
  path = args.query_file.read().strip()
 
  try:
- path = jsonpath.compile(path)
+ path = jsonpath.JSONPathEnvironment(
+ unicode_escape=not args.no_unicode_escape
+ ).compile(path)
  except JSONPathSyntaxError as err:
  if args.debug:
  raise

jsonpath/env.py

@@ -70,8 +70,10 @@ class attributes `root_token`, `self_token` and `filter_context_token`.
  ## Class attributes
 
  Arguments:
- filter_caching (bool): If `True`, filter expressions will be cached where
- possible.
+ filter_caching (bool): If `True`, filter expressions will be cached
+ where possible.
+ unicode_escape: If `True`, decode UTF-16 escape sequences found in
+ JSONPath string literals.
 
  Attributes:
  filter_context_token (str): The pattern used to select extra filter context
@@ -113,8 +115,18 @@ class attributes `root_token`, `self_token` and `filter_context_token`.
  parser_class: Type[Parser] = Parser
  match_class: Type[JSONPathMatch] = JSONPathMatch
 
- def __init__(self, *, filter_caching: bool = True) -> None:
- self.filter_caching = filter_caching
+ def __init__(
+ self,
+ *,
+ filter_caching: bool = True,
+ unicode_escape: bool = True,
+ ) -> None:
+ self.filter_caching: bool = filter_caching
+ """Enable or disable filter expression caching."""
+
+ self.unicode_escape: bool = unicode_escape
+ """Enable or disable decoding of UTF-16 escape sequences found in
+ JSONPath string literals."""
 
  self.lexer: Lexer = self.lexer_class(env=self)
  """The lexer bound to this environment."""

jsonpath/parse.py

@@ -347,7 +347,7 @@ def parse_slice(self, stream: TokenStream) -> SliceSelector:
  step=step,
  )
 
- def parse_selector_list(self, stream: TokenStream) -> ListSelector:
+ def parse_selector_list(self, stream: TokenStream) -> ListSelector: # noqa: PLR0912
  """Parse a comma separated list JSONPath selectors from a stream of tokens."""
  tok = stream.next_token()
  list_items: List[
@@ -391,13 +391,16 @@ def parse_selector_list(self, stream: TokenStream) -> ListSelector:
  token=stream.current,
  )
 
- name = (
- codecs.decode(
- stream.current.value.replace("\\/", "/"), "unicode-escape"
+ if self.env.unicode_escape:
+ name = (
+ codecs.decode(
+ stream.current.value.replace("\\/", "/"), "unicode-escape"
+ )
+ .encode("utf-16", "surrogatepass")
+ .decode("utf-16")
  )
- .encode("utf-16", "surrogatepass")
- .decode("utf-16")
- )
+ else:
+ name = stream.current.value
 
  list_items.append(
  PropertySelector(

tests/test_env.py

@@ -145,3 +145,16 @@ def test_no_match_compound_path(env: JSONPathEnvironment) -> None:
  """Test that we get `None` if there are no matches in a compound path."""
  match = env.match("$.other | $.foo", {"some": 1, "thing": 2})
  assert match is None
+
+
+def test_no_unicode_escape() -> None:
+ """Test that we can disable decoding of UTF-16 escape sequences."""
+ document = {"𝄞": "A"}
+ selector = '$["\\uD834\\uDD1E"]'
+
+ env = JSONPathEnvironment(unicode_escape=True)
+ assert env.findall(selector, document) == ["A"]
+
+ env = JSONPathEnvironment(unicode_escape=False)
+ assert env.findall(selector, document) == []
+ assert env.findall(selector, {"\\uD834\\uDD1E": "B"}) == ["B"]