#42 - fixed parsing raw string literals

nanoporetech · Oct 27, 2024 · b8dcfa8 · b8dcfa8
1 parent 4ee2325
commit b8dcfa8
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 1 deletion.
diff --git a/src/Lexer.ts b/src/Lexer.ts
@@ -223,7 +223,12 @@ class StreamLexer {
     }
     this._current += 1;
     const literal = stream.slice(start + 1, this._current - 1);
-    return literal.replace(`\\'`, `'`);
+
+    // emulating es2021: String.prototype.replaceAll()
+    return literal
+      .split(`\\\\`).join(`\\`)
+      .split(`\\'`).join(`'`)
+      ;
   }
 
   private consumeNumber(stream: string): LexerToken {

diff --git a/test/jmespath.spec.js b/test/jmespath.spec.js
@@ -50,6 +50,18 @@ describe('tokenize', () => {
   it('should tokenize json literals', () => {
     expect(tokenize('`true`')).toMatchObject([{ type: 'Literal', value: true, start: 0 }]);
   });
+  it('should tokenize raw strings', () => {
+    expect(tokenize("'raw-string'")).toMatchObject([{ type: 'Literal', value: "raw-string", start: 0 }]);
+  });
+  it('should tokenize raw strings single quote', () => {
+    expect(tokenize("'\\''")).toMatchObject([{ type: 'Literal', value: "'", start: 0 }]);
+  });
+  it('should tokenize raw strings surrounding quotes', () => {
+    expect(tokenize("'\\'raw-string\\''")).toMatchObject([{ type: 'Literal', value: "'raw-string'", start: 0 }]);
+  });
+  it('should tokenize raw strings backslash characters', () => {
+    expect(tokenize("'\\\\'")).toMatchObject([{ type: 'Literal', value: "\\", start: 0 }]);
+  });
   it('should not requiring surrounding quotes for strings', () => {
     expect(tokenize('`foo`')).toMatchObject([{ type: 'Literal', value: 'foo', start: 0 }]);
   });