fixing the lexer canIUse() API method and use it (and the new `lexe…

…r.fastLex()` API) in the parser whenever appropriate.
GerHobbelt · Nov 12, 2017 · 0b712cd · 0b712cd
1 parent 63bb8cf
commit 0b712cd
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 8 deletions.
diff --git a/lib/jison-parser-kernel.js b/lib/jison-parser-kernel.js
@@ -680,7 +680,7 @@ function parse(input, parseParams) {
 
 //_lexer_without_token_stack:
 
-    function lex() {
+    function stdLex() {
         var token = lexer.lex();
         // if token isn't its numeric value, convert
         if (typeof token !== 'number') {
@@ -690,6 +690,18 @@ function parse(input, parseParams) {
         return token || EOF;
     }
 
+    function fastLex() {
+        var token = lexer.fastLex();
+        // if token isn't its numeric value, convert
+        if (typeof token !== 'number') {
+            token = self.symbols_[token] || token;
+        }
+
+        return token || EOF;
+    }
+
+    var lex = stdLex;
+
 //_lexer_with_token_stack:
 
     // lex function that supports token stacks
@@ -786,6 +798,17 @@ function parse(input, parseParams) {
 
         lexer.setInput(input, sharedState_yy);
 
+        // NOTE: we *assume* no lexer pre/post handlers are set up *after* 
+        // this initial `setInput()` call: hence we can now check and decide
+        // whether we'll go with the standard, slower, lex() API or the
+        // `fast_lex()` one:
+        if (typeof lexer.canIUse === 'function') {
+            var lexerInfo = lexer.canIUse();
+            if (lexerInfo.fastLex && typeof fastLex === 'function') {
+                lex = fastLex;
+            }
+        } 
+
         yyloc = lexer.yylloc;
         lstack[sp] = yyloc;
         vstack[sp] = null;

diff --git a/lib/jison.js b/lib/jison.js
@@ -7240,7 +7240,7 @@ parser.parse = `function parse(input, parseParams) {
 
 //_lexer_without_token_stack:
 
-    function lex() {
+    function stdLex() {
         var token = lexer.lex();
         // if token isn't its numeric value, convert
         if (typeof token !== 'number') {
@@ -7250,6 +7250,18 @@ parser.parse = `function parse(input, parseParams) {
         return token || EOF;
     }
 
+    function fastLex() {
+        var token = lexer.fastLex();
+        // if token isn't its numeric value, convert
+        if (typeof token !== 'number') {
+            token = self.symbols_[token] || token;
+        }
+
+        return token || EOF;
+    }
+
+    var lex = stdLex;
+
 //_lexer_with_token_stack:
 
     // lex function that supports token stacks
@@ -7346,6 +7358,17 @@ parser.parse = `function parse(input, parseParams) {
 
         lexer.setInput(input, sharedState_yy);
 
+        // NOTE: we *assume* no lexer pre/post handlers are set up *after* 
+        // this initial \`setInput()\` call: hence we can now check and decide
+        // whether we'll go with the standard, slower, lex() API or the
+        // \`fast_lex()\` one:
+        if (typeof lexer.canIUse === 'function') {
+            var lexerInfo = lexer.canIUse();
+            if (lexerInfo.fastLex && typeof fastLex === 'function') {
+                lex = fastLex;
+            }
+        } 
+
         yyloc = lexer.yylloc;
         lstack[sp] = yyloc;
         vstack[sp] = null;

diff --git a/packages/jison-lex/jison-lexer-kernel.js b/packages/jison-lex/jison-lexer-kernel.js
@@ -1123,16 +1123,16 @@
      */
     canIUse: function lexer_canIUse() {
         var rv = {
-            fast_lex: !(
+            fastLex: !(
                 typeof this.pre_lex === 'function' ||
                 typeof this.options.pre_lex === 'function' ||
                 (this.yy && typeof this.yy.pre_lex === 'function') ||
                 (this.yy && typeof this.yy.post_lex === 'function') ||
                 typeof this.options.post_lex === 'function' ||
                 typeof this.post_lex === 'function'
-            ),
+            ) && typeof this.fastLex === 'function',
         };
-        return r;
+        return rv;
     },
 
 

diff --git a/packages/jison-lex/regexp-lexer.js b/packages/jison-lex/regexp-lexer.js
@@ -2416,16 +2416,16 @@ return `{
      */
     canIUse: function lexer_canIUse() {
         var rv = {
-            fast_lex: !(
+            fastLex: !(
                 typeof this.pre_lex === 'function' ||
                 typeof this.options.pre_lex === 'function' ||
                 (this.yy && typeof this.yy.pre_lex === 'function') ||
                 (this.yy && typeof this.yy.post_lex === 'function') ||
                 typeof this.options.post_lex === 'function' ||
                 typeof this.post_lex === 'function'
-            ),
+            ) && typeof this.fastLex === 'function',
         };
-        return r;
+        return rv;
     },