Skip to content

Commit

Permalink
First bit of work for zaach#341: lexer run-time optimizations. For th…
Browse files Browse the repository at this point in the history 
…at to deliver, the lexer *generator* will need information about which features are used, e.g. extended error handling, location tracking (with or without `yylloc.range` range tracking), etc.
  • Loading branch information
@GerHobbelt
GerHobbelt committed Jan 31, 2017
1 parent 95f52df commit 22b521a
Show file tree
Hide file tree
Showing 4 changed files with 68 additions and 18 deletions.
2 changes: 1 addition & 1 deletion lib/jison.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ generator.constructor = function Jison_Generator(grammar, lexGrammarStr, opt) {
this.processGrammar(grammar);

if (grammar.lex) {
this.lexer = new Lexer(grammar.lex, null, this.terminals_);
this.lexer = new Lexer(grammar.lex, null, this.terminals_, this);
}
};

Expand Down
35 changes: 30 additions & 5 deletions lib/util/regexp-lexer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1731,12 +1731,12 @@ function generateErrorClass() {
var jisonLexerErrorDefinition = generateErrorClass();


function RegExpLexer(dict, input, tokens) {
function RegExpLexer(dict, input, tokens, build_options) {
var opts;
var dump = false;

function test_me(tweak_cb, description, src_exception, ex_callback) {
opts = processGrammar(dict, tokens);
opts = processGrammar(dict, tokens, build_options);
opts.__in_rules_failure_analysis_mode__ = false;
if (tweak_cb) {
tweak_cb();
Expand Down Expand Up @@ -2679,15 +2679,40 @@ function expandParseArguments(parseFn, options) {


// generate lexer source from a grammar
function generate(dict, tokens) {
var opt = processGrammar(dict, tokens);
function generate(dict, tokens, build_options) {
var opt = processGrammar(dict, tokens, build_options);

return generateFromOpts(opt);
}

// process the grammar and build final data structures and functions
function processGrammar(dict, tokens) {
function processGrammar(dict, tokens, build_options) {
build_options = build_options || {};
var opts = {};
// include the knowledge passed through `build_options` about which lexer
// features will actually be *used* by the environment (which in 99.9%
// of cases is a jison *parser*):
//
// (this stuff comes striaght from the jison Optimization Analysis.)
//
opts.actionsAreAllDefault = build_options.actionsAreAllDefault;
opts.actionsUseYYLENG = build_options.actionsUseYYLENG;
opts.actionsUseYYLINENO = build_options.actionsUseYYLINENO;
opts.actionsUseYYTEXT = build_options.actionsUseYYTEXT;
opts.actionsUseYYLOC = build_options.actionsUseYYLOC;
opts.actionsUseParseError = build_options.actionsUseParseError;
opts.actionsUseYYERROR = build_options.actionsUseYYERROR;
opts.actionsUseYYERROK = build_options.actionsUseYYERROK;
opts.actionsUseYYCLEARIN = build_options.actionsUseYYCLEARIN;
opts.actionsUseValueTracking = build_options.actionsUseValueTracking;
opts.actionsUseValueAssignment = build_options.actionsUseValueAssignment;
opts.actionsUseLocationTracking = build_options.actionsUseLocationTracking;
opts.actionsUseLocationAssignment = build_options.actionsUseLocationAssignment;
opts.actionsUseYYSTACK = build_options.actionsUseYYSTACK;
opts.actionsUseYYSSTACK = build_options.actionsUseYYSSTACK;
opts.actionsUseYYSTACKPOINTER = build_options.actionsUseYYSTACKPOINTER;
opts.hasErrorRecovery = build_options.hasErrorRecovery;

if (typeof dict === 'string') {
dict = lexParser.parse(dict);
}
Expand Down
2 changes: 1 addition & 1 deletion modules/jison-lex
Submodule jison-lex updated from 3e2d5b to 9dc761
47 changes: 36 additions & 11 deletions web/content/assets/js/jison.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@ generator.constructor = function Jison_Generator(grammar, lexGrammarStr, opt) {
this.processGrammar(grammar);

if (grammar.lex) {
this.lexer = new Lexer(grammar.lex, null, this.terminals_);
this.lexer = new Lexer(grammar.lex, null, this.terminals_, this);
}
};

Expand Down Expand Up @@ -17083,12 +17083,12 @@ function generateErrorClass() {
var jisonLexerErrorDefinition = generateErrorClass();


function RegExpLexer(dict, input, tokens) {
function RegExpLexer(dict, input, tokens, build_options) {
var opts;
var dump = false;

function test_me(tweak_cb, description, src_exception, ex_callback) {
opts = processGrammar(dict, tokens);
opts = processGrammar(dict, tokens, build_options);
opts.__in_rules_failure_analysis_mode__ = false;
if (tweak_cb) {
tweak_cb();
Expand Down Expand Up @@ -18031,15 +18031,40 @@ function expandParseArguments(parseFn, options) {


// generate lexer source from a grammar
function generate(dict, tokens) {
var opt = processGrammar(dict, tokens);
function generate(dict, tokens, build_options) {
var opt = processGrammar(dict, tokens, build_options);

return generateFromOpts(opt);
}

// process the grammar and build final data structures and functions
function processGrammar(dict, tokens) {
function processGrammar(dict, tokens, build_options) {
build_options = build_options || {};
var opts = {};
// include the knowledge passed through `build_options` about which lexer
// features will actually be *used* by the environment (which in 99.9%
// of cases is a jison *parser*):
//
// (this stuff comes striaght from the jison Optimization Analysis.)
//
opts.actionsAreAllDefault = build_options.actionsAreAllDefault;
opts.actionsUseYYLENG = build_options.actionsUseYYLENG;
opts.actionsUseYYLINENO = build_options.actionsUseYYLINENO;
opts.actionsUseYYTEXT = build_options.actionsUseYYTEXT;
opts.actionsUseYYLOC = build_options.actionsUseYYLOC;
opts.actionsUseParseError = build_options.actionsUseParseError;
opts.actionsUseYYERROR = build_options.actionsUseYYERROR;
opts.actionsUseYYERROK = build_options.actionsUseYYERROK;
opts.actionsUseYYCLEARIN = build_options.actionsUseYYCLEARIN;
opts.actionsUseValueTracking = build_options.actionsUseValueTracking;
opts.actionsUseValueAssignment = build_options.actionsUseValueAssignment;
opts.actionsUseLocationTracking = build_options.actionsUseLocationTracking;
opts.actionsUseLocationAssignment = build_options.actionsUseLocationAssignment;
opts.actionsUseYYSTACK = build_options.actionsUseYYSTACK;
opts.actionsUseYYSSTACK = build_options.actionsUseYYSSTACK;
opts.actionsUseYYSTACKPOINTER = build_options.actionsUseYYSTACKPOINTER;
opts.hasErrorRecovery = build_options.hasErrorRecovery;

if (typeof dict === 'string') {
dict = lexParser.parse(dict);
}
Expand Down Expand Up @@ -27760,14 +27785,14 @@ module.exports={
"node": ">=4.0"
},
"dependencies": {
"ebnf-parser": "GerHobbelt/ebnf-parser#master",
"jison-lex": "GerHobbelt/jison-lex#master",
"lex-parser": "GerHobbelt/lex-parser#master",
"ebnf-parser": "github:GerHobbelt/ebnf-parser#master",
"jison-lex": "github:GerHobbelt/jison-lex#master",
"lex-parser": "github:GerHobbelt/lex-parser#master",
"recast": "0.11.20",
"jscodeshift": "0.3.30",
"json5": "0.5.1",
"nomnom": "GerHobbelt/nomnom#master",
"xregexp": "GerHobbelt/xregexp#master"
"nomnom": "github:GerHobbelt/nomnom#master",
"xregexp": "github:GerHobbelt/xregexp#master"
},
"devDependencies": {
"browserify": "14.0.0",
Expand Down

0 comments on commit 22b521a

Please sign in to comment.