Skip to content

Commit

Permalink
Update patterns to fix some warnings in the tokenizer
Browse files Browse the repository at this point in the history
  • Loading branch information
@duckdoom4
duckdoom4 committed Jul 12, 2024
1 parent 9e56d94 commit 43008f6
Show file tree
Hide file tree
Showing 15 changed files with 500 additions and 406 deletions.
14 changes: 7 additions & 7 deletions src/tokenizer/atl-token-patterns.g.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
// THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
// DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
// ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
// Last generated: 12/07/2024 09:35:29 (UTC+0)
// Last generated: 12/07/2024 18:50:56 (UTC+0)

import { KeywordTokenType, EntityTokenType, MetaTokenType, CharacterTokenType } from "./renpy-tokens";
import { TokenPattern } from "./token-pattern-types";
Expand Down Expand Up @@ -173,7 +173,7 @@ export const atlExpression: TokenPattern = {
{
debugName: "atlExpression.patterns![1]",

begin: /\b(with)\b[ \t]*/dg,
begin: /\b(with)\b/dg,
beginCaptures: {
1: { token: KeywordTokenType.With, /*keyword.with.renpy*/ },
},
Expand All @@ -189,7 +189,7 @@ export const atlKeywords: TokenPattern = {
debugName: "atlKeywords.patterns![0]",

// https://www.renpy.org/doc/html/atl.html#animation-statement
match: /^[ \t]*(animation)\b/dgm,
match: /(?<=^[ \t]*)(animation)\b/dgm,
captures: {
1: { token: KeywordTokenType.Animation, /*keyword.animation.renpy*/ },
},
Expand All @@ -198,7 +198,7 @@ export const atlKeywords: TokenPattern = {
debugName: "atlKeywords.patterns![1]",

// https://www.renpy.org/doc/html/atl.html#pass-statement
match: /^[ \t]*(pass)\b/dgm,
match: /(?<=^[ \t]*)(pass)\b/dgm,
captures: {
1: { token: KeywordTokenType.Pass, /*keyword.control.flow.pass.renpy*/ },
},
Expand All @@ -207,7 +207,7 @@ export const atlKeywords: TokenPattern = {
debugName: "atlKeywords.patterns![2]",

// https://www.renpy.org/doc/html/atl.html#repeat-statement and https://www.renpy.org/doc/html/atl.html#time-statement
begin: /^[ \t]*(?:(repeat)|(time)|(pause))\b[ \t]*/dgm,
begin: /(?<=^[ \t]*)(?:(repeat)|(time)|(pause))\b/dgm,
beginCaptures: {
1: { token: KeywordTokenType.Repeat, /*keyword.control.flow.repeat.renpy*/ },
2: { token: KeywordTokenType.Time, /*keyword.control.flow.time.renpy*/ },
Expand All @@ -221,7 +221,7 @@ export const atlKeywords: TokenPattern = {

// https://www.renpy.org/doc/html/atl.html#expression-statement
contentToken: MetaTokenType.ATLWith, /*meta.atl.with.renpy*/
begin: /^[ \t]*(with)\b[ \t]*/dgm,
begin: /(?<=^[ \t]*)(with)\b/dgm,
beginCaptures: {
1: { token: KeywordTokenType.With, /*keyword.with.renpy*/ },
},
Expand All @@ -233,7 +233,7 @@ export const atlKeywords: TokenPattern = {

// https://www.renpy.org/doc/html/atl.html#contains-statement
contentToken: MetaTokenType.ATLContains, /*meta.atl.contains.renpy*/
begin: /^[ \t]*(contains)\b[ \t]*/dgm,
begin: /(?<=^[ \t]*)(contains)\b/dgm,
beginCaptures: {
1: { token: KeywordTokenType.Contains, /*keyword.control.flow.contains.renpy*/ },
},
Expand Down
5 changes: 5 additions & 0 deletions src/tokenizer/debug-decorator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -446,8 +446,12 @@ ${(decoration.hoverMessage as MarkdownString).value}`,
case MetaTokenType.CameraStatement:
case MetaTokenType.SceneStatement:
case MetaTokenType.ShowStatement:
case MetaTokenType.HideStatement:
case MetaTokenType.ImageStatement:
case MetaTokenType.CallStatement:
case MetaTokenType.CallArguments:
case MetaTokenType.FromClause:
case MetaTokenType.FromArguments:
case MetaTokenType.JumpStatement:
case MetaTokenType.PlayAudioStatement:
case MetaTokenType.QueueAudioStatement:
Expand Down Expand Up @@ -482,6 +486,7 @@ ${(decoration.hoverMessage as MarkdownString).value}`,
case MetaTokenType.OnlayerParameters:
case MetaTokenType.WithParameters:
case MetaTokenType.ZorderParameters:
case MetaTokenType.PauseStatement:
case MetaTokenType.PauseParameters:
case MetaTokenType.ATLBlock:
case MetaTokenType.ATLChoiceBlock:
Expand Down
138 changes: 70 additions & 68 deletions src/tokenizer/python-token-patterns.g.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
// THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
// DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
// ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
// Last generated: 12/07/2024 09:35:29 (UTC+0)
// Last generated: 12/07/2024 18:44:34 (UTC+0)

import { MetaTokenType, KeywordTokenType, EntityTokenType, CharacterTokenType, OperatorTokenType, LiteralTokenType } from "./renpy-tokens";
import { MetaTokenType, KeywordTokenType, CharacterTokenType, EntityTokenType, OperatorTokenType, LiteralTokenType } from "./renpy-tokens";
import { TokenPattern } from "./token-pattern-types";

export const python: TokenPattern = {
Expand Down Expand Up @@ -40,15 +40,6 @@ export const semicolon: TokenPattern = {

export const comments: TokenPattern = {
patterns: [
{
debugName: "comments.patterns![0]",

token: MetaTokenType.Comment, /*comment.line.number-sign.renpy*/
match: /^[ \t]*(#[ \t]*(?:end)?region)\b.*$/dgm,
captures: {
1: { token: MetaTokenType.CommentRegionTag, /*punctuation.definition.tag.region*/ },
},
},
{
debugName: "comments.patterns![1]",

Expand Down Expand Up @@ -241,9 +232,25 @@ export const statementKeyword: TokenPattern = {
]
};

export const newLine: TokenPattern = {
debugName: "newLine",

token: CharacterTokenType.NewLine, /*punctuation.new-line.renpy*/
match: /\r\n|\r|\n/g,
};

export const whitespace: TokenPattern = {
debugName: "whitespace",

token: CharacterTokenType.Whitespace, /*punctuation.whitespace.python*/
match: /[ \t]+/g,
};

export const expressionBare: TokenPattern = {
// valid Python expressions w/o comments and line continuation
patterns: [
whitespace,
newLine,
]
};

Expand Down Expand Up @@ -711,7 +718,7 @@ export const classDeclaration: TokenPattern = {
debugName: "classDeclaration.patterns![0]",

token: MetaTokenType.ClassDefinition, /*meta.class.python*/
begin: /\s*(class)\s+(?=[a-zA-Z_]\w*\s*(:|\())/dg,
begin: /\s*(class)\s+(?=[a-zA-Z_]\w*\s*(?::|\())/dg,
beginCaptures: {
1: { token: KeywordTokenType.Class, /*storage.type.class.python*/ },
},
Expand Down Expand Up @@ -1091,7 +1098,7 @@ export const decorator: TokenPattern = {
debugName: "decorator",

token: MetaTokenType.FunctionDefinition, /*meta.function.decorator.python*/
begin: /^\s*((@))\s*(?=[a-zA-Z_]\w*)/dgm,
begin: /(?<=^[ \t]*)(@)\s*(?=[a-zA-Z_]\w*)/dgm,
beginCaptures: {
1: { token: EntityTokenType.FunctionName, /*entity.name.function.decorator.python*/ },
2: { token: CharacterTokenType.AtSymbol, /*punctuation.definition.decorator.python*/ },
Expand All @@ -1111,9 +1118,9 @@ export const decoratorName: TokenPattern = {
debugName: "decoratorName.patterns![2]",

token: EntityTokenType.FunctionName, /*entity.name.function.decorator.python*/
match: /([a-zA-Z_]\w*)|(\.)/dg,
match: /(?:[a-zA-Z_]\w*)|(\.)/dg,
captures: {
2: { token: CharacterTokenType.Dot, /*punctuation.separator.dot.python*/ },
1: { token: CharacterTokenType.Dot, /*punctuation.separator.dot.python*/ },
},
},
lineContinuation,
Expand Down Expand Up @@ -1545,27 +1552,6 @@ export const regexpCharecterSetEscapes: TokenPattern = {
]
};

export const codetags: TokenPattern = {
debugName: "codetags",

match: /(?:\b(NOTE|XXX|HACK|FIXME|BUG|TODO)\b)/dg,
captures: {
1: { token: MetaTokenType.CommentCodeTag, /*keyword.codetag.notation.python*/ },
},
};

export const commentsBase: TokenPattern = {
debugName: "commentsBase",

token: MetaTokenType.Comment, /*comment.line.number-sign.python*/
begin: /(\#)/dg,
beginCaptures: {
1: { token: CharacterTokenType.Hashtag, /*punctuation.definition.comment.python*/ },
},
end: /($)/gm,
patterns: [codetags]
};

export const commentsStringSingleThree: TokenPattern = {
debugName: "commentsStringSingleThree",

Expand All @@ -1575,7 +1561,8 @@ export const commentsStringSingleThree: TokenPattern = {
1: { token: CharacterTokenType.Hashtag, /*punctuation.definition.comment.python*/ },
},
end: /($|(?='''))/gm,
patterns: [codetags]
patterns: [
]
};

export const commentsStringDoubleThree: TokenPattern = {
Expand All @@ -1587,7 +1574,8 @@ export const commentsStringDoubleThree: TokenPattern = {
1: { token: CharacterTokenType.Hashtag, /*punctuation.definition.comment.python*/ },
},
end: /($|(?="""))/gm,
patterns: [codetags]
patterns: [
]
};

export const singleOneRegexpExpression: TokenPattern = {
Expand Down Expand Up @@ -1663,7 +1651,8 @@ export const singleOneRegexpComments: TokenPattern = {
1: { token: MetaTokenType.CommentEnd, /*punctuation.comment.end.regexp*/ },
2: { token: MetaTokenType.Invalid, /*invalid.illegal.newline.python*/ },
},
patterns: [codetags]
patterns: [
]
};

export const singleOneRegexpLookahead: TokenPattern = {
Expand Down Expand Up @@ -1853,7 +1842,8 @@ export const singleThreeRegexpComments: TokenPattern = {
1: { token: MetaTokenType.CommentEnd, /*punctuation.comment.end.regexp*/ },
2: { token: MetaTokenType.Invalid, /*invalid.illegal.newline.python*/ },
},
patterns: [codetags]
patterns: [
]
};

export const singleThreeRegexpLookahead: TokenPattern = {
Expand Down Expand Up @@ -2060,7 +2050,8 @@ export const doubleOneRegexpComments: TokenPattern = {
1: { token: MetaTokenType.CommentEnd, /*punctuation.comment.end.regexp*/ },
2: { token: MetaTokenType.Invalid, /*invalid.illegal.newline.python*/ },
},
patterns: [codetags]
patterns: [
]
};

export const doubleOneRegexpLookahead: TokenPattern = {
Expand Down Expand Up @@ -2250,7 +2241,8 @@ export const doubleThreeRegexpComments: TokenPattern = {
1: { token: MetaTokenType.CommentEnd, /*punctuation.comment.end.regexp*/ },
2: { token: MetaTokenType.Invalid, /*invalid.illegal.newline.python*/ },
},
patterns: [codetags]
patterns: [
]
};

export const doubleThreeRegexpLookahead: TokenPattern = {
Expand Down Expand Up @@ -2518,7 +2510,7 @@ export const stringRawBinQuotedSingleLine: TokenPattern = {
export const stringQuotedSingleLine: TokenPattern = {
debugName: "stringQuotedSingleLine",

token: LiteralTokenType.String, /*string.quoted.single.python*/
token: LiteralTokenType.String, /*string.quoted.single.test.python*/
begin: /(?:\b([rR])(?=[uU]))?([uU])?(['"])/dg,
beginCaptures: {
1: { token: MetaTokenType.Invalid, /*invalid.illegal.prefix.python*/ },
Expand All @@ -2538,8 +2530,8 @@ export const stringSingleBadBrace1FormattingUnicode: TokenPattern = {
debugName: "stringSingleBadBrace1FormattingUnicode",

// template using {% ... %}
begin: /(?=\{%(.*?(?!(['"])|((?<!\\)\n)))%\})/g,
end: /(?=(['"])|((?<!\\)\n))/g,
begin: /(?=\{%(?:.*?(?!['"]|(?:(?<!\\)\n)))%\})/g,
end: /(?=['"]|(?:(?<!\\)\n))/g,
patterns: [
escapeSequenceUnicode,
escapeSequence,
Expand All @@ -2551,29 +2543,17 @@ export const stringSingleBadBrace1FormattingRaw: TokenPattern = {
debugName: "stringSingleBadBrace1FormattingRaw",

// template using {% ... %}
begin: /(?=\{%(.*?(?!(['"])|((?<!\\)\n)))%\})/g,
end: /(?=(['"])|((?<!\\)\n))/g,
begin: /(?=\{%(?:.*?(?!['"]|(?:(?<!\\)\n)))%\})/g,
end: /(?=['"]|(?:(?<!\\)\n))/g,
patterns: [stringConsumeEscape]
};

export const stringSingleBadBrace2FormattingUnicode: TokenPattern = {
debugName: "stringSingleBadBrace2FormattingUnicode",

// odd format or format-like syntax
begin: /(?!\{\{)(?=\{(\w*?(?!(['"])|((?<!\\)\n))[^!:\.\[}\w]).*?(?!(['"])|((?<!\\)\n))\})/g,
end: /(?=(['"])|((?<!\\)\n))/g,
patterns: [
escapeSequenceUnicode,
stringEntity,
]
};

export const stringSingleBadBrace2FormattingRaw: TokenPattern = {
debugName: "stringSingleBadBrace2FormattingRaw",

// odd format or format-like syntax
begin: /(?!\{\{)(?=\{(\w*?(?!(['"])|((?<!\\)\n))[^!:\.\[}\w]).*?(?!(['"])|((?<!\\)\n))\})/g,
end: /(?=(['"])|((?<!\\)\n))/g,
begin: /(?!\{\{)(?=\{(?:\w*?(?!['"]|(?:(?<!\\)\n))[^!:\.\[}\w]).*?(?!['"]|(?:(?<!\\)\n))\})/g,
end: /(?=['"]|(?:(?<!\\)\n))/g,
patterns: [
stringConsumeEscape,
stringFormatting,
Expand Down Expand Up @@ -3069,14 +3049,37 @@ export const fstringTerminatorMultiTail: TokenPattern = {
// Push pattern references that were not defined on include
python.patterns!.push(statement, expression);
statement.patterns!.push(importStatement, classDeclaration, functionDeclaration, generator, statementKeyword, assignmentOperator, decorator, docstringStatement, semicolon);
comments.patterns!.push(commentsBase);
docstringStatement.patterns!.push(docstring);
docstring.patterns![0].patterns!.push(docstringPrompt, codetags, docstringGutsUnicode);
docstring.patterns![1].patterns!.push(stringConsumeEscape, docstringPrompt, codetags);
docstring.patterns![2].patterns!.push(codetags, docstringGutsUnicode);
docstring.patterns![3].patterns!.push(stringConsumeEscape, codetags);
docstring.patterns![0].patterns!.push(docstringPrompt, docstringGutsUnicode);
docstring.patterns![1].patterns!.push(stringConsumeEscape, docstringPrompt);
docstring.patterns![2].patterns!.push(docstringGutsUnicode);
docstring.patterns![3].patterns!.push(stringConsumeEscape);
docstringGutsUnicode.patterns!.push(escapeSequenceUnicode, escapeSequence, stringLineContinuation);
expressionBare.patterns!.push(backticks, illegalAnno, literal, regexp, string, lambda, generator, illegalOperator, operator, curlyBraces, itemAccess, list, oddFunctionCall, roundBraces, functionCall, builtinFunctions, builtinTypes, builtinExceptions, magicNames, specialNames, illegalNames, specialVariables, ellipsis, punctuation, lineContinuation);
expressionBare.patterns!.splice(0, 0, backticks);
expressionBare.patterns!.splice(1, 0, illegalAnno);
expressionBare.patterns!.splice(2, 0, literal);
expressionBare.patterns!.splice(3, 0, regexp);
expressionBare.patterns!.splice(4, 0, string);
expressionBare.patterns!.splice(5, 0, lambda);
expressionBare.patterns!.splice(6, 0, generator);
expressionBare.patterns!.splice(7, 0, illegalOperator);
expressionBare.patterns!.splice(8, 0, operator);
expressionBare.patterns!.splice(9, 0, curlyBraces);
expressionBare.patterns!.splice(10, 0, itemAccess);
expressionBare.patterns!.splice(11, 0, list);
expressionBare.patterns!.splice(12, 0, oddFunctionCall);
expressionBare.patterns!.splice(13, 0, roundBraces);
expressionBare.patterns!.splice(14, 0, functionCall);
expressionBare.patterns!.splice(15, 0, builtinFunctions);
expressionBare.patterns!.splice(16, 0, builtinTypes);
expressionBare.patterns!.splice(17, 0, builtinExceptions);
expressionBare.patterns!.splice(18, 0, magicNames);
expressionBare.patterns!.splice(19, 0, specialNames);
expressionBare.patterns!.splice(20, 0, illegalNames);
expressionBare.patterns!.splice(21, 0, specialVariables);
expressionBare.patterns!.splice(22, 0, ellipsis);
expressionBare.patterns!.splice(23, 0, punctuation);
expressionBare.patterns!.splice(24, 0, lineContinuation);
expressionBase.patterns!.push(lineContinuation);
expression.patterns!.splice(1, 0, memberAccess);
memberAccess.patterns!.push(functionCall, memberAccessBase, memberAccessAttribute);
Expand Down Expand Up @@ -3173,7 +3176,6 @@ doubleThreeRegexpExpression.patterns!.splice(12, 0, doubleThreeRegexpParentheses
stringRawQuotedSingleLine.patterns!.splice(0, 0, stringSingleBadBrace1FormattingRaw);
stringRawQuotedSingleLine.patterns!.splice(1, 0, stringSingleBadBrace2FormattingRaw);
stringQuotedSingleLine.patterns!.splice(0, 0, stringSingleBadBrace1FormattingUnicode);
stringQuotedSingleLine.patterns!.splice(1, 0, stringSingleBadBrace2FormattingUnicode);
stringRawQuotedMultiLine.patterns!.splice(0, 0, stringMultiBadBrace1FormattingRaw);
stringRawQuotedMultiLine.patterns!.splice(1, 0, stringMultiBadBrace2FormattingRaw);
stringQuotedMultiLine.patterns!.splice(0, 0, stringMultiBadBrace1FormattingUnicode);
Expand Down
Loading

0 comments on commit 43008f6

Please sign in to comment.