From 43008f6c2040057949e5759f2c6c49af3bc86aeb Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Fri, 12 Jul 2024 20:59:29 +0200 Subject: [PATCH] Update patterns to fix some warnings in the tokenizer --- src/tokenizer/atl-token-patterns.g.ts | 14 +- src/tokenizer/debug-decorator.ts | 5 + src/tokenizer/python-token-patterns.g.ts | 138 +++++++------ src/tokenizer/renpy-token-patterns.g.ts | 252 +++++++++++++---------- src/tokenizer/renpy-tokens.ts | 4 + src/tokenizer/screen-token-patterns.g.ts | 28 ++- src/tokenizer/style-token-patterns.g.ts | 32 +-- src/tokenizer/token-definitions.ts | 6 +- src/tokenizer/token-patterns.g.ts | 36 +++- syntax-to-token-pattern.py | 2 + syntaxes/renpy.atl.tmLanguage.json | 12 +- syntaxes/renpy.python.tmLanguage.json | 110 ++++------ syntaxes/renpy.screen.tmLanguage.json | 28 +-- syntaxes/renpy.style.tmLanguage.json | 32 ++- syntaxes/renpy.tmLanguage.json | 207 +++++++++++-------- 15 files changed, 500 insertions(+), 406 deletions(-) diff --git a/src/tokenizer/atl-token-patterns.g.ts b/src/tokenizer/atl-token-patterns.g.ts index 949c479..dd3703d 100644 --- a/src/tokenizer/atl-token-patterns.g.ts +++ b/src/tokenizer/atl-token-patterns.g.ts @@ -5,7 +5,7 @@ // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT. // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED. -// Last generated: 12/07/2024 09:35:29 (UTC+0) +// Last generated: 12/07/2024 18:50:56 (UTC+0) import { KeywordTokenType, EntityTokenType, MetaTokenType, CharacterTokenType } from "./renpy-tokens"; import { TokenPattern } from "./token-pattern-types"; @@ -173,7 +173,7 @@ export const atlExpression: TokenPattern = { { debugName: "atlExpression.patterns![1]", - begin: /\b(with)\b[ \t]*/dg, + begin: /\b(with)\b/dg, beginCaptures: { 1: { token: KeywordTokenType.With, /*keyword.with.renpy*/ }, }, @@ -189,7 +189,7 @@ export const atlKeywords: TokenPattern = { debugName: "atlKeywords.patterns![0]", // https://www.renpy.org/doc/html/atl.html#animation-statement - match: /^[ \t]*(animation)\b/dgm, + match: /(?<=^[ \t]*)(animation)\b/dgm, captures: { 1: { token: KeywordTokenType.Animation, /*keyword.animation.renpy*/ }, }, @@ -198,7 +198,7 @@ export const atlKeywords: TokenPattern = { debugName: "atlKeywords.patterns![1]", // https://www.renpy.org/doc/html/atl.html#pass-statement - match: /^[ \t]*(pass)\b/dgm, + match: /(?<=^[ \t]*)(pass)\b/dgm, captures: { 1: { token: KeywordTokenType.Pass, /*keyword.control.flow.pass.renpy*/ }, }, @@ -207,7 +207,7 @@ export const atlKeywords: TokenPattern = { debugName: "atlKeywords.patterns![2]", // https://www.renpy.org/doc/html/atl.html#repeat-statement and https://www.renpy.org/doc/html/atl.html#time-statement - begin: /^[ \t]*(?:(repeat)|(time)|(pause))\b[ \t]*/dgm, + begin: /(?<=^[ \t]*)(?:(repeat)|(time)|(pause))\b/dgm, beginCaptures: { 1: { token: KeywordTokenType.Repeat, /*keyword.control.flow.repeat.renpy*/ }, 2: { token: KeywordTokenType.Time, /*keyword.control.flow.time.renpy*/ }, @@ -221,7 +221,7 @@ export const atlKeywords: TokenPattern = { // https://www.renpy.org/doc/html/atl.html#expression-statement contentToken: MetaTokenType.ATLWith, /*meta.atl.with.renpy*/ - begin: /^[ \t]*(with)\b[ \t]*/dgm, + begin: /(?<=^[ \t]*)(with)\b/dgm, beginCaptures: { 1: { token: KeywordTokenType.With, /*keyword.with.renpy*/ }, }, @@ -233,7 +233,7 @@ export const atlKeywords: TokenPattern = { // https://www.renpy.org/doc/html/atl.html#contains-statement contentToken: MetaTokenType.ATLContains, /*meta.atl.contains.renpy*/ - begin: /^[ \t]*(contains)\b[ \t]*/dgm, + begin: /(?<=^[ \t]*)(contains)\b/dgm, beginCaptures: { 1: { token: KeywordTokenType.Contains, /*keyword.control.flow.contains.renpy*/ }, }, diff --git a/src/tokenizer/debug-decorator.ts b/src/tokenizer/debug-decorator.ts index 317d147..9ff1409 100644 --- a/src/tokenizer/debug-decorator.ts +++ b/src/tokenizer/debug-decorator.ts @@ -446,8 +446,12 @@ ${(decoration.hoverMessage as MarkdownString).value}`, case MetaTokenType.CameraStatement: case MetaTokenType.SceneStatement: case MetaTokenType.ShowStatement: + case MetaTokenType.HideStatement: case MetaTokenType.ImageStatement: case MetaTokenType.CallStatement: + case MetaTokenType.CallArguments: + case MetaTokenType.FromClause: + case MetaTokenType.FromArguments: case MetaTokenType.JumpStatement: case MetaTokenType.PlayAudioStatement: case MetaTokenType.QueueAudioStatement: @@ -482,6 +486,7 @@ ${(decoration.hoverMessage as MarkdownString).value}`, case MetaTokenType.OnlayerParameters: case MetaTokenType.WithParameters: case MetaTokenType.ZorderParameters: + case MetaTokenType.PauseStatement: case MetaTokenType.PauseParameters: case MetaTokenType.ATLBlock: case MetaTokenType.ATLChoiceBlock: diff --git a/src/tokenizer/python-token-patterns.g.ts b/src/tokenizer/python-token-patterns.g.ts index a586e8e..4819511 100644 --- a/src/tokenizer/python-token-patterns.g.ts +++ b/src/tokenizer/python-token-patterns.g.ts @@ -5,9 +5,9 @@ // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT. // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED. -// Last generated: 12/07/2024 09:35:29 (UTC+0) +// Last generated: 12/07/2024 18:44:34 (UTC+0) -import { MetaTokenType, KeywordTokenType, EntityTokenType, CharacterTokenType, OperatorTokenType, LiteralTokenType } from "./renpy-tokens"; +import { MetaTokenType, KeywordTokenType, CharacterTokenType, EntityTokenType, OperatorTokenType, LiteralTokenType } from "./renpy-tokens"; import { TokenPattern } from "./token-pattern-types"; export const python: TokenPattern = { @@ -40,15 +40,6 @@ export const semicolon: TokenPattern = { export const comments: TokenPattern = { patterns: [ - { - debugName: "comments.patterns![0]", - - token: MetaTokenType.Comment, /*comment.line.number-sign.renpy*/ - match: /^[ \t]*(#[ \t]*(?:end)?region)\b.*$/dgm, - captures: { - 1: { token: MetaTokenType.CommentRegionTag, /*punctuation.definition.tag.region*/ }, - }, - }, { debugName: "comments.patterns![1]", @@ -241,9 +232,25 @@ export const statementKeyword: TokenPattern = { ] }; +export const newLine: TokenPattern = { + debugName: "newLine", + + token: CharacterTokenType.NewLine, /*punctuation.new-line.renpy*/ + match: /\r\n|\r|\n/g, +}; + +export const whitespace: TokenPattern = { + debugName: "whitespace", + + token: CharacterTokenType.Whitespace, /*punctuation.whitespace.python*/ + match: /[ \t]+/g, +}; + export const expressionBare: TokenPattern = { // valid Python expressions w/o comments and line continuation patterns: [ + whitespace, + newLine, ] }; @@ -711,7 +718,7 @@ export const classDeclaration: TokenPattern = { debugName: "classDeclaration.patterns![0]", token: MetaTokenType.ClassDefinition, /*meta.class.python*/ - begin: /\s*(class)\s+(?=[a-zA-Z_]\w*\s*(:|\())/dg, + begin: /\s*(class)\s+(?=[a-zA-Z_]\w*\s*(?::|\())/dg, beginCaptures: { 1: { token: KeywordTokenType.Class, /*storage.type.class.python*/ }, }, @@ -1091,7 +1098,7 @@ export const decorator: TokenPattern = { debugName: "decorator", token: MetaTokenType.FunctionDefinition, /*meta.function.decorator.python*/ - begin: /^\s*((@))\s*(?=[a-zA-Z_]\w*)/dgm, + begin: /(?<=^[ \t]*)(@)\s*(?=[a-zA-Z_]\w*)/dgm, beginCaptures: { 1: { token: EntityTokenType.FunctionName, /*entity.name.function.decorator.python*/ }, 2: { token: CharacterTokenType.AtSymbol, /*punctuation.definition.decorator.python*/ }, @@ -1111,9 +1118,9 @@ export const decoratorName: TokenPattern = { debugName: "decoratorName.patterns![2]", token: EntityTokenType.FunctionName, /*entity.name.function.decorator.python*/ - match: /([a-zA-Z_]\w*)|(\.)/dg, + match: /(?:[a-zA-Z_]\w*)|(\.)/dg, captures: { - 2: { token: CharacterTokenType.Dot, /*punctuation.separator.dot.python*/ }, + 1: { token: CharacterTokenType.Dot, /*punctuation.separator.dot.python*/ }, }, }, lineContinuation, @@ -1545,27 +1552,6 @@ export const regexpCharecterSetEscapes: TokenPattern = { ] }; -export const codetags: TokenPattern = { - debugName: "codetags", - - match: /(?:\b(NOTE|XXX|HACK|FIXME|BUG|TODO)\b)/dg, - captures: { - 1: { token: MetaTokenType.CommentCodeTag, /*keyword.codetag.notation.python*/ }, - }, -}; - -export const commentsBase: TokenPattern = { - debugName: "commentsBase", - - token: MetaTokenType.Comment, /*comment.line.number-sign.python*/ - begin: /(\#)/dg, - beginCaptures: { - 1: { token: CharacterTokenType.Hashtag, /*punctuation.definition.comment.python*/ }, - }, - end: /($)/gm, - patterns: [codetags] -}; - export const commentsStringSingleThree: TokenPattern = { debugName: "commentsStringSingleThree", @@ -1575,7 +1561,8 @@ export const commentsStringSingleThree: TokenPattern = { 1: { token: CharacterTokenType.Hashtag, /*punctuation.definition.comment.python*/ }, }, end: /($|(?='''))/gm, - patterns: [codetags] + patterns: [ + ] }; export const commentsStringDoubleThree: TokenPattern = { @@ -1587,7 +1574,8 @@ export const commentsStringDoubleThree: TokenPattern = { 1: { token: CharacterTokenType.Hashtag, /*punctuation.definition.comment.python*/ }, }, end: /($|(?="""))/gm, - patterns: [codetags] + patterns: [ + ] }; export const singleOneRegexpExpression: TokenPattern = { @@ -1663,7 +1651,8 @@ export const singleOneRegexpComments: TokenPattern = { 1: { token: MetaTokenType.CommentEnd, /*punctuation.comment.end.regexp*/ }, 2: { token: MetaTokenType.Invalid, /*invalid.illegal.newline.python*/ }, }, - patterns: [codetags] + patterns: [ + ] }; export const singleOneRegexpLookahead: TokenPattern = { @@ -1853,7 +1842,8 @@ export const singleThreeRegexpComments: TokenPattern = { 1: { token: MetaTokenType.CommentEnd, /*punctuation.comment.end.regexp*/ }, 2: { token: MetaTokenType.Invalid, /*invalid.illegal.newline.python*/ }, }, - patterns: [codetags] + patterns: [ + ] }; export const singleThreeRegexpLookahead: TokenPattern = { @@ -2060,7 +2050,8 @@ export const doubleOneRegexpComments: TokenPattern = { 1: { token: MetaTokenType.CommentEnd, /*punctuation.comment.end.regexp*/ }, 2: { token: MetaTokenType.Invalid, /*invalid.illegal.newline.python*/ }, }, - patterns: [codetags] + patterns: [ + ] }; export const doubleOneRegexpLookahead: TokenPattern = { @@ -2250,7 +2241,8 @@ export const doubleThreeRegexpComments: TokenPattern = { 1: { token: MetaTokenType.CommentEnd, /*punctuation.comment.end.regexp*/ }, 2: { token: MetaTokenType.Invalid, /*invalid.illegal.newline.python*/ }, }, - patterns: [codetags] + patterns: [ + ] }; export const doubleThreeRegexpLookahead: TokenPattern = { @@ -2518,7 +2510,7 @@ export const stringRawBinQuotedSingleLine: TokenPattern = { export const stringQuotedSingleLine: TokenPattern = { debugName: "stringQuotedSingleLine", - token: LiteralTokenType.String, /*string.quoted.single.python*/ + token: LiteralTokenType.String, /*string.quoted.single.test.python*/ begin: /(?:\b([rR])(?=[uU]))?([uU])?(['"])/dg, beginCaptures: { 1: { token: MetaTokenType.Invalid, /*invalid.illegal.prefix.python*/ }, @@ -2538,8 +2530,8 @@ export const stringSingleBadBrace1FormattingUnicode: TokenPattern = { debugName: "stringSingleBadBrace1FormattingUnicode", // template using {% ... %} - begin: /(?=\{%(.*?(?!(['"])|((? = { ShowStatement: { name: "ShowStatement", value: MetaTokenType.ShowStatement }, HideStatement: { name: "HideStatement", value: MetaTokenType.HideStatement }, - JumpStatement: { name: "JumpStatement", value: MetaTokenType.JumpStatement }, CallStatement: { name: "CallStatement", value: MetaTokenType.CallStatement }, + CallArguments: { name: "CallArguments", value: MetaTokenType.CallArguments }, + FromClause: { name: "FromClause", value: MetaTokenType.FromClause }, + FromArguments: { name: "FromArguments", value: MetaTokenType.FromArguments }, + JumpStatement: { name: "JumpStatement", value: MetaTokenType.JumpStatement }, PlayAudioStatement: { name: "PlayAudioStatement", value: MetaTokenType.PlayAudioStatement }, QueueAudioStatement: { name: "QueueAudioStatement", value: MetaTokenType.QueueAudioStatement }, @@ -703,6 +706,7 @@ const tokenTypeDefinitions: EnumToString = { OnlayerParameters: { name: "OnlayerParameters", value: MetaTokenType.OnlayerParameters }, WithParameters: { name: "WithParameters", value: MetaTokenType.WithParameters }, ZorderParameters: { name: "ZorderParameters", value: MetaTokenType.ZorderParameters }, + PauseStatement: { name: "PauseStatement", value: MetaTokenType.PauseStatement }, PauseParameters: { name: "PauseParameters", value: MetaTokenType.PauseParameters }, ATLBlock: { name: "ATLBlock", value: MetaTokenType.ATLBlock }, diff --git a/src/tokenizer/token-patterns.g.ts b/src/tokenizer/token-patterns.g.ts index 4d57214..c40b98b 100644 --- a/src/tokenizer/token-patterns.g.ts +++ b/src/tokenizer/token-patterns.g.ts @@ -3,13 +3,13 @@ // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT. // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED. -// Last generated: 12/07/2024 09:35:29 (UTC+0) +// Last generated: 12/07/2024 18:44:34 (UTC+0) +import * as AtlPatterns from "./atl-token-patterns.g"; import * as RenpyPatterns from "./renpy-token-patterns.g"; -import * as ScreenPatterns from "./screen-token-patterns.g"; import * as StylePatterns from "./style-token-patterns.g"; -import * as AtlPatterns from "./atl-token-patterns.g"; import * as PythonPatterns from "./python-token-patterns.g"; +import * as ScreenPatterns from "./screen-token-patterns.g"; // Push all RenpyPatterns external includes RenpyPatterns.literal.patterns!.push(PythonPatterns.literal); @@ -27,10 +27,10 @@ RenpyPatterns.define.patterns!.splice(2, 0, PythonPatterns.expression); RenpyPatterns.define.patterns![3].patterns!.splice(0, 0, PythonPatterns.expression); RenpyPatterns.defaultStatement.patterns!.splice(1, 0, PythonPatterns.expression); RenpyPatterns.defaultStatement.patterns![2].patterns!.splice(0, 0, PythonPatterns.expression); -RenpyPatterns.oneLinePython.patterns!.splice(2, 0, PythonPatterns.expression); +RenpyPatterns.oneLinePython.patterns!.splice(1, 0, PythonPatterns.expression); RenpyPatterns.sayStatements.patterns![0].endCaptures![3].patterns!.push(PythonPatterns.functionArguments); RenpyPatterns.sayStatements.patterns![1].endCaptures![3].patterns!.push(PythonPatterns.functionArguments); -RenpyPatterns.conditionals.captures![2].patterns!.push(PythonPatterns.expression); +RenpyPatterns.conditionals.captures![2].patterns!.splice(0, 0, PythonPatterns.expression); RenpyPatterns.labelName.patterns!.splice(0, 0, PythonPatterns.builtinPossibleCallables); RenpyPatterns.labelCall.patterns!.splice(0, 0, PythonPatterns.specialVariables); RenpyPatterns.labelCall.patterns!.push(PythonPatterns.functionArguments); @@ -38,10 +38,10 @@ RenpyPatterns.labelDefName.patterns!.splice(0, 0, PythonPatterns.builtinPossible RenpyPatterns.label.captures![2].patterns!.splice(1, 0, PythonPatterns.parameters); RenpyPatterns.returnStatements.patterns!.push(PythonPatterns.expression); RenpyPatterns.callJumpExpression.patterns!.push(PythonPatterns.expression); -RenpyPatterns.callPass.patterns!.push(PythonPatterns.functionArguments); +RenpyPatterns.callPass.patterns!.splice(0, 0, PythonPatterns.functionArguments); RenpyPatterns.menuOption.beginCaptures![3].patterns!.splice(0, 0, PythonPatterns.functionArguments); -RenpyPatterns.menuOption.beginCaptures![3].patterns![1].captures![2].patterns!.push(PythonPatterns.expressionBare); -RenpyPatterns.menuSet.captures![2].patterns!.push(PythonPatterns.expressionBare); +RenpyPatterns.menuOption.beginCaptures![3].patterns![1].patterns!.push(PythonPatterns.expression); +RenpyPatterns.menuSet.patterns!.push(PythonPatterns.expression); RenpyPatterns.menu.beginCaptures![3].patterns!.push(PythonPatterns.functionArguments); RenpyPatterns.audioParams.patterns!.push(PythonPatterns.number); RenpyPatterns.play.patterns![0].patterns!.splice(2, 0, PythonPatterns.expression); @@ -111,19 +111,33 @@ ScreenPatterns.screenSimpleExpression.patterns!.push(RenpyPatterns.whitespace); ScreenPatterns.screenText.patterns!.push(RenpyPatterns.atStatement); ScreenPatterns.screenText.patterns![0].patterns!.push(AtlPatterns.atl); ScreenPatterns.screenText.patterns![1].patterns!.splice(0, 0, RenpyPatterns.strings); -ScreenPatterns.screen.patterns![0].beginCaptures![3].patterns!.push(PythonPatterns.parameters, RenpyPatterns.invalidToken); +ScreenPatterns.screen.patterns![0].patterns!.splice(1, 0, PythonPatterns.parameters); +ScreenPatterns.screen.patterns![0].patterns!.push(RenpyPatterns.fallbackPatterns); ScreenPatterns.screenFallback.patterns!.push(RenpyPatterns.basePatterns); ScreenPatterns.screenLanguage.patterns!.splice(0, 0, RenpyPatterns.conditionals); // Push all StylePatterns external includes StylePatterns.styleBlockTester.patterns![0].patterns!.push(RenpyPatterns.fallbackPatterns); -StylePatterns.styleClause.patterns![2].captures![2].patterns!.push(RenpyPatterns.name); +StylePatterns.styleClause.patterns![0].patterns!.push(RenpyPatterns.fallbackPatterns); +StylePatterns.styleClause.patterns![2].patterns!.push(RenpyPatterns.name, RenpyPatterns.fallbackPatterns); StylePatterns.styleClause.patterns![4].patterns!.push(RenpyPatterns.simpleExpression); StylePatterns.styleClause.patterns![5].patterns!.push(RenpyPatterns.simpleExpression); StylePatterns.style.patterns!.push(RenpyPatterns.fallbackPatterns); // Push all PythonPatterns external includes +PythonPatterns.comments.patterns!.splice(0, 0, RenpyPatterns.regionComment); +PythonPatterns.comments.patterns!.push(RenpyPatterns.commentsBase); +PythonPatterns.docstring.patterns![0].patterns!.splice(1, 0, RenpyPatterns.codetags); +PythonPatterns.docstring.patterns![1].patterns!.push(RenpyPatterns.codetags); +PythonPatterns.docstring.patterns![2].patterns!.splice(0, 0, RenpyPatterns.codetags); +PythonPatterns.docstring.patterns![3].patterns!.push(RenpyPatterns.codetags); PythonPatterns.stringUnicodeGuts.patterns!.splice(0, 0, RenpyPatterns.stringsInterior); +PythonPatterns.commentsStringSingleThree.patterns!.push(RenpyPatterns.codetags); +PythonPatterns.commentsStringDoubleThree.patterns!.push(RenpyPatterns.codetags); +PythonPatterns.singleOneRegexpComments.patterns!.push(RenpyPatterns.codetags); +PythonPatterns.singleThreeRegexpComments.patterns!.push(RenpyPatterns.codetags); +PythonPatterns.doubleOneRegexpComments.patterns!.push(RenpyPatterns.codetags); +PythonPatterns.doubleThreeRegexpComments.patterns!.push(RenpyPatterns.codetags); -export { RenpyPatterns, ScreenPatterns, StylePatterns, AtlPatterns, PythonPatterns }; \ No newline at end of file +export { AtlPatterns, RenpyPatterns, StylePatterns, PythonPatterns, ScreenPatterns }; \ No newline at end of file diff --git a/syntax-to-token-pattern.py b/syntax-to-token-pattern.py index fd3fea5..a00425c 100644 --- a/syntax-to-token-pattern.py +++ b/syntax-to-token-pattern.py @@ -57,6 +57,8 @@ def get_parts(range: slice) -> str: return "KeywordTokenType.Class" elif get_part(2) == "style": return "KeywordTokenType.Style" + elif get_part(2) == "screen": + return "KeywordTokenType.Screen" elif get_part(2) == "imaginary": return "MetaTokenType.ImaginaryNumberStorageType" elif get_part(2) == "number": diff --git a/syntaxes/renpy.atl.tmLanguage.json b/syntaxes/renpy.atl.tmLanguage.json index 29f6d1a..7e3bbb4 100644 --- a/syntaxes/renpy.atl.tmLanguage.json +++ b/syntaxes/renpy.atl.tmLanguage.json @@ -154,7 +154,7 @@ "patterns": [ { "include": "#atl-simple-expression" }, { - "begin": "\\b(with)\\b[ \\t]*", + "begin": "\\b(with)\\b", "beginCaptures": { "1": { "name": "keyword.with.renpy" } }, @@ -168,21 +168,21 @@ "patterns": [ { "comment": "https://www.renpy.org/doc/html/atl.html#animation-statement", - "match": "^[ \\t]*(animation)\\b", + "match": "(?<=^[ \\t]*)(animation)\\b", "captures": { "1": { "name": "keyword.animation.renpy" } } }, { "comment": "https://www.renpy.org/doc/html/atl.html#pass-statement", - "match": "^[ \\t]*(pass)\\b", + "match": "(?<=^[ \\t]*)(pass)\\b", "captures": { "1": { "name": "keyword.control.flow.pass.renpy" } } }, { "comment": "https://www.renpy.org/doc/html/atl.html#repeat-statement and https://www.renpy.org/doc/html/atl.html#time-statement", - "begin": "^[ \\t]*(?:(repeat)|(time)|(pause))\\b[ \\t]*", + "begin": "(?<=^[ \\t]*)(?:(repeat)|(time)|(pause))\\b", "beginCaptures": { "1": { "name": "keyword.control.flow.repeat.renpy" }, "2": { "name": "keyword.control.flow.time.renpy" }, @@ -194,7 +194,7 @@ { "comment": "https://www.renpy.org/doc/html/atl.html#expression-statement", "contentName": "meta.atl.with.renpy", - "begin": "^[ \\t]*(with)\\b[ \\t]*", + "begin": "(?<=^[ \\t]*)(with)\\b", "beginCaptures": { "1": { "name": "keyword.with.renpy" } }, @@ -204,7 +204,7 @@ { "comment": "https://www.renpy.org/doc/html/atl.html#contains-statement", "contentName": "meta.atl.contains.renpy", - "begin": "^[ \\t]*(contains)\\b[ \\t]*", + "begin": "(?<=^[ \\t]*)(contains)\\b", "beginCaptures": { "1": { "name": "keyword.control.flow.contains.renpy" } }, diff --git a/syntaxes/renpy.python.tmLanguage.json b/syntaxes/renpy.python.tmLanguage.json index 892e5ea..03b450d 100644 --- a/syntaxes/renpy.python.tmLanguage.json +++ b/syntaxes/renpy.python.tmLanguage.json @@ -64,13 +64,7 @@ }, "comments": { "patterns": [ - { - "name": "comment.line.number-sign.renpy", - "match": "^[ \\t]*(#[ \\t]*(?:end)?region)\\b.*$", - "captures": { - "1": { "name": "punctuation.definition.tag.region" } - } - }, + { "include": "source.renpy#region-comment" }, { "name": "comment.line.number-sign.python", "contentName": "meta.typehint.comment.python", @@ -99,7 +93,7 @@ } ] }, - { "include": "#comments-base" } + { "include": "source.renpy#comments-base" } ] }, "docstring-statement": { @@ -120,7 +114,7 @@ "endCaptures": { "1": { "name": "punctuation.definition.string.end.python" } }, - "patterns": [ { "include": "#docstring-prompt" }, { "include": "#codetags" }, { "include": "#docstring-guts-unicode" } ] + "patterns": [ { "include": "#docstring-prompt" }, { "include": "source.renpy#codetags" }, { "include": "#docstring-guts-unicode" } ] }, { "name": "string.quoted.docstring.raw.multi.python", @@ -133,7 +127,7 @@ "endCaptures": { "1": { "name": "punctuation.definition.string.end.python" } }, - "patterns": [ { "include": "#string-consume-escape" }, { "include": "#docstring-prompt" }, { "include": "#codetags" } ] + "patterns": [ { "include": "#string-consume-escape" }, { "include": "#docstring-prompt" }, { "include": "source.renpy#codetags" } ] }, { "name": "string.quoted.docstring.single.python", @@ -146,7 +140,7 @@ "1": { "name": "punctuation.definition.string.end.python" }, "2": { "name": "invalid.illegal.newline.python" } }, - "patterns": [ { "include": "#codetags" }, { "include": "#docstring-guts-unicode" } ] + "patterns": [ { "include": "source.renpy#codetags" }, { "include": "#docstring-guts-unicode" } ] }, { "name": "string.quoted.docstring.raw.single.python", @@ -160,7 +154,7 @@ "1": { "name": "punctuation.definition.string.end.python" }, "2": { "name": "invalid.illegal.newline.python" } }, - "patterns": [ { "include": "#string-consume-escape" }, { "include": "#codetags" } ] + "patterns": [ { "include": "#string-consume-escape" }, { "include": "source.renpy#codetags" } ] } ] }, @@ -212,6 +206,14 @@ } ] }, + "new-line": { + "match": "\\r\\n|\\r|\\n", + "name": "punctuation.new-line.renpy" + }, + "whitespace": { + "match": "[ \\t]+", + "name": "punctuation.whitespace.python" + }, "expression-bare": { "comment": "valid Python expressions w/o comments and line continuation", "patterns": [ @@ -239,7 +241,9 @@ { "include": "#special-variables" }, { "include": "#ellipsis" }, { "include": "#punctuation" }, - { "include": "#line-continuation" } + { "include": "#line-continuation" }, + { "include": "#whitespace" }, + { "include": "#new-line" } ] }, "expression-base": { @@ -306,10 +310,7 @@ "endCaptures": { "0": { "name": "punctuation.definition.list.end.python" } }, - "patterns": [ - { "include": "#expression" }, - { "include": "#whitespace" } - ] + "patterns": [ { "include": "#expression" } ] }, "odd-function-call": { "comment": "A bit obscured function call where there may have been an arbitrary number of other operations to get the function.E.g. \"arr[idx](args)\"", @@ -330,10 +331,7 @@ "endCaptures": { "0": { "name": "punctuation.parenthesis.end.python" } }, - "patterns": [ - { "include": "#expression" }, - { "include": "#whitespace" } - ] + "patterns": [ { "include": "#expression" } ] }, "line-continuation": { "patterns": [ @@ -381,10 +379,6 @@ } ] }, - "whitespace": { - "match": "[ \\t]+", - "name": "punctuation.whitespace.python" - }, "literal": { "patterns": [ { @@ -607,7 +601,7 @@ "patterns": [ { "name": "meta.class.python", - "begin": "\\s*(class)\\s+(?=[[:alpha:]_]\\w*\\s*(:|\\())", + "begin": "\\s*(class)\\s+(?=[[:alpha:]_]\\w*\\s*(?::|\\())", "end": "(:)", "beginCaptures": { "1": { "name": "storage.type.class.python" } @@ -916,7 +910,7 @@ }, "decorator": { "name": "meta.function.decorator.python", - "begin": "^\\s*((@))\\s*(?=[[:alpha:]_]\\w*)", + "begin": "(?<=^[ \\t]*)(@)\\s*(?=[[:alpha:]_]\\w*)", "end": "(\\))(?:(.*?)(?=\\s*(?:\\#|$)))|(?=\\n|\\#)", "beginCaptures": { "1": { "name": "entity.name.function.decorator.python" }, @@ -934,9 +928,9 @@ { "include": "#illegal-object-name" }, { "name": "entity.name.function.decorator.python", - "match": "([[:alpha:]_]\\w*)|(\\.)", + "match": "(?:[[:alpha:]_]\\w*)|(\\.)", "captures": { - "2": { "name": "punctuation.separator.dot.python" } + "1": { "name": "punctuation.separator.dot.python" } } }, { "include": "#line-continuation" }, @@ -1234,21 +1228,6 @@ { "include": "#regexp-escape-catchall" } ] }, - "codetags": { - "match": "(?:\\b(NOTE|XXX|HACK|FIXME|BUG|TODO)\\b)", - "captures": { - "1": { "name": "keyword.codetag.notation.python" } - } - }, - "comments-base": { - "name": "comment.line.number-sign.python", - "begin": "(\\#)", - "beginCaptures": { - "1": { "name": "punctuation.definition.comment.python" } - }, - "end": "($)", - "patterns": [ { "include": "#codetags" } ] - }, "comments-string-single-three": { "name": "comment.line.number-sign.python", "begin": "(\\#)", @@ -1256,7 +1235,7 @@ "1": { "name": "punctuation.definition.comment.python" } }, "end": "($|(?='''))", - "patterns": [ { "include": "#codetags" } ] + "patterns": [ { "include": "source.renpy#codetags" } ] }, "comments-string-double-three": { "name": "comment.line.number-sign.python", @@ -1265,7 +1244,7 @@ "1": { "name": "punctuation.definition.comment.python" } }, "end": "($|(?=\"\"\"))", - "patterns": [ { "include": "#codetags" } ] + "patterns": [ { "include": "source.renpy#codetags" } ] }, "single-one-regexp-expression": { "patterns": [ @@ -1337,7 +1316,7 @@ "1": { "name": "punctuation.comment.end.regexp" }, "2": { "name": "invalid.illegal.newline.python" } }, - "patterns": [ { "include": "#codetags" } ] + "patterns": [ { "include": "source.renpy#codetags" } ] }, "single-one-regexp-lookahead": { "begin": "(\\()\\?=", @@ -1499,7 +1478,7 @@ "1": { "name": "punctuation.comment.end.regexp" }, "2": { "name": "invalid.illegal.newline.python" } }, - "patterns": [ { "include": "#codetags" } ] + "patterns": [ { "include": "source.renpy#codetags" } ] }, "single-three-regexp-lookahead": { "begin": "(\\()\\?=", @@ -1660,7 +1639,7 @@ "1": { "name": "punctuation.comment.end.regexp" }, "2": { "name": "invalid.illegal.newline.python" } }, - "patterns": [ { "include": "#codetags" } ] + "patterns": [ { "include": "source.renpy#codetags" } ] }, "double-one-regexp-lookahead": { "begin": "(\\()\\?=", @@ -1822,7 +1801,7 @@ "1": { "name": "punctuation.comment.end.regexp" }, "2": { "name": "invalid.illegal.newline.python" } }, - "patterns": [ { "include": "#codetags" } ] + "patterns": [ { "include": "source.renpy#codetags" } ] }, "double-three-regexp-lookahead": { "begin": "(\\()\\?=", @@ -2021,7 +2000,7 @@ "patterns": [ { "include": "#string-raw-bin-guts" } ] }, "string-quoted-single-line": { - "name": "string.quoted.single.python", + "name": "string.quoted.single.test.python", "begin": "(?:\\b([rR])(?=[uU]))?([uU])?(['\"])", "end": "(\\3)|((?