Merge pull request #425 from renpy/more-bug-fixes

More bug fixes

src/tokenizer/atl-token-patterns.g.ts

@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 12/07/2024 18:50:56 (UTC+0)
+// Last generated: 14/07/2024 12:28:58 (UTC+0)
 
 import { KeywordTokenType, EntityTokenType, MetaTokenType, CharacterTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
@@ -432,19 +432,17 @@ export const atlStatements: TokenPattern = {
 };
 
 export const atlBlockTester: TokenPattern = {
-    patterns: [
-        {
-            debugName: "atlBlockTester.patterns![0]",
+    debugName: "atlBlockTester",
 
-            contentToken: MetaTokenType.ATLBlock, /*meta.atl-block.renpy*/
-            begin: /(?<=(^[ \t]*)(?:camera|image|show|scene|transform|on|block|parallel|contains|choice)\b.*?)(:)/dgm,
-            beginCaptures: {
-                2: { token: CharacterTokenType.Colon, /*punctuation.section.atl.begin.renpy*/ },
-            },
-            // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
-            end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm,
-            patterns: []
-        }]
+    contentToken: MetaTokenType.ATLBlock, /*meta.atl-block.renpy*/
+    begin: /(?<=(^[ \t]*)(?:camera|image|show|scene|transform|on|block|parallel|contains|choice)\b.*?)(:)/dgm,
+    beginCaptures: {
+        2: { token: CharacterTokenType.Colon, /*punctuation.section.atl.begin.renpy*/ },
+    },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
+    end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm,
+    patterns: [
+    ]
 };
 
 export const transform: TokenPattern = {

src/tokenizer/debug-decorator.ts

@@ -242,7 +242,7 @@ async function updateDecorations() {
             range: range,
             hoverMessage: {
                 language: "text",
-                value: `Token: ${tokenTypeToStringMap[token.type]}(id: ${token.type})
+                value: `${token.isMetaToken() ? "MetaToken" : "Token"}: ${tokenTypeToStringMap[token.type]}(id: ${token.type})
 Start: {Line: ${range.start.line + 1}, Char: ${range.start.character + 1}}
 End: {Line: ${range.end.line + 1}, Char: ${range.end.character + 1}}
 Content: {${content?.replaceAll("\n", "\\n")}}`,
@@ -327,6 +327,7 @@ ${(decoration.hoverMessage as MarkdownString).value}`,
             case KeywordTokenType.Other:
             case KeywordTokenType.OtherPython:
             case KeywordTokenType.OtherAudio:
+            case KeywordTokenType.Layer:
             case KeywordTokenType.Take: // Renpy style sub-expression keywords
             case KeywordTokenType.Del:
             case KeywordTokenType.Clear:

src/tokenizer/python-token-patterns.g.ts

@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 12/07/2024 18:44:34 (UTC+0)
+// Last generated: 14/07/2024 12:28:58 (UTC+0)
 
 import { MetaTokenType, KeywordTokenType, CharacterTokenType, EntityTokenType, OperatorTokenType, LiteralTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
@@ -85,7 +85,7 @@ export const docstringStatement: TokenPattern = {
     debugName: "docstringStatement",
 
     // the string either terminates correctly or by the beginning of a new line (this is for single line docstrings that aren't terminated) AND it's not followed by another docstring
-    begin: /^(?=\s*[rR]?(\'\'\'|\"\"\"|\'|\"))/gm,
+    begin: /(?<=^[ \t]*)(?=[rR]?(\'\'\'|\"\"\"|\'|\"))/gm,
     // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /((?<=\1)|^)(?!\s*[rR]?(\'\'\'|\"\"\"|\'|\"))/gm,
     patterns: [
@@ -351,7 +351,7 @@ export const oddFunctionCall: TokenPattern = {
 
     // A bit obscured function call where there may have been an arbitrary number of other operations to get the function.E.g. "arr[idx](args)"
     token: MetaTokenType.FunctionCall, /*meta.function-call.python*/
-    begin: /(?<=\]|\))\s*(?=\()/g,
+    begin: /(?<=\]|\)|")\s*(?=\()/g,
     end: /(\))/dg,
     endCaptures: {
         1: { token: CharacterTokenType.CloseParentheses, /*punctuation.definition.arguments.end.python*/ },
@@ -482,7 +482,7 @@ export const numberHex: TokenPattern = {
     debugName: "numberHex",
 
     token: MetaTokenType.ConstantNumeric, /*constant.numeric.hex.python*/
-    match: /(?<![\w.])(0[xX])(_?[0-9a-fA-F])+\b/dg,
+    match: /(?<![\w.])(0[xX])(?:_?[0-9a-fA-F])+\b/dg,
     captures: {
         1: { token: MetaTokenType.NumberStorageType, /*storage.type.number.python*/ },
     },
@@ -492,7 +492,7 @@ export const numberOct: TokenPattern = {
     debugName: "numberOct",
 
     token: MetaTokenType.ConstantNumeric, /*constant.numeric.oct.python*/
-    match: /(?<![\w.])(0[oO])(_?[0-7])+\b/dg,
+    match: /(?<![\w.])(0[oO])(?:_?[0-7])+\b/dg,
     captures: {
         1: { token: MetaTokenType.NumberStorageType, /*storage.type.number.python*/ },
     },
@@ -502,7 +502,7 @@ export const numberBin: TokenPattern = {
     debugName: "numberBin",
 
     token: MetaTokenType.ConstantNumeric, /*constant.numeric.bin.python*/
-    match: /(?<![\w.])(0[bB])(_?[01])+\b/dg,
+    match: /(?<![\w.])(0[bB])(?:_?[01])+\b/dg,
     captures: {
         1: { token: MetaTokenType.NumberStorageType, /*storage.type.number.python*/ },
     },
@@ -513,9 +513,9 @@ export const numberLong: TokenPattern = {
 
     // this is to support python2 syntax for long ints
     token: MetaTokenType.ConstantNumeric, /*constant.numeric.bin.python*/
-    match: /(?<![\w.])([1-9][0-9]*|0)([lL])\b/dg,
+    match: /(?<![\w.])(?:[1-9][0-9]*|0)([lL])\b/dg,
     captures: {
-        2: { token: MetaTokenType.NumberStorageType, /*storage.type.number.python*/ },
+        1: { token: MetaTokenType.NumberStorageType, /*storage.type.number.python*/ },
     },
 };
 
@@ -625,7 +625,7 @@ export const stringFormatting: TokenPattern = {
     debugName: "stringFormatting",
 
     token: MetaTokenType.FormatPercent, /*meta.format.percent.python*/
-    match: /(%(\([\w\s]*\))?[-+#0 ]*(\d+|\*)?(\.(\d+|\*))?([hlL])?[diouxXeEfFgGcrsab%])/dg,
+    match: /(%(?:\([\w\s]*\))?[-+#0 ]*(?:\d+|\*)?(?:\.(?:\d+|\*))?(?:[hlL])?[diouxXeEfFgGcrsab%])/dg,
     captures: {
         1: { token: MetaTokenType.Placeholder, /*constant.character.format.placeholder.other.python*/ },
     },
@@ -1211,10 +1211,11 @@ export const functionArguments: TokenPattern = {
         {
             debugName: "functionArguments.patterns![4]",
 
-            match: /\b([a-zA-Z_]\w*)\s*(=)(?!=)/dg,
+            match: /\b([a-zA-Z_]\w*)(\s+)?(=)(?!=)/dg,
             captures: {
                 1: { token: EntityTokenType.Identifier, /*variable.parameter.function-call.python*/ },
-                2: { token: OperatorTokenType.Assignment, /*keyword.operator.assignment.python*/ },
+                2: { token: CharacterTokenType.Whitespace, /*punctuation.whitespace.python*/ },
+                3: { token: OperatorTokenType.Assignment, /*keyword.operator.assignment.python*/ },
             },
         },
         {
@@ -2510,7 +2511,7 @@ export const stringRawBinQuotedSingleLine: TokenPattern = {
 export const stringQuotedSingleLine: TokenPattern = {
     debugName: "stringQuotedSingleLine",
 
-    token: LiteralTokenType.String, /*string.quoted.single.test.python*/
+    token: LiteralTokenType.String, /*string.quoted.single.python*/
     begin: /(?:\b([rR])(?=[uU]))?([uU])?(['"])/dg,
     beginCaptures: {
         1: { token: MetaTokenType.Invalid, /*invalid.illegal.prefix.python*/ },