Fix missing token definitions and resolved errors

The generated files depend on back references in end patterns that are used at runtime to match against begin patterns Added a check for Unicode usage and add the flag if needed
renpy · Jul 12, 2024 · fbcda49 · fbcda49
1 parent bcb3105
commit fbcda49
Show file tree

Hide file tree

Showing 10 changed files with 304 additions and 170 deletions.
diff --git a/src/tokenizer/atl-token-patterns.g.ts b/src/tokenizer/atl-token-patterns.g.ts
@@ -1,12 +1,11 @@
-// @ts-nocheck: Generated file
 /* eslint-disable no-useless-escape */
 /* eslint-disable no-useless-backreference */
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 10/07/2024 12:47:55 (UTC+0)
+// Last generated: 12/07/2024 09:35:29 (UTC+0)
 
 import { KeywordTokenType, EntityTokenType, MetaTokenType, CharacterTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
@@ -442,6 +441,7 @@ export const atlBlockTester: TokenPattern = {
             beginCaptures: {
                 2: { token: CharacterTokenType.Colon, /*punctuation.section.atl.begin.renpy*/ },
             },
+            // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
             end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm,
             patterns: []
         }]

diff --git a/src/tokenizer/debug-decorator.ts b/src/tokenizer/debug-decorator.ts
@@ -461,10 +461,15 @@ ${(decoration.hoverMessage as MarkdownString).value}`,
             case MetaTokenType.ScreenStatement:
             case MetaTokenType.ScreenSensitive:
             case MetaTokenType.ScreenFrame:
+            case MetaTokenType.ScreenFrameStatement:
             case MetaTokenType.ScreenWindow:
+            case MetaTokenType.ScreenWindowStatement:
             case MetaTokenType.ScreenText:
             case MetaTokenType.ScreenBlock:
+            case MetaTokenType.ScreenVboxStatement:
+            case MetaTokenType.ScreenHboxStatement:
             case MetaTokenType.StyleStatement:
+            case MetaTokenType.StyleParameters:
             case MetaTokenType.StyleBlock:
             case MetaTokenType.NarratorSayStatement:
             case MetaTokenType.SayStatement:
@@ -486,7 +491,15 @@ ${(decoration.hoverMessage as MarkdownString).value}`,
             case MetaTokenType.ATLEvent:
             case MetaTokenType.ATLFunction:
             case MetaTokenType.ATLWarper:
-            case MetaTokenType.ATLOn:
+            case MetaTokenType.ATLOnStatement:
+            case MetaTokenType.ATLOnParameters:
+            case MetaTokenType.ATLChoiceStatement:
+            case MetaTokenType.ATLChoiceParameters:
+            case MetaTokenType.ATLBlockStatement:
+            case MetaTokenType.ATLParallelStatement:
+            case MetaTokenType.ATLContainsStatement:
+            case MetaTokenType.TransformStatement:
+            case MetaTokenType.TransformParameters:
             case MetaTokenType.MemberAccess:
             case MetaTokenType.ItemAccess:
             case MetaTokenType.IndexedName:
@@ -501,6 +514,7 @@ ${(decoration.hoverMessage as MarkdownString).value}`,
             case MetaTokenType.FunctionCall:
             case MetaTokenType.FunctionCallGeneric:
             case MetaTokenType.Fstring:
+            case MetaTokenType.InterpolateFlags:
             case MetaTokenType.ControlFlowKeyword:
             case MetaTokenType.LogicalOperatorKeyword:
             case MetaTokenType.Operator:
@@ -603,6 +617,7 @@ ${(decoration.hoverMessage as MarkdownString).value}`,
             case CharacterTokenType.DollarSymbol:
             case CharacterTokenType.AtSymbol:
             case CharacterTokenType.EqualsSymbol:
+            case CharacterTokenType.ExclamationMark:
             case CharacterTokenType.Quote:
             case CharacterTokenType.DoubleQuote:
             case CharacterTokenType.BackQuote:

diff --git a/src/tokenizer/python-token-patterns.g.ts b/src/tokenizer/python-token-patterns.g.ts
@@ -1,12 +1,11 @@
-// @ts-nocheck: Generated file
 /* eslint-disable no-useless-escape */
 /* eslint-disable no-useless-backreference */
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 10/07/2024 13:49:44 (UTC+0)
+// Last generated: 12/07/2024 09:35:29 (UTC+0)
 
 import { MetaTokenType, KeywordTokenType, EntityTokenType, CharacterTokenType, OperatorTokenType, LiteralTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
@@ -96,6 +95,7 @@ export const docstringStatement: TokenPattern = {
 
     // the string either terminates correctly or by the beginning of a new line (this is for single line docstrings that aren't terminated) AND it's not followed by another docstring
     begin: /^(?=\s*[rR]?(\'\'\'|\"\"\"|\'|\"))/gm,
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /((?<=\1)|^)(?!\s*[rR]?(\'\'\'|\"\"\"|\'|\"))/gm,
     patterns: [
     ]
@@ -111,6 +111,7 @@ export const docstring: TokenPattern = {
             beginCaptures: {
                 1: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
             },
+            // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
             end: /(\1)/dg,
             endCaptures: {
                 1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -127,6 +128,7 @@ export const docstring: TokenPattern = {
                 1: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
                 2: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
             },
+            // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
             end: /(\2)/dg,
             endCaptures: {
                 1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -142,6 +144,7 @@ export const docstring: TokenPattern = {
             beginCaptures: {
                 1: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
             },
+            // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
             end: /(\1)|(\n)/dg,
             endCaptures: {
                 1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -159,6 +162,7 @@ export const docstring: TokenPattern = {
                 1: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
                 2: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
             },
+            // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
             end: /(\2)|(\n)/dg,
             endCaptures: {
                 1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -2466,6 +2470,7 @@ export const stringRawQuotedSingleLine: TokenPattern = {
         2: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
         3: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\4)|((?<!\\)\n)/dg,
     endCaptures: {
         1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -2483,6 +2488,7 @@ export const stringBinQuotedSingleLine: TokenPattern = {
         1: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
         2: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\2)|((?<!\\)\n)/dg,
     endCaptures: {
         1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -2500,6 +2506,7 @@ export const stringRawBinQuotedSingleLine: TokenPattern = {
         1: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
         2: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\2)|((?<!\\)\n)/dg,
     endCaptures: {
         1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -2518,6 +2525,7 @@ export const stringQuotedSingleLine: TokenPattern = {
         2: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
         3: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\3)|((?<!\\)\n)/dg,
     endCaptures: {
         1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -2582,6 +2590,7 @@ export const stringRawQuotedMultiLine: TokenPattern = {
         2: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
         3: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\4)/dg,
     endCaptures: {
         1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -2599,6 +2608,7 @@ export const stringBinQuotedMultiLine: TokenPattern = {
         1: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
         2: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\2)/dg,
     endCaptures: {
         1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -2616,6 +2626,7 @@ export const stringRawBinQuotedMultiLine: TokenPattern = {
         1: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
         2: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\2)/dg,
     endCaptures: {
         1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -2634,6 +2645,7 @@ export const stringQuotedMultiLine: TokenPattern = {
         2: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
         3: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\3)/dg,
     endCaptures: {
         1: { token: MetaTokenType.StringEnd, /*punctuation.definition.string.end.python*/ },
@@ -2698,6 +2710,7 @@ export const fstringFnormQuotedSingleLine: TokenPattern = {
         2: { token: MetaTokenType.Invalid, /*invalid.illegal.prefix.python*/ },
         3: { token: LiteralTokenType.String, /*punctuation.definition.string.begin.python string.interpolated.python string.quoted.single.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\3)|((?<!\\)\n)/dg,
     endCaptures: {
         1: { token: LiteralTokenType.String, /*punctuation.definition.string.end.python string.interpolated.python string.quoted.single.python*/ },
@@ -2719,6 +2732,7 @@ export const fstringNormfQuotedSingleLine: TokenPattern = {
         2: { token: MetaTokenType.StringStorageType, /*string.interpolated.python string.quoted.single.python storage.type.string.python*/ },
         3: { token: MetaTokenType.StringBegin, /*string.quoted.single.python punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\3)|((?<!\\)\n)/dg,
     endCaptures: {
         1: { token: LiteralTokenType.String, /*punctuation.definition.string.end.python string.interpolated.python string.quoted.single.python*/ },
@@ -2739,6 +2753,7 @@ export const fstringRawQuotedSingleLine: TokenPattern = {
         1: { token: MetaTokenType.StringStorageType, /*string.interpolated.python string.quoted.raw.single.python storage.type.string.python*/ },
         2: { token: MetaTokenType.StringBegin, /*string.quoted.raw.single.python punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\2)|((?<!\\)\n)/dg,
     endCaptures: {
         1: { token: LiteralTokenType.String, /*punctuation.definition.string.end.python string.interpolated.python string.quoted.raw.single.python*/ },
@@ -2878,6 +2893,7 @@ export const fstringFnormQuotedMultiLine: TokenPattern = {
         2: { token: MetaTokenType.Invalid, /*invalid.illegal.prefix.python*/ },
         3: { token: LiteralTokenType.String, /*punctuation.definition.string.begin.python string.interpolated.python string.quoted.multi.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\3)/dg,
     endCaptures: {
         1: { token: LiteralTokenType.String, /*punctuation.definition.string.end.python string.interpolated.python string.quoted.multi.python*/ },
@@ -2899,6 +2915,7 @@ export const fstringNormfQuotedMultiLine: TokenPattern = {
         2: { token: MetaTokenType.StringStorageType, /*string.interpolated.python string.quoted.multi.python storage.type.string.python*/ },
         3: { token: MetaTokenType.StringBegin, /*string.quoted.multi.python punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\3)/dg,
     endCaptures: {
         1: { token: LiteralTokenType.String, /*punctuation.definition.string.end.python string.interpolated.python string.quoted.multi.python*/ },
@@ -2919,6 +2936,7 @@ export const fstringRawQuotedMultiLine: TokenPattern = {
         1: { token: MetaTokenType.StringStorageType, /*string.interpolated.python string.quoted.raw.multi.python storage.type.string.python*/ },
         2: { token: MetaTokenType.StringBegin, /*string.quoted.raw.multi.python punctuation.definition.string.begin.python*/ },
     },
+    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
     end: /(\2)/dg,
     endCaptures: {
         1: { token: LiteralTokenType.String, /*punctuation.definition.string.end.python string.interpolated.python string.quoted.raw.multi.python*/ },