Tokenizer updates (#317)

* WIP: Tokenizer update

* Split ATL patterns into it's own file

* Update typescript file generator

* replace with auto-generated ATL patterns

* Added sort function for vector

* WIP: auto generate python and renpy patterns

Something is still causing the tokenizer to misalign when using the generated patterns. I'll try to fix it tomorrow

* Switch last token to be the important one

This is to match the behaviour of textmate, where the top most token is the last token added

* Added missing token cases

* Update tokenizer to match textmate behaviour on range patterns

This allows the end pattern of a range to extend based on the content pattern matches

* Move to separate functions

* WIP optimizations

* Fix bugs

* Code cleanup and fixes

* Delete all token position update code

* Update pattern generator

Due to the way typescript initializes const object, we can't have circular references to patterns.
To fix that, we just push all external dependencies in a shared file after all other files have been loaded.

* Fix some bugs in the syntax patterns

Remove use of \G as javascript's regex engine doesn't support it
Other fixes to make sure we can tokenize the python grammar file
Change some renpy keyword to display as control flow keywords

* Ignore generated files

* Delete this file as it is no longer used

* Fix custom color style injection

* Fixed merge error

* Use dataclass annotation

* Don't display unmatched tokens to the end user

Will add a special rule in our development area for displaying these in red

.prettierignore

@@ -3,4 +3,7 @@
 **/dist
 
 # Ignore all markdown files:
-*.md
+*.md
+
+# Ignore all generated typescript files:
+*.g.ts

examples/tokenizer_tests.rpy

@@ -0,0 +1,24 @@
+init python hide early in Namespace:
+
+    def sampleFunction(name, delay, position=(0,0)):
+        """
+        This is a sample function.
+        """
+# test
+        renpy.pause(delay)
+        return name
+
+    class Inventory:
+        """
+        This is a fake inventory class.
+        """
+
+        def __init__(self):
+            self.items = []
+
+        def add(self, item):
+            """Add an item to the inventory."""
+            self.items.append(item);
+            return
+
+

examples/unit_test.rpy

@@ -837,7 +837,7 @@
         queue sound "woof.mp3" volume 0.75
         queue sound "woof.mp3" volume 1.0
 
-        define audio.woof = "woof.mp23
+        define audio.woof = "woof.mp3"
 
         # ...
 
@@ -1853,7 +1853,7 @@
         image big hello world = Text("Hello World", style="big")
 
         screen hello_world:
-            text "Hello, World" style "big"
+            text "Hello, World" style "big" at 0# Comment
 
         style my_text is text:
             size 40
@@ -3555,9 +3555,9 @@
     #region Customizing the Keymap
         # see https://www.renpy.org/doc/html/keymap.html
 
-        init:
-            $ config.keymap['dismiss'].append('t')
-            $ config.keymap['dismiss'].remove('K_SPACE')
+        init python:
+            config.keymap['dismiss'].append('t')
+            config.keymap['dismiss'].remove('K_SPACE')
 
             config.keymap = dict(
 
@@ -3678,59 +3678,59 @@
 
             )
 
-        config.pad_bindings = {
-            "pad_leftshoulder_press" : [ "rollback", ],
-            "pad_lefttrigger_pos" : [ "rollback", ],
-            "pad_back_press" : [ "rollback", ],
+            config.pad_bindings = {
+                "pad_leftshoulder_press" : [ "rollback", ],
+                "pad_lefttrigger_pos" : [ "rollback", ],
+                "pad_back_press" : [ "rollback", ],
 
-            "repeat_pad_leftshoulder_press" : [ "rollback", ],
-            "repeat_pad_lefttrigger_pos" : [ "rollback", ],
-            "repeat_pad_back_press" : [ "rollback", ],
+                "repeat_pad_leftshoulder_press" : [ "rollback", ],
+                "repeat_pad_lefttrigger_pos" : [ "rollback", ],
+                "repeat_pad_back_press" : [ "rollback", ],
 
-            "pad_guide_press" : [ "game_menu", ],
-            "pad_start_press" : [ "game_menu", ],
+                "pad_guide_press" : [ "game_menu", ],
+                "pad_start_press" : [ "game_menu", ],
 
-            "pad_y_press" : [ "hide_windows", ],
+                "pad_y_press" : [ "hide_windows", ],
 
-            "pad_rightshoulder_press" : [ "rollforward", ],
-            "repeat_pad_rightshoulder_press" : [ "rollforward", ],
+                "pad_rightshoulder_press" : [ "rollforward", ],
+                "repeat_pad_rightshoulder_press" : [ "rollforward", ],
 
-            "pad_righttrigger_pos" : [ "dismiss", "button_select", "bar_activate", "bar_deactivate" ],
-            "pad_a_press" : [ "dismiss", "button_select", "bar_activate", "bar_deactivate"],
-            "pad_b_press" : [ "button_alternate" ],
+                "pad_righttrigger_pos" : [ "dismiss", "button_select", "bar_activate", "bar_deactivate" ],
+                "pad_a_press" : [ "dismiss", "button_select", "bar_activate", "bar_deactivate"],
+                "pad_b_press" : [ "button_alternate" ],
 
-            "pad_dpleft_press" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
-            "pad_leftx_neg" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
-            "pad_rightx_neg" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
+                "pad_dpleft_press" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
+                "pad_leftx_neg" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
+                "pad_rightx_neg" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
 
-            "pad_dpright_press" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
-            "pad_leftx_pos" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
-            "pad_rightx_pos" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
+                "pad_dpright_press" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
+                "pad_leftx_pos" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
+                "pad_rightx_pos" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
 
-            "pad_dpup_press" : [ "focus_up", "bar_up", "viewport_uparrow" ],
-            "pad_lefty_neg" : [ "focus_up", "bar_up", "viewport_uparrow" ],
-            "pad_righty_neg" : [ "focus_up", "bar_up", "viewport_uparrow" ],
+                "pad_dpup_press" : [ "focus_up", "bar_up", "viewport_uparrow" ],
+                "pad_lefty_neg" : [ "focus_up", "bar_up", "viewport_uparrow" ],
+                "pad_righty_neg" : [ "focus_up", "bar_up", "viewport_uparrow" ],
 
-            "pad_dpdown_press" : [ "focus_down", "bar_down", "viewport_downarrow" ],
-            "pad_lefty_pos" : [ "focus_down", "bar_down", "viewport_downarrow" ],
-            "pad_righty_pos" : [ "focus_down", "bar_down", "viewport_downarrow" ],
+                "pad_dpdown_press" : [ "focus_down", "bar_down", "viewport_downarrow" ],
+                "pad_lefty_pos" : [ "focus_down", "bar_down", "viewport_downarrow" ],
+                "pad_righty_pos" : [ "focus_down", "bar_down", "viewport_downarrow" ],
 
-            "repeat_pad_dpleft_press" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
-            "repeat_pad_leftx_neg" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
-            "repeat_pad_rightx_neg" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
+                "repeat_pad_dpleft_press" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
+                "repeat_pad_leftx_neg" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
+                "repeat_pad_rightx_neg" : [ "focus_left", "bar_left", "viewport_leftarrow" ],
 
-            "repeat_pad_dpright_press" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
-            "repeat_pad_leftx_pos" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
-            "repeat_pad_rightx_pos" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
+                "repeat_pad_dpright_press" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
+                "repeat_pad_leftx_pos" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
+                "repeat_pad_rightx_pos" : [ "focus_right", "bar_right", "viewport_rightarrow" ],
 
-            "repeat_pad_dpup_press" : [ "focus_up", "bar_up", "viewport_uparrow" ],
-            "repeat_pad_lefty_neg" : [ "focus_up", "bar_up", "viewport_uparrow" ],
-            "repeat_pad_righty_neg" : [ "focus_up", "bar_up", "viewport_uparrow" ],
+                "repeat_pad_dpup_press" : [ "focus_up", "bar_up", "viewport_uparrow" ],
+                "repeat_pad_lefty_neg" : [ "focus_up", "bar_up", "viewport_uparrow" ],
+                "repeat_pad_righty_neg" : [ "focus_up", "bar_up", "viewport_uparrow" ],
 
-            "repeat_pad_dpdown_press" : [ "focus_down", "bar_down", "viewport_downarrow" ],
-            "repeat_pad_lefty_pos" : [ "focus_down", "bar_down", "viewport_downarrow" ],
-            "repeat_pad_righty_pos" : [ "focus_down", "bar_down", "viewport_downarrow" ],
-        }
+                "repeat_pad_dpdown_press" : [ "focus_down", "bar_down", "viewport_downarrow" ],
+                "repeat_pad_lefty_pos" : [ "focus_down", "bar_down", "viewport_downarrow" ],
+                "repeat_pad_righty_pos" : [ "focus_down", "bar_down", "viewport_downarrow" ],
+            }
 
     #endregion Customizing the Keymap
 

package.json

@@ -71,6 +71,10 @@
       {
         "scopeName": "source.renpy.python",
         "path": "./syntaxes/renpy.python.tmLanguage.json"
+      },
+      {
+        "scopeName": "source.renpy.atl",
+        "path": "./syntaxes/renpy.atl.tmLanguage.json"
       }
     ],
     "snippets": [

src/color.ts

@@ -1,9 +1,13 @@
 // Color conversion methods for Color provider
 import { CancellationToken, Color, ColorInformation, ColorPresentation, DocumentColorProvider, Range, TextDocument, TextEdit } from "vscode";
+import { ValueEqualsSet } from "./utilities/hashset";
+import { tokenizeDocument } from "./tokenizer/tokenizer";
+import { LiteralTokenType } from "./tokenizer/renpy-tokens";
+import { TextMateRule, injectCustomTextmateTokens } from "./decorator";
+/*import { tokenizeDocument } from "./tokenizer/tokenizer";
 import { injectCustomTextmateTokens, TextMateRule } from "./decorator";
 import { LiteralTokenType } from "./tokenizer/renpy-tokens";
-import { tokenizeDocument } from "./tokenizer/tokenizer";
-import { ValueEqualsSet } from "./utilities/hashset";
+import { ValueEqualsSet } from "./utilities/hashset";*/
 
 export class RenpyColorProvider implements DocumentColorProvider {
     public provideDocumentColors(document: TextDocument, token: CancellationToken): Thenable<ColorInformation[]> {
@@ -112,15 +116,15 @@ export function getColorPresentations(color: Color, document: TextDocument, rang
 }
 
 export function injectCustomColorStyles(document: TextDocument) {
+    // Disabled until filter is added to the tree class
     const documentTokens = tokenizeDocument(document);
-
     // TODO: Should probably make sure this constant is actually part of a tag, but for now this is fine.
-    const colorTags = documentTokens.filter((x) => x.tokenType === LiteralTokenType.Color);
+    const colorTags = documentTokens.filter((x) => x.token?.tokenType === LiteralTokenType.Color);
     const colorRules = new ValueEqualsSet<TextMateRule>();
 
     // Build the new rules for this file
-    colorTags.forEach((color) => {
-        const lowerColor = document.getText(color.getRange()).toLowerCase();
+    colorTags.forEach((colorNode) => {
+        const lowerColor = document.getText(colorNode.token?.getVSCodeRange()).toLowerCase();
         const newRule = new TextMateRule(`renpy.meta.color.${lowerColor}`, { foreground: lowerColor });
         colorRules.add(newRule);
     });