feat: move delimiter parsing to the scanner

Co-Authored-By: ObserverOfTime <[email protected]>

.github/workflows/ci.yml

@@ -36,7 +36,20 @@ jobs:
         uses: tree-sitter/[email protected]
         with:
           node-version: ${{vars.NODE_VERSION}}
+      - name: Check for scanner changes
+        uses: tj-actions/changed-files@v42
+        id: scanner-check
+        if: runner.os == 'Linux'
+        with:
+          files: src/scanner.c
+      - name: Fuzz scanner
+        uses: tree-sitter/fuzz-action@v4
+        if: steps.scanner-check.outputs.any_changed == 'true'
+        with:
+          corpus: examples
       - name: Run tests
         uses: tree-sitter/[email protected]
         with:
           test-library: ${{runner.os == 'Linux'}}
+          corpus-files: examples/*
+          invalid-files: examples/invalid.tst

examples/attributes.tst

@@ -0,0 +1,39 @@
+=========================
+Test that will be skipped
+:skip
+=========================
+
+int main() {}
+
+-------------------------
+
+====================================
+Test that will run on Linux or macOS
+
+:platform(linux)
+:platform(macos)
+====================================
+
+int main() {}
+
+------------------------------------
+
+========================================================================
+Test that expects an error, and will fail fast if there's no parse error
+:fail-fast
+:error
+========================================================================
+
+int main ( {}
+
+------------------------------------------------------------------------
+
+=================================================
+Test that will parse with both Typescript and TSX
+:language(typescript)
+:language(tsx)
+=================================================
+
+console.log('Hello, world!');
+
+-------------------------------------------------

examples/invalid.tst

@@ -0,0 +1,9 @@
+==================|||
+Invalid suffix
+==================|||
+
+foo
+
+---||
+
+(foo)

examples/simple.tst

@@ -0,0 +1,17 @@
+==================
+Return statements
+==================
+
+func x() int {
+  return 1;
+}
+
+---
+
+(source_file
+  (function_definition
+    (identifier)
+    (parameter_list)
+    (primitive_type)
+    (block
+      (return_statement (number)))))

examples/suffix.tst

@@ -0,0 +1,15 @@
+==================|||
+Basic module
+==================|||
+
+---- MODULE Test ----
+increment(n) == n + 1
+====
+
+---|||
+
+(source_file
+  (module (identifier)
+    (operator (identifier)
+      (parameter_list (identifier))
+      (plus (identifier_ref) (number)))))

grammar.js

@@ -1,63 +1,69 @@
 /// <reference types="./types/dsl" />
 // @ts-check
 
-const I = token.immediate;
-
 module.exports = grammar({
   name: "test",
 
-  extras: _ => [/\r?\n/],
+  externals: $ => [
+    $._equals_begin,
+    $._equals_end,
+    $._dashes,
+  ],
+
+  extras: _ => [],
 
   rules: {
-    file: $ => repeat($.test),
+    file: $ => repeat(
+      choice($.test, $._eol)
+    ),
 
-    test: $ => seq(
+    test: $ => prec.right(seq(
       $.header,
-      alias(repeat1(/./), $.input),
+      alias($._body, $.input),
       alias($._dashes, $.separator),
-      alias(repeat(/./), $.output)
-    ),
+      optional(alias($._body, $.output)),
+    )),
+
+    _line: _ => /[^\r\n]+/,
+
+    _body: $ => repeat1(choice($._line, $._eol)),
 
     header: $ => seq(
-      alias($._equals, $.separator),
-      alias(repeat1(/[^\r\n]/), $.name),
+      alias($._equals_begin, $.separator),
+      alias($._line, $.name),
       optional($.attributes),
-      alias($._equals, $.separator)
+      alias($._equals_end, $.separator),
     ),
 
     attributes: $ => repeat1(
-      field("attribute", choice(
-        $.skip,
-        $.error,
-        $.fail_fast,
-        $.language,
-        $.platform,
-      ))
+      choice($.attribute, $._eol)
     ),
 
-    skip: _ => ":skip",
-
-    error: _ => ":error",
-
-    fail_fast: _ => ":fail-fast",
+    attribute: $ => choice(
+      ":skip",
+      ":error",
+      ":fail-fast",
+      $._language,
+      $._platform,
+    ),
 
-    language: $ => seq(
+    _language: $ => seq(
       ":language",
-      I("("),
-      alias($._lang, $.parameter),
-      I(")")
+      "(",
+      field("language", alias($._lang, $.parameter)),
+      ")"
     ),
 
-    platform: $ => seq(
+    _platform: $ => seq(
       ":platform",
-      I("("),
-      alias($._os, $.parameter),
-      I(")")
+      "(",
+      field("platform", alias($._os, $.parameter)),
+      ")"
     ),
 
-    _lang: _ => I(repeat1(/[\w.-]/)),
+    _lang: _ => token(repeat1(/[\w.-]/)),
 
-    _os: _ => I(choice(
+    _os: _ => token(choice(
       "linux",
       "macos",
       "ios",
@@ -70,8 +76,6 @@ module.exports = grammar({
       "windows",
     )),
 
-    _equals: $ => token(prec(1, repeat1("="))),
-
-    _dashes: $ => token(prec(1, seq("---", repeat("-"))))
+    _eol: _ => /[\r\n]|\r\n/
   }
 });

package.json

@@ -37,7 +37,8 @@
     {
       "scope": "text.test",
       "highlights": "queries/highlights.scm",
-      "injections": "queries/injections.scm"
+      "injections": "queries/injections.scm",
+      "file-types": ["tst"]
     }
   ]
 }