From f921c5408c04be801dabfad2518a17189487cae6 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Mon, 18 Nov 2024 16:28:37 +0200 Subject: [PATCH] feat: update to tree-sitter 0.24 --- .editorconfig | 21 ++- .gitattributes | 32 +++- .github/workflows/publish.yml | 8 + .gitignore | 12 +- CMakeLists.txt | 58 ++++++++ Cargo.lock | 96 ++++++++++++ Cargo.toml | 11 +- Makefile | 74 +++++----- Package.resolved | 16 ++ Package.swift | 39 +---- README.md | 3 +- binding.gyp | 20 ++- bindings/c/tree-sitter-test.pc.in | 15 +- bindings/go/binding.go | 2 + bindings/go/binding_test.go | 4 +- bindings/go/go.mod | 5 - bindings/node/binding.cc | 3 +- bindings/node/binding_test.js | 6 +- bindings/node/index.d.ts | 1 - bindings/python/test_binding.py | 16 -- bindings/python/tests/test_binding.py | 11 ++ bindings/python/tree_sitter_test/__init__.py | 32 +++- bindings/python/tree_sitter_test/__init__.pyi | 7 +- bindings/python/tree_sitter_test/binding.c | 4 +- bindings/rust/build.rs | 9 +- bindings/rust/lib.rs | 27 ++-- go.mod | 7 + go.sum | 36 +++++ package-lock.json | 138 +++++++++--------- package.json | 57 ++++---- pyproject.toml | 15 +- setup.py | 37 +++-- src/grammar.json | 1 + src/node-types.json | 1 + src/parser.c | 104 ++++++------- src/tree_sitter/alloc.h | 8 +- src/tree_sitter/parser.h | 52 ++++++- tree-sitter.json | 28 ++++ 38 files changed, 677 insertions(+), 339 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 Cargo.lock create mode 100644 Package.resolved delete mode 100644 bindings/go/go.mod delete mode 100644 bindings/python/test_binding.py create mode 100644 bindings/python/tests/test_binding.py create mode 100644 go.mod create mode 100644 go.sum create mode 100644 tree-sitter.json diff --git a/.editorconfig b/.editorconfig index d3a8b5b..6e7a8c6 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,9 +2,10 @@ root = true [*] charset = utf-8 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true + +[*.lua] +indent_style = space +indent_size = 4 [*.{json,toml,yml,gyp}] indent_style = space @@ -14,14 +15,18 @@ indent_size = 2 indent_style = space indent_size = 2 -[*.rs] +[*.scm] indent_style = space -indent_size = 4 +indent_size = 2 [*.{c,cc,h}] indent_style = space indent_size = 4 +[*.rs] +indent_style = space +indent_size = 4 + [*.{py,pyi}] indent_style = space indent_size = 4 @@ -37,3 +42,9 @@ indent_size = 8 [Makefile] indent_style = tab indent_size = 8 + +[parser.c] +indent_size = 2 + +[{alloc,array,parser}.h] +indent_size = 2 diff --git a/.gitattributes b/.gitattributes index ffb52ab..7e2cae0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,11 +1,37 @@ -* text eol=lf +* text=auto eol=lf +# Generated source files src/*.json linguist-generated src/parser.c linguist-generated src/tree_sitter/* linguist-generated -bindings/** linguist-generated +# C bindings +bindings/c/* linguist-generated +CMakeLists.txt linguist-generated +Makefile linguist-generated + +# Rust bindings +bindings/rust/* linguist-generated +Cargo.toml linguist-generated +Cargo.lock linguist-generated + +# Node.js bindings +bindings/node/* linguist-generated binding.gyp linguist-generated +package.json linguist-generated +package-lock.json linguist-generated + +# Python bindings +bindings/python/** linguist-generated setup.py linguist-generated -Makefile linguist-generated +pyproject.toml linguist-generated + +# Go bindings +bindings/go/* linguist-generated +go.mod linguist-generated +go.sum linguist-generated + +# Swift bindings +bindings/swift/** linguist-generated Package.swift linguist-generated +Package.resolved linguist-generated diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 455e952..f5cdd95 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,6 +9,14 @@ concurrency: cancel-in-progress: true jobs: + github: + uses: tree-sitter/workflows/.github/workflows/release.yml@main + with: + attestations: true + permissions: + contents: write + id-token: write + attestations: write npm: uses: tree-sitter/workflows/.github/workflows/package-npm.yml@main secrets: diff --git a/.gitignore b/.gitignore index cc69035..308fcab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,15 @@ # Rust artifacts -Cargo.lock target/ # Node artifacts build/ prebuilds/ node_modules/ -*.tgz # Swift artifacts .build/ -Package.resolved # Go artifacts -go.sum _obj/ # Python artifacts @@ -30,7 +26,15 @@ dist/ *.dll *.pc +# Example dirs +/examples/*/ + # Grammar volatiles *.wasm *.obj *.o + +# Archives +*.tar.gz +*.tgz +*.zip diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..b2e3ebb --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,58 @@ +cmake_minimum_required(VERSION 3.13) + +project(tree-sitter-test + VERSION "0.1.0" + DESCRIPTION "TS corpus test grammar for tree-sitter" + HOMEPAGE_URL "https://github.com/tree-sitter-grammars/tree-sitter-test" + LANGUAGES C) + +option(BUILD_SHARED_LIBS "Build using shared libraries" ON) +option(TREE_SITTER_REUSE_ALLOCATOR "Reuse the library allocator" OFF) + +set(TREE_SITTER_ABI_VERSION 15 CACHE STRING "Tree-sitter ABI version") +if(NOT ${TREE_SITTER_ABI_VERSION} MATCHES "^[0-9]+$") + unset(TREE_SITTER_ABI_VERSION CACHE) + message(FATAL_ERROR "TREE_SITTER_ABI_VERSION must be an integer") +endif() + +find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI") + +add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" + COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json + --abi=${TREE_SITTER_ABI_VERSION} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Generating parser.c") + +add_library(tree-sitter-test src/parser.c) +if(EXISTS src/scanner.c) + target_sources(tree-sitter-test PRIVATE src/scanner.c) +endif() +target_include_directories(tree-sitter-test PRIVATE src) + +target_compile_definitions(tree-sitter-test PRIVATE + $<$:TREE_SITTER_REUSE_ALLOCATOR> + $<$:TREE_SITTER_DEBUG>) + +set_target_properties(tree-sitter-test + PROPERTIES + C_STANDARD 11 + POSITION_INDEPENDENT_CODE ON + SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}" + DEFINE_SYMBOL "") + +configure_file(bindings/c/tree-sitter-test.pc.in + "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-test.pc" @ONLY) + +include(GNUInstallDirs) + +install(FILES bindings/c/tree-sitter-test.h + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter") +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-test.pc" + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig") +install(TARGETS tree-sitter-test + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") + +add_custom_target(ts-test "${TREE_SITTER_CLI}" test + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "tree-sitter test") diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..296f755 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,96 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "cc" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +dependencies = [ + "shlex", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "tree-sitter" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67baf55e7e1b6806063b1e51041069c90afff16afcbbccd278d899f9d84bca4" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ddffe35a0e5eeeadf13ff7350af564c6e73993a24db62caee1822b185c2600" + +[[package]] +name = "tree-sitter-test" +version = "0.1.0" +dependencies = [ + "cc", + "tree-sitter", + "tree-sitter-language", +] diff --git a/Cargo.toml b/Cargo.toml index a47e136..d4881b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,12 +2,12 @@ name = "tree-sitter-test" description = "TS corpus test grammar for tree-sitter" version = "0.1.0" +authors = ["ObserverOfTime "] license = "MIT" readme = "README.md" keywords = ["incremental", "parsing", "tree-sitter", "test"] -categories = ["parsing", "text-editors"] +categories = ["parser-implementations", "parsing", "text-editors"] repository = "https://github.com/tree-sitter-grammars/tree-sitter-test" -authors = ["ObserverOfTime "] edition = "2021" autoexamples = false @@ -18,7 +18,10 @@ include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"] path = "bindings/rust/lib.rs" [dependencies] -tree-sitter = ">=0.21.0" +tree-sitter-language = "0.1" [build-dependencies] -cc = "1.0.87" +cc = "1.1.22" + +[dev-dependencies] +tree-sitter = "0.24.4" diff --git a/Makefile b/Makefile index 9772308..28ec2cb 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,16 @@ -VERSION := 0.1.0 +ifeq ($(OS),Windows_NT) +$(error Windows is not supported) +endif LANGUAGE_NAME := tree-sitter-test +HOMEPAGE_URL := https://github.com/tree-sitter-grammars/tree-sitter-test +VERSION := 0.1.0 # repository SRC_DIR := src -PARSER_URL := https://github.com/tree-sitter-grammars/tree-sitter-test - TS ?= tree-sitter -# ABI versioning -SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION))) -SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION))) - # install directory layout PREFIX ?= /usr/local INCLUDEDIR ?= $(PREFIX)/include @@ -20,34 +18,29 @@ LIBDIR ?= $(PREFIX)/lib PCLIBDIR ?= $(LIBDIR)/pkgconfig # source/object files -SRCS := $(wildcard $(SRC_DIR)/*.c) -OBJS := $(patsubst %.c,%.o,$(SRCS)) +PARSER := $(SRC_DIR)/parser.c +EXTRAS := $(filter-out $(PARSER),$(wildcard $(SRC_DIR)/*.c)) +OBJS := $(patsubst %.c,%.o,$(PARSER) $(EXTRAS)) # flags -ARFLAGS := rcs +ARFLAGS ?= rcs override CFLAGS += -I$(SRC_DIR) -std=c11 -fPIC +# ABI versioning +SONAME_MAJOR = $(shell sed -n 's/\#define LANGUAGE_VERSION //p' $(PARSER)) +SONAME_MINOR = $(word 1,$(subst ., ,$(VERSION))) + # OS-specific bits -ifeq ($(OS),Windows_NT) - $(error "Windows is not supported") -else ifeq ($(shell uname),Darwin) +ifeq ($(shell uname),Darwin) SOEXT = dylib - SOEXTVER_MAJOR = $(SONAME_MAJOR).dylib - SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).dylib - LINKSHARED := $(LINKSHARED)-dynamiclib -Wl, - ifneq ($(ADDITIONAL_LIBS),) - LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS), - endif - LINKSHARED := $(LINKSHARED)-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SONAME_MAJOR).dylib,-rpath,@executable_path/../Frameworks + SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) + SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) + LINKSHARED = -dynamiclib -Wl,-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SOEXTVER),-rpath,@executable_path/../Frameworks else SOEXT = so - SOEXTVER_MAJOR = so.$(SONAME_MAJOR) - SOEXTVER = so.$(SONAME_MAJOR).$(SONAME_MINOR) - LINKSHARED := $(LINKSHARED)-shared -Wl, - ifneq ($(ADDITIONAL_LIBS),) - LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS) - endif - LINKSHARED := $(LINKSHARED)-soname,lib$(LANGUAGE_NAME).so.$(SONAME_MAJOR) + SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR) + SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR) + LINKSHARED = -shared -Wl,-soname,lib$(LANGUAGE_NAME).$(SOEXTVER) endif ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) PCLIBDIR := $(PREFIX)/libdata/pkgconfig @@ -65,22 +58,21 @@ ifneq ($(STRIP),) endif $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in - sed -e 's|@URL@|$(PARSER_URL)|' \ - -e 's|@VERSION@|$(VERSION)|' \ - -e 's|@LIBDIR@|$(LIBDIR)|' \ - -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ - -e 's|@REQUIRES@|$(REQUIRES)|' \ - -e 's|@ADDITIONAL_LIBS@|$(ADDITIONAL_LIBS)|' \ - -e 's|=$(PREFIX)|=$${prefix}|' \ - -e 's|@PREFIX@|$(PREFIX)|' $< > $@ - -$(SRC_DIR)/parser.c: grammar.js - $(TS) generate --no-bindings + sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \ + -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ + -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \ + -e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \ + -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ + -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ + +$(PARSER): $(SRC_DIR)/grammar.json + $(TS) generate $^ install: all - install -Dm644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h - install -Dm644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc - install -Dm755 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a + install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)' + install -m644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h + install -m644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc + install -m644 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) diff --git a/Package.resolved b/Package.resolved new file mode 100644 index 0000000..9e0a023 --- /dev/null +++ b/Package.resolved @@ -0,0 +1,16 @@ +{ + "object": { + "pins": [ + { + "package": "SwiftTreeSitter", + "repositoryURL": "https://github.com/ChimeHQ/SwiftTreeSitter", + "state": { + "branch": null, + "revision": "2599e95310b3159641469d8a21baf2d3d200e61f", + "version": "0.8.0" + } + } + ] + }, + "version": 1 +} diff --git a/Package.swift b/Package.swift index 1697a6d..f36ab99 100644 --- a/Package.swift +++ b/Package.swift @@ -1,9 +1,15 @@ // swift-tools-version:5.3 + +import Foundation import PackageDescription +var sources = ["src/parser.c"] +if FileManager.default.fileExists(atPath: "src/scanner.c") { + sources.append("src/scanner.c") +} + let package = Package( name: "TreeSitterTest", - platforms: [.macOS(.v10_13), .iOS(.v11)], products: [ .library(name: "TreeSitterTest", targets: ["TreeSitterTest"]), ], @@ -15,36 +21,7 @@ let package = Package( name: "TreeSitterTest", dependencies: [], path: ".", - exclude: [ - "Cargo.toml", - "Makefile", - "binding.gyp", - "bindings/c", - "bindings/go", - "bindings/node", - "bindings/python", - "bindings/rust", - "prebuilds", - "node_modules", - "grammar.js", - "package.json", - "package-lock.json", - "pyproject.toml", - "setup.py", - "test", - "examples", - "ftplugin", - "ftdetect", - "plugin", - ".editorconfig", - ".github", - ".gitignore", - ".gitattributes", - ], - sources: [ - "src/parser.c", - "src/scanner.c" - ], + sources: sources, resources: [ .copy("queries") ], diff --git a/README.md b/README.md index 6a40f6d..e3ad67a 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,7 @@ Here's how you can install it using [lazy.nvim](https://github.com/folke/lazy.nv ```lua { "tree-sitter-grammars/tree-sitter-test", - -- compile on your own on Windows - build = "make parser/test.so", + build = "mkdir parser && tree-sitter build -o parser/test.so", ft = "test", init = function() -- toggle full-width rules for test separators diff --git a/binding.gyp b/binding.gyp index a7d3bb9..29e59bf 100644 --- a/binding.gyp +++ b/binding.gyp @@ -11,10 +11,24 @@ "sources": [ "bindings/node/binding.cc", "src/parser.c", - "src/scanner.c", ], - "cflags_c": [ - "-std=c11", + "variables": { + "has_scanner": "::New(env, tree_sitter_test()); language.TypeTag(&LANGUAGE_TYPE_TAG); exports["language"] = language; diff --git a/bindings/node/binding_test.js b/bindings/node/binding_test.js index afede30..55becac 100644 --- a/bindings/node/binding_test.js +++ b/bindings/node/binding_test.js @@ -1,9 +1,9 @@ -/// - const assert = require("node:assert"); const { test } = require("node:test"); +const Parser = require("tree-sitter"); + test("can load grammar", () => { - const parser = new (require("tree-sitter"))(); + const parser = new Parser(); assert.doesNotThrow(() => parser.setLanguage(require("."))); }); diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index efe259e..528e060 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -19,7 +19,6 @@ type NodeInfo = }); type Language = { - name: string; language: unknown; nodeTypeInfo: NodeInfo[]; }; diff --git a/bindings/python/test_binding.py b/bindings/python/test_binding.py deleted file mode 100644 index 27dc7c5..0000000 --- a/bindings/python/test_binding.py +++ /dev/null @@ -1,16 +0,0 @@ -from unittest import TestCase - -import tree_sitter as ts -import tree_sitter_test as binding - - -class TestLanguage(TestCase): - def setUp(self): - self.parser = ts.Parser() - - def test_can_load_grammar(self): - try: - language = ts.Language(binding.language(), "test") - self.parser.set_language(language) - except Exception as exc: - self.fail(f"Error loading Test grammar: {exc}") diff --git a/bindings/python/tests/test_binding.py b/bindings/python/tests/test_binding.py new file mode 100644 index 0000000..6b331ee --- /dev/null +++ b/bindings/python/tests/test_binding.py @@ -0,0 +1,11 @@ +from unittest import TestCase + +import tree_sitter, tree_sitter_test + + +class TestLanguage(TestCase): + def test_can_load_grammar(self): + try: + tree_sitter.Language(tree_sitter_test.language()) + except Exception: + self.fail("Error loading Test grammar") diff --git a/bindings/python/tree_sitter_test/__init__.py b/bindings/python/tree_sitter_test/__init__.py index f577120..4d2c6dc 100644 --- a/bindings/python/tree_sitter_test/__init__.py +++ b/bindings/python/tree_sitter_test/__init__.py @@ -1,3 +1,33 @@ -"TS corpus test grammar for tree-sitter" +"""TS corpus test grammar for tree-sitter""" + +from importlib.resources import files as _files from ._binding import language + + +def _get_query(name, file): + query = _files(f"{__package__}.queries") / file + globals()[name] = query.read_text() + return globals()[name] + + +def __getattr__(name): + if name == "HIGHLIGHTS_QUERY": + return _get_query("HIGHLIGHTS_QUERY", "highlights.scm") + if name == "INJECTIONS_QUERY": + return _get_query("INJECTIONS_QUERY", "injections.scm") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = [ + "language", + "HIGHLIGHTS_QUERY", + "INJECTIONS_QUERY", +] + + +def __dir__(): + return sorted(__all__ + [ + "__all__", "__builtins__", "__cached__", "__doc__", "__file__", + "__loader__", "__name__", "__package__", "__path__", "__spec__", + ]) diff --git a/bindings/python/tree_sitter_test/__init__.pyi b/bindings/python/tree_sitter_test/__init__.pyi index 5416666..48688e6 100644 --- a/bindings/python/tree_sitter_test/__init__.pyi +++ b/bindings/python/tree_sitter_test/__init__.pyi @@ -1 +1,6 @@ -def language() -> int: ... +from typing import Final + +HIGHLIGHTS_QUERY: Final[str] +INJECTIONS_QUERY: Final[str] + +def language() -> object: ... diff --git a/bindings/python/tree_sitter_test/binding.c b/bindings/python/tree_sitter_test/binding.c index 6352f87..be361c1 100644 --- a/bindings/python/tree_sitter_test/binding.c +++ b/bindings/python/tree_sitter_test/binding.c @@ -4,8 +4,8 @@ typedef struct TSLanguage TSLanguage; TSLanguage *tree_sitter_test(void); -static PyObject* _binding_language(PyObject *self, PyObject *args) { - return PyLong_FromVoidPtr(tree_sitter_test()); +static PyObject* _binding_language(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { + return PyCapsule_New(tree_sitter_test(), "tree_sitter.Language", NULL); } static PyMethodDef methods[] = { diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs index da068d1..47369ff 100644 --- a/bindings/rust/build.rs +++ b/bindings/rust/build.rs @@ -4,13 +4,18 @@ fn main() { let mut c_config = cc::Build::new(); c_config.std("c11").include(src_dir); + #[cfg(target_env = "msvc")] + c_config.flag("-utf-8"); + let parser_path = src_dir.join("parser.c"); c_config.file(&parser_path); println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); let scanner_path = src_dir.join("scanner.c"); - c_config.file(&scanner_path); - println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); + if scanner_path.exists() { + c_config.file(&scanner_path); + println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); + } c_config.compile("tree-sitter-test"); } diff --git a/bindings/rust/lib.rs b/bindings/rust/lib.rs index 86574e0..a2ba0b1 100644 --- a/bindings/rust/lib.rs +++ b/bindings/rust/lib.rs @@ -1,6 +1,6 @@ -//! This crate provides Test language support for the [tree-sitter][] parsing library. +//! This crate provides TS corpus test support for the [tree-sitter][] parsing library. //! -//! Typically, you will use the [language][language func] function to add this language to a +//! Typically, you will use the [LANGUAGE][] constant to add this language to a //! tree-sitter [Parser][], and then use the parser to parse some code: //! //! ``` @@ -24,28 +24,27 @@ //! (return_statement (number))))) //! "#; //! let mut parser = tree_sitter::Parser::new(); -//! parser.set_language(&tree_sitter_test::language()).expect("Error loading Test grammar"); +//! let language = tree_sitter_test::LANGUAGE; +//! parser +//! .set_language(&language.into()) +//! .expect("Error loading Test parser"); //! let tree = parser.parse(code, None).unwrap(); //! assert!(!tree.root_node().has_error()); //! ``` //! -//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html -//! [language func]: fn.language.html //! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html //! [tree-sitter]: https://tree-sitter.github.io/ -use tree_sitter::Language; +use tree_sitter_language::LanguageFn; extern "C" { - fn tree_sitter_test() -> Language; + fn tree_sitter_test() -> *const (); } -/// Get the tree-sitter [Language][] for this grammar. +/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar. /// -/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html -pub fn language() -> Language { - unsafe { tree_sitter_test() } -} +/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html +pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_test) }; /// The content of the [`node-types.json`][] file for this grammar. /// @@ -64,7 +63,7 @@ mod tests { fn test_can_load_grammar() { let mut parser = tree_sitter::Parser::new(); parser - .set_language(&super::language()) - .expect("Error loading Test grammar"); + .set_language(&super::LANGUAGE.into()) + .expect("Error loading Test parser"); } } diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..08b4548 --- /dev/null +++ b/go.mod @@ -0,0 +1,7 @@ +module github.com/tree-sitter-grammars/tree-sitter-test + +go 1.22 + +require github.com/tree-sitter/go-tree-sitter v0.24.0 + +require github.com/mattn/go-pointer v0.0.1 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0a48bbe --- /dev/null +++ b/go.sum @@ -0,0 +1,36 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tree-sitter/go-tree-sitter v0.24.0 h1:kRZb6aBNfcI/u0Qh8XEt3zjNVnmxTisDBN+kXK0xRYQ= +github.com/tree-sitter/go-tree-sitter v0.24.0/go.mod h1:x681iFVoLMEwOSIHA1chaLkXlroXEN7WY+VHGFaoDbk= +github.com/tree-sitter/tree-sitter-c v0.21.5-0.20240818205408-927da1f210eb h1:A8425heRM8mylnv4H58FPUiH+aYivyitre0PzxrfmWs= +github.com/tree-sitter/tree-sitter-c v0.21.5-0.20240818205408-927da1f210eb/go.mod h1:dOF6gtQiF9UwNh995T5OphYmtIypkjsp3ap7r9AN/iA= +github.com/tree-sitter/tree-sitter-cpp v0.22.4-0.20240818224355-b1a4e2b25148 h1:AfFPZwtwGN01BW1jDdqBVqscTwetvMpydqYZz57RSlc= +github.com/tree-sitter/tree-sitter-cpp v0.22.4-0.20240818224355-b1a4e2b25148/go.mod h1:Bh6U3viD57rFXRYIQ+kmiYtr+1Bx0AceypDLJJSyi9s= +github.com/tree-sitter/tree-sitter-embedded-template v0.21.1-0.20240819044651-ffbf64942c33 h1:TwqSV3qLp3tKSqirGLRHnjFk9Tc2oy57LIl+FQ4GjI4= +github.com/tree-sitter/tree-sitter-embedded-template v0.21.1-0.20240819044651-ffbf64942c33/go.mod h1:CvCKCt3v04Ufos1zZnNCelBDeCGRpPucaN8QczoUsN4= +github.com/tree-sitter/tree-sitter-go v0.21.3-0.20240818010209-8c0f0e7a6012 h1:Xvxck3tE5FW7F7bTS97iNM2ADMyCMJztVqn5HYKdJGo= +github.com/tree-sitter/tree-sitter-go v0.21.3-0.20240818010209-8c0f0e7a6012/go.mod h1:T40D0O1cPvUU/+AmiXVXy1cncYQT6wem4Z0g4SfAYvY= +github.com/tree-sitter/tree-sitter-html v0.20.5-0.20240818004741-d11201a263d0 h1:c46K6uh5Dz00zJeU9BfjXdb8I+E4RkUdfnWJpQADXFo= +github.com/tree-sitter/tree-sitter-html v0.20.5-0.20240818004741-d11201a263d0/go.mod h1:hcNt/kOJHcIcuMvouE7LJcYdeFUFbVpBJ6d4wmOA+tU= +github.com/tree-sitter/tree-sitter-java v0.21.1-0.20240824015150-576d8097e495 h1:jrt4qbJVEFs4H93/ITxygHc6u0TGqAkkate7TQ4wFSA= +github.com/tree-sitter/tree-sitter-java v0.21.1-0.20240824015150-576d8097e495/go.mod h1:oyaR7fLnRV0hT9z6qwE9GkaeTom/hTDwK3H2idcOJFc= +github.com/tree-sitter/tree-sitter-javascript v0.21.5-0.20240818005344-15887341e5b5 h1:om4X9AVg3asL8gxNJDcz4e/Wp+VpQj1PY3uJXKr6EOg= +github.com/tree-sitter/tree-sitter-javascript v0.21.5-0.20240818005344-15887341e5b5/go.mod h1:nNqgPoV/h9uYWk6kYEFdEAhNVOacpfpRW5SFmdaP4tU= +github.com/tree-sitter/tree-sitter-json v0.21.1-0.20240818005659-bdd69eb8c8a5 h1:pfV3G3k7NCKqKk8THBmyuh2zA33lgYHS3GVrzRR8ry4= +github.com/tree-sitter/tree-sitter-json v0.21.1-0.20240818005659-bdd69eb8c8a5/go.mod h1:GbMKRjLfk0H+PI7nLi1Sx5lHf5wCpLz9al8tQYSxpEk= +github.com/tree-sitter/tree-sitter-php v0.22.9-0.20240819002312-a552625b56c1 h1:ZXZMDwE+IhUtGug4Brv6NjJWUU3rfkZBKpemf6RY8/g= +github.com/tree-sitter/tree-sitter-php v0.22.9-0.20240819002312-a552625b56c1/go.mod h1:UKCLuYnJ312Mei+3cyTmGOHzn0YAnaPRECgJmHtzrqs= +github.com/tree-sitter/tree-sitter-python v0.21.1-0.20240818005537-55a9b8a4fbfb h1:EXEM82lFM7JjJb6qiKZXkpIDaCcbV2obNn82ghwj9lw= +github.com/tree-sitter/tree-sitter-python v0.21.1-0.20240818005537-55a9b8a4fbfb/go.mod h1:lXCF1nGG5Dr4J3BTS0ObN4xJCCICiSu/b+Xe/VqMV7g= +github.com/tree-sitter/tree-sitter-ruby v0.21.1-0.20240818211811-7dbc1e2d0e2d h1:fcYCvoXdcP1uRQYXqJHRy6Hec+uKScQdKVtMwK9JeCI= +github.com/tree-sitter/tree-sitter-ruby v0.21.1-0.20240818211811-7dbc1e2d0e2d/go.mod h1:T1nShQ4v5AJtozZ8YyAS4uzUtDAJj/iv4YfwXSbUHzg= +github.com/tree-sitter/tree-sitter-rust v0.21.3-0.20240818005432-2b43eafe6447 h1:o9alBu1J/WjrcTKEthYtXmdkDc5OVXD+PqlvnEZ0Lzc= +github.com/tree-sitter/tree-sitter-rust v0.21.3-0.20240818005432-2b43eafe6447/go.mod h1:1Oh95COkkTn6Ezp0vcMbvfhRP5gLeqqljR0BYnBzWvc= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/package-lock.json b/package-lock.json index 9389758..882fc70 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,19 +10,19 @@ "hasInstallScript": true, "license": "MIT", "dependencies": { - "node-addon-api": "^8.0.0", - "node-gyp-build": "^4.8.0" + "node-addon-api": "^8.2.1", + "node-gyp-build": "^4.8.2" }, "devDependencies": { - "prebuildify": "^6.0.0", - "tree-sitter-cli": "^0.22.2" + "prebuildify": "^6.0.1", + "tree-sitter-cli": "^0.24.4" }, "peerDependencies": { "tree-sitter": "^0.21.1" }, "peerDependenciesMeta": { - "tree_sitter": { - "optional": true + "tree-sitter": { + "optional": false } } }, @@ -44,13 +44,15 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "MIT" }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "dev": true, + "license": "MIT", "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -76,6 +78,7 @@ "url": "https://feross.org/support" } ], + "license": "MIT", "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -85,31 +88,25 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", "dev": true, + "license": "MIT", "dependencies": { "once": "^1.4.0" } }, - "node_modules/execspawn": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/execspawn/-/execspawn-1.0.1.tgz", - "integrity": "sha512-s2k06Jy9i8CUkYe0+DxRlvtkZoOkwwfhB+Xxo5HGUtrISVW2m98jO2tr67DGRFxZwkjQqloA3v/tNtjhBRBieg==", - "dev": true, - "dependencies": { - "util-extend": "^1.0.1" - } - }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/ieee754": { "version": "1.2.1", @@ -129,31 +126,22 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "BSD-3-Clause" }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } + "license": "ISC" }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", "dev": true, + "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -162,13 +150,15 @@ "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/node-abi": { - "version": "3.56.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.56.0.tgz", - "integrity": "sha512-fZjdhDOeRcaS+rcpve7XuwHBmktS1nS1gzgghwKUQQ8nTy2FdSDr6ZT8k6YhvlJeHmmQMYiT/IH9hfco5zeW2Q==", + "version": "3.71.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.71.0.tgz", + "integrity": "sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==", "dev": true, + "license": "MIT", "dependencies": { "semver": "^7.3.5" }, @@ -177,17 +167,19 @@ } }, "node_modules/node-addon-api": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.0.0.tgz", - "integrity": "sha512-ipO7rsHEBqa9STO5C5T10fj732ml+5kLN1cAG8/jdHd56ldQeGj3Q7+scUS+VHK/qy1zLEwC4wMK5+yM0btPvw==", + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.2.2.tgz", + "integrity": "sha512-9emqXAKhVoNrQ792nLI/wpzPpJ/bj/YXxW0CvAau1+RdGBcCRF1Dmz7719zgVsQNrzHl9Tzn3ImZ4qWFarWL0A==", + "license": "MIT", "engines": { "node": "^18 || ^20 || >= 21" } }, "node_modules/node-gyp-build": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.0.tgz", - "integrity": "sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og==", + "version": "4.8.3", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.3.tgz", + "integrity": "sha512-EMS95CMJzdoSKoIiXo8pxKoL8DYxwIZXYlLmgPb8KUv794abpnLK6ynsCAWNliOjREKruYKdzbh76HHYUHX7nw==", + "license": "MIT", "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", @@ -199,6 +191,7 @@ "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-3.1.0.tgz", "integrity": "sha512-Dbl4A/VfiVGLgQv29URL9xshU8XDY1GeLy+fsaZ1AA8JDSfjvr5P5+pzRbWqRSBxk6/DW7MIh8lTM/PaGnP2kg==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^3.0.0" }, @@ -211,6 +204,7 @@ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dev": true, + "license": "ISC", "dependencies": { "wrappy": "1" } @@ -220,17 +214,18 @@ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/prebuildify": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/prebuildify/-/prebuildify-6.0.0.tgz", - "integrity": "sha512-DEvK4C3tcimIp7Pzqbs036n9i6CTKGp1XVEpMnr4wV3enKU5sBogPP+lP3KZw7993i42bXnsd5eIxAXQ566Cqw==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/prebuildify/-/prebuildify-6.0.1.tgz", + "integrity": "sha512-8Y2oOOateom/s8dNBsGIcnm6AxPmLH4/nanQzL5lQMU+sC0CMhzARZHizwr36pUPLdvBnOkCNQzxg4djuFSgIw==", "dev": true, + "license": "MIT", "dependencies": { - "execspawn": "^1.0.1", "minimist": "^1.2.5", "mkdirp-classic": "^0.5.3", "node-abi": "^3.3.0", @@ -243,10 +238,11 @@ } }, "node_modules/pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", "dev": true, + "license": "MIT", "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -257,6 +253,7 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "dev": true, + "license": "MIT", "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -284,16 +281,15 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "MIT" }, "node_modules/semver": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", - "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, + "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -306,6 +302,7 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "dev": true, + "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" } @@ -315,6 +312,7 @@ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", "dev": true, + "license": "MIT", "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -327,6 +325,7 @@ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", "dev": true, + "license": "MIT", "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -343,6 +342,7 @@ "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.21.1.tgz", "integrity": "sha512-7dxoA6kYvtgWw80265MyqJlkRl4yawIjO7S5MigytjELkX43fV2WsAXzsNfO7sBpPPCF5Gp0+XzHk0DwLCq3xQ==", "hasInstallScript": true, + "license": "MIT", "peer": true, "dependencies": { "node-addon-api": "^8.0.0", @@ -350,38 +350,32 @@ } }, "node_modules/tree-sitter-cli": { - "version": "0.22.2", - "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.22.2.tgz", - "integrity": "sha512-ecqccEp27XMFXgjLMEEU71vK9JCWAC7fqSTTxcs5P1tnEnaaf4GkHz/wfo4lJ9l3rfxcTDPxN84tHAoitIQqdA==", + "version": "0.24.4", + "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.24.4.tgz", + "integrity": "sha512-I4sdtDidnujYL0tR0Re9q0UJt5KrITf2m+GMHjT4LH6IC6kpM6eLzSR7RS36Z4t5ZQBjDHvg2QUJHAWQi3P2TA==", "dev": true, "hasInstallScript": true, + "license": "MIT", "bin": { "tree-sitter": "cli.js" + }, + "engines": { + "node": ">=12.0.0" } }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true - }, - "node_modules/util-extend": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/util-extend/-/util-extend-1.0.3.tgz", - "integrity": "sha512-mLs5zAK+ctllYBj+iAQvlDCwoxU/WDOUaJkcFudeiAX6OajC6BKXJUa9a+tbtkC11dz2Ufb7h0lyvIOVn4LADA==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true - }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "dev": true, + "license": "ISC" } } } diff --git a/package.json b/package.json index ed576df..c6cdbdb 100644 --- a/package.json +++ b/package.json @@ -2,58 +2,55 @@ "name": "tree-sitter-test", "version": "0.1.0", "description": "TS corpus test grammar for tree-sitter", - "repository": "github:tree-sitter-grammars/tree-sitter-test", - "author": "ObserverOfTime ", + "repository": "https://github.com/tree-sitter-grammars/tree-sitter-test", + "license": "MIT", + "author": { + "name": "ObserverOfTime", + "email": "chronobserver@disroot.org" + }, "contributors": [ - "Amaan Qureshi " + { + "name": "Amaan Quresehi", + "email": "amaanq12@gmail.com" + } ], - "license": "MIT", "main": "bindings/node", "types": "bindings/node", - "files": [ - "grammar.js", - "binding.gyp", - "prebuilds/**", - "bindings/node/*", - "queries/**", - "src/**" - ], "keywords": [ "incremental", "parsing", "tree-sitter", "test" ], + "files": [ + "grammar.js", + "binding.gyp", + "prebuilds/**", + "bindings/node/*", + "queries/*", + "src/**", + "*.wasm" + ], "dependencies": { - "node-addon-api": "^8.0.0", - "node-gyp-build": "^4.8.0" + "node-addon-api": "^8.2.1", + "node-gyp-build": "^4.8.2" }, "devDependencies": { - "tree-sitter-cli": "^0.22.2", - "prebuildify": "^6.0.0" + "prebuildify": "^6.0.1", + "tree-sitter-cli": "^0.24.4" }, "peerDependencies": { "tree-sitter": "^0.21.1" }, "peerDependenciesMeta": { - "tree_sitter": { + "tree-sitter": { "optional": true } }, "scripts": { "install": "node-gyp-build", - "prestart": "tree-sitter build --wasm -o tree-sitter-test.wasm", + "prestart": "tree-sitter build --wasm", "start": "tree-sitter playground", - "test": "node --test" - }, - "tree-sitter": [ - { - "scope": "text.test", - "highlights": "queries/test/highlights.scm", - "injections": "queries/test/injections.scm", - "file-types": [ - "test" - ] - } - ] + "test": "node --test bindings/node/*_test.js" + } } diff --git a/pyproject.toml b/pyproject.toml index eea3586..7f2a54a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,26 +9,21 @@ version = "0.1.0" keywords = ["incremental", "parsing", "tree-sitter", "test"] classifiers = [ "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Topic :: Software Development :: Compilers", "Topic :: Text Processing :: Linguistic", - "Typing :: Typed" + "Typing :: Typed", ] -requires-python = ">=3.8" +authors = [{ name = "ObserverOfTime", email = "chronobserver@disroot.org" }] +requires-python = ">=3.9" license.text = "MIT" readme = "README.md" -[[project.authors]] -name = "ObserverOfTime" -email = "chronobserver@disroot.org" - [project.urls] Homepage = "https://github.com/tree-sitter-grammars/tree-sitter-test" -Sponsor = "https://github.com/sponsors/ObserverOfTime" [project.optional-dependencies] -core = ["tree-sitter~=0.21"] +core = ["tree-sitter~=0.22"] [tool.cibuildwheel] -build = "cp38-*" +build = "cp39-*" build-frontend = "build" diff --git a/setup.py b/setup.py index bd95a9b..cee87f0 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,28 @@ -from os.path import isdir, join +from os import path from platform import system from setuptools import Extension, find_packages, setup from setuptools.command.build import build from wheel.bdist_wheel import bdist_wheel +sources = [ + "bindings/python/tree_sitter_test/binding.c", + "src/parser.c", +] +if path.exists("src/scanner.c"): + sources.extend("src/scanner.c") + +if system() != "Windows": + cflags = ["-std=c11", "-fvisibility=hidden"] +else: + cflags = ["/std:c11", "/utf-8"] + class Build(build): def run(self): - if isdir("queries"): - dest = join(self.build_lib, "tree_sitter_test", "queries") - self.copy_tree("queries", dest) + if path.isdir("queries"): + dest = path.join(self.build_lib, "tree_sitter_test", "queries") + self.copy_tree("queries/test", dest) super().run() @@ -18,7 +30,7 @@ class BdistWheel(bdist_wheel): def get_tag(self): python, abi, platform = super().get_tag() if python.startswith("cp"): - python, abi = "cp38", "abi3" + python, abi = "cp39", "abi3" return python, abi, platform @@ -33,17 +45,12 @@ def get_tag(self): ext_modules=[ Extension( name="_binding", - sources=[ - "bindings/python/tree_sitter_test/binding.c", - "src/parser.c", - "src/scanner.c", - ], - extra_compile_args=( - ["-std=c11"] if system() != 'Windows' else [] - ), + sources=sources, + extra_compile_args=cflags, define_macros=[ - ("Py_LIMITED_API", "0x03080000"), - ("PY_SSIZE_T_CLEAN", None) + ("Py_LIMITED_API", "0x03090000"), + ("PY_SSIZE_T_CLEAN", None), + ("TREE_SITTER_HIDE_SYMBOLS", None), ], include_dirs=["src"], py_limited_api=True, diff --git a/src/grammar.json b/src/grammar.json index 52a6436..94b1e9e 100644 --- a/src/grammar.json +++ b/src/grammar.json @@ -1,4 +1,5 @@ { + "$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json", "name": "test", "rules": { "file": { diff --git a/src/node-types.json b/src/node-types.json index 2d8fb52..cdad097 100644 --- a/src/node-types.json +++ b/src/node-types.json @@ -43,6 +43,7 @@ { "type": "file", "named": true, + "root": true, "fields": {}, "children": { "multiple": true, diff --git a/src/parser.c b/src/parser.c index 99f9750..bf5bef5 100644 --- a/src/parser.c +++ b/src/parser.c @@ -314,21 +314,23 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { switch (state) { case 0: if (eof) ADVANCE(76); - if (lookahead == '\n') ADVANCE(129); - if (lookahead == '\r') ADVANCE(130); - if (lookahead == '(') ADVANCE(82); - if (lookahead == ')') ADVANCE(83); - if (lookahead == ':') ADVANCE(16); - if (lookahead == 'a') ADVANCE(105); - if (lookahead == 'd') ADVANCE(116); - if (lookahead == 'f') ADVANCE(117); - if (lookahead == 'i') ADVANCE(110); - if (lookahead == 'l') ADVANCE(100); - if (lookahead == 'm') ADVANCE(85); - if (lookahead == 'n') ADVANCE(93); - if (lookahead == 'o') ADVANCE(115); - if (lookahead == 's') ADVANCE(111); - if (lookahead == 'w') ADVANCE(102); + ADVANCE_MAP( + '\n', 129, + '\r', 130, + '(', 82, + ')', 83, + ':', 16, + 'a', 105, + 'd', 116, + 'f', 117, + 'i', 110, + 'l', 100, + 'm', 85, + 'n', 93, + 'o', 115, + 's', 111, + 'w', 102, + ); if (lookahead == '-' || lookahead == '.' || ('0' <= lookahead && lookahead <= '9') || @@ -355,16 +357,18 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { if (lookahead == 'a') ADVANCE(12); END_STATE(); case 7: - if (lookahead == 'a') ADVANCE(41); - if (lookahead == 'd') ADVANCE(62); - if (lookahead == 'f') ADVANCE(59); - if (lookahead == 'i') ADVANCE(47); - if (lookahead == 'l') ADVANCE(31); - if (lookahead == 'm') ADVANCE(6); - if (lookahead == 'n') ADVANCE(19); - if (lookahead == 'o') ADVANCE(54); - if (lookahead == 's') ADVANCE(52); - if (lookahead == 'w') ADVANCE(33); + ADVANCE_MAP( + 'a', 41, + 'd', 62, + 'f', 59, + 'i', 47, + 'l', 31, + 'm', 6, + 'n', 19, + 'o', 54, + 's', 52, + 'w', 33, + ); END_STATE(); case 8: if (lookahead == 'a') ADVANCE(26); @@ -1387,7 +1391,7 @@ static const uint32_t ts_small_parse_table_map[] = { static const TSParseActionEntry ts_parse_actions[] = { [0] = {.entry = {.count = 0, .reusable = false}}, [1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(), - [3] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file, 0), + [3] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file, 0, 0, 0), [5] = {.entry = {.count = 1, .reusable = true}}, SHIFT(10), [7] = {.entry = {.count = 1, .reusable = true}}, SHIFT(25), [9] = {.entry = {.count = 1, .reusable = true}}, SHIFT(7), @@ -1395,34 +1399,34 @@ static const TSParseActionEntry ts_parse_actions[] = { [13] = {.entry = {.count = 1, .reusable = true}}, SHIFT(26), [15] = {.entry = {.count = 1, .reusable = true}}, SHIFT(4), [17] = {.entry = {.count = 1, .reusable = true}}, SHIFT(21), - [19] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_attributes_repeat1, 2), SHIFT_REPEAT(7), - [22] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_attributes_repeat1, 2), SHIFT_REPEAT(29), - [25] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_attributes_repeat1, 2), SHIFT_REPEAT(26), - [28] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_attributes_repeat1, 2), SHIFT_REPEAT(3), - [31] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_attributes_repeat1, 2), + [19] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_attributes_repeat1, 2, 0, 0), SHIFT_REPEAT(7), + [22] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_attributes_repeat1, 2, 0, 0), SHIFT_REPEAT(29), + [25] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_attributes_repeat1, 2, 0, 0), SHIFT_REPEAT(26), + [28] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_attributes_repeat1, 2, 0, 0), SHIFT_REPEAT(3), + [31] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_attributes_repeat1, 2, 0, 0), [33] = {.entry = {.count = 1, .reusable = true}}, SHIFT(3), - [35] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_attributes, 1), - [37] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__platform, 4, .production_id = 7), - [39] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__language, 4, .production_id = 6), - [41] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_attribute, 1), - [43] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_attribute, 1, .production_id = 3), - [45] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_attribute, 1, .production_id = 4), - [47] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file, 1), + [35] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_attributes, 1, 0, 0), + [37] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__platform, 4, 0, 7), + [39] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__language, 4, 0, 6), + [41] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_attribute, 1, 0, 0), + [43] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_attribute, 1, 0, 3), + [45] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_attribute, 1, 0, 4), + [47] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file, 1, 0, 0), [49] = {.entry = {.count = 1, .reusable = true}}, SHIFT(11), - [51] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_repeat1, 2), - [53] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_file_repeat1, 2), SHIFT_REPEAT(11), - [56] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_file_repeat1, 2), SHIFT_REPEAT(25), - [59] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_test, 4, .production_id = 5), + [51] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_repeat1, 2, 0, 0), + [53] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_file_repeat1, 2, 0, 0), SHIFT_REPEAT(11), + [56] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_file_repeat1, 2, 0, 0), SHIFT_REPEAT(25), + [59] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_test, 4, 0, 5), [61] = {.entry = {.count = 1, .reusable = true}}, SHIFT(15), - [63] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym__body, 2), - [65] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym__body, 2), SHIFT_REPEAT(15), - [68] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_test, 3, .production_id = 1), - [70] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym__body, 1), + [63] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym__body, 2, 0, 0), + [65] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym__body, 2, 0, 0), SHIFT_REPEAT(15), + [68] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_test, 3, 0, 1), + [70] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym__body, 1, 0, 0), [72] = {.entry = {.count = 1, .reusable = true}}, SHIFT(19), [74] = {.entry = {.count = 1, .reusable = true}}, SHIFT(14), - [76] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym__body, 2), SHIFT_REPEAT(19), - [79] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_header, 5, .production_id = 2), - [81] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_header, 4, .production_id = 2), + [76] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym__body, 2, 0, 0), SHIFT_REPEAT(19), + [79] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_header, 5, 0, 2), + [81] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_header, 4, 0, 2), [83] = {.entry = {.count = 1, .reusable = true}}, SHIFT(20), [85] = {.entry = {.count = 1, .reusable = true}}, SHIFT(27), [87] = {.entry = {.count = 1, .reusable = true}}, SHIFT(28), @@ -1481,7 +1485,7 @@ void tree_sitter_test_external_scanner_deserialize(void *, const char *, unsigne #define TS_PUBLIC __attribute__((visibility("default"))) #endif -TS_PUBLIC const TSLanguage *tree_sitter_test() { +TS_PUBLIC const TSLanguage *tree_sitter_test(void) { static const TSLanguage language = { .version = LANGUAGE_VERSION, .symbol_count = SYMBOL_COUNT, diff --git a/src/tree_sitter/alloc.h b/src/tree_sitter/alloc.h index 1f4466d..1abdd12 100644 --- a/src/tree_sitter/alloc.h +++ b/src/tree_sitter/alloc.h @@ -12,10 +12,10 @@ extern "C" { // Allow clients to override allocation functions #ifdef TREE_SITTER_REUSE_ALLOCATOR -extern void *(*ts_current_malloc)(size_t); -extern void *(*ts_current_calloc)(size_t, size_t); -extern void *(*ts_current_realloc)(void *, size_t); -extern void (*ts_current_free)(void *); +extern void *(*ts_current_malloc)(size_t size); +extern void *(*ts_current_calloc)(size_t count, size_t size); +extern void *(*ts_current_realloc)(void *ptr, size_t size); +extern void (*ts_current_free)(void *ptr); #ifndef ts_malloc #define ts_malloc ts_current_malloc diff --git a/src/tree_sitter/parser.h b/src/tree_sitter/parser.h index 17b4fde..799f599 100644 --- a/src/tree_sitter/parser.h +++ b/src/tree_sitter/parser.h @@ -47,6 +47,7 @@ struct TSLexer { uint32_t (*get_column)(TSLexer *); bool (*is_at_included_range_start)(const TSLexer *); bool (*eof)(const TSLexer *); + void (*log)(const TSLexer *, const char *, ...); }; typedef enum { @@ -86,6 +87,11 @@ typedef union { } entry; } TSParseActionEntry; +typedef struct { + int32_t start; + int32_t end; +} TSCharacterRange; + struct TSLanguage { uint32_t version; uint32_t symbol_count; @@ -125,6 +131,24 @@ struct TSLanguage { const TSStateId *primary_state_ids; }; +static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { + uint32_t index = 0; + uint32_t size = len - index; + while (size > 1) { + uint32_t half_size = size / 2; + uint32_t mid_index = index + half_size; + TSCharacterRange *range = &ranges[mid_index]; + if (lookahead >= range->start && lookahead <= range->end) { + return true; + } else if (lookahead > range->end) { + index = mid_index; + } + size -= half_size; + } + TSCharacterRange *range = &ranges[index]; + return (lookahead >= range->start && lookahead <= range->end); +} + /* * Lexer Macros */ @@ -154,6 +178,17 @@ struct TSLanguage { goto next_state; \ } +#define ADVANCE_MAP(...) \ + { \ + static const uint16_t map[] = { __VA_ARGS__ }; \ + for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ + if (map[i] == lookahead) { \ + state = map[i + 1]; \ + goto next_state; \ + } \ + } \ + } + #define SKIP(state_value) \ { \ skip = true; \ @@ -203,14 +238,15 @@ struct TSLanguage { } \ }} -#define REDUCE(symbol_val, child_count_val, ...) \ - {{ \ - .reduce = { \ - .type = TSParseActionTypeReduce, \ - .symbol = symbol_val, \ - .child_count = child_count_val, \ - __VA_ARGS__ \ - }, \ +#define REDUCE(symbol_name, children, precedence, prod_id) \ + {{ \ + .reduce = { \ + .type = TSParseActionTypeReduce, \ + .symbol = symbol_name, \ + .child_count = children, \ + .dynamic_precedence = precedence, \ + .production_id = prod_id \ + }, \ }} #define RECOVER() \ diff --git a/tree-sitter.json b/tree-sitter.json new file mode 100644 index 0000000..fe5c98b --- /dev/null +++ b/tree-sitter.json @@ -0,0 +1,28 @@ +{ + "grammars": [ + { + "name": "test", + "camelcase": "Test", + "path": ".", + "scope": "text.test", + "file-types": [ + "test" + ], + "injection-regex": "^test$" + } + ], + "metadata": { + "version": "0.1.0", + "license": "MIT", + "description": "TS corpus test grammar for tree-sitter", + "authors": [ + { + "name": "ObserverOfTime", + "email": "chronobserver@disroot.org" + } + ], + "links": { + "repository": "https://github.com/tree-sitter-grammars/tree-sitter-test" + } + } +}