Merge pull request #518 from tursodatabase/lucio/vendor-parser

Vendored sqlite3-parser and remove patch

Cargo.toml

@@ -12,15 +12,13 @@ members = [
   "libsql-sys-tmp",
 
   "vendored/rusqlite",
+  "vendored/sqlite3-parser"
 ]
 
 [profile.release]
 codegen-units = 1
 panic = "abort"
 
-[patch.crates-io]
-sqlite3-parser = { git = "https://github.com/LucioFranco/lemon-rs" }
-
 [workspace.dependencies]
 rusqlite = { path = "vendored/rusqlite", version = "0.29", default-features = false, features = [
     "libsql-experimental",

libsql-server/Cargo.toml

@@ -48,7 +48,7 @@ serde_json = { version = "1.0.91", features = ["preserve_order"] }
 sha2 = "0.10"
 sha256 = "1.1.3"
 sqld-libsql-bindings = { version = "0", path = "../libsql-sys-tmp" }
-sqlite3-parser = { version = "0.11.0", default-features = false, features = [ "YYNOERRORRECOVERY" ] }
+sqlite3-parser = { path = "../vendored/sqlite3-parser", version = "0.11.0", default-features = false, features = [ "YYNOERRORRECOVERY" ] }
 tempfile = "3.3.0"
 thiserror = "1.0.38"
 tokio = { version = "1.22.2", features = ["rt-multi-thread", "net", "io-std", "io-util", "time", "macros", "sync", "fs", "signal"] }

libsql/Cargo.toml

@@ -37,10 +37,9 @@ tower-http = { version = "0.4.4", features = ["trace", "util"], optional = true
 prost = { version = "0.12", optional = true }
 http = { version = "0.2", optional = true }
 
-sqlite3-parser = { version = "0.11", optional = true }
+sqlite3-parser = { path = "../vendored/sqlite3-parser", version = "0.11", optional = true }
 fallible-iterator = { version = "0.3", optional = true }
 
-
 [dev-dependencies]
 arbitrary = { version = "1.3.0", features = ["derive_arbitrary"] }
 criterion = { version = "0.5", features = ["html_reports", "async", "async_futures", "async_tokio"] }

vendored/sqlite3-parser/.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "cargo" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

vendored/sqlite3-parser/.github/workflows/rust.yml

@@ -0,0 +1,21 @@
+name: CI
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build
+      run: cargo build
+    - name: Run tests
+      run: cargo test

vendored/sqlite3-parser/.gitignore

@@ -0,0 +1,6 @@
+target
+rlemon
+*.h
+*.out
+Cargo.lock
+cmake-build-debug

vendored/sqlite3-parser/CMakeLists.txt

@@ -0,0 +1,5 @@
+cmake_minimum_required(VERSION 3.6)
+project(rlemon)
+
+set(SOURCE_FILES third_party/lemon/lemon.c)
+add_executable(rlemon ${SOURCE_FILES})

vendored/sqlite3-parser/Cargo.toml

@@ -0,0 +1,45 @@
+[package]
+name = "sqlite3-parser"
+version = "0.11.0"
+edition = "2021"
+authors = ["gwenn"]
+description = "SQL parser (as understood by SQLite)"
+documentation = "http://docs.rs/sqlite3-parser"
+repository = "https://github.com/gwenn/lemon-rs"
+readme = "README.md"
+categories = ["parser-implementations"]
+keywords = ["sql", "parser", "scanner", "tokenizer"]
+license = "Apache-2.0/MIT"
+build = "build.rs" # Lemon preprocessing
+
+[badges]
+maintenance = { status = "experimental" }
+
+[features]
+# FIXME: specific to one parser, not global
+# FIXME(lucio): Disable all features except the default ones.
+# YYTRACKMAXSTACKDEPTH = []
+YYNOERRORRECOVERY = []
+# YYSTACKDYNAMIC = []
+# YYCOVERAGE = []
+# NDEBUG = []
+default = ["YYNOERRORRECOVERY"]
+
+[dependencies]
+phf = { version = "0.11", features = ["uncased"] }
+log = "0.4"
+memchr = "2.0"
+fallible-iterator = "0.3"
+smallvec = ">=1.6.1"
+bitflags = "2.0"
+uncased = "0.9"
+indexmap = "2.0"
+
+[dev-dependencies]
+env_logger = { version = "0.10", default-features = false }
+
+[build-dependencies]
+cc = "1.0"
+phf_shared = { version = "0.11", features = ["uncased"] }
+phf_codegen = "0.11"
+uncased = "0.9"

vendored/sqlite3-parser/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org>

vendored/sqlite3-parser/README.md

@@ -0,0 +1,74 @@
+[![Build Status](https://github.com/gwenn/lemon-rs/workflows/CI/badge.svg)](https://github.com/gwenn/lemon-rs/actions)
+[![Latest Version](https://img.shields.io/crates/v/sqlite3-parser.svg)](https://crates.io/crates/sqlite3-parser)
+[![Docs](https://docs.rs/sqlite3-parser/badge.svg)](https://docs.rs/sqlite3-parser)
+[![dependency status](https://deps.rs/repo/github/gwenn/lemon-rs/status.svg)](https://deps.rs/repo/github/gwenn/lemon-rs)
+
+[LEMON parser generator](https://www.sqlite.org/src/doc/trunk/doc/lemon.html) modified to generate Rust code.
+
+Lemon source and SQLite3 grammar were last synced as of May 2022.
+
+## Unsupported
+
+### Unsupported Grammar syntax
+
+* `%token_destructor`: Code to execute to destroy token data
+* `%default_destructor`: Code for the default non-terminal destructor
+* `%destructor`: Code which executes whenever this symbol is
+  popped from the stack during error processing
+
+https://www.codeproject.com/Articles/1056460/Generating-a-High-Speed-Parser-Part-Lemon
+https://www.sqlite.org/lemon.html
+
+### SQLite
+
+[SQLite lexer](http://www.sqlite.org/src/artifact?ci=trunk&filename=src/tokenize.c) and [SQLite parser](http://www.sqlite.org/src/artifact?ci=trunk&filename=src/parse.y) have been ported from C to Rust.
+The parser generates an AST.
+
+Lexer/Parser:
+  - Keep track of position (line, column).
+  - Streamable (stop at the end of statement).
+  - Resumable (restart after the end of statement).
+
+Lexer and parser have been tested with the following scripts:
+  * https://github.com/bkiers/sqlite-parser/tree/master/src/test/resources
+  * https://github.com/codeschool/sqlite-parser/tree/master/test/sql/official-suite which can be updated with script in https://github.com/codeschool/sqlite-parser/tree/master/test/misc
+
+TODO:
+  - [ ] Check generated AST (reparse/reinject)
+  - [ ] [If a keyword in double quotes is used in a context where it cannot be resolved to an identifier but where a string literal is allowed, then the token is understood to be a string literal instead of an identifier.](https://sqlite.org/lang_keywords.html)
+  - [ ] Tests
+  - [ ] Do not panic while parsing
+  - [ ] CREATE VIRTUAL TABLE args
+  - [ ] Zero copy (at least tokens)
+
+### Unsupported by Rust
+
+* `#line` directive
+
+## API change
+
+* No `ParseAlloc`/`ParseFree` anymore 
+
+## Features not tested
+
+* NDEBUG
+* YYNOERRORRECOVERY
+* YYERRORSYMBOL
+
+## To be fixed
+
+* RHS are moved. Maybe it is not a problem if they are always used once.
+  Just add a check in lemon...
+* `%extra_argument` is not supported.
+* Terminal symbols generated by lemon should be dumped in a specified file.
+
+## Raison d'être
+
+* [lemon_rust](https://github.com/rodrigorc/lemon_rust) does the same thing
+but with an old version of `lemon`. And it seems not possible to use `yystack`
+as a stack because items may be access randomly and the `top+1` item can be used.
+
+* [lalrpop](https://github.com/nikomatsakis/lalrpop) would be the perfect
+alternative but it does not support fallback/streaming
+(see [this](https://github.com/nikomatsakis/lalrpop/issues/156) issue)
+and compilation/generation is slow.

vendored/sqlite3-parser/Sync.md

@@ -0,0 +1,7 @@
+When some changes happen in the official SQLite repository,
+they can be applied locally:
+ - $SQLITE/tool/lemon.c => $RLEMON/third_party/lemon.c
+ - $SQLITE/tool/lempar.c => $RLEMON/third_party/lempar.rs
+ - $SQLITE/tool/mkkeywordhash.c => $RLEMON/src/dialect/mod.rs
+ - $SQLITE/src/tokenize.c => $RLEMON/src/lexer/sql/mod.rs
+ - $SQLITE/src/parse.y => $RLEMON/src/parser/parse.y (and $RLEMON/src/dialect/token.rs, $RLEMON/src/dialect/mod.rs)