From 246838a69f68f4cee9a46ec59d6557d77492f392 Mon Sep 17 00:00:00 2001 From: Agaev Guseyn <60943542+agscpp@users.noreply.github.com> Date: Tue, 17 Sep 2024 00:04:21 +0300 Subject: [PATCH 01/80] Fix parsing of negative values (#1419) Co-authored-by: Agaev Huseyn --- src/parser/mod.rs | 53 ++++++++++++++++++++----------------- tests/sqlparser_common.rs | 12 +++++++++ tests/sqlparser_postgres.rs | 20 ++++++++++++++ 3 files changed, 60 insertions(+), 25 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 8d920b2ce..e2f4dd508 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -7411,6 +7411,7 @@ impl<'a> Parser<'a> { } } + /// Parse an unsigned numeric literal pub fn parse_number_value(&mut self) -> Result { match self.parse_value()? { v @ Value::Number(_, _) => Ok(v), @@ -7422,6 +7423,26 @@ impl<'a> Parser<'a> { } } + /// Parse a numeric literal as an expression. Returns a [`Expr::UnaryOp`] if the number is signed, + /// otherwise returns a [`Expr::Value`] + pub fn parse_number(&mut self) -> Result { + let next_token = self.next_token(); + match next_token.token { + Token::Plus => Ok(Expr::UnaryOp { + op: UnaryOperator::Plus, + expr: Box::new(Expr::Value(self.parse_number_value()?)), + }), + Token::Minus => Ok(Expr::UnaryOp { + op: UnaryOperator::Minus, + expr: Box::new(Expr::Value(self.parse_number_value()?)), + }), + _ => { + self.prev_token(); + Ok(Expr::Value(self.parse_number_value()?)) + } + } + } + fn parse_introduced_string_value(&mut self) -> Result { let next_token = self.next_token(); let location = next_token.location; @@ -11741,30 +11762,20 @@ impl<'a> Parser<'a> { //[ INCREMENT [ BY ] increment ] if self.parse_keywords(&[Keyword::INCREMENT]) { if self.parse_keywords(&[Keyword::BY]) { - sequence_options.push(SequenceOptions::IncrementBy( - Expr::Value(self.parse_number_value()?), - true, - )); + sequence_options.push(SequenceOptions::IncrementBy(self.parse_number()?, true)); } else { - sequence_options.push(SequenceOptions::IncrementBy( - Expr::Value(self.parse_number_value()?), - false, - )); + sequence_options.push(SequenceOptions::IncrementBy(self.parse_number()?, false)); } } //[ MINVALUE minvalue | NO MINVALUE ] if self.parse_keyword(Keyword::MINVALUE) { - sequence_options.push(SequenceOptions::MinValue(Some(Expr::Value( - self.parse_number_value()?, - )))); + sequence_options.push(SequenceOptions::MinValue(Some(self.parse_number()?))); } else if self.parse_keywords(&[Keyword::NO, Keyword::MINVALUE]) { sequence_options.push(SequenceOptions::MinValue(None)); } //[ MAXVALUE maxvalue | NO MAXVALUE ] if self.parse_keywords(&[Keyword::MAXVALUE]) { - sequence_options.push(SequenceOptions::MaxValue(Some(Expr::Value( - self.parse_number_value()?, - )))); + sequence_options.push(SequenceOptions::MaxValue(Some(self.parse_number()?))); } else if self.parse_keywords(&[Keyword::NO, Keyword::MAXVALUE]) { sequence_options.push(SequenceOptions::MaxValue(None)); } @@ -11772,22 +11783,14 @@ impl<'a> Parser<'a> { //[ START [ WITH ] start ] if self.parse_keywords(&[Keyword::START]) { if self.parse_keywords(&[Keyword::WITH]) { - sequence_options.push(SequenceOptions::StartWith( - Expr::Value(self.parse_number_value()?), - true, - )); + sequence_options.push(SequenceOptions::StartWith(self.parse_number()?, true)); } else { - sequence_options.push(SequenceOptions::StartWith( - Expr::Value(self.parse_number_value()?), - false, - )); + sequence_options.push(SequenceOptions::StartWith(self.parse_number()?, false)); } } //[ CACHE cache ] if self.parse_keywords(&[Keyword::CACHE]) { - sequence_options.push(SequenceOptions::Cache(Expr::Value( - self.parse_number_value()?, - ))); + sequence_options.push(SequenceOptions::Cache(self.parse_number()?)); } // [ [ NO ] CYCLE ] if self.parse_keywords(&[Keyword::NO, Keyword::CYCLE]) { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index ff45d7b6e..5525650f2 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -2819,6 +2819,18 @@ fn parse_window_function_null_treatment_arg() { ); } +#[test] +fn parse_negative_value() { + let sql1 = "SELECT -1"; + one_statement_parses_to(sql1, "SELECT -1"); + + let sql2 = "CREATE SEQUENCE name INCREMENT -10 MINVALUE -1000 MAXVALUE 15 START -100;"; + one_statement_parses_to( + sql2, + "CREATE SEQUENCE name INCREMENT -10 MINVALUE -1000 MAXVALUE 15 START -100", + ); +} + #[test] fn parse_create_table() { let sql = "CREATE TABLE uk_cities (\ diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index ec1311f2c..91d365692 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -277,6 +277,26 @@ fn parse_create_sequence() { "CREATE TEMPORARY SEQUENCE IF NOT EXISTS name3 INCREMENT 1 NO MINVALUE MAXVALUE 20 OWNED BY NONE", ); + let sql7 = "CREATE SEQUENCE name4 + AS BIGINT + INCREMENT -15 + MINVALUE - 2000 MAXVALUE -50 + START WITH - 60"; + pg().one_statement_parses_to( + sql7, + "CREATE SEQUENCE name4 AS BIGINT INCREMENT -15 MINVALUE -2000 MAXVALUE -50 START WITH -60", + ); + + let sql8 = "CREATE SEQUENCE name5 + AS BIGINT + INCREMENT +10 + MINVALUE + 30 MAXVALUE +5000 + START WITH + 45"; + pg().one_statement_parses_to( + sql8, + "CREATE SEQUENCE name5 AS BIGINT INCREMENT +10 MINVALUE +30 MAXVALUE +5000 START WITH +45", + ); + assert!(matches!( pg().parse_sql_statements("CREATE SEQUENCE foo INCREMENT 1 NO MINVALUE NO"), Err(ParserError::ParserError(_)) From 1c505ce736350e4e746d64168867d75d1a4cd6a7 Mon Sep 17 00:00:00 2001 From: hulk Date: Thu, 19 Sep 2024 18:56:00 +0800 Subject: [PATCH 02/80] Allow to use ON CLUSTER cluster_name in TRUNCATE syntax (#1428) --- src/ast/mod.rs | 9 +++++++++ src/parser/mod.rs | 3 +++ tests/sqlparser_common.rs | 21 +++++++++++++++++++++ tests/sqlparser_postgres.rs | 7 +++++-- 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 6c851906c..4e3cbcdba 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2172,6 +2172,11 @@ pub enum Statement { /// Postgres-specific option /// [ CASCADE | RESTRICT ] cascade: Option, + /// ClickHouse-specific option + /// [ ON CLUSTER cluster_name ] + /// + /// [ClickHouse](https://clickhouse.com/docs/en/sql-reference/statements/truncate/) + on_cluster: Option, }, /// ```sql /// MSCK @@ -3293,6 +3298,7 @@ impl fmt::Display for Statement { only, identity, cascade, + on_cluster, } => { let table = if *table { "TABLE " } else { "" }; let only = if *only { "ONLY " } else { "" }; @@ -3321,6 +3327,9 @@ impl fmt::Display for Statement { write!(f, " PARTITION ({})", display_comma_separated(parts))?; } } + if let Some(on_cluster) = on_cluster { + write!(f, " ON CLUSTER {on_cluster}")?; + } Ok(()) } Statement::AttachDatabase { diff --git a/src/parser/mod.rs b/src/parser/mod.rs index e2f4dd508..4e5ca2b5a 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -708,6 +708,8 @@ impl<'a> Parser<'a> { }; }; + let on_cluster = self.parse_optional_on_cluster()?; + Ok(Statement::Truncate { table_names, partitions, @@ -715,6 +717,7 @@ impl<'a> Parser<'a> { only, identity, cascade, + on_cluster, }) } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 5525650f2..c4a52938d 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -10804,3 +10804,24 @@ fn test_extract_seconds_single_quote_err() { "sql parser error: Expected: date/time field, found: 'seconds'" ); } + +#[test] +fn test_truncate_table_with_on_cluster() { + let sql = "TRUNCATE TABLE t ON CLUSTER cluster_name"; + match all_dialects().verified_stmt(sql) { + Statement::Truncate { on_cluster, .. } => { + assert_eq!(on_cluster, Some(Ident::new("cluster_name"))); + } + _ => panic!("Expected: TRUNCATE TABLE statement"), + } + + // Omit ON CLUSTER is allowed + all_dialects().verified_stmt("TRUNCATE TABLE t"); + + assert_eq!( + ParserError::ParserError("Expected: identifier, found: EOF".to_string()), + all_dialects() + .parse_sql_statements("TRUNCATE TABLE t ON CLUSTER") + .unwrap_err() + ); +} diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 91d365692..069b61d05 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -4010,6 +4010,7 @@ fn parse_truncate() { only: false, identity: None, cascade: None, + on_cluster: None, }, truncate ); @@ -4032,7 +4033,8 @@ fn parse_truncate_with_options() { table: true, only: true, identity: Some(TruncateIdentityOption::Restart), - cascade: Some(TruncateCascadeOption::Cascade) + cascade: Some(TruncateCascadeOption::Cascade), + on_cluster: None, }, truncate ); @@ -4063,7 +4065,8 @@ fn parse_truncate_with_table_list() { table: true, only: false, identity: Some(TruncateIdentityOption::Restart), - cascade: Some(TruncateCascadeOption::Cascade) + cascade: Some(TruncateCascadeOption::Cascade), + on_cluster: None, }, truncate ); From 04a53e57538524a9cda8ca7304312113849aec52 Mon Sep 17 00:00:00 2001 From: Siyuan Huang <73871299+kysshsy@users.noreply.github.com> Date: Thu, 19 Sep 2024 23:28:02 +0800 Subject: [PATCH 03/80] feat: support explain options (#1426) --- src/ast/mod.rs | 48 +++++++++++ src/dialect/duckdb.rs | 6 ++ src/dialect/generic.rs | 4 + src/dialect/mod.rs | 4 + src/dialect/postgresql.rs | 5 ++ src/parser/mod.rs | 45 ++++++++++- tests/sqlparser_common.rs | 163 +++++++++++++++++++++++++++++++++++++- 7 files changed, 268 insertions(+), 7 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 4e3cbcdba..94dabb059 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -3032,6 +3032,8 @@ pub enum Statement { statement: Box, /// Optional output format of explain format: Option, + /// Postgres style utility options, `(analyze, verbose true)` + options: Option>, }, /// ```sql /// SAVEPOINT @@ -3219,6 +3221,7 @@ impl fmt::Display for Statement { analyze, statement, format, + options, } => { write!(f, "{describe_alias} ")?; @@ -3234,6 +3237,10 @@ impl fmt::Display for Statement { write!(f, "FORMAT {format} ")?; } + if let Some(options) = options { + write!(f, "({}) ", display_comma_separated(options))?; + } + write!(f, "{statement}") } Statement::Query(s) => write!(f, "{s}"), @@ -7125,6 +7132,47 @@ where } } +/// Represents a single PostgreSQL utility option. +/// +/// A utility option is a key-value pair where the key is an identifier (IDENT) and the value +/// can be one of the following: +/// - A number with an optional sign (`+` or `-`). Example: `+10`, `-10.2`, `3` +/// - A non-keyword string. Example: `option1`, `'option2'`, `"option3"` +/// - keyword: `TRUE`, `FALSE`, `ON` (`off` is also accept). +/// - Empty. Example: `ANALYZE` (identifier only) +/// +/// Utility options are used in various PostgreSQL DDL statements, including statements such as +/// `CLUSTER`, `EXPLAIN`, `VACUUM`, and `REINDEX`. These statements format options as `( option [, ...] )`. +/// +/// [CLUSTER](https://www.postgresql.org/docs/current/sql-cluster.html) +/// [EXPLAIN](https://www.postgresql.org/docs/current/sql-explain.html) +/// [VACUUM](https://www.postgresql.org/docs/current/sql-vacuum.html) +/// [REINDEX](https://www.postgresql.org/docs/current/sql-reindex.html) +/// +/// For example, the `EXPLAIN` AND `VACUUM` statements with options might look like this: +/// ```sql +/// EXPLAIN (ANALYZE, VERBOSE TRUE, FORMAT TEXT) SELECT * FROM my_table; +/// +/// VACCUM (VERBOSE, ANALYZE ON, PARALLEL 10) my_table; +/// ``` +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct UtilityOption { + pub name: Ident, + pub arg: Option, +} + +impl Display for UtilityOption { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(ref arg) = self.arg { + write!(f, "{} {}", self.name, arg) + } else { + write!(f, "{}", self.name) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/dialect/duckdb.rs b/src/dialect/duckdb.rs index 1fc211685..811f25f72 100644 --- a/src/dialect/duckdb.rs +++ b/src/dialect/duckdb.rs @@ -55,4 +55,10 @@ impl Dialect for DuckDbDialect { fn support_map_literal_syntax(&self) -> bool { true } + + // DuckDB is compatible with PostgreSQL syntax for this statement, + // although not all features may be implemented. + fn supports_explain_with_utility_options(&self) -> bool { + true + } } diff --git a/src/dialect/generic.rs b/src/dialect/generic.rs index c8f1c00d9..1c638d8b0 100644 --- a/src/dialect/generic.rs +++ b/src/dialect/generic.rs @@ -90,4 +90,8 @@ impl Dialect for GenericDialect { fn supports_create_index_with_clause(&self) -> bool { true } + + fn supports_explain_with_utility_options(&self) -> bool { + true + } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 6b80243ff..cf0af6329 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -536,6 +536,10 @@ pub trait Dialect: Debug + Any { fn require_interval_qualifier(&self) -> bool { false } + + fn supports_explain_with_utility_options(&self) -> bool { + false + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index eba3a6989..5a7db0216 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -166,6 +166,11 @@ impl Dialect for PostgreSqlDialect { fn supports_create_index_with_clause(&self) -> bool { true } + + /// see + fn supports_explain_with_utility_options(&self) -> bool { + true + } } pub fn parse_comment(parser: &mut Parser) -> Result { diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 4e5ca2b5a..751101fea 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1277,6 +1277,29 @@ impl<'a> Parser<'a> { } } + pub fn parse_utility_options(&mut self) -> Result, ParserError> { + self.expect_token(&Token::LParen)?; + let options = self.parse_comma_separated(Self::parse_utility_option)?; + self.expect_token(&Token::RParen)?; + + Ok(options) + } + + fn parse_utility_option(&mut self) -> Result { + let name = self.parse_identifier(false)?; + + let next_token = self.peek_token(); + if next_token == Token::Comma || next_token == Token::RParen { + return Ok(UtilityOption { name, arg: None }); + } + let arg = self.parse_expr()?; + + Ok(UtilityOption { + name, + arg: Some(arg), + }) + } + fn try_parse_expr_sub_query(&mut self) -> Result, ParserError> { if self .parse_one_of_keywords(&[Keyword::SELECT, Keyword::WITH]) @@ -8464,11 +8487,24 @@ impl<'a> Parser<'a> { &mut self, describe_alias: DescribeAlias, ) -> Result { - let analyze = self.parse_keyword(Keyword::ANALYZE); - let verbose = self.parse_keyword(Keyword::VERBOSE); + let mut analyze = false; + let mut verbose = false; let mut format = None; - if self.parse_keyword(Keyword::FORMAT) { - format = Some(self.parse_analyze_format()?); + let mut options = None; + + // Note: DuckDB is compatible with PostgreSQL syntax for this statement, + // although not all features may be implemented. + if describe_alias == DescribeAlias::Explain + && self.dialect.supports_explain_with_utility_options() + && self.peek_token().token == Token::LParen + { + options = Some(self.parse_utility_options()?) + } else { + analyze = self.parse_keyword(Keyword::ANALYZE); + verbose = self.parse_keyword(Keyword::VERBOSE); + if self.parse_keyword(Keyword::FORMAT) { + format = Some(self.parse_analyze_format()?); + } } match self.maybe_parse(|parser| parser.parse_statement()) { @@ -8481,6 +8517,7 @@ impl<'a> Parser<'a> { verbose, statement: Box::new(statement), format, + options, }), _ => { let hive_format = diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index c4a52938d..2f001e17e 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -4268,22 +4268,26 @@ fn parse_scalar_function_in_projection() { } fn run_explain_analyze( + dialect: TestedDialects, query: &str, expected_verbose: bool, expected_analyze: bool, expected_format: Option, + exepcted_options: Option>, ) { - match verified_stmt(query) { + match dialect.verified_stmt(query) { Statement::Explain { describe_alias: _, analyze, verbose, statement, format, + options, } => { assert_eq!(verbose, expected_verbose); assert_eq!(analyze, expected_analyze); assert_eq!(format, expected_format); + assert_eq!(options, exepcted_options); assert_eq!("SELECT sqrt(id) FROM foo", statement.to_string()); } _ => panic!("Unexpected Statement, must be Explain"), @@ -4328,47 +4332,73 @@ fn explain_desc() { #[test] fn parse_explain_analyze_with_simple_select() { // Describe is an alias for EXPLAIN - run_explain_analyze("DESCRIBE SELECT sqrt(id) FROM foo", false, false, None); + run_explain_analyze( + all_dialects(), + "DESCRIBE SELECT sqrt(id) FROM foo", + false, + false, + None, + None, + ); - run_explain_analyze("EXPLAIN SELECT sqrt(id) FROM foo", false, false, None); run_explain_analyze( + all_dialects(), + "EXPLAIN SELECT sqrt(id) FROM foo", + false, + false, + None, + None, + ); + run_explain_analyze( + all_dialects(), "EXPLAIN VERBOSE SELECT sqrt(id) FROM foo", true, false, None, + None, ); run_explain_analyze( + all_dialects(), "EXPLAIN ANALYZE SELECT sqrt(id) FROM foo", false, true, None, + None, ); run_explain_analyze( + all_dialects(), "EXPLAIN ANALYZE VERBOSE SELECT sqrt(id) FROM foo", true, true, None, + None, ); run_explain_analyze( + all_dialects(), "EXPLAIN ANALYZE FORMAT GRAPHVIZ SELECT sqrt(id) FROM foo", false, true, Some(AnalyzeFormat::GRAPHVIZ), + None, ); run_explain_analyze( + all_dialects(), "EXPLAIN ANALYZE VERBOSE FORMAT JSON SELECT sqrt(id) FROM foo", true, true, Some(AnalyzeFormat::JSON), + None, ); run_explain_analyze( + all_dialects(), "EXPLAIN VERBOSE FORMAT TEXT SELECT sqrt(id) FROM foo", true, false, Some(AnalyzeFormat::TEXT), + None, ); } @@ -10825,3 +10855,130 @@ fn test_truncate_table_with_on_cluster() { .unwrap_err() ); } + +#[test] +fn parse_explain_with_option_list() { + run_explain_analyze( + all_dialects_where(|d| d.supports_explain_with_utility_options()), + "EXPLAIN (ANALYZE false, VERBOSE true) SELECT sqrt(id) FROM foo", + false, + false, + None, + Some(vec![ + UtilityOption { + name: Ident::new("ANALYZE"), + arg: Some(Expr::Value(Value::Boolean(false))), + }, + UtilityOption { + name: Ident::new("VERBOSE"), + arg: Some(Expr::Value(Value::Boolean(true))), + }, + ]), + ); + + run_explain_analyze( + all_dialects_where(|d| d.supports_explain_with_utility_options()), + "EXPLAIN (ANALYZE ON, VERBOSE OFF) SELECT sqrt(id) FROM foo", + false, + false, + None, + Some(vec![ + UtilityOption { + name: Ident::new("ANALYZE"), + arg: Some(Expr::Identifier(Ident::new("ON"))), + }, + UtilityOption { + name: Ident::new("VERBOSE"), + arg: Some(Expr::Identifier(Ident::new("OFF"))), + }, + ]), + ); + + run_explain_analyze( + all_dialects_where(|d| d.supports_explain_with_utility_options()), + r#"EXPLAIN (FORMAT1 TEXT, FORMAT2 'JSON', FORMAT3 "XML", FORMAT4 YAML) SELECT sqrt(id) FROM foo"#, + false, + false, + None, + Some(vec![ + UtilityOption { + name: Ident::new("FORMAT1"), + arg: Some(Expr::Identifier(Ident::new("TEXT"))), + }, + UtilityOption { + name: Ident::new("FORMAT2"), + arg: Some(Expr::Value(Value::SingleQuotedString("JSON".to_string()))), + }, + UtilityOption { + name: Ident::new("FORMAT3"), + arg: Some(Expr::Identifier(Ident::with_quote('"', "XML"))), + }, + UtilityOption { + name: Ident::new("FORMAT4"), + arg: Some(Expr::Identifier(Ident::new("YAML"))), + }, + ]), + ); + + run_explain_analyze( + all_dialects_where(|d| d.supports_explain_with_utility_options()), + r#"EXPLAIN (NUM1 10, NUM2 +10.1, NUM3 -10.2) SELECT sqrt(id) FROM foo"#, + false, + false, + None, + Some(vec![ + UtilityOption { + name: Ident::new("NUM1"), + arg: Some(Expr::Value(Value::Number("10".parse().unwrap(), false))), + }, + UtilityOption { + name: Ident::new("NUM2"), + arg: Some(Expr::UnaryOp { + op: UnaryOperator::Plus, + expr: Box::new(Expr::Value(Value::Number("10.1".parse().unwrap(), false))), + }), + }, + UtilityOption { + name: Ident::new("NUM3"), + arg: Some(Expr::UnaryOp { + op: UnaryOperator::Minus, + expr: Box::new(Expr::Value(Value::Number("10.2".parse().unwrap(), false))), + }), + }, + ]), + ); + + let utility_options = vec![ + UtilityOption { + name: Ident::new("ANALYZE"), + arg: None, + }, + UtilityOption { + name: Ident::new("VERBOSE"), + arg: Some(Expr::Value(Value::Boolean(true))), + }, + UtilityOption { + name: Ident::new("WAL"), + arg: Some(Expr::Identifier(Ident::new("OFF"))), + }, + UtilityOption { + name: Ident::new("FORMAT"), + arg: Some(Expr::Identifier(Ident::new("YAML"))), + }, + UtilityOption { + name: Ident::new("USER_DEF_NUM"), + arg: Some(Expr::UnaryOp { + op: UnaryOperator::Minus, + expr: Box::new(Expr::Value(Value::Number("100.1".parse().unwrap(), false))), + }), + }, + ]; + run_explain_analyze ( + all_dialects_where(|d| d.supports_explain_with_utility_options()), + "EXPLAIN (ANALYZE, VERBOSE true, WAL OFF, FORMAT YAML, USER_DEF_NUM -100.1) SELECT sqrt(id) FROM foo", + false, + false, + None, + Some(utility_options), + ); +} From 71318df8b9e168b5d7d288d11b9e13a3c58ff55d Mon Sep 17 00:00:00 2001 From: Fischer <89784872+Fischer0522@users.noreply.github.com> Date: Sat, 21 Sep 2024 00:39:17 +0800 Subject: [PATCH 04/80] chore: remove redundant punctuation (#1434) --- src/parser/mod.rs | 2 +- tests/sqlparser_clickhouse.rs | 12 ++++++------ tests/sqlparser_common.rs | 4 ++-- tests/sqlparser_databricks.rs | 2 +- tests/sqlparser_mssql.rs | 2 +- tests/sqlparser_snowflake.rs | 8 ++++---- tests/sqlparser_sqlite.rs | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 751101fea..da1b4d2a6 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1264,7 +1264,7 @@ impl<'a> Parser<'a> { self.prev_token(); self.parse_duckdb_struct_literal() } - _ => self.expected("an expression:", next_token), + _ => self.expected("an expression", next_token), }?; if self.parse_keyword(Keyword::COLLATE) { diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index f89da7ee9..ae769c763 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -276,13 +276,13 @@ fn parse_alter_table_attach_and_detach_partition() { clickhouse_and_generic() .parse_sql_statements(format!("ALTER TABLE t0 {operation} PARTITION").as_str()) .unwrap_err(), - ParserError("Expected: an expression:, found: EOF".to_string()) + ParserError("Expected: an expression, found: EOF".to_string()) ); assert_eq!( clickhouse_and_generic() .parse_sql_statements(format!("ALTER TABLE t0 {operation} PART").as_str()) .unwrap_err(), - ParserError("Expected: an expression:, found: EOF".to_string()) + ParserError("Expected: an expression, found: EOF".to_string()) ); } } @@ -355,7 +355,7 @@ fn parse_alter_table_add_projection() { clickhouse_and_generic() .parse_sql_statements("ALTER TABLE t0 ADD PROJECTION my_name (SELECT)") .unwrap_err(), - ParserError("Expected: an expression:, found: )".to_string()) + ParserError("Expected: an expression, found: )".to_string()) ); } @@ -498,13 +498,13 @@ fn parse_optimize_table() { clickhouse_and_generic() .parse_sql_statements("OPTIMIZE TABLE t0 DEDUPLICATE BY") .unwrap_err(), - ParserError("Expected: an expression:, found: EOF".to_string()) + ParserError("Expected: an expression, found: EOF".to_string()) ); assert_eq!( clickhouse_and_generic() .parse_sql_statements("OPTIMIZE TABLE t0 PARTITION") .unwrap_err(), - ParserError("Expected: an expression:, found: EOF".to_string()) + ParserError("Expected: an expression, found: EOF".to_string()) ); assert_eq!( clickhouse_and_generic() @@ -1479,7 +1479,7 @@ fn parse_freeze_and_unfreeze_partition() { clickhouse_and_generic() .parse_sql_statements(format!("ALTER TABLE t0 {operation_name} PARTITION").as_str()) .unwrap_err(), - ParserError("Expected: an expression:, found: EOF".to_string()) + ParserError("Expected: an expression, found: EOF".to_string()) ); assert_eq!( clickhouse_and_generic() diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 2f001e17e..ac89fc1ca 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -2051,7 +2051,7 @@ fn parse_tuple_invalid() { let sql = "select (), 2"; let res = parse_sql_statements(sql); assert_eq!( - ParserError::ParserError("Expected: an expression:, found: )".to_string()), + ParserError::ParserError("Expected: an expression, found: )".to_string()), res.unwrap_err() ); } @@ -9556,7 +9556,7 @@ fn parse_projection_trailing_comma() { trailing_commas .parse_sql_statements("SELECT * FROM track ORDER BY milliseconds,") .unwrap_err(), - ParserError::ParserError("Expected: an expression:, found: EOF".to_string()) + ParserError::ParserError("Expected: an expression, found: EOF".to_string()) ); assert_eq!( diff --git a/tests/sqlparser_databricks.rs b/tests/sqlparser_databricks.rs index ee0cf2d7d..095ab0b30 100644 --- a/tests/sqlparser_databricks.rs +++ b/tests/sqlparser_databricks.rs @@ -64,7 +64,7 @@ fn test_databricks_exists() { let res = databricks().parse_sql_statements("SELECT EXISTS ("); assert_eq!( // TODO: improve this error message... - ParserError::ParserError("Expected: an expression:, found: EOF".to_string()), + ParserError::ParserError("Expected: an expression, found: EOF".to_string()), res.unwrap_err(), ); } diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 0ab160f56..97114aed0 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -487,7 +487,7 @@ fn parse_convert() { let error_sql = "SELECT CONVERT(INT, 'foo',) FROM T"; assert_eq!( - ParserError::ParserError("Expected: an expression:, found: )".to_owned()), + ParserError::ParserError("Expected: an expression, found: )".to_owned()), ms().parse_sql_statements(error_sql).unwrap_err() ); } diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index d0876fc50..5be5bfd3a 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -1230,13 +1230,13 @@ fn parse_snowflake_declare_result_set() { let error_sql = "DECLARE res RESULTSET DEFAULT"; assert_eq!( - ParserError::ParserError("Expected: an expression:, found: EOF".to_owned()), + ParserError::ParserError("Expected: an expression, found: EOF".to_owned()), snowflake().parse_sql_statements(error_sql).unwrap_err() ); let error_sql = "DECLARE res RESULTSET :="; assert_eq!( - ParserError::ParserError("Expected: an expression:, found: EOF".to_owned()), + ParserError::ParserError("Expected: an expression, found: EOF".to_owned()), snowflake().parse_sql_statements(error_sql).unwrap_err() ); } @@ -1328,13 +1328,13 @@ fn parse_snowflake_declare_variable() { let error_sql = "DECLARE profit INT DEFAULT"; assert_eq!( - ParserError::ParserError("Expected: an expression:, found: EOF".to_owned()), + ParserError::ParserError("Expected: an expression, found: EOF".to_owned()), snowflake().parse_sql_statements(error_sql).unwrap_err() ); let error_sql = "DECLARE profit DEFAULT"; assert_eq!( - ParserError::ParserError("Expected: an expression:, found: EOF".to_owned()), + ParserError::ParserError("Expected: an expression, found: EOF".to_owned()), snowflake().parse_sql_statements(error_sql).unwrap_err() ); } diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index ce11b015a..8e2afb8d7 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -432,7 +432,7 @@ fn invalid_empty_list() { let sql = "SELECT * FROM t1 WHERE a IN (,,)"; let sqlite = sqlite_with_options(ParserOptions::new().with_trailing_commas(true)); assert_eq!( - "sql parser error: Expected: an expression:, found: ,", + "sql parser error: Expected: an expression, found: ,", sqlite.parse_sql_statements(sql).unwrap_err().to_string() ); } From fb42425d51162cd57e5a0cbd49eafc5743d60daf Mon Sep 17 00:00:00 2001 From: Aleksei Piianin Date: Fri, 20 Sep 2024 18:44:24 +0200 Subject: [PATCH 05/80] MS SQL Server: add support for IDENTITY column option (#1432) --- src/ast/ddl.rs | 28 ++++++++++ src/ast/mod.rs | 4 +- src/parser/mod.rs | 14 +++++ tests/sqlparser_mssql.rs | 111 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 2 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index b5444b8da..3cb754dfc 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -1050,6 +1050,20 @@ impl fmt::Display for ColumnOptionDef { } } +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct IdentityProperty { + pub seed: Expr, + pub increment: Expr, +} + +impl fmt::Display for IdentityProperty { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}, {}", self.seed, self.increment) + } +} + /// `ColumnOption`s are modifiers that follow a column definition in a `CREATE /// TABLE` statement. #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -1120,6 +1134,13 @@ pub enum ColumnOption { /// [1]: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#view_column_option_list /// [2]: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#column_option_list Options(Vec), + /// MS SQL Server specific: Creates an identity column in a table. + /// Syntax + /// ```sql + /// IDENTITY [ (seed , increment) ] + /// ``` + /// [MS SQL Server]: https://learn.microsoft.com/en-us/sql/t-sql/statements/create-table-transact-sql-identity-property + Identity(Option), } impl fmt::Display for ColumnOption { @@ -1221,6 +1242,13 @@ impl fmt::Display for ColumnOption { Options(options) => { write!(f, "OPTIONS({})", display_comma_separated(options)) } + Identity(parameters) => { + write!(f, "IDENTITY")?; + if let Some(parameters) = parameters { + write!(f, "({parameters})")?; + } + Ok(()) + } } } } diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 94dabb059..4eba1e59f 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -36,8 +36,8 @@ pub use self::dcl::{AlterRoleOperation, ResetConfig, RoleOption, SetConfigValue, pub use self::ddl::{ AlterColumnOperation, AlterIndexOperation, AlterTableOperation, ClusteredBy, ColumnDef, ColumnOption, ColumnOptionDef, ConstraintCharacteristics, Deduplicate, DeferrableInitial, - GeneratedAs, GeneratedExpressionMode, IndexOption, IndexType, KeyOrIndexDisplay, Owner, - Partition, ProcedureParam, ReferentialAction, TableConstraint, + GeneratedAs, GeneratedExpressionMode, IdentityProperty, IndexOption, IndexType, + KeyOrIndexDisplay, Owner, Partition, ProcedureParam, ReferentialAction, TableConstraint, UserDefinedTypeCompositeAttributeDef, UserDefinedTypeRepresentation, ViewColumnDef, }; pub use self::dml::{CreateIndex, CreateTable, Delete, Insert}; diff --git a/src/parser/mod.rs b/src/parser/mod.rs index da1b4d2a6..5cce97767 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -6069,6 +6069,20 @@ impl<'a> Parser<'a> { && dialect_of!(self is MySqlDialect | SQLiteDialect | DuckDbDialect | GenericDialect) { self.parse_optional_column_option_as() + } else if self.parse_keyword(Keyword::IDENTITY) + && dialect_of!(self is MsSqlDialect | GenericDialect) + { + let property = if self.consume_token(&Token::LParen) { + let seed = self.parse_number()?; + self.expect_token(&Token::Comma)?; + let increment = self.parse_number()?; + self.expect_token(&Token::RParen)?; + + Some(IdentityProperty { seed, increment }) + } else { + None + }; + Ok(Some(ColumnOption::Identity(property))) } else { Ok(None) } diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 97114aed0..9df8ae351 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -908,6 +908,117 @@ fn parse_create_table_with_invalid_options() { } } +#[test] +fn parse_create_table_with_identity_column() { + let with_column_options = [ + ( + r#"CREATE TABLE mytable (columnA INT IDENTITY NOT NULL)"#, + vec![ + ColumnOptionDef { + name: None, + option: ColumnOption::Identity(None), + }, + ColumnOptionDef { + name: None, + option: ColumnOption::NotNull, + }, + ], + ), + ( + r#"CREATE TABLE mytable (columnA INT IDENTITY(1, 1) NOT NULL)"#, + vec![ + ColumnOptionDef { + name: None, + #[cfg(not(feature = "bigdecimal"))] + option: ColumnOption::Identity(Some(IdentityProperty { + seed: Expr::Value(Value::Number("1".to_string(), false)), + increment: Expr::Value(Value::Number("1".to_string(), false)), + })), + #[cfg(feature = "bigdecimal")] + option: ColumnOption::Identity(Some(IdentityProperty { + seed: Expr::Value(Value::Number(bigdecimal::BigDecimal::from(1), false)), + increment: Expr::Value(Value::Number( + bigdecimal::BigDecimal::from(1), + false, + )), + })), + }, + ColumnOptionDef { + name: None, + option: ColumnOption::NotNull, + }, + ], + ), + ]; + + for (sql, column_options) in with_column_options { + assert_eq!( + ms_and_generic().verified_stmt(sql), + Statement::CreateTable(CreateTable { + or_replace: false, + temporary: false, + external: false, + global: None, + if_not_exists: false, + transient: false, + volatile: false, + name: ObjectName(vec![Ident { + value: "mytable".to_string(), + quote_style: None, + },],), + columns: vec![ColumnDef { + name: Ident { + value: "columnA".to_string(), + quote_style: None, + }, + data_type: Int(None,), + collation: None, + options: column_options, + },], + constraints: vec![], + hive_distribution: HiveDistributionStyle::NONE, + hive_formats: Some(HiveFormat { + row_format: None, + serde_properties: None, + storage: None, + location: None, + },), + table_properties: vec![], + with_options: vec![], + file_format: None, + location: None, + query: None, + without_rowid: false, + like: None, + clone: None, + engine: None, + comment: None, + auto_increment_offset: None, + default_charset: None, + collation: None, + on_commit: None, + on_cluster: None, + primary_key: None, + order_by: None, + partition_by: None, + cluster_by: None, + clustered_by: None, + options: None, + strict: false, + copy_grants: false, + enable_schema_evolution: None, + change_tracking: None, + data_retention_time_in_days: None, + max_data_extension_time_in_days: None, + default_ddl_collation: None, + with_aggregation_policy: None, + with_row_access_policy: None, + with_tags: None, + }), + ); + } +} + fn ms() -> TestedDialects { TestedDialects { dialects: vec![Box::new(MsSqlDialect {})], From 9934f3d93100720a2b15290be6f767fb675bec11 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 21 Sep 2024 06:23:28 -0400 Subject: [PATCH 06/80] Update to ASF header / add when missing (#1437) --- CHANGELOG.md | 19 ++++++ Cargo.toml | 17 +++++ HEADER | 23 ++++--- SECURITY.md | 19 ++++++ derive/Cargo.toml | 17 +++++ derive/README.md | 19 ++++++ derive/src/lib.rs | 79 ++++++++++++++++------ docs/benchmarking.md | 19 ++++++ docs/custom_sql_parser.md | 19 ++++++ docs/fuzzing.md | 19 ++++++ docs/releasing.md | 19 ++++++ examples/cli.rs | 23 ++++--- examples/parse_select.rs | 23 ++++--- fuzz/Cargo.toml | 17 +++++ fuzz/fuzz_targets/fuzz_parse_sql.rs | 17 +++++ rustfmt.toml | 17 +++++ sqlparser_bench/Cargo.toml | 17 +++++ sqlparser_bench/benches/sqlparser_bench.rs | 23 ++++--- src/ast/data_type.rs | 23 ++++--- src/ast/dcl.rs | 23 ++++--- src/ast/ddl.rs | 23 ++++--- src/ast/dml.rs | 23 ++++--- src/ast/helpers/mod.rs | 16 +++++ src/ast/helpers/stmt_create_table.rs | 17 +++++ src/ast/helpers/stmt_data_loading.rs | 23 ++++--- src/ast/mod.rs | 23 ++++--- src/ast/operator.rs | 23 ++++--- src/ast/query.rs | 23 ++++--- src/ast/trigger.rs | 23 ++++--- src/ast/value.rs | 23 ++++--- src/ast/visitor.rs | 23 ++++--- src/dialect/ansi.rs | 23 ++++--- src/dialect/bigquery.rs | 23 ++++--- src/dialect/clickhouse.rs | 23 ++++--- src/dialect/databricks.rs | 17 +++++ src/dialect/duckdb.rs | 23 ++++--- src/dialect/generic.rs | 23 ++++--- src/dialect/hive.rs | 23 ++++--- src/dialect/mod.rs | 23 ++++--- src/dialect/mssql.rs | 23 ++++--- src/dialect/mysql.rs | 23 ++++--- src/dialect/postgresql.rs | 17 +++++ src/dialect/redshift.rs | 23 ++++--- src/dialect/snowflake.rs | 23 ++++--- src/dialect/sqlite.rs | 23 ++++--- src/keywords.rs | 23 ++++--- src/lib.rs | 23 ++++--- src/test_utils.rs | 23 ++++--- src/tokenizer.rs | 23 ++++--- tests/sqlparser_bigquery.rs | 23 ++++--- tests/sqlparser_clickhouse.rs | 23 ++++--- tests/sqlparser_common.rs | 23 ++++--- tests/sqlparser_custom_dialect.rs | 23 ++++--- tests/sqlparser_databricks.rs | 17 +++++ tests/sqlparser_duckdb.rs | 23 ++++--- tests/sqlparser_hive.rs | 23 ++++--- tests/sqlparser_mssql.rs | 23 ++++--- tests/sqlparser_mysql.rs | 23 ++++--- tests/sqlparser_postgres.rs | 23 ++++--- tests/sqlparser_redshift.rs | 23 ++++--- tests/sqlparser_regression.rs | 23 ++++--- tests/sqlparser_snowflake.rs | 23 ++++--- tests/sqlparser_sqlite.rs | 23 ++++--- tests/test_utils/mod.rs | 23 ++++--- 64 files changed, 1006 insertions(+), 427 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c77755d59..07142602d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ + + # Changelog All notable changes to this project will be documented in this file. diff --git a/Cargo.toml b/Cargo.toml index 2448b67ce..99546465b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [package] name = "sqlparser" description = "Extensible SQL Lexer and Parser with support for ANSI SQL:2011" diff --git a/HEADER b/HEADER index 79fe4603f..6e778edd7 100644 --- a/HEADER +++ b/HEADER @@ -1,11 +1,16 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. \ No newline at end of file +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md index 0636b8fc2..662ab74f0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,3 +1,22 @@ + + # Security Policy ## Reporting a Vulnerability diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 02eb7c880..0c5852c4c 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [package] name = "sqlparser_derive" description = "proc macro for sqlparser" diff --git a/derive/README.md b/derive/README.md index ffb5d266e..aa70e7c71 100644 --- a/derive/README.md +++ b/derive/README.md @@ -1,3 +1,22 @@ + + # SQL Parser Derive Macro ## Visit diff --git a/derive/src/lib.rs b/derive/src/lib.rs index d19696aa4..5ad1607f9 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,31 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use proc_macro2::TokenStream; use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::spanned::Spanned; use syn::{ parse::{Parse, ParseStream}, - parse_macro_input, parse_quote, Attribute, Data, DeriveInput, - Fields, GenericParam, Generics, Ident, Index, LitStr, Meta, Token + parse_macro_input, parse_quote, Attribute, Data, DeriveInput, Fields, GenericParam, Generics, + Ident, Index, LitStr, Meta, Token, }; - /// Implementation of `[#derive(Visit)]` #[proc_macro_derive(VisitMut, attributes(visit))] pub fn derive_visit_mut(input: proc_macro::TokenStream) -> proc_macro::TokenStream { - derive_visit(input, &VisitType { - visit_trait: quote!(VisitMut), - visitor_trait: quote!(VisitorMut), - modifier: Some(quote!(mut)), - }) + derive_visit( + input, + &VisitType { + visit_trait: quote!(VisitMut), + visitor_trait: quote!(VisitorMut), + modifier: Some(quote!(mut)), + }, + ) } /// Implementation of `[#derive(Visit)]` #[proc_macro_derive(Visit, attributes(visit))] pub fn derive_visit_immutable(input: proc_macro::TokenStream) -> proc_macro::TokenStream { - derive_visit(input, &VisitType { - visit_trait: quote!(Visit), - visitor_trait: quote!(Visitor), - modifier: None, - }) + derive_visit( + input, + &VisitType { + visit_trait: quote!(Visit), + visitor_trait: quote!(Visitor), + modifier: None, + }, + ) } struct VisitType { @@ -34,15 +56,16 @@ struct VisitType { modifier: Option, } -fn derive_visit( - input: proc_macro::TokenStream, - visit_type: &VisitType, -) -> proc_macro::TokenStream { +fn derive_visit(input: proc_macro::TokenStream, visit_type: &VisitType) -> proc_macro::TokenStream { // Parse the input tokens into a syntax tree. let input = parse_macro_input!(input as DeriveInput); let name = input.ident; - let VisitType { visit_trait, visitor_trait, modifier } = visit_type; + let VisitType { + visit_trait, + visitor_trait, + modifier, + } = visit_type; let attributes = Attributes::parse(&input.attrs); // Add a bound `T: Visit` to every type parameter T. @@ -87,7 +110,10 @@ impl Parse for WithIdent { let mut result = WithIdent { with: None }; let ident = input.parse::()?; if ident != "with" { - return Err(syn::Error::new(ident.span(), "Expected identifier to be `with`")); + return Err(syn::Error::new( + ident.span(), + "Expected identifier to be `with`", + )); } input.parse::()?; let s = input.parse::()?; @@ -131,17 +157,26 @@ impl Attributes { } // Add a bound `T: Visit` to every type parameter T. -fn add_trait_bounds(mut generics: Generics, VisitType{visit_trait, ..}: &VisitType) -> Generics { +fn add_trait_bounds(mut generics: Generics, VisitType { visit_trait, .. }: &VisitType) -> Generics { for param in &mut generics.params { if let GenericParam::Type(ref mut type_param) = *param { - type_param.bounds.push(parse_quote!(sqlparser::ast::#visit_trait)); + type_param + .bounds + .push(parse_quote!(sqlparser::ast::#visit_trait)); } } generics } // Generate the body of the visit implementation for the given type -fn visit_children(data: &Data, VisitType{visit_trait, modifier, ..}: &VisitType) -> TokenStream { +fn visit_children( + data: &Data, + VisitType { + visit_trait, + modifier, + .. + }: &VisitType, +) -> TokenStream { match data { Data::Struct(data) => match &data.fields { Fields::Named(fields) => { diff --git a/docs/benchmarking.md b/docs/benchmarking.md index feae53c84..bba67bc4d 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -1,3 +1,22 @@ + + # Benchmarking Run `cargo bench` in the project `sqlparser_bench` execute the queries. diff --git a/docs/custom_sql_parser.md b/docs/custom_sql_parser.md index 055ddbaa5..9e9a0dd87 100644 --- a/docs/custom_sql_parser.md +++ b/docs/custom_sql_parser.md @@ -1,3 +1,22 @@ + + # Writing a Custom SQL Parser I have explored many different ways of building this library to make it easy to extend it for custom SQL dialects. Most of my attempts ended in failure but I have now found a workable solution. It is not without downsides but this seems to be the most pragmatic solution. diff --git a/docs/fuzzing.md b/docs/fuzzing.md index 72d3d96eb..cc4ca4d78 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -1,3 +1,22 @@ + + # Fuzzing ## Installing `honggfuzz` diff --git a/docs/releasing.md b/docs/releasing.md index 13ae8290f..c1b85a20c 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,3 +1,22 @@ + + # Releasing ## Prerequisites diff --git a/examples/cli.rs b/examples/cli.rs index 72f963b1e..8a5d6501e 100644 --- a/examples/cli.rs +++ b/examples/cli.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] diff --git a/examples/parse_select.rs b/examples/parse_select.rs index 71fe1fa1e..1d977fb05 100644 --- a/examples/parse_select.rs +++ b/examples/parse_select.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 72ab86ef6..c162854d4 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [package] name = "fuzz" version = "0.1.0" diff --git a/fuzz/fuzz_targets/fuzz_parse_sql.rs b/fuzz/fuzz_targets/fuzz_parse_sql.rs index 629fa360b..446b036cd 100644 --- a/fuzz/fuzz_targets/fuzz_parse_sql.rs +++ b/fuzz/fuzz_targets/fuzz_parse_sql.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use honggfuzz::fuzz; use sqlparser::dialect::GenericDialect; use sqlparser::parser::Parser; diff --git a/rustfmt.toml b/rustfmt.toml index edd0eecba..b9037cc0b 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # We use rustfmt's default settings to format the source code \ No newline at end of file diff --git a/sqlparser_bench/Cargo.toml b/sqlparser_bench/Cargo.toml index f2cd93288..9c33658a2 100644 --- a/sqlparser_bench/Cargo.toml +++ b/sqlparser_bench/Cargo.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [package] name = "sqlparser_bench" version = "0.1.0" diff --git a/sqlparser_bench/benches/sqlparser_bench.rs b/sqlparser_bench/benches/sqlparser_bench.rs index 5293c0f50..27c58b450 100644 --- a/sqlparser_bench/benches/sqlparser_bench.rs +++ b/sqlparser_bench/benches/sqlparser_bench.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use criterion::{criterion_group, criterion_main, Criterion}; use sqlparser::dialect::GenericDialect; diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index f3ebd16da..bc48341c4 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #[cfg(not(feature = "std"))] use alloc::{boxed::Box, format, string::String, vec::Vec}; diff --git a/src/ast/dcl.rs b/src/ast/dcl.rs index 1b0a77095..d47476ffa 100644 --- a/src/ast/dcl.rs +++ b/src/ast/dcl.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //! AST types specific to GRANT/REVOKE/ROLE variants of [`Statement`](crate::ast::Statement) //! (commonly referred to as Data Control Language, or DCL) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 3cb754dfc..e0441d07a 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //! AST types specific to CREATE/ALTER variants of [`Statement`](crate::ast::Statement) //! (commonly referred to as Data Definition Language, or DDL) diff --git a/src/ast/dml.rs b/src/ast/dml.rs index c0e58e21a..8121f2c5b 100644 --- a/src/ast/dml.rs +++ b/src/ast/dml.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #[cfg(not(feature = "std"))] use alloc::{boxed::Box, string::String, vec::Vec}; diff --git a/src/ast/helpers/mod.rs b/src/ast/helpers/mod.rs index b54e59b6d..d6924ab88 100644 --- a/src/ast/helpers/mod.rs +++ b/src/ast/helpers/mod.rs @@ -1,2 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. pub mod stmt_create_table; pub mod stmt_data_loading; diff --git a/src/ast/helpers/stmt_create_table.rs b/src/ast/helpers/stmt_create_table.rs index 82532b291..364969c40 100644 --- a/src/ast/helpers/stmt_create_table.rs +++ b/src/ast/helpers/stmt_create_table.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #[cfg(not(feature = "std"))] use alloc::{boxed::Box, format, string::String, vec, vec::Vec}; diff --git a/src/ast/helpers/stmt_data_loading.rs b/src/ast/helpers/stmt_data_loading.rs index a259e664b..a17f8c68c 100644 --- a/src/ast/helpers/stmt_data_loading.rs +++ b/src/ast/helpers/stmt_data_loading.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //! AST types specific to loading and unloading syntax, like one available in Snowflake which //! contains: STAGE ddl operations, PUT upload or COPY INTO diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 4eba1e59f..00898939c 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //! SQL Abstract Syntax Tree (AST) types #[cfg(not(feature = "std"))] diff --git a/src/ast/operator.rs b/src/ast/operator.rs index db6ed0564..c3bb379d6 100644 --- a/src/ast/operator.rs +++ b/src/ast/operator.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use core::fmt; diff --git a/src/ast/query.rs b/src/ast/query.rs index c52d01105..ec0198674 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #[cfg(not(feature = "std"))] use alloc::{boxed::Box, vec::Vec}; diff --git a/src/ast/trigger.rs b/src/ast/trigger.rs index a0913db94..cf1c8c466 100644 --- a/src/ast/trigger.rs +++ b/src/ast/trigger.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //! SQL Abstract Syntax Tree (AST) for triggers. use super::*; diff --git a/src/ast/value.rs b/src/ast/value.rs index 17cdb839d..30d956a07 100644 --- a/src/ast/value.rs +++ b/src/ast/value.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #[cfg(not(feature = "std"))] use alloc::string::String; diff --git a/src/ast/visitor.rs b/src/ast/visitor.rs index 1b8a43802..418e0a299 100644 --- a/src/ast/visitor.rs +++ b/src/ast/visitor.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //! Recursive visitors for ast Nodes. See [`Visitor`] for more details. diff --git a/src/dialect/ansi.rs b/src/dialect/ansi.rs index 61ae5829e..32ba7b32a 100644 --- a/src/dialect/ansi.rs +++ b/src/dialect/ansi.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use crate::dialect::Dialect; diff --git a/src/dialect/bigquery.rs b/src/dialect/bigquery.rs index 3bce6702b..96633552b 100644 --- a/src/dialect/bigquery.rs +++ b/src/dialect/bigquery.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use crate::dialect::Dialect; diff --git a/src/dialect/clickhouse.rs b/src/dialect/clickhouse.rs index 09735cbee..ee8ee94d2 100644 --- a/src/dialect/clickhouse.rs +++ b/src/dialect/clickhouse.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use crate::dialect::Dialect; diff --git a/src/dialect/databricks.rs b/src/dialect/databricks.rs index d3661444b..4924e8077 100644 --- a/src/dialect/databricks.rs +++ b/src/dialect/databricks.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::dialect::Dialect; /// A [`Dialect`] for [Databricks SQL](https://www.databricks.com/) diff --git a/src/dialect/duckdb.rs b/src/dialect/duckdb.rs index 811f25f72..e1b8db118 100644 --- a/src/dialect/duckdb.rs +++ b/src/dialect/duckdb.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use crate::dialect::Dialect; diff --git a/src/dialect/generic.rs b/src/dialect/generic.rs index 1c638d8b0..bea56d0ef 100644 --- a/src/dialect/generic.rs +++ b/src/dialect/generic.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use crate::dialect::Dialect; diff --git a/src/dialect/hive.rs b/src/dialect/hive.rs index b32d44cb9..63642b33c 100644 --- a/src/dialect/hive.rs +++ b/src/dialect/hive.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use crate::dialect::Dialect; diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index cf0af6329..619b54713 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. mod ansi; mod bigquery; diff --git a/src/dialect/mssql.rs b/src/dialect/mssql.rs index ec247092f..a9d296be3 100644 --- a/src/dialect/mssql.rs +++ b/src/dialect/mssql.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use crate::dialect::Dialect; diff --git a/src/dialect/mysql.rs b/src/dialect/mysql.rs index b8c4631fd..2b5e42e59 100644 --- a/src/dialect/mysql.rs +++ b/src/dialect/mysql.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #[cfg(not(feature = "std"))] use alloc::boxed::Box; diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index 5a7db0216..945c4fcc0 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/src/dialect/redshift.rs b/src/dialect/redshift.rs index bd4dc817c..3bfdec3b0 100644 --- a/src/dialect/redshift.rs +++ b/src/dialect/redshift.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use crate::dialect::Dialect; use core::iter::Peekable; diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index 4f37004b1..e14746d9a 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #[cfg(not(feature = "std"))] use crate::alloc::string::ToString; diff --git a/src/dialect/sqlite.rs b/src/dialect/sqlite.rs index cc08d7961..d3813d91c 100644 --- a/src/dialect/sqlite.rs +++ b/src/dialect/sqlite.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use crate::ast::Statement; use crate::dialect::Dialect; diff --git a/src/keywords.rs b/src/keywords.rs index 68eaf050b..d384062f2 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //! This module defines //! 1) a list of constants for every keyword diff --git a/src/lib.rs b/src/lib.rs index ba0132ed8..6c8987b63 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //! # SQL Parser for Rust //! diff --git a/src/test_utils.rs b/src/test_utils.rs index 5c05ec996..eb0352353 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. /// This module contains internal utilities used for testing the library. /// While technically public, the library's users are not supposed to rely diff --git a/src/tokenizer.rs b/src/tokenizer.rs index be11a3140..6d0c86ff2 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //! SQL Tokenizer //! diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index e051baa8b..55afe473b 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #[macro_use] mod test_utils; diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index ae769c763..e30c33678 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] //! Test SQL syntax specific to ClickHouse. diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index ac89fc1ca..9aa76882a 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] //! Test SQL syntax, which all sqlparser dialects must parse in the same way. diff --git a/tests/sqlparser_custom_dialect.rs b/tests/sqlparser_custom_dialect.rs index 5b29047a4..e9ca82aba 100644 --- a/tests/sqlparser_custom_dialect.rs +++ b/tests/sqlparser_custom_dialect.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. //! Test the ability for dialects to override parsing diff --git a/tests/sqlparser_databricks.rs b/tests/sqlparser_databricks.rs index 095ab0b30..7dcfee68a 100644 --- a/tests/sqlparser_databricks.rs +++ b/tests/sqlparser_databricks.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use sqlparser::ast::*; use sqlparser::dialect::{DatabricksDialect, GenericDialect}; use sqlparser::parser::ParserError; diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index 12368a88c..4703f4b60 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #[macro_use] mod test_utils; diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 5426dfbc4..069500bf6 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 9df8ae351..5a2ef9e87 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] //! Test SQL syntax specific to Microsoft's T-SQL. The parser based on the diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 33587c35a..87468b933 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] //! Test SQL syntax specific to MySQL. The parser based on the generic dialect diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 069b61d05..9ba3e5dbc 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] //! Test SQL syntax specific to PostgreSQL. The parser based on the diff --git a/tests/sqlparser_redshift.rs b/tests/sqlparser_redshift.rs index 440116e02..eeba37957 100644 --- a/tests/sqlparser_redshift.rs +++ b/tests/sqlparser_redshift.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #[macro_use] mod test_utils; diff --git a/tests/sqlparser_regression.rs b/tests/sqlparser_regression.rs index e869e0932..55e03c45b 100644 --- a/tests/sqlparser_regression.rs +++ b/tests/sqlparser_regression.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 5be5bfd3a..50c5f740e 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] //! Test SQL syntax specific to Snowflake. The parser based on the diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index 8e2afb8d7..d7fd3b896 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #![warn(clippy::all)] //! Test SQL syntax specific to SQLite. The parser based on the diff --git a/tests/test_utils/mod.rs b/tests/test_utils/mod.rs index d3a56ce46..4bb0b1151 100644 --- a/tests/test_utils/mod.rs +++ b/tests/test_utils/mod.rs @@ -1,14 +1,19 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. // Re-export everything from `src/test_utils.rs`. pub use sqlparser::test_utils::*; From 2403f79f55d2e2bb6281f787198fc029d9775440 Mon Sep 17 00:00:00 2001 From: Thomas Dagenais Date: Sat, 21 Sep 2024 06:25:14 -0400 Subject: [PATCH 07/80] Some small optimizations (#1424) --- src/ast/helpers/stmt_data_loading.rs | 9 ++++++--- src/ast/mod.rs | 25 ++++++++----------------- src/parser/mod.rs | 6 +++--- 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/ast/helpers/stmt_data_loading.rs b/src/ast/helpers/stmt_data_loading.rs index a17f8c68c..cda6c6ea4 100644 --- a/src/ast/helpers/stmt_data_loading.rs +++ b/src/ast/helpers/stmt_data_loading.rs @@ -108,11 +108,14 @@ impl fmt::Display for StageParamsObject { impl fmt::Display for DataLoadingOptions { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { if !self.options.is_empty() { + let mut first = false; for option in &self.options { - write!(f, "{}", option)?; - if !option.eq(self.options.last().unwrap()) { - write!(f, " ")?; + if !first { + first = true; + } else { + f.write_str(" ")?; } + write!(f, "{}", option)?; } } Ok(()) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 00898939c..9b7a66650 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -4511,30 +4511,21 @@ impl fmt::Display for Statement { options, query, } => { - if table_flag.is_some() { - write!( - f, - "CACHE {table_flag} TABLE {table_name}", - table_flag = table_flag.clone().unwrap(), - table_name = table_name, - )?; + if let Some(table_flag) = table_flag { + write!(f, "CACHE {table_flag} TABLE {table_name}")?; } else { - write!(f, "CACHE TABLE {table_name}",)?; + write!(f, "CACHE TABLE {table_name}")?; } if !options.is_empty() { write!(f, " OPTIONS({})", display_comma_separated(options))?; } - let has_query = query.is_some(); - if *has_as && has_query { - write!(f, " AS {query}", query = query.clone().unwrap()) - } else if !has_as && has_query { - write!(f, " {query}", query = query.clone().unwrap()) - } else if *has_as && !has_query { - write!(f, " AS") - } else { - Ok(()) + match (*has_as, query) { + (true, Some(query)) => write!(f, " AS {query}"), + (true, None) => f.write_str(" AS"), + (false, Some(query)) => write!(f, " {query}"), + (false, None) => Ok(()), } } Statement::UNCache { diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 5cce97767..5d57347cf 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -10552,12 +10552,12 @@ impl<'a> Parser<'a> { return parser_err!("Unsupported statement REPLACE", self.peek_token().location); } - let insert = &mut self.parse_insert()?; - if let Statement::Insert(Insert { replace_into, .. }) = insert { + let mut insert = self.parse_insert()?; + if let Statement::Insert(Insert { replace_into, .. }) = &mut insert { *replace_into = true; } - Ok(insert.clone()) + Ok(insert) } /// Parse an INSERT statement, returning a `Box`ed SetExpr From affe8b549884a351ead4f35aa8bdf4cae8c93e4b Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 22 Sep 2024 07:32:37 -0400 Subject: [PATCH 08/80] Fix `codestyle` CI check (#1438) --- .github/workflows/rust.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 146ea3120..4aab9cee7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -10,11 +10,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup Rust Toolchain uses: ./.github/actions/setup-builder - with: - # Note that `nightly` is required for `license_template_path`, as - # it's an unstable feature. - rust-version: nightly - - run: cargo +nightly fmt -- --check --config-path <(echo 'license_template_path = "HEADER"') + - run: cargo fmt -- --check lint: runs-on: ubuntu-latest From 8a534c0e279dde2fad06e55423fba9cbffcca0a2 Mon Sep 17 00:00:00 2001 From: hulk Date: Thu, 26 Sep 2024 01:32:04 +0800 Subject: [PATCH 09/80] Implements CREATE POLICY syntax for PostgreSQL (#1440) --- src/ast/mod.rs | 85 +++++++++++++++++++++++++++ src/keywords.rs | 2 + src/parser/mod.rs | 118 +++++++++++++++++++++++++++++++++----- tests/sqlparser_common.rs | 102 ++++++++++++++++++++++++++++++++ 4 files changed, 292 insertions(+), 15 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 9b7a66650..83646d298 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2135,6 +2135,35 @@ pub enum FromTable { WithoutKeyword(Vec), } +/// Policy type for a `CREATE POLICY` statement. +/// ```sql +/// AS [ PERMISSIVE | RESTRICTIVE ] +/// ``` +/// [PostgreSQL](https://www.postgresql.org/docs/current/sql-createpolicy.html) +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum CreatePolicyType { + Permissive, + Restrictive, +} + +/// Policy command for a `CREATE POLICY` statement. +/// ```sql +/// FOR [ALL | SELECT | INSERT | UPDATE | DELETE] +/// ``` +/// [PostgreSQL](https://www.postgresql.org/docs/current/sql-createpolicy.html) +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum CreatePolicyCommand { + All, + Select, + Insert, + Update, + Delete, +} + /// A top-level statement (SELECT, INSERT, CREATE, etc.) #[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -2375,6 +2404,20 @@ pub enum Statement { options: Vec, }, /// ```sql + /// CREATE POLICY + /// ``` + /// See [PostgreSQL](https://www.postgresql.org/docs/current/sql-createpolicy.html) + CreatePolicy { + name: Ident, + #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))] + table_name: ObjectName, + policy_type: Option, + command: Option, + to: Option>, + using: Option, + with_check: Option, + }, + /// ```sql /// ALTER TABLE /// ``` AlterTable { @@ -4052,6 +4095,48 @@ impl fmt::Display for Statement { write!(f, " )")?; Ok(()) } + Statement::CreatePolicy { + name, + table_name, + policy_type, + command, + to, + using, + with_check, + } => { + write!(f, "CREATE POLICY {name} ON {table_name}")?; + + if let Some(policy_type) = policy_type { + match policy_type { + CreatePolicyType::Permissive => write!(f, " AS PERMISSIVE")?, + CreatePolicyType::Restrictive => write!(f, " AS RESTRICTIVE")?, + } + } + + if let Some(command) = command { + match command { + CreatePolicyCommand::All => write!(f, " FOR ALL")?, + CreatePolicyCommand::Select => write!(f, " FOR SELECT")?, + CreatePolicyCommand::Insert => write!(f, " FOR INSERT")?, + CreatePolicyCommand::Update => write!(f, " FOR UPDATE")?, + CreatePolicyCommand::Delete => write!(f, " FOR DELETE")?, + } + } + + if let Some(to) = to { + write!(f, " TO {}", display_comma_separated(to))?; + } + + if let Some(using) = using { + write!(f, " USING ({using})")?; + } + + if let Some(with_check) = with_check { + write!(f, " WITH CHECK ({with_check})")?; + } + + Ok(()) + } Statement::AlterTable { name, if_exists, diff --git a/src/keywords.rs b/src/keywords.rs index d384062f2..49c6ce20f 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -568,6 +568,7 @@ define_keywords!( PERCENTILE_DISC, PERCENT_RANK, PERIOD, + PERMISSIVE, PERSISTENT, PIVOT, PLACING, @@ -634,6 +635,7 @@ define_keywords!( RESTART, RESTRICT, RESTRICTED, + RESTRICTIVE, RESULT, RESULTSET, RETAIN, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 5d57347cf..4c3f8788d 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -32,6 +32,7 @@ use IsLateral::*; use IsOptional::*; use crate::ast::helpers::stmt_create_table::{CreateTableBuilder, CreateTableConfiguration}; +use crate::ast::Statement::CreatePolicy; use crate::ast::*; use crate::dialect::*; use crate::keywords::{Keyword, ALL_KEYWORDS}; @@ -3569,6 +3570,8 @@ impl<'a> Parser<'a> { } else if self.parse_keyword(Keyword::MATERIALIZED) || self.parse_keyword(Keyword::VIEW) { self.prev_token(); self.parse_create_view(or_replace, temporary) + } else if self.parse_keyword(Keyword::POLICY) { + self.parse_create_policy() } else if self.parse_keyword(Keyword::EXTERNAL) { self.parse_create_external_table(or_replace) } else if self.parse_keyword(Keyword::FUNCTION) { @@ -4762,6 +4765,105 @@ impl<'a> Parser<'a> { }) } + pub fn parse_owner(&mut self) -> Result { + let owner = match self.parse_one_of_keywords(&[Keyword::CURRENT_USER, Keyword::CURRENT_ROLE, Keyword::SESSION_USER]) { + Some(Keyword::CURRENT_USER) => Owner::CurrentUser, + Some(Keyword::CURRENT_ROLE) => Owner::CurrentRole, + Some(Keyword::SESSION_USER) => Owner::SessionUser, + Some(_) => unreachable!(), + None => { + match self.parse_identifier(false) { + Ok(ident) => Owner::Ident(ident), + Err(e) => { + return Err(ParserError::ParserError(format!("Expected: CURRENT_USER, CURRENT_ROLE, SESSION_USER or identifier after OWNER TO. {e}"))) + } + } + }, + }; + Ok(owner) + } + + /// ```sql + /// CREATE POLICY name ON table_name [ AS { PERMISSIVE | RESTRICTIVE } ] + /// [ FOR { ALL | SELECT | INSERT | UPDATE | DELETE } ] + /// [ TO { role_name | PUBLIC | CURRENT_USER | CURRENT_ROLE | SESSION_USER } [, ...] ] + /// [ USING ( using_expression ) ] + /// [ WITH CHECK ( with_check_expression ) ] + /// ``` + /// + /// [PostgreSQL Documentation](https://www.postgresql.org/docs/current/sql-createpolicy.html) + pub fn parse_create_policy(&mut self) -> Result { + let name = self.parse_identifier(false)?; + self.expect_keyword(Keyword::ON)?; + let table_name = self.parse_object_name(false)?; + + let policy_type = if self.parse_keyword(Keyword::AS) { + let keyword = + self.expect_one_of_keywords(&[Keyword::PERMISSIVE, Keyword::RESTRICTIVE])?; + Some(match keyword { + Keyword::PERMISSIVE => CreatePolicyType::Permissive, + Keyword::RESTRICTIVE => CreatePolicyType::Restrictive, + _ => unreachable!(), + }) + } else { + None + }; + + let command = if self.parse_keyword(Keyword::FOR) { + let keyword = self.expect_one_of_keywords(&[ + Keyword::ALL, + Keyword::SELECT, + Keyword::INSERT, + Keyword::UPDATE, + Keyword::DELETE, + ])?; + Some(match keyword { + Keyword::ALL => CreatePolicyCommand::All, + Keyword::SELECT => CreatePolicyCommand::Select, + Keyword::INSERT => CreatePolicyCommand::Insert, + Keyword::UPDATE => CreatePolicyCommand::Update, + Keyword::DELETE => CreatePolicyCommand::Delete, + _ => unreachable!(), + }) + } else { + None + }; + + let to = if self.parse_keyword(Keyword::TO) { + Some(self.parse_comma_separated(|p| p.parse_owner())?) + } else { + None + }; + + let using = if self.parse_keyword(Keyword::USING) { + self.expect_token(&Token::LParen)?; + let expr = self.parse_expr()?; + self.expect_token(&Token::RParen)?; + Some(expr) + } else { + None + }; + + let with_check = if self.parse_keywords(&[Keyword::WITH, Keyword::CHECK]) { + self.expect_token(&Token::LParen)?; + let expr = self.parse_expr()?; + self.expect_token(&Token::RParen)?; + Some(expr) + } else { + None + }; + + Ok(CreatePolicy { + name, + table_name, + policy_type, + command, + to, + using, + with_check, + }) + } + pub fn parse_drop(&mut self) -> Result { // MySQL dialect supports `TEMPORARY` let temporary = dialect_of!(self is MySqlDialect | GenericDialect | DuckDbDialect) @@ -6941,21 +7043,7 @@ impl<'a> Parser<'a> { } else if dialect_of!(self is PostgreSqlDialect | GenericDialect) && self.parse_keywords(&[Keyword::OWNER, Keyword::TO]) { - let new_owner = match self.parse_one_of_keywords(&[Keyword::CURRENT_USER, Keyword::CURRENT_ROLE, Keyword::SESSION_USER]) { - Some(Keyword::CURRENT_USER) => Owner::CurrentUser, - Some(Keyword::CURRENT_ROLE) => Owner::CurrentRole, - Some(Keyword::SESSION_USER) => Owner::SessionUser, - Some(_) => unreachable!(), - None => { - match self.parse_identifier(false) { - Ok(ident) => Owner::Ident(ident), - Err(e) => { - return Err(ParserError::ParserError(format!("Expected: CURRENT_USER, CURRENT_ROLE, SESSION_USER or identifier after OWNER TO. {e}"))) - } - } - }, - }; - + let new_owner = self.parse_owner()?; AlterTableOperation::OwnerTo { new_owner } } else if dialect_of!(self is ClickHouseDialect|GenericDialect) && self.parse_keyword(Keyword::ATTACH) diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 9aa76882a..711070034 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -10987,3 +10987,105 @@ fn parse_explain_with_option_list() { Some(utility_options), ); } + +#[test] +fn test_create_policy() { + let sql = concat!( + "CREATE POLICY my_policy ON my_table ", + "AS PERMISSIVE FOR SELECT ", + "TO my_role, CURRENT_USER ", + "USING (c0 = 1) ", + "WITH CHECK (true)" + ); + + match all_dialects().verified_stmt(sql) { + Statement::CreatePolicy { + name, + table_name, + to, + using, + with_check, + .. + } => { + assert_eq!(name.to_string(), "my_policy"); + assert_eq!(table_name.to_string(), "my_table"); + assert_eq!( + to, + Some(vec![ + Owner::Ident(Ident::new("my_role")), + Owner::CurrentUser + ]) + ); + assert_eq!( + using, + Some(Expr::BinaryOp { + left: Box::new(Expr::Identifier(Ident::new("c0"))), + op: BinaryOperator::Eq, + right: Box::new(Expr::Value(Value::Number("1".parse().unwrap(), false))), + }) + ); + assert_eq!(with_check, Some(Expr::Value(Value::Boolean(true)))); + } + _ => unreachable!(), + } + + // USING with SELECT query + all_dialects().verified_stmt(concat!( + "CREATE POLICY my_policy ON my_table ", + "AS PERMISSIVE FOR SELECT ", + "TO my_role, CURRENT_USER ", + "USING (c0 IN (SELECT column FROM t0)) ", + "WITH CHECK (true)" + )); + // omit AS / FOR / TO / USING / WITH CHECK clauses is allowed + all_dialects().verified_stmt("CREATE POLICY my_policy ON my_table"); + + // missing table name + assert_eq!( + all_dialects() + .parse_sql_statements("CREATE POLICY my_policy") + .unwrap_err() + .to_string(), + "sql parser error: Expected: ON, found: EOF" + ); + // missing policy type + assert_eq!( + all_dialects() + .parse_sql_statements("CREATE POLICY my_policy ON my_table AS") + .unwrap_err() + .to_string(), + "sql parser error: Expected: one of PERMISSIVE or RESTRICTIVE, found: EOF" + ); + // missing FOR command + assert_eq!( + all_dialects() + .parse_sql_statements("CREATE POLICY my_policy ON my_table FOR") + .unwrap_err() + .to_string(), + "sql parser error: Expected: one of ALL or SELECT or INSERT or UPDATE or DELETE, found: EOF" + ); + // missing TO owners + assert_eq!( + all_dialects() + .parse_sql_statements("CREATE POLICY my_policy ON my_table TO") + .unwrap_err() + .to_string(), + "sql parser error: Expected: CURRENT_USER, CURRENT_ROLE, SESSION_USER or identifier after OWNER TO. sql parser error: Expected: identifier, found: EOF" + ); + // missing USING expression + assert_eq!( + all_dialects() + .parse_sql_statements("CREATE POLICY my_policy ON my_table USING") + .unwrap_err() + .to_string(), + "sql parser error: Expected: (, found: EOF" + ); + // missing WITH CHECK expression + assert_eq!( + all_dialects() + .parse_sql_statements("CREATE POLICY my_policy ON my_table WITH CHECK") + .unwrap_err() + .to_string(), + "sql parser error: Expected: (, found: EOF" + ); +} From 08fc6d381397760156fc247985b25fa85741bf47 Mon Sep 17 00:00:00 2001 From: Eason <30045503+Eason0729@users.noreply.github.com> Date: Sun, 29 Sep 2024 17:58:00 +0800 Subject: [PATCH 10/80] make `parse_expr_with_alias` public (#1444) Co-authored-by: Andrew Lamb Co-authored-by: Ifeanyi Ubah --- src/parser/mod.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 4c3f8788d..b163b4cfb 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -10377,8 +10377,31 @@ impl<'a> Parser<'a> { Ok(ExprWithAlias { expr, alias }) } + /// Parses an expression with an optional alias - fn parse_expr_with_alias(&mut self) -> Result { + /// Examples: + + /// ```sql + /// SUM(price) AS total_price + /// ``` + + /// ```sql + /// SUM(price) + /// ``` + /// + /// Example + /// ``` + /// # use sqlparser::parser::{Parser, ParserError}; + /// # use sqlparser::dialect::GenericDialect; + /// # fn main() ->Result<(), ParserError> { + /// let sql = r#"SUM("a") as "b""#; + /// let mut parser = Parser::new(&GenericDialect).try_with_sql(sql)?; + /// let expr_with_alias = parser.parse_expr_with_alias()?; + /// assert_eq!(Some("b".to_string()), expr_with_alias.alias.map(|x|x.value)); + /// # Ok(()) + /// # } + + pub fn parse_expr_with_alias(&mut self) -> Result { let expr = self.parse_expr()?; let alias = if self.parse_keyword(Keyword::AS) { Some(self.parse_identifier(false)?) From 2af981e4e6ad00d5550758b17fa3218f6e0a2b09 Mon Sep 17 00:00:00 2001 From: hulk Date: Sun, 29 Sep 2024 17:58:34 +0800 Subject: [PATCH 11/80] Implements DROP POLICY syntax for PostgreSQL (#1445) --- src/ast/mod.rs | 26 ++++++++++++++++++++++++ src/parser/mod.rs | 42 +++++++++++++++++++++++++++++---------- tests/sqlparser_common.rs | 41 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 11 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 83646d298..03db24fcd 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2546,6 +2546,16 @@ pub enum Statement { name: Ident, storage_specifier: Option, }, + ///```sql + /// DROP POLICY + /// ``` + /// See [PostgreSQL](https://www.postgresql.org/docs/current/sql-droppolicy.html) + DropPolicy { + if_exists: bool, + name: Ident, + table_name: ObjectName, + option: Option, + }, /// ```sql /// DECLARE /// ``` @@ -4258,6 +4268,22 @@ impl fmt::Display for Statement { } Ok(()) } + Statement::DropPolicy { + if_exists, + name, + table_name, + option, + } => { + write!(f, "DROP POLICY")?; + if *if_exists { + write!(f, " IF EXISTS")?; + } + write!(f, " {name} ON {table_name}")?; + if let Some(option) = option { + write!(f, " {option}")?; + } + Ok(()) + } Statement::Discard { object_type } => { write!(f, "DISCARD {object_type}")?; Ok(()) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b163b4cfb..404a16c48 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -4887,6 +4887,8 @@ impl<'a> Parser<'a> { ObjectType::Stage } else if self.parse_keyword(Keyword::FUNCTION) { return self.parse_drop_function(); + } else if self.parse_keyword(Keyword::POLICY) { + return self.parse_drop_policy(); } else if self.parse_keyword(Keyword::PROCEDURE) { return self.parse_drop_procedure(); } else if self.parse_keyword(Keyword::SECRET) { @@ -4928,6 +4930,14 @@ impl<'a> Parser<'a> { }) } + fn parse_optional_referential_action(&mut self) -> Option { + match self.parse_one_of_keywords(&[Keyword::CASCADE, Keyword::RESTRICT]) { + Some(Keyword::CASCADE) => Some(ReferentialAction::Cascade), + Some(Keyword::RESTRICT) => Some(ReferentialAction::Restrict), + _ => None, + } + } + /// ```sql /// DROP FUNCTION [ IF EXISTS ] name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] [, ...] /// [ CASCADE | RESTRICT ] @@ -4935,11 +4945,7 @@ impl<'a> Parser<'a> { fn parse_drop_function(&mut self) -> Result { let if_exists = self.parse_keywords(&[Keyword::IF, Keyword::EXISTS]); let func_desc = self.parse_comma_separated(Parser::parse_function_desc)?; - let option = match self.parse_one_of_keywords(&[Keyword::CASCADE, Keyword::RESTRICT]) { - Some(Keyword::CASCADE) => Some(ReferentialAction::Cascade), - Some(Keyword::RESTRICT) => Some(ReferentialAction::Restrict), - _ => None, - }; + let option = self.parse_optional_referential_action(); Ok(Statement::DropFunction { if_exists, func_desc, @@ -4947,6 +4953,25 @@ impl<'a> Parser<'a> { }) } + /// ```sql + /// DROP POLICY [ IF EXISTS ] name ON table_name [ CASCADE | RESTRICT ] + /// ``` + /// + /// [PostgreSQL Documentation](https://www.postgresql.org/docs/current/sql-droppolicy.html) + fn parse_drop_policy(&mut self) -> Result { + let if_exists = self.parse_keywords(&[Keyword::IF, Keyword::EXISTS]); + let name = self.parse_identifier(false)?; + self.expect_keyword(Keyword::ON)?; + let table_name = self.parse_object_name(false)?; + let option = self.parse_optional_referential_action(); + Ok(Statement::DropPolicy { + if_exists, + name, + table_name, + option, + }) + } + /// ```sql /// DROP PROCEDURE [ IF EXISTS ] name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] [, ...] /// [ CASCADE | RESTRICT ] @@ -4954,12 +4979,7 @@ impl<'a> Parser<'a> { fn parse_drop_procedure(&mut self) -> Result { let if_exists = self.parse_keywords(&[Keyword::IF, Keyword::EXISTS]); let proc_desc = self.parse_comma_separated(Parser::parse_function_desc)?; - let option = match self.parse_one_of_keywords(&[Keyword::CASCADE, Keyword::RESTRICT]) { - Some(Keyword::CASCADE) => Some(ReferentialAction::Cascade), - Some(Keyword::RESTRICT) => Some(ReferentialAction::Restrict), - Some(_) => unreachable!(), // parse_one_of_keywords does not return other keywords - None => None, - }; + let option = self.parse_optional_referential_action(); Ok(Statement::DropProcedure { if_exists, proc_desc, diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 711070034..48c4aafe1 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11089,3 +11089,44 @@ fn test_create_policy() { "sql parser error: Expected: (, found: EOF" ); } + +#[test] +fn test_drop_policy() { + let sql = "DROP POLICY IF EXISTS my_policy ON my_table RESTRICT"; + match all_dialects().verified_stmt(sql) { + Statement::DropPolicy { + if_exists, + name, + table_name, + option, + } => { + assert_eq!(if_exists, true); + assert_eq!(name.to_string(), "my_policy"); + assert_eq!(table_name.to_string(), "my_table"); + assert_eq!(option, Some(ReferentialAction::Restrict)); + } + _ => unreachable!(), + } + + // omit IF EXISTS is allowed + all_dialects().verified_stmt("DROP POLICY my_policy ON my_table CASCADE"); + // omit option is allowed + all_dialects().verified_stmt("DROP POLICY my_policy ON my_table"); + + // missing table name + assert_eq!( + all_dialects() + .parse_sql_statements("DROP POLICY my_policy") + .unwrap_err() + .to_string(), + "sql parser error: Expected: ON, found: EOF" + ); + // Wrong option name + assert_eq!( + all_dialects() + .parse_sql_statements("DROP POLICY my_policy ON my_table WRONG") + .unwrap_err() + .to_string(), + "sql parser error: Expected: end of statement, found: WRONG" + ); +} From 73dc8a352d7d9edad3def308042f4db2a3c9a027 Mon Sep 17 00:00:00 2001 From: Heran Lin Date: Sun, 29 Sep 2024 18:17:19 +0800 Subject: [PATCH 12/80] Support `DROP DATABASE` (#1443) --- src/ast/mod.rs | 2 ++ src/parser/mod.rs | 4 +++- tests/sqlparser_common.rs | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 03db24fcd..19354ab96 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -5691,6 +5691,7 @@ pub enum ObjectType { View, Index, Schema, + Database, Role, Sequence, Stage, @@ -5703,6 +5704,7 @@ impl fmt::Display for ObjectType { ObjectType::View => "VIEW", ObjectType::Index => "INDEX", ObjectType::Schema => "SCHEMA", + ObjectType::Database => "DATABASE", ObjectType::Role => "ROLE", ObjectType::Sequence => "SEQUENCE", ObjectType::Stage => "STAGE", diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 404a16c48..e5fd53554 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -4881,6 +4881,8 @@ impl<'a> Parser<'a> { ObjectType::Role } else if self.parse_keyword(Keyword::SCHEMA) { ObjectType::Schema + } else if self.parse_keyword(Keyword::DATABASE) { + ObjectType::Database } else if self.parse_keyword(Keyword::SEQUENCE) { ObjectType::Sequence } else if self.parse_keyword(Keyword::STAGE) { @@ -4897,7 +4899,7 @@ impl<'a> Parser<'a> { return self.parse_drop_trigger(); } else { return self.expected( - "TABLE, VIEW, INDEX, ROLE, SCHEMA, FUNCTION, PROCEDURE, STAGE, TRIGGER, SECRET or SEQUENCE after DROP", + "TABLE, VIEW, INDEX, ROLE, SCHEMA, DATABASE, FUNCTION, PROCEDURE, STAGE, TRIGGER, SECRET or SEQUENCE after DROP", self.peek_token(), ); }; diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 48c4aafe1..e066451df 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -6734,6 +6734,43 @@ fn parse_create_database_ine() { } } +#[test] +fn parse_drop_database() { + let sql = "DROP DATABASE mycatalog.mydb"; + match verified_stmt(sql) { + Statement::Drop { + names, + object_type, + if_exists, + .. + } => { + assert_eq!( + vec!["mycatalog.mydb"], + names.iter().map(ToString::to_string).collect::>() + ); + assert_eq!(ObjectType::Database, object_type); + assert!(!if_exists); + } + _ => unreachable!(), + } +} + +#[test] +fn parse_drop_database_if_exists() { + let sql = "DROP DATABASE IF EXISTS mydb"; + match verified_stmt(sql) { + Statement::Drop { + object_type, + if_exists, + .. + } => { + assert_eq!(ObjectType::Database, object_type); + assert!(if_exists); + } + _ => unreachable!(), + } +} + #[test] fn parse_create_view() { let sql = "CREATE VIEW myschema.myview AS SELECT foo FROM bar"; From 51cbd5a3e69f9cc903f91d7df335658094b5e811 Mon Sep 17 00:00:00 2001 From: hulk Date: Sun, 29 Sep 2024 18:17:31 +0800 Subject: [PATCH 13/80] Implements ALTER POLICY syntax for PostgreSQL (#1446) Co-authored-by: Andrew Lamb --- src/ast/ddl.rs | 43 +++++++++++++++++++++ src/ast/mod.rs | 28 +++++++++++--- src/parser/alter.rs | 65 ++++++++++++++++++++++++++++++- src/parser/mod.rs | 2 + tests/sqlparser_common.rs | 81 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 213 insertions(+), 6 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index e0441d07a..4578ae8f1 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -234,6 +234,49 @@ pub enum AlterTableOperation { OwnerTo { new_owner: Owner }, } +/// An `ALTER Policy` (`Statement::AlterPolicy`) operation +/// +/// [PostgreSQL Documentation](https://www.postgresql.org/docs/current/sql-altertable.html) +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum AlterPolicyOperation { + Rename { + new_name: Ident, + }, + Apply { + to: Option>, + using: Option, + with_check: Option, + }, +} + +impl fmt::Display for AlterPolicyOperation { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + AlterPolicyOperation::Rename { new_name } => { + write!(f, " RENAME TO {new_name}") + } + AlterPolicyOperation::Apply { + to, + using, + with_check, + } => { + if let Some(to) = to { + write!(f, " TO {}", display_comma_separated(to))?; + } + if let Some(using) = using { + write!(f, " USING ({using})")?; + } + if let Some(with_check) = with_check { + write!(f, " WITH CHECK ({with_check})")?; + } + Ok(()) + } + } + } +} + #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 19354ab96..a11fa63f4 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -39,11 +39,12 @@ pub use self::data_type::{ }; pub use self::dcl::{AlterRoleOperation, ResetConfig, RoleOption, SetConfigValue, Use}; pub use self::ddl::{ - AlterColumnOperation, AlterIndexOperation, AlterTableOperation, ClusteredBy, ColumnDef, - ColumnOption, ColumnOptionDef, ConstraintCharacteristics, Deduplicate, DeferrableInitial, - GeneratedAs, GeneratedExpressionMode, IdentityProperty, IndexOption, IndexType, - KeyOrIndexDisplay, Owner, Partition, ProcedureParam, ReferentialAction, TableConstraint, - UserDefinedTypeCompositeAttributeDef, UserDefinedTypeRepresentation, ViewColumnDef, + AlterColumnOperation, AlterIndexOperation, AlterPolicyOperation, AlterTableOperation, + ClusteredBy, ColumnDef, ColumnOption, ColumnOptionDef, ConstraintCharacteristics, Deduplicate, + DeferrableInitial, GeneratedAs, GeneratedExpressionMode, IdentityProperty, IndexOption, + IndexType, KeyOrIndexDisplay, Owner, Partition, ProcedureParam, ReferentialAction, + TableConstraint, UserDefinedTypeCompositeAttributeDef, UserDefinedTypeRepresentation, + ViewColumnDef, }; pub use self::dml::{CreateIndex, CreateTable, Delete, Insert}; pub use self::operator::{BinaryOperator, UnaryOperator}; @@ -2459,6 +2460,16 @@ pub enum Statement { operation: AlterRoleOperation, }, /// ```sql + /// ALTER POLICY ON [] + /// ``` + /// (Postgresql-specific) + AlterPolicy { + name: Ident, + #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))] + table_name: ObjectName, + operation: AlterPolicyOperation, + }, + /// ```sql /// ATTACH DATABASE 'path/to/file' AS alias /// ``` /// (SQLite-specific) @@ -4197,6 +4208,13 @@ impl fmt::Display for Statement { Statement::AlterRole { name, operation } => { write!(f, "ALTER ROLE {name} {operation}") } + Statement::AlterPolicy { + name, + table_name, + operation, + } => { + write!(f, "ALTER POLICY {name} ON {table_name}{operation}") + } Statement::Drop { object_type, if_exists, diff --git a/src/parser/alter.rs b/src/parser/alter.rs index 7bf99af66..28fdaf764 100644 --- a/src/parser/alter.rs +++ b/src/parser/alter.rs @@ -17,7 +17,10 @@ use alloc::vec; use super::{Parser, ParserError}; use crate::{ - ast::{AlterRoleOperation, Expr, Password, ResetConfig, RoleOption, SetConfigValue, Statement}, + ast::{ + AlterPolicyOperation, AlterRoleOperation, Expr, Password, ResetConfig, RoleOption, + SetConfigValue, Statement, + }, dialect::{MsSqlDialect, PostgreSqlDialect}, keywords::Keyword, tokenizer::Token, @@ -36,6 +39,66 @@ impl<'a> Parser<'a> { )) } + /// Parse ALTER POLICY statement + /// ```sql + /// ALTER POLICY policy_name ON table_name [ RENAME TO new_name ] + /// or + /// ALTER POLICY policy_name ON table_name + /// [ TO { role_name | PUBLIC | CURRENT_ROLE | CURRENT_USER | SESSION_USER } [, ...] ] + /// [ USING ( using_expression ) ] + /// [ WITH CHECK ( check_expression ) ] + /// ``` + /// + /// [PostgreSQL](https://www.postgresql.org/docs/current/sql-alterpolicy.html) + pub fn parse_alter_policy(&mut self) -> Result { + let name = self.parse_identifier(false)?; + self.expect_keyword(Keyword::ON)?; + let table_name = self.parse_object_name(false)?; + + if self.parse_keyword(Keyword::RENAME) { + self.expect_keyword(Keyword::TO)?; + let new_name = self.parse_identifier(false)?; + Ok(Statement::AlterPolicy { + name, + table_name, + operation: AlterPolicyOperation::Rename { new_name }, + }) + } else { + let to = if self.parse_keyword(Keyword::TO) { + Some(self.parse_comma_separated(|p| p.parse_owner())?) + } else { + None + }; + + let using = if self.parse_keyword(Keyword::USING) { + self.expect_token(&Token::LParen)?; + let expr = self.parse_expr()?; + self.expect_token(&Token::RParen)?; + Some(expr) + } else { + None + }; + + let with_check = if self.parse_keywords(&[Keyword::WITH, Keyword::CHECK]) { + self.expect_token(&Token::LParen)?; + let expr = self.parse_expr()?; + self.expect_token(&Token::RParen)?; + Some(expr) + } else { + None + }; + Ok(Statement::AlterPolicy { + name, + table_name, + operation: AlterPolicyOperation::Apply { + to, + using, + with_check, + }, + }) + } + } + fn parse_mssql_alter_role(&mut self) -> Result { let role_name = self.parse_identifier(false)?; diff --git a/src/parser/mod.rs b/src/parser/mod.rs index e5fd53554..a60dea3b8 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -7140,6 +7140,7 @@ impl<'a> Parser<'a> { Keyword::TABLE, Keyword::INDEX, Keyword::ROLE, + Keyword::POLICY, ])?; match object_type { Keyword::VIEW => self.parse_alter_view(), @@ -7191,6 +7192,7 @@ impl<'a> Parser<'a> { }) } Keyword::ROLE => self.parse_alter_role(), + Keyword::POLICY => self.parse_alter_policy(), // unreachable because expect_one_of_keywords used above _ => unreachable!(), } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index e066451df..29352981a 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11167,3 +11167,84 @@ fn test_drop_policy() { "sql parser error: Expected: end of statement, found: WRONG" ); } + +#[test] +fn test_alter_policy() { + match verified_stmt("ALTER POLICY old_policy ON my_table RENAME TO new_policy") { + Statement::AlterPolicy { + name, + table_name, + operation, + .. + } => { + assert_eq!(name.to_string(), "old_policy"); + assert_eq!(table_name.to_string(), "my_table"); + assert_eq!( + operation, + AlterPolicyOperation::Rename { + new_name: Ident::new("new_policy") + } + ); + } + _ => unreachable!(), + } + + match verified_stmt(concat!( + "ALTER POLICY my_policy ON my_table TO CURRENT_USER ", + "USING ((SELECT c0)) WITH CHECK (c0 > 0)" + )) { + Statement::AlterPolicy { + name, table_name, .. + } => { + assert_eq!(name.to_string(), "my_policy"); + assert_eq!(table_name.to_string(), "my_table"); + } + _ => unreachable!(), + } + + // omit TO / USING / WITH CHECK clauses is allowed + verified_stmt("ALTER POLICY my_policy ON my_table"); + + // mixing RENAME and APPLY expressions + assert_eq!( + parse_sql_statements("ALTER POLICY old_policy ON my_table TO public RENAME TO new_policy") + .unwrap_err() + .to_string(), + "sql parser error: Expected: end of statement, found: RENAME" + ); + assert_eq!( + parse_sql_statements("ALTER POLICY old_policy ON my_table RENAME TO new_policy TO public") + .unwrap_err() + .to_string(), + "sql parser error: Expected: end of statement, found: TO" + ); + // missing TO in RENAME TO + assert_eq!( + parse_sql_statements("ALTER POLICY old_policy ON my_table RENAME") + .unwrap_err() + .to_string(), + "sql parser error: Expected: TO, found: EOF" + ); + // missing new name in RENAME TO + assert_eq!( + parse_sql_statements("ALTER POLICY old_policy ON my_table RENAME TO") + .unwrap_err() + .to_string(), + "sql parser error: Expected: identifier, found: EOF" + ); + + // missing the expression in USING + assert_eq!( + parse_sql_statements("ALTER POLICY my_policy ON my_table USING") + .unwrap_err() + .to_string(), + "sql parser error: Expected: (, found: EOF" + ); + // missing the expression in WITH CHECK + assert_eq!( + parse_sql_statements("ALTER POLICY my_policy ON my_table WITH CHECK") + .unwrap_err() + .to_string(), + "sql parser error: Expected: (, found: EOF" + ); +} From ce2686a16997c17fd7729e378e21cc463d2e97d4 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 30 Sep 2024 07:08:13 -0400 Subject: [PATCH 14/80] Add a note discouraging new use of `dialect_of` macro (#1448) --- src/dialect/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 619b54713..90dffe82d 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -60,6 +60,13 @@ use alloc::boxed::Box; /// Convenience check if a [`Parser`] uses a certain dialect. /// +/// Note: when possible please the new style, adding a method to the [`Dialect`] +/// trait rather than using this macro. +/// +/// The benefits of adding a method on `Dialect` over this macro are: +/// 1. user defined [`Dialect`]s can customize the parsing behavior +/// 2. The differences between dialects can be clearly documented in the trait +/// /// `dialect_of!(parser Is SQLiteDialect | GenericDialect)` evaluates /// to `true` if `parser.dialect` is one of the [`Dialect`]s specified. macro_rules! dialect_of { From 1e0460a7dfad4a6767d718a9d849b23c8b61ee80 Mon Sep 17 00:00:00 2001 From: Joshua Warner Date: Mon, 30 Sep 2024 10:40:07 -0700 Subject: [PATCH 15/80] Expand handling of `LIMIT 1, 2` handling to include sqlite (#1447) --- src/dialect/clickhouse.rs | 4 ++++ src/dialect/generic.rs | 4 ++++ src/dialect/mod.rs | 7 ++++++- src/dialect/mysql.rs | 4 ++++ src/dialect/sqlite.rs | 4 ++++ src/parser/mod.rs | 2 +- src/test_utils.rs | 41 ++++++++++++++++++++++----------------- tests/sqlparser_common.rs | 19 ++++++++++++++---- 8 files changed, 61 insertions(+), 24 deletions(-) diff --git a/src/dialect/clickhouse.rs b/src/dialect/clickhouse.rs index ee8ee94d2..0c8f08040 100644 --- a/src/dialect/clickhouse.rs +++ b/src/dialect/clickhouse.rs @@ -46,4 +46,8 @@ impl Dialect for ClickHouseDialect { fn require_interval_qualifier(&self) -> bool { true } + + fn supports_limit_comma(&self) -> bool { + true + } } diff --git a/src/dialect/generic.rs b/src/dialect/generic.rs index bea56d0ef..8bec45b28 100644 --- a/src/dialect/generic.rs +++ b/src/dialect/generic.rs @@ -99,4 +99,8 @@ impl Dialect for GenericDialect { fn supports_explain_with_utility_options(&self) -> bool { true } + + fn supports_limit_comma(&self) -> bool { + true + } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 90dffe82d..29ad0b98a 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -67,7 +67,7 @@ use alloc::boxed::Box; /// 1. user defined [`Dialect`]s can customize the parsing behavior /// 2. The differences between dialects can be clearly documented in the trait /// -/// `dialect_of!(parser Is SQLiteDialect | GenericDialect)` evaluates +/// `dialect_of!(parser is SQLiteDialect | GenericDialect)` evaluates /// to `true` if `parser.dialect` is one of the [`Dialect`]s specified. macro_rules! dialect_of { ( $parsed_dialect: ident is $($dialect_type: ty)|+ ) => { @@ -323,6 +323,11 @@ pub trait Dialect: Debug + Any { false } + /// Does the dialect support parsing `LIMIT 1, 2` as `LIMIT 2 OFFSET 1`? + fn supports_limit_comma(&self) -> bool { + false + } + /// Does the dialect support trailing commas in the projection list? fn supports_projection_trailing_commas(&self) -> bool { self.supports_trailing_commas() diff --git a/src/dialect/mysql.rs b/src/dialect/mysql.rs index 2b5e42e59..d1bf33345 100644 --- a/src/dialect/mysql.rs +++ b/src/dialect/mysql.rs @@ -93,6 +93,10 @@ impl Dialect for MySqlDialect { fn require_interval_qualifier(&self) -> bool { true } + + fn supports_limit_comma(&self) -> bool { + true + } } /// `LOCK TABLES` diff --git a/src/dialect/sqlite.rs b/src/dialect/sqlite.rs index d3813d91c..5c563bf4a 100644 --- a/src/dialect/sqlite.rs +++ b/src/dialect/sqlite.rs @@ -73,4 +73,8 @@ impl Dialect for SQLiteDialect { fn supports_in_empty_list(&self) -> bool { true } + + fn supports_limit_comma(&self) -> bool { + true + } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a60dea3b8..a4445e04d 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8739,7 +8739,7 @@ impl<'a> Parser<'a> { offset = Some(self.parse_offset()?) } - if dialect_of!(self is GenericDialect | MySqlDialect | ClickHouseDialect) + if self.dialect.supports_limit_comma() && limit.is_some() && offset.is_none() && self.consume_token(&Token::Comma) diff --git a/src/test_utils.rs b/src/test_utils.rs index eb0352353..e588b3506 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -47,6 +47,14 @@ pub struct TestedDialects { } impl TestedDialects { + /// Create a TestedDialects with default options and the given dialects. + pub fn new(dialects: Vec>) -> Self { + Self { + dialects, + options: None, + } + } + fn new_parser<'a>(&self, dialect: &'a dyn Dialect) -> Parser<'a> { let parser = Parser::new(dialect); if let Some(options) = &self.options { @@ -211,24 +219,21 @@ impl TestedDialects { /// Returns all available dialects. pub fn all_dialects() -> TestedDialects { - let all_dialects = vec![ - Box::new(GenericDialect {}) as Box, - Box::new(PostgreSqlDialect {}) as Box, - Box::new(MsSqlDialect {}) as Box, - Box::new(AnsiDialect {}) as Box, - Box::new(SnowflakeDialect {}) as Box, - Box::new(HiveDialect {}) as Box, - Box::new(RedshiftSqlDialect {}) as Box, - Box::new(MySqlDialect {}) as Box, - Box::new(BigQueryDialect {}) as Box, - Box::new(SQLiteDialect {}) as Box, - Box::new(DuckDbDialect {}) as Box, - Box::new(DatabricksDialect {}) as Box, - ]; - TestedDialects { - dialects: all_dialects, - options: None, - } + TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(MsSqlDialect {}), + Box::new(AnsiDialect {}), + Box::new(SnowflakeDialect {}), + Box::new(HiveDialect {}), + Box::new(RedshiftSqlDialect {}), + Box::new(MySqlDialect {}), + Box::new(BigQueryDialect {}), + Box::new(SQLiteDialect {}), + Box::new(DuckDbDialect {}), + Box::new(DatabricksDialect {}), + Box::new(ClickHouseDialect {}), + ]) } /// Returns all dialects matching the given predicate. diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 29352981a..2677c193c 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -6832,7 +6832,9 @@ fn parse_create_view_with_options() { #[test] fn parse_create_view_with_columns() { let sql = "CREATE VIEW v (has, cols) AS SELECT 1, 2"; - match verified_stmt(sql) { + // TODO: why does this fail for ClickHouseDialect? (#1449) + // match all_dialects().verified_stmt(sql) { + match all_dialects_except(|d| d.is::()).verified_stmt(sql) { Statement::CreateView { name, columns, @@ -8624,17 +8626,26 @@ fn verified_expr(query: &str) -> Expr { #[test] fn parse_offset_and_limit() { - let sql = "SELECT foo FROM bar LIMIT 2 OFFSET 2"; + let sql = "SELECT foo FROM bar LIMIT 1 OFFSET 2"; let expect = Some(Offset { value: Expr::Value(number("2")), rows: OffsetRows::None, }); let ast = verified_query(sql); assert_eq!(ast.offset, expect); - assert_eq!(ast.limit, Some(Expr::Value(number("2")))); + assert_eq!(ast.limit, Some(Expr::Value(number("1")))); // different order is OK - one_statement_parses_to("SELECT foo FROM bar OFFSET 2 LIMIT 2", sql); + one_statement_parses_to("SELECT foo FROM bar OFFSET 2 LIMIT 1", sql); + + // mysql syntax is ok for some dialects + TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(MySqlDialect {}), + Box::new(SQLiteDialect {}), + Box::new(ClickHouseDialect {}), + ]) + .one_statement_parses_to("SELECT foo FROM bar LIMIT 2, 1", sql); // expressions are allowed let sql = "SELECT foo FROM bar LIMIT 1 + 2 OFFSET 3 * 4"; From 32a126b27c75a8248bde75401b8fbdc157e3dfc2 Mon Sep 17 00:00:00 2001 From: hulk Date: Sat, 5 Oct 2024 04:03:02 +0800 Subject: [PATCH 16/80] Fix always uses CommentDef::WithoutEq while parsing the inline comment (#1453) --- src/dialect/snowflake.rs | 14 ++++---------- src/parser/mod.rs | 27 +++++++++++++++++++++------ tests/sqlparser_common.rs | 32 ++++++++++++++++++++++++++++++++ tests/sqlparser_mysql.rs | 6 +++--- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index e14746d9a..f256c9b53 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -22,9 +22,7 @@ use crate::ast::helpers::stmt_data_loading::{ DataLoadingOption, DataLoadingOptionType, DataLoadingOptions, StageLoadSelectItem, StageParamsObject, }; -use crate::ast::{ - CommentDef, Ident, ObjectName, RowAccessPolicy, Statement, Tag, WrappedCollection, -}; +use crate::ast::{Ident, ObjectName, RowAccessPolicy, Statement, Tag, WrappedCollection}; use crate::dialect::{Dialect, Precedence}; use crate::keywords::Keyword; use crate::parser::{Parser, ParserError}; @@ -210,13 +208,9 @@ pub fn parse_create_table( builder = builder.copy_grants(true); } Keyword::COMMENT => { - parser.expect_token(&Token::Eq)?; - let next_token = parser.next_token(); - let comment = match next_token.token { - Token::SingleQuotedString(str) => Some(CommentDef::WithEq(str)), - _ => parser.expected("comment", next_token)?, - }; - builder = builder.comment(comment); + // Rewind the COMMENT keyword + parser.prev_token(); + builder = builder.comment(parser.parse_optional_inline_comment()?); } Keyword::AS => { let query = parser.parse_boxed_query()?; diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a4445e04d..028bae705 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -5871,12 +5871,9 @@ impl<'a> Parser<'a> { // Excludes Hive dialect here since it has been handled after table column definitions. if !dialect_of!(self is HiveDialect) && self.parse_keyword(Keyword::COMMENT) { - let _ = self.consume_token(&Token::Eq); - let next_token = self.next_token(); - comment = match next_token.token { - Token::SingleQuotedString(str) => Some(CommentDef::WithoutEq(str)), - _ => self.expected("comment", next_token)?, - } + // rewind the COMMENT keyword + self.prev_token(); + comment = self.parse_optional_inline_comment()? }; // Parse optional `AS ( query )` @@ -5957,6 +5954,24 @@ impl<'a> Parser<'a> { }) } + pub fn parse_optional_inline_comment(&mut self) -> Result, ParserError> { + let comment = if self.parse_keyword(Keyword::COMMENT) { + let has_eq = self.consume_token(&Token::Eq); + let next_token = self.next_token(); + match next_token.token { + Token::SingleQuotedString(str) => Some(if has_eq { + CommentDef::WithEq(str) + } else { + CommentDef::WithoutEq(str) + }), + _ => self.expected("comment", next_token)?, + } + } else { + None + }; + Ok(comment) + } + pub fn parse_optional_procedure_parameters( &mut self, ) -> Result>, ParserError> { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 2677c193c..a06b47a56 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -47,6 +47,7 @@ mod test_utils; #[cfg(test)] use pretty_assertions::assert_eq; +use sqlparser::ast::ColumnOption::Comment; use sqlparser::ast::Expr::{Identifier, UnaryOp}; use sqlparser::test_utils::all_dialects_except; @@ -9841,6 +9842,37 @@ fn test_comment_hash_syntax() { dialects.verified_only_select_with_canonical(sql, canonical); } +#[test] +fn test_parse_inline_comment() { + let sql = + "CREATE TABLE t0 (id INT COMMENT 'comment without equal') COMMENT = 'comment with equal'"; + // Hive dialect doesn't support `=` in table comment, please refer: + // [Hive](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-CreateTable) + match all_dialects_except(|d| d.is::()).verified_stmt(sql) { + Statement::CreateTable(CreateTable { + columns, comment, .. + }) => { + assert_eq!( + columns, + vec![ColumnDef { + name: Ident::new("id".to_string()), + data_type: DataType::Int(None), + collation: None, + options: vec![ColumnOptionDef { + name: None, + option: Comment("comment without equal".to_string()), + }] + }] + ); + assert_eq!( + comment.unwrap(), + CommentDef::WithEq("comment with equal".to_string()) + ); + } + _ => unreachable!(), + } +} + #[test] fn test_buffer_reuse() { let d = GenericDialect {}; diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 87468b933..19dbda21f 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -713,11 +713,11 @@ fn parse_create_table_primary_and_unique_key_characteristic_test() { #[test] fn parse_create_table_comment() { - let canonical = "CREATE TABLE foo (bar INT) COMMENT 'baz'"; + let without_equal = "CREATE TABLE foo (bar INT) COMMENT 'baz'"; let with_equal = "CREATE TABLE foo (bar INT) COMMENT = 'baz'"; - for sql in [canonical, with_equal] { - match mysql().one_statement_parses_to(sql, canonical) { + for sql in [without_equal, with_equal] { + match mysql().verified_stmt(sql) { Statement::CreateTable(CreateTable { name, comment, .. }) => { assert_eq!(name.to_string(), "foo"); assert_eq!(comment.expect("Should exist").to_string(), "baz"); From e849f7f14350b480fbbca6618da9ae7cea2df5a2 Mon Sep 17 00:00:00 2001 From: Yoav Cohen <59807311+yoavcloud@users.noreply.github.com> Date: Fri, 4 Oct 2024 22:03:38 +0200 Subject: [PATCH 17/80] Add support for the LIKE ANY and ILIKE ANY pattern-matching condition (#1456) --- src/ast/mod.rs | 20 ++++++++++++++++---- src/parser/mod.rs | 2 ++ tests/sqlparser_common.rs | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index a11fa63f4..57cd401d3 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -613,6 +613,9 @@ pub enum Expr { /// `[NOT] LIKE [ESCAPE ]` Like { negated: bool, + // Snowflake supports the ANY keyword to match against a list of patterns + // https://docs.snowflake.com/en/sql-reference/functions/like_any + any: bool, expr: Box, pattern: Box, escape_char: Option, @@ -620,6 +623,9 @@ pub enum Expr { /// `ILIKE` (case-insensitive `LIKE`) ILike { negated: bool, + // Snowflake supports the ANY keyword to match against a list of patterns + // https://docs.snowflake.com/en/sql-reference/functions/like_any + any: bool, expr: Box, pattern: Box, escape_char: Option, @@ -1242,20 +1248,23 @@ impl fmt::Display for Expr { expr, pattern, escape_char, + any, } => match escape_char { Some(ch) => write!( f, - "{} {}LIKE {} ESCAPE '{}'", + "{} {}LIKE {}{} ESCAPE '{}'", expr, if *negated { "NOT " } else { "" }, + if *any { "ANY " } else { "" }, pattern, ch ), _ => write!( f, - "{} {}LIKE {}", + "{} {}LIKE {}{}", expr, if *negated { "NOT " } else { "" }, + if *any { "ANY " } else { "" }, pattern ), }, @@ -1264,20 +1273,23 @@ impl fmt::Display for Expr { expr, pattern, escape_char, + any, } => match escape_char { Some(ch) => write!( f, - "{} {}ILIKE {} ESCAPE '{}'", + "{} {}ILIKE {}{} ESCAPE '{}'", expr, if *negated { "NOT " } else { "" }, + if *any { "ANY" } else { "" }, pattern, ch ), _ => write!( f, - "{} {}ILIKE {}", + "{} {}ILIKE {}{}", expr, if *negated { "NOT " } else { "" }, + if *any { "ANY " } else { "" }, pattern ), }, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 028bae705..88c3bd19c 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -2749,6 +2749,7 @@ impl<'a> Parser<'a> { } else if self.parse_keyword(Keyword::LIKE) { Ok(Expr::Like { negated, + any: self.parse_keyword(Keyword::ANY), expr: Box::new(expr), pattern: Box::new( self.parse_subexpr(self.dialect.prec_value(Precedence::Like))?, @@ -2758,6 +2759,7 @@ impl<'a> Parser<'a> { } else if self.parse_keyword(Keyword::ILIKE) { Ok(Expr::ILike { negated, + any: self.parse_keyword(Keyword::ANY), expr: Box::new(expr), pattern: Box::new( self.parse_subexpr(self.dialect.prec_value(Precedence::Like))?, diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index a06b47a56..5d5a17ca5 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -1550,6 +1550,7 @@ fn parse_not_precedence() { negated: true, pattern: Box::new(Expr::Value(Value::SingleQuotedString("b".into()))), escape_char: None, + any: false, }), }, ); @@ -1580,6 +1581,7 @@ fn parse_null_like() { SelectItem::ExprWithAlias { expr: Expr::Like { expr: Box::new(Expr::Identifier(Ident::new("column1"))), + any: false, negated: false, pattern: Box::new(Expr::Value(Value::Null)), escape_char: None, @@ -1595,6 +1597,7 @@ fn parse_null_like() { SelectItem::ExprWithAlias { expr: Expr::Like { expr: Box::new(Expr::Value(Value::Null)), + any: false, negated: false, pattern: Box::new(Expr::Identifier(Ident::new("column1"))), escape_char: None, @@ -1622,6 +1625,7 @@ fn parse_ilike() { negated, pattern: Box::new(Expr::Value(Value::SingleQuotedString("%a".to_string()))), escape_char: None, + any: false, }, select.selection.unwrap() ); @@ -1638,6 +1642,7 @@ fn parse_ilike() { negated, pattern: Box::new(Expr::Value(Value::SingleQuotedString("%a".to_string()))), escape_char: Some('^'.to_string()), + any: false, }, select.selection.unwrap() ); @@ -1655,6 +1660,7 @@ fn parse_ilike() { negated, pattern: Box::new(Expr::Value(Value::SingleQuotedString("%a".to_string()))), escape_char: None, + any: false, })), select.selection.unwrap() ); @@ -1677,6 +1683,7 @@ fn parse_like() { negated, pattern: Box::new(Expr::Value(Value::SingleQuotedString("%a".to_string()))), escape_char: None, + any: false, }, select.selection.unwrap() ); @@ -1693,6 +1700,7 @@ fn parse_like() { negated, pattern: Box::new(Expr::Value(Value::SingleQuotedString("%a".to_string()))), escape_char: Some('^'.to_string()), + any: false, }, select.selection.unwrap() ); @@ -1710,6 +1718,7 @@ fn parse_like() { negated, pattern: Box::new(Expr::Value(Value::SingleQuotedString("%a".to_string()))), escape_char: None, + any: false, })), select.selection.unwrap() ); @@ -10130,6 +10139,7 @@ fn test_selective_aggregation() { expr: Box::new(Expr::Identifier(Ident::new("name"))), pattern: Box::new(Expr::Value(Value::SingleQuotedString("a%".to_owned()))), escape_char: None, + any: false, })), null_treatment: None, over: None, @@ -11291,3 +11301,11 @@ fn test_alter_policy() { "sql parser error: Expected: (, found: EOF" ); } + +#[test] +fn test_select_where_with_like_or_ilike_any() { + verified_stmt(r#"SELECT * FROM x WHERE a ILIKE ANY '%abc%'"#); + verified_stmt(r#"SELECT * FROM x WHERE a LIKE ANY '%abc%'"#); + verified_stmt(r#"SELECT * FROM x WHERE a ILIKE ANY ('%Jo%oe%', 'T%e')"#); + verified_stmt(r#"SELECT * FROM x WHERE a LIKE ANY ('%Jo%oe%', 'T%e')"#); +} From 8ccb87a8355858ccb06ffc20484657cd7980d78c Mon Sep 17 00:00:00 2001 From: Maxwell Knight <107858496+MaxwellKnight@users.noreply.github.com> Date: Fri, 4 Oct 2024 23:07:26 +0300 Subject: [PATCH 18/80] added ability to parse extension to parse_comment inside postgres dialect (#1451) --- src/ast/mod.rs | 2 ++ src/dialect/postgresql.rs | 4 ++++ tests/sqlparser_postgres.rs | 15 +++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 57cd401d3..8c4bc25af 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -1855,6 +1855,7 @@ impl fmt::Display for ShowCreateObject { pub enum CommentObject { Column, Table, + Extension, } impl fmt::Display for CommentObject { @@ -1862,6 +1863,7 @@ impl fmt::Display for CommentObject { match self { CommentObject::Column => f.write_str("COLUMN"), CommentObject::Table => f.write_str("TABLE"), + CommentObject::Extension => f.write_str("EXTENSION"), } } } diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index 945c4fcc0..2a66705bb 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -205,6 +205,10 @@ pub fn parse_comment(parser: &mut Parser) -> Result { let object_name = parser.parse_object_name(false)?; (CommentObject::Table, object_name) } + Token::Word(w) if w.keyword == Keyword::EXTENSION => { + let object_name = parser.parse_object_name(false)?; + (CommentObject::Extension, object_name) + } _ => parser.expected("comment object_type", token)?, }; diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 9ba3e5dbc..735b87b5d 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -2902,6 +2902,21 @@ fn parse_comments() { _ => unreachable!(), } + match pg().verified_stmt("COMMENT ON EXTENSION plpgsql IS 'comment'") { + Statement::Comment { + object_type, + object_name, + comment: Some(comment), + if_exists, + } => { + assert_eq!("comment", comment); + assert_eq!("plpgsql", object_name.to_string()); + assert_eq!(CommentObject::Extension, object_type); + assert!(!if_exists); + } + _ => unreachable!(), + } + match pg().verified_stmt("COMMENT ON TABLE public.tab IS 'comment'") { Statement::Comment { object_type, From 84348d483eafaf4b608c9869ab03984852c7a4d2 Mon Sep 17 00:00:00 2001 From: Aleksei Piianin Date: Mon, 7 Oct 2024 22:20:23 +0200 Subject: [PATCH 19/80] Snowflake: support of views column comment (#1441) --- src/ast/ddl.rs | 9 +++------ src/ast/mod.rs | 12 +++++------ src/parser/mod.rs | 9 ++++++--- tests/sqlparser_bigquery.rs | 4 ++-- tests/sqlparser_snowflake.rs | 39 ++++++++++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 17 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 4578ae8f1..6eafc4da0 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -1040,6 +1040,7 @@ impl fmt::Display for ColumnDef { /// ```sql /// name /// age OPTIONS(description = "age column", tag = "prod") +/// amount COMMENT 'The total amount for the order line' /// created_at DateTime64 /// ``` #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -1048,7 +1049,7 @@ impl fmt::Display for ColumnDef { pub struct ViewColumnDef { pub name: Ident, pub data_type: Option, - pub options: Option>, + pub options: Option>, } impl fmt::Display for ViewColumnDef { @@ -1058,11 +1059,7 @@ impl fmt::Display for ViewColumnDef { write!(f, " {}", data_type)?; } if let Some(options) = self.options.as_ref() { - write!( - f, - " OPTIONS({})", - display_comma_separated(options.as_slice()) - )?; + write!(f, " {}", display_comma_separated(options.as_slice()))?; } Ok(()) } diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 8c4bc25af..995370f5b 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -3930,6 +3930,12 @@ impl fmt::Display for Statement { .map(|to| format!(" TO {to}")) .unwrap_or_default() )?; + if !columns.is_empty() { + write!(f, " ({})", display_comma_separated(columns))?; + } + if matches!(options, CreateTableOptions::With(_)) { + write!(f, " {options}")?; + } if let Some(comment) = comment { write!( f, @@ -3937,12 +3943,6 @@ impl fmt::Display for Statement { value::escape_single_quote_string(comment) )?; } - if matches!(options, CreateTableOptions::With(_)) { - write!(f, " {options}")?; - } - if !columns.is_empty() { - write!(f, " ({})", display_comma_separated(columns))?; - } if !cluster_by.is_empty() { write!(f, " CLUSTER BY ({})", display_comma_separated(cluster_by))?; } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 88c3bd19c..4e67524ab 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8361,11 +8361,14 @@ impl<'a> Parser<'a> { /// Parses a column definition within a view. fn parse_view_column(&mut self) -> Result { let name = self.parse_identifier(false)?; - let options = if dialect_of!(self is BigQueryDialect | GenericDialect) - && self.parse_keyword(Keyword::OPTIONS) + let options = if (dialect_of!(self is BigQueryDialect | GenericDialect) + && self.parse_keyword(Keyword::OPTIONS)) + || (dialect_of!(self is SnowflakeDialect | GenericDialect) + && self.parse_keyword(Keyword::COMMENT)) { self.prev_token(); - Some(self.parse_options(Keyword::OPTIONS)?) + self.parse_optional_column_option()? + .map(|option| vec![option]) } else { None }; diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index 55afe473b..63517fe57 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -272,10 +272,10 @@ fn parse_create_view_with_options() { ViewColumnDef { name: Ident::new("age"), data_type: None, - options: Some(vec![SqlOption::KeyValue { + options: Some(vec![ColumnOption::Options(vec![SqlOption::KeyValue { key: Ident::new("description"), value: Expr::Value(Value::DoubleQuotedString("field age".to_string())), - }]) + }])]), }, ], columns diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 50c5f740e..e73b6999a 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -2405,3 +2405,42 @@ fn parse_use() { ); } } + +#[test] +fn view_comment_option_should_be_after_column_list() { + for sql in [ + "CREATE OR REPLACE VIEW v (a) COMMENT = 'Comment' AS SELECT a FROM t", + "CREATE OR REPLACE VIEW v (a COMMENT 'a comment', b, c COMMENT 'c comment') COMMENT = 'Comment' AS SELECT a FROM t", + "CREATE OR REPLACE VIEW v (a COMMENT 'a comment', b, c COMMENT 'c comment') WITH (foo = bar) COMMENT = 'Comment' AS SELECT a FROM t", + ] { + snowflake_and_generic() + .verified_stmt(sql); + } +} + +#[test] +fn parse_view_column_descriptions() { + let sql = "CREATE OR REPLACE VIEW v (a COMMENT 'Comment', b) AS SELECT a, b FROM table1"; + + match snowflake_and_generic().verified_stmt(sql) { + Statement::CreateView { name, columns, .. } => { + assert_eq!(name.to_string(), "v"); + assert_eq!( + columns, + vec![ + ViewColumnDef { + name: Ident::new("a"), + data_type: None, + options: Some(vec![ColumnOption::Comment("Comment".to_string())]), + }, + ViewColumnDef { + name: Ident::new("b"), + data_type: None, + options: None, + } + ] + ); + } + _ => unreachable!(), + }; +} From 8badcdc2008bea4b5973c9dd12351a7cef29d8ab Mon Sep 17 00:00:00 2001 From: nucccc <44038808+nucccc@users.noreply.github.com> Date: Mon, 7 Oct 2024 22:20:55 +0200 Subject: [PATCH 20/80] Add SQLite "ON CONFLICT" column option in CREATE TABLE statements (#1442) Co-authored-by: hulk Co-authored-by: Ifeanyi Ubah --- src/ast/ddl.rs | 8 ++++++++ src/parser/mod.rs | 13 +++++++++++++ tests/sqlparser_sqlite.rs | 41 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 6eafc4da0..0677e63bf 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -33,6 +33,7 @@ use crate::ast::{ display_comma_separated, display_separated, DataType, Expr, Ident, MySQLColumnPosition, ObjectName, OrderByExpr, ProjectionSelect, SequenceOptions, SqlOption, Value, }; +use crate::keywords::Keyword; use crate::tokenizer::Token; /// An `ALTER TABLE` (`Statement::AlterTable`) operation @@ -1186,6 +1187,9 @@ pub enum ColumnOption { /// ``` /// [MS SQL Server]: https://learn.microsoft.com/en-us/sql/t-sql/statements/create-table-transact-sql-identity-property Identity(Option), + /// SQLite specific: ON CONFLICT option on column definition + /// + OnConflict(Keyword), } impl fmt::Display for ColumnOption { @@ -1294,6 +1298,10 @@ impl fmt::Display for ColumnOption { } Ok(()) } + OnConflict(keyword) => { + write!(f, "ON CONFLICT {:?}", keyword)?; + Ok(()) + } } } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 4e67524ab..39173b63c 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -6224,6 +6224,19 @@ impl<'a> Parser<'a> { None }; Ok(Some(ColumnOption::Identity(property))) + } else if dialect_of!(self is SQLiteDialect | GenericDialect) + && self.parse_keywords(&[Keyword::ON, Keyword::CONFLICT]) + { + // Support ON CONFLICT for SQLite + Ok(Some(ColumnOption::OnConflict( + self.expect_one_of_keywords(&[ + Keyword::ROLLBACK, + Keyword::ABORT, + Keyword::FAIL, + Keyword::IGNORE, + Keyword::REPLACE, + ])?, + ))) } else { Ok(None) } diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index d7fd3b896..b5427fb3c 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -22,6 +22,7 @@ #[macro_use] mod test_utils; +use sqlparser::keywords::Keyword; use test_utils::*; use sqlparser::ast::SelectItem::UnnamedExpr; @@ -281,6 +282,46 @@ fn parse_create_table_gencol() { sqlite_and_generic().verified_stmt("CREATE TABLE t1 (a INT, b INT AS (a * 2) STORED)"); } +#[test] +fn parse_create_table_on_conflict_col() { + for keyword in [ + Keyword::ROLLBACK, + Keyword::ABORT, + Keyword::FAIL, + Keyword::IGNORE, + Keyword::REPLACE, + ] { + let sql = format!("CREATE TABLE t1 (a INT, b INT ON CONFLICT {:?})", keyword); + match sqlite_and_generic().verified_stmt(&sql) { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!( + vec![ColumnOptionDef { + name: None, + option: ColumnOption::OnConflict(keyword), + }], + columns[1].options + ); + } + _ => unreachable!(), + } + } +} + +#[test] +fn test_parse_create_table_on_conflict_col_err() { + let sql_err = "CREATE TABLE t1 (a INT, b INT ON CONFLICT BOH)"; + let err = sqlite_and_generic() + .parse_sql_statements(sql_err) + .unwrap_err(); + assert_eq!( + err, + ParserError::ParserError( + "Expected: one of ROLLBACK or ABORT or FAIL or IGNORE or REPLACE, found: BOH" + .to_string() + ) + ); +} + #[test] fn parse_create_table_untyped() { sqlite().verified_stmt("CREATE TABLE t1 (a, b AS (a * 2), c NOT NULL)"); From ac956dc963b9a857202b62b50c99f35313efbce2 Mon Sep 17 00:00:00 2001 From: David Caldwell Date: Tue, 8 Oct 2024 09:26:32 -0700 Subject: [PATCH 21/80] Add support for ASC and DESC in CREATE TABLE column constraints for SQLite. (#1462) --- src/dialect/generic.rs | 4 ++++ src/dialect/mod.rs | 4 ++++ src/dialect/sqlite.rs | 4 ++++ src/parser/mod.rs | 14 ++++++++++++++ tests/sqlparser_sqlite.rs | 37 +++++++++++++++++++++++++++++++++++++ 5 files changed, 63 insertions(+) diff --git a/src/dialect/generic.rs b/src/dialect/generic.rs index 8bec45b28..92d720a0b 100644 --- a/src/dialect/generic.rs +++ b/src/dialect/generic.rs @@ -103,4 +103,8 @@ impl Dialect for GenericDialect { fn supports_limit_comma(&self) -> bool { true } + + fn supports_asc_desc_in_column_definition(&self) -> bool { + true + } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 29ad0b98a..744f5a8c8 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -557,6 +557,10 @@ pub trait Dialect: Debug + Any { fn supports_explain_with_utility_options(&self) -> bool { false } + + fn supports_asc_desc_in_column_definition(&self) -> bool { + false + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/sqlite.rs b/src/dialect/sqlite.rs index 5c563bf4a..95717f9fd 100644 --- a/src/dialect/sqlite.rs +++ b/src/dialect/sqlite.rs @@ -77,4 +77,8 @@ impl Dialect for SQLiteDialect { fn supports_limit_comma(&self) -> bool { true } + + fn supports_asc_desc_in_column_definition(&self) -> bool { + true + } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 39173b63c..34574a6b3 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -6192,6 +6192,20 @@ impl<'a> Parser<'a> { Ok(Some(ColumnOption::DialectSpecific(vec![ Token::make_keyword("AUTOINCREMENT"), ]))) + } else if self.parse_keyword(Keyword::ASC) + && self.dialect.supports_asc_desc_in_column_definition() + { + // Support ASC for SQLite + Ok(Some(ColumnOption::DialectSpecific(vec![ + Token::make_keyword("ASC"), + ]))) + } else if self.parse_keyword(Keyword::DESC) + && self.dialect.supports_asc_desc_in_column_definition() + { + // Support DESC for SQLite + Ok(Some(ColumnOption::DialectSpecific(vec![ + Token::make_keyword("DESC"), + ]))) } else if self.parse_keywords(&[Keyword::ON, Keyword::UPDATE]) && dialect_of!(self is MySqlDialect | GenericDialect) { diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index b5427fb3c..d3e670e32 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -238,6 +238,43 @@ fn parse_create_table_auto_increment() { } } +#[test] +fn parse_create_table_primary_key_asc_desc() { + let expected_column_def = |kind| ColumnDef { + name: "bar".into(), + data_type: DataType::Int(None), + collation: None, + options: vec![ + ColumnOptionDef { + name: None, + option: ColumnOption::Unique { + is_primary: true, + characteristics: None, + }, + }, + ColumnOptionDef { + name: None, + option: ColumnOption::DialectSpecific(vec![Token::make_keyword(kind)]), + }, + ], + }; + + let sql = "CREATE TABLE foo (bar INT PRIMARY KEY ASC)"; + match sqlite_and_generic().verified_stmt(sql) { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!(vec![expected_column_def("ASC")], columns); + } + _ => unreachable!(), + } + let sql = "CREATE TABLE foo (bar INT PRIMARY KEY DESC)"; + match sqlite_and_generic().verified_stmt(sql) { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!(vec![expected_column_def("DESC")], columns); + } + _ => unreachable!(), + } +} + #[test] fn parse_create_sqlite_quote() { let sql = "CREATE TABLE `PRIMARY` (\"KEY\" INT, [INDEX] INT)"; From 7905fb49054c28538a69dabd802c7bad6ef86fe8 Mon Sep 17 00:00:00 2001 From: hulk Date: Wed, 9 Oct 2024 00:27:07 +0800 Subject: [PATCH 22/80] Add support of `EXPLAIN QUERY PLAN` syntax for SQLite dialect (#1458) --- src/ast/mod.rs | 9 +++++++++ src/keywords.rs | 1 + src/parser/mod.rs | 4 ++++ tests/sqlparser_common.rs | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 995370f5b..bc559a660 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -3111,6 +3111,11 @@ pub enum Statement { analyze: bool, // Display additional information regarding the plan. verbose: bool, + /// `EXPLAIN QUERY PLAN` + /// Display the query plan without running the query. + /// + /// [SQLite](https://sqlite.org/lang_explain.html) + query_plan: bool, /// A SQL query that specifies what to explain statement: Box, /// Optional output format of explain @@ -3302,12 +3307,16 @@ impl fmt::Display for Statement { describe_alias, verbose, analyze, + query_plan, statement, format, options, } => { write!(f, "{describe_alias} ")?; + if *query_plan { + write!(f, "QUERY PLAN ")?; + } if *analyze { write!(f, "ANALYZE ")?; } diff --git a/src/keywords.rs b/src/keywords.rs index 49c6ce20f..ecf4bd474 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -572,6 +572,7 @@ define_keywords!( PERSISTENT, PIVOT, PLACING, + PLAN, PLANS, POLICY, PORTION, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 34574a6b3..cc60bbbd7 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8662,6 +8662,7 @@ impl<'a> Parser<'a> { ) -> Result { let mut analyze = false; let mut verbose = false; + let mut query_plan = false; let mut format = None; let mut options = None; @@ -8672,6 +8673,8 @@ impl<'a> Parser<'a> { && self.peek_token().token == Token::LParen { options = Some(self.parse_utility_options()?) + } else if self.parse_keywords(&[Keyword::QUERY, Keyword::PLAN]) { + query_plan = true; } else { analyze = self.parse_keyword(Keyword::ANALYZE); verbose = self.parse_keyword(Keyword::VERBOSE); @@ -8688,6 +8691,7 @@ impl<'a> Parser<'a> { describe_alias, analyze, verbose, + query_plan, statement: Box::new(statement), format, options, diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 5d5a17ca5..7068ffc30 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -4295,6 +4295,7 @@ fn run_explain_analyze( describe_alias: _, analyze, verbose, + query_plan, statement, format, options, @@ -4303,6 +4304,7 @@ fn run_explain_analyze( assert_eq!(analyze, expected_analyze); assert_eq!(format, expected_format); assert_eq!(options, exepcted_options); + assert!(!query_plan); assert_eq!("SELECT sqrt(id) FROM foo", statement.to_string()); } _ => panic!("Unexpected Statement, must be Explain"), @@ -4417,6 +4419,36 @@ fn parse_explain_analyze_with_simple_select() { ); } +#[test] +fn parse_explain_query_plan() { + match all_dialects().verified_stmt("EXPLAIN QUERY PLAN SELECT sqrt(id) FROM foo") { + Statement::Explain { + query_plan, + analyze, + verbose, + statement, + .. + } => { + assert!(query_plan); + assert!(!analyze); + assert!(!verbose); + assert_eq!("SELECT sqrt(id) FROM foo", statement.to_string()); + } + _ => unreachable!(), + } + + // omit QUERY PLAN should be good + all_dialects().verified_stmt("EXPLAIN SELECT sqrt(id) FROM foo"); + + // missing PLAN keyword should return error + assert_eq!( + ParserError::ParserError("Expected: end of statement, found: SELECT".to_string()), + all_dialects() + .parse_sql_statements("EXPLAIN QUERY SELECT sqrt(id) FROM foo") + .unwrap_err() + ); +} + #[test] fn parse_named_argument_function() { let sql = "SELECT FUN(a => '1', b => '2') FROM foo"; From c01e054fd85a4dcd16f27ca949e72d6a226d2f66 Mon Sep 17 00:00:00 2001 From: David Caldwell Date: Wed, 9 Oct 2024 06:59:00 -0700 Subject: [PATCH 23/80] Add "DROP TYPE" support. (#1461) --- src/ast/mod.rs | 2 ++ src/parser/mod.rs | 4 ++- tests/sqlparser_common.rs | 54 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index bc559a660..39f9def8e 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -5736,6 +5736,7 @@ pub enum ObjectType { Role, Sequence, Stage, + Type, } impl fmt::Display for ObjectType { @@ -5749,6 +5750,7 @@ impl fmt::Display for ObjectType { ObjectType::Role => "ROLE", ObjectType::Sequence => "SEQUENCE", ObjectType::Stage => "STAGE", + ObjectType::Type => "TYPE", }) } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index cc60bbbd7..c9e66e1f3 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -4889,6 +4889,8 @@ impl<'a> Parser<'a> { ObjectType::Sequence } else if self.parse_keyword(Keyword::STAGE) { ObjectType::Stage + } else if self.parse_keyword(Keyword::TYPE) { + ObjectType::Type } else if self.parse_keyword(Keyword::FUNCTION) { return self.parse_drop_function(); } else if self.parse_keyword(Keyword::POLICY) { @@ -4901,7 +4903,7 @@ impl<'a> Parser<'a> { return self.parse_drop_trigger(); } else { return self.expected( - "TABLE, VIEW, INDEX, ROLE, SCHEMA, DATABASE, FUNCTION, PROCEDURE, STAGE, TRIGGER, SECRET or SEQUENCE after DROP", + "TABLE, VIEW, INDEX, ROLE, SCHEMA, DATABASE, FUNCTION, PROCEDURE, STAGE, TRIGGER, SECRET, SEQUENCE, or TYPE after DROP", self.peek_token(), ); }; diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 7068ffc30..15132903c 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -9690,6 +9690,60 @@ fn parse_create_type() { ); } +#[test] +fn parse_drop_type() { + let sql = "DROP TYPE abc"; + match verified_stmt(sql) { + Statement::Drop { + names, + object_type, + if_exists, + cascade, + .. + } => { + assert_eq_vec(&["abc"], &names); + assert_eq!(ObjectType::Type, object_type); + assert!(!if_exists); + assert!(!cascade); + } + _ => unreachable!(), + }; + + let sql = "DROP TYPE IF EXISTS def, magician, quaternion"; + match verified_stmt(sql) { + Statement::Drop { + names, + object_type, + if_exists, + cascade, + .. + } => { + assert_eq_vec(&["def", "magician", "quaternion"], &names); + assert_eq!(ObjectType::Type, object_type); + assert!(if_exists); + assert!(!cascade); + } + _ => unreachable!(), + } + + let sql = "DROP TYPE IF EXISTS my_type CASCADE"; + match verified_stmt(sql) { + Statement::Drop { + names, + object_type, + if_exists, + cascade, + .. + } => { + assert_eq_vec(&["my_type"], &names); + assert_eq!(ObjectType::Type, object_type); + assert!(if_exists); + assert!(cascade); + } + _ => unreachable!(), + } +} + #[test] fn parse_call() { all_dialects().verified_stmt("CALL my_procedure()"); From 0a941b87dccde2faa9775366f39062202018eef8 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 10 Oct 2024 01:01:06 +0800 Subject: [PATCH 24/80] chore: Add asf.yaml (#1463) Co-authored-by: Andrew Lamb --- .asf.yaml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .asf.yaml diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 000000000..8ad476f12 --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This file controls the settings of this repository +# +# See more details at +# https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features + +notifications: + commits: commits@datafusion.apache.org + issues: github@datafusion.apache.org + pullrequests: github@datafusion.apache.org +github: + description: "Extensible SQL Lexer and Parser for Rust" + labels: + - big-data + - rust + - sql + enabled_merge_buttons: + squash: true + merge: false + rebase: false + features: + issues: true From a4fa9e08b7a73c2d6efc360f9d481352e3521b7f Mon Sep 17 00:00:00 2001 From: Yoav Cohen <59807311+yoavcloud@users.noreply.github.com> Date: Wed, 9 Oct 2024 23:47:14 +0200 Subject: [PATCH 25/80] Add support for quantified comparison predicates (ALL/ANY/SOME) (#1459) --- src/ast/mod.rs | 26 ++++++++++++++++++-- src/parser/mod.rs | 51 ++++++++++++++++++++++++--------------- tests/sqlparser_common.rs | 9 +++++++ 3 files changed, 64 insertions(+), 22 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 39f9def8e..2fbe91afc 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -646,12 +646,16 @@ pub enum Expr { regexp: bool, }, /// `ANY` operation e.g. `foo > ANY(bar)`, comparison operator is one of `[=, >, <, =>, =<, !=]` + /// AnyOp { left: Box, compare_op: BinaryOperator, right: Box, + // ANY and SOME are synonymous: https://docs.cloudera.com/cdw-runtime/cloud/using-hiveql/topics/hive_comparison_predicates.html + is_some: bool, }, /// `ALL` operation e.g. `foo > ALL(bar)`, comparison operator is one of `[=, >, <, =>, =<, !=]` + /// AllOp { left: Box, compare_op: BinaryOperator, @@ -1332,12 +1336,30 @@ impl fmt::Display for Expr { left, compare_op, right, - } => write!(f, "{left} {compare_op} ANY({right})"), + is_some, + } => { + let add_parens = !matches!(right.as_ref(), Expr::Subquery(_)); + write!( + f, + "{left} {compare_op} {}{}{right}{}", + if *is_some { "SOME" } else { "ANY" }, + if add_parens { "(" } else { "" }, + if add_parens { ")" } else { "" }, + ) + } Expr::AllOp { left, compare_op, right, - } => write!(f, "{left} {compare_op} ALL({right})"), + } => { + let add_parens = !matches!(right.as_ref(), Expr::Subquery(_)); + write!( + f, + "{left} {compare_op} ALL{}{right}{}", + if add_parens { "(" } else { "" }, + if add_parens { ")" } else { "" }, + ) + } Expr::UnaryOp { op, expr } => { if op == &UnaryOperator::PGPostfixFactorial { write!(f, "{expr}{op}") diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c9e66e1f3..cd9be1d8f 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1302,13 +1302,9 @@ impl<'a> Parser<'a> { } fn try_parse_expr_sub_query(&mut self) -> Result, ParserError> { - if self - .parse_one_of_keywords(&[Keyword::SELECT, Keyword::WITH]) - .is_none() - { + if !self.peek_sub_query() { return Ok(None); } - self.prev_token(); Ok(Some(Expr::Subquery(self.parse_boxed_query()?))) } @@ -1334,12 +1330,7 @@ impl<'a> Parser<'a> { // Snowflake permits a subquery to be passed as an argument without // an enclosing set of parens if it's the only argument. - if dialect_of!(self is SnowflakeDialect) - && self - .parse_one_of_keywords(&[Keyword::WITH, Keyword::SELECT]) - .is_some() - { - self.prev_token(); + if dialect_of!(self is SnowflakeDialect) && self.peek_sub_query() { let subquery = self.parse_boxed_query()?; self.expect_token(&Token::RParen)?; return Ok(Expr::Function(Function { @@ -2639,10 +2630,21 @@ impl<'a> Parser<'a> { }; if let Some(op) = regular_binary_operator { - if let Some(keyword) = self.parse_one_of_keywords(&[Keyword::ANY, Keyword::ALL]) { + if let Some(keyword) = + self.parse_one_of_keywords(&[Keyword::ANY, Keyword::ALL, Keyword::SOME]) + { self.expect_token(&Token::LParen)?; - let right = self.parse_subexpr(precedence)?; - self.expect_token(&Token::RParen)?; + let right = if self.peek_sub_query() { + // We have a subquery ahead (SELECT\WITH ...) need to rewind and + // use the parenthesis for parsing the subquery as an expression. + self.prev_token(); // LParen + self.parse_subexpr(precedence)? + } else { + // Non-subquery expression + let right = self.parse_subexpr(precedence)?; + self.expect_token(&Token::RParen)?; + right + }; if !matches!( op, @@ -2667,10 +2669,11 @@ impl<'a> Parser<'a> { compare_op: op, right: Box::new(right), }, - Keyword::ANY => Expr::AnyOp { + Keyword::ANY | Keyword::SOME => Expr::AnyOp { left: Box::new(expr), compare_op: op, right: Box::new(right), + is_some: keyword == Keyword::SOME, }, _ => unreachable!(), }) @@ -10507,11 +10510,7 @@ impl<'a> Parser<'a> { vec![] }; PivotValueSource::Any(order_by) - } else if self - .parse_one_of_keywords(&[Keyword::SELECT, Keyword::WITH]) - .is_some() - { - self.prev_token(); + } else if self.peek_sub_query() { PivotValueSource::Subquery(self.parse_query()?) } else { PivotValueSource::List(self.parse_comma_separated(Self::parse_expr_with_alias)?) @@ -12177,6 +12176,18 @@ impl<'a> Parser<'a> { pub fn into_tokens(self) -> Vec { self.tokens } + + /// Returns true if the next keyword indicates a sub query, i.e. SELECT or WITH + fn peek_sub_query(&mut self) -> bool { + if self + .parse_one_of_keywords(&[Keyword::SELECT, Keyword::WITH]) + .is_some() + { + self.prev_token(); + return true; + } + false + } } impl Word { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 15132903c..5327880a4 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -1907,6 +1907,7 @@ fn parse_binary_any() { left: Box::new(Expr::Identifier(Ident::new("a"))), compare_op: BinaryOperator::Eq, right: Box::new(Expr::Identifier(Ident::new("b"))), + is_some: false, }), select.projection[0] ); @@ -11395,3 +11396,11 @@ fn test_select_where_with_like_or_ilike_any() { verified_stmt(r#"SELECT * FROM x WHERE a ILIKE ANY ('%Jo%oe%', 'T%e')"#); verified_stmt(r#"SELECT * FROM x WHERE a LIKE ANY ('%Jo%oe%', 'T%e')"#); } + +#[test] +fn test_any_some_all_comparison() { + verified_stmt("SELECT c1 FROM tbl WHERE c1 = ANY(SELECT c2 FROM tbl)"); + verified_stmt("SELECT c1 FROM tbl WHERE c1 >= ALL(SELECT c2 FROM tbl)"); + verified_stmt("SELECT c1 FROM tbl WHERE c1 <> SOME(SELECT c2 FROM tbl)"); + verified_stmt("SELECT 1 = ANY(WITH x AS (SELECT 1) SELECT * FROM x)"); +} From 749b061fbfcdd997f0097152ec4bf9b7376b8c4e Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Fri, 11 Oct 2024 17:15:18 +0200 Subject: [PATCH 26/80] MySQL dialect: Add support for hash comments (#1466) --- src/tokenizer.rs | 5 +++-- tests/sqlparser_common.rs | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 6d0c86ff2..4186ec824 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -43,7 +43,8 @@ use sqlparser_derive::{Visit, VisitMut}; use crate::ast::DollarQuotedString; use crate::dialect::Dialect; use crate::dialect::{ - BigQueryDialect, DuckDbDialect, GenericDialect, PostgreSqlDialect, SnowflakeDialect, + BigQueryDialect, DuckDbDialect, GenericDialect, MySqlDialect, PostgreSqlDialect, + SnowflakeDialect, }; use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX}; @@ -1140,7 +1141,7 @@ impl<'a> Tokenizer<'a> { } '{' => self.consume_and_return(chars, Token::LBrace), '}' => self.consume_and_return(chars, Token::RBrace), - '#' if dialect_of!(self is SnowflakeDialect | BigQueryDialect) => { + '#' if dialect_of!(self is SnowflakeDialect | BigQueryDialect | MySqlDialect) => { chars.next(); // consume the '#', starting a snowflake single-line comment let comment = self.tokenize_single_line_comment(chars); Ok(Some(Token::Whitespace(Whitespace::SingleLineComment { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 5327880a4..7140109b2 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -9923,7 +9923,11 @@ fn test_release_savepoint() { #[test] fn test_comment_hash_syntax() { let dialects = TestedDialects { - dialects: vec![Box::new(BigQueryDialect {}), Box::new(SnowflakeDialect {})], + dialects: vec![ + Box::new(BigQueryDialect {}), + Box::new(SnowflakeDialect {}), + Box::new(MySqlDialect {}), + ], options: None, }; let sql = r#" From 7c20d4ae1f8a07c39a8e0e45c0cfe4134138953f Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Mon, 14 Oct 2024 19:14:40 +0200 Subject: [PATCH 27/80] Fix #1469 (SET ROLE regression) (#1474) --- src/parser/mod.rs | 40 +++++++++++++++++++++++---------------- tests/sqlparser_common.rs | 24 +++++++++++++++++++++++ 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index cd9be1d8f..b4c0487b4 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -9416,27 +9416,35 @@ impl<'a> Parser<'a> { } } + /// Parse a `SET ROLE` statement. Expects SET to be consumed already. + fn parse_set_role(&mut self, modifier: Option) -> Result { + self.expect_keyword(Keyword::ROLE)?; + let context_modifier = match modifier { + Some(Keyword::LOCAL) => ContextModifier::Local, + Some(Keyword::SESSION) => ContextModifier::Session, + _ => ContextModifier::None, + }; + + let role_name = if self.parse_keyword(Keyword::NONE) { + None + } else { + Some(self.parse_identifier(false)?) + }; + Ok(Statement::SetRole { + context_modifier, + role_name, + }) + } + pub fn parse_set(&mut self) -> Result { let modifier = self.parse_one_of_keywords(&[Keyword::SESSION, Keyword::LOCAL, Keyword::HIVEVAR]); if let Some(Keyword::HIVEVAR) = modifier { self.expect_token(&Token::Colon)?; - } else if self.parse_keyword(Keyword::ROLE) { - let context_modifier = match modifier { - Some(Keyword::LOCAL) => ContextModifier::Local, - Some(Keyword::SESSION) => ContextModifier::Session, - _ => ContextModifier::None, - }; - - let role_name = if self.parse_keyword(Keyword::NONE) { - None - } else { - Some(self.parse_identifier(false)?) - }; - return Ok(Statement::SetRole { - context_modifier, - role_name, - }); + } else if let Some(set_role_stmt) = + self.maybe_parse(|parser| parser.parse_set_role(modifier)) + { + return Ok(set_role_stmt); } let variables = if self.parse_keywords(&[Keyword::TIME, Keyword::ZONE]) { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 7140109b2..55ab3ddc3 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -7665,6 +7665,30 @@ fn parse_set_variable() { one_statement_parses_to("SET SOMETHING TO '1'", "SET SOMETHING = '1'"); } +#[test] +fn parse_set_role_as_variable() { + match verified_stmt("SET role = 'foobar'") { + Statement::SetVariable { + local, + hivevar, + variables, + value, + } => { + assert!(!local); + assert!(!hivevar); + assert_eq!( + variables, + OneOrManyWithParens::One(ObjectName(vec!["role".into()])) + ); + assert_eq!( + value, + vec![Expr::Value(Value::SingleQuotedString("foobar".into()))] + ); + } + _ => unreachable!(), + } +} + #[test] fn parse_double_colon_cast_at_timezone() { let sql = "SELECT '2001-01-01T00:00:00.000Z'::TIMESTAMP AT TIME ZONE 'Europe/Brussels' FROM t"; From 1dd7d26fbbc53ccbc1a77bbe4d713d44f814b6eb Mon Sep 17 00:00:00 2001 From: Yoav Cohen <59807311+yoavcloud@users.noreply.github.com> Date: Sun, 20 Oct 2024 20:12:39 +0200 Subject: [PATCH 28/80] Add support for parsing MsSql alias with equals (#1467) --- src/dialect/mod.rs | 11 +++++++++++ src/dialect/mssql.rs | 4 ++++ src/parser/mod.rs | 18 ++++++++++++++++++ tests/sqlparser_common.rs | 17 +++++++++++++++++ 4 files changed, 50 insertions(+) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 744f5a8c8..871055685 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -561,6 +561,17 @@ pub trait Dialect: Debug + Any { fn supports_asc_desc_in_column_definition(&self) -> bool { false } + + /// Returns true if this dialect supports treating the equals operator `=` within a `SelectItem` + /// as an alias assignment operator, rather than a boolean expression. + /// For example: the following statements are equivalent for such a dialect: + /// ```sql + /// SELECT col_alias = col FROM tbl; + /// SELECT col_alias AS col FROM tbl; + /// ``` + fn supports_eq_alias_assigment(&self) -> bool { + false + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/mssql.rs b/src/dialect/mssql.rs index a9d296be3..cace372c0 100644 --- a/src/dialect/mssql.rs +++ b/src/dialect/mssql.rs @@ -49,4 +49,8 @@ impl Dialect for MsSqlDialect { fn supports_connect_by(&self) -> bool { true } + + fn supports_eq_alias_assigment(&self) -> bool { + true + } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b4c0487b4..842e85c12 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -11181,6 +11181,24 @@ impl<'a> Parser<'a> { self.peek_token().location ) } + Expr::BinaryOp { + left, + op: BinaryOperator::Eq, + right, + } if self.dialect.supports_eq_alias_assigment() + && matches!(left.as_ref(), Expr::Identifier(_)) => + { + let Expr::Identifier(alias) = *left else { + return parser_err!( + "BUG: expected identifier expression as alias", + self.peek_token().location + ); + }; + Ok(SelectItem::ExprWithAlias { + expr: *right, + alias, + }) + } expr => self .parse_optional_alias(keywords::RESERVED_FOR_COLUMN_ALIAS) .map(|alias| match alias { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 55ab3ddc3..55725eeca 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11432,3 +11432,20 @@ fn test_any_some_all_comparison() { verified_stmt("SELECT c1 FROM tbl WHERE c1 <> SOME(SELECT c2 FROM tbl)"); verified_stmt("SELECT 1 = ANY(WITH x AS (SELECT 1) SELECT * FROM x)"); } + +#[test] +fn test_alias_equal_expr() { + let dialects = all_dialects_where(|d| d.supports_eq_alias_assigment()); + let sql = r#"SELECT some_alias = some_column FROM some_table"#; + let expected = r#"SELECT some_column AS some_alias FROM some_table"#; + let _ = dialects.one_statement_parses_to(sql, expected); + + let sql = r#"SELECT some_alias = (a*b) FROM some_table"#; + let expected = r#"SELECT (a * b) AS some_alias FROM some_table"#; + let _ = dialects.one_statement_parses_to(sql, expected); + + let dialects = all_dialects_where(|d| !d.supports_eq_alias_assigment()); + let sql = r#"SELECT x = (a * b) FROM some_table"#; + let expected = r#"SELECT x = (a * b) FROM some_table"#; + let _ = dialects.one_statement_parses_to(sql, expected); +} From 3421e1e4d432990889f4f9ece418420dbe9dc2d5 Mon Sep 17 00:00:00 2001 From: Aleksei Piianin Date: Sun, 20 Oct 2024 20:13:25 +0200 Subject: [PATCH 29/80] Snowflake: support for extended column options in `CREATE TABLE` (#1454) --- src/ast/ddl.rs | 244 +++++++++++++++++++++++++-- src/ast/mod.rs | 11 +- src/dialect/mod.rs | 15 +- src/dialect/snowflake.rs | 122 ++++++++++++-- src/keywords.rs | 2 + src/parser/mod.rs | 28 +++- tests/sqlparser_mssql.rs | 31 ++-- tests/sqlparser_snowflake.rs | 315 +++++++++++++++++++++++++++++++++++ 8 files changed, 723 insertions(+), 45 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 0677e63bf..49f6ae4bd 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -31,7 +31,7 @@ use sqlparser_derive::{Visit, VisitMut}; use crate::ast::value::escape_single_quote_string; use crate::ast::{ display_comma_separated, display_separated, DataType, Expr, Ident, MySQLColumnPosition, - ObjectName, OrderByExpr, ProjectionSelect, SequenceOptions, SqlOption, Value, + ObjectName, OrderByExpr, ProjectionSelect, SequenceOptions, SqlOption, Tag, Value, }; use crate::keywords::Keyword; use crate::tokenizer::Token; @@ -1096,17 +1096,221 @@ impl fmt::Display for ColumnOptionDef { } } +/// Identity is a column option for defining an identity or autoincrement column in a `CREATE TABLE` statement. +/// Syntax +/// ```sql +/// { IDENTITY | AUTOINCREMENT } [ (seed , increment) | START num INCREMENT num ] [ ORDER | NOORDER ] +/// ``` +/// [MS SQL Server]: https://learn.microsoft.com/en-us/sql/t-sql/statements/create-table-transact-sql-identity-property +/// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum IdentityPropertyKind { + /// An identity property declared via the `AUTOINCREMENT` key word + /// Example: + /// ```sql + /// AUTOINCREMENT(100, 1) NOORDER + /// AUTOINCREMENT START 100 INCREMENT 1 ORDER + /// ``` + /// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table + Autoincrement(IdentityProperty), + /// An identity property declared via the `IDENTITY` key word + /// Example, [MS SQL Server] or [Snowflake]: + /// ```sql + /// IDENTITY(100, 1) + /// ``` + /// [Snowflake] + /// ```sql + /// IDENTITY(100, 1) ORDER + /// IDENTITY START 100 INCREMENT 1 NOORDER + /// ``` + /// [MS SQL Server]: https://learn.microsoft.com/en-us/sql/t-sql/statements/create-table-transact-sql-identity-property + /// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table + Identity(IdentityProperty), +} + +impl fmt::Display for IdentityPropertyKind { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let (command, property) = match self { + IdentityPropertyKind::Identity(property) => ("IDENTITY", property), + IdentityPropertyKind::Autoincrement(property) => ("AUTOINCREMENT", property), + }; + write!(f, "{command}")?; + if let Some(parameters) = &property.parameters { + write!(f, "{parameters}")?; + } + if let Some(order) = &property.order { + write!(f, "{order}")?; + } + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct IdentityProperty { + pub parameters: Option, + pub order: Option, +} + +/// A format of parameters of identity column. +/// +/// It is [Snowflake] specific. +/// Syntax +/// ```sql +/// (seed , increment) | START num INCREMENT num +/// ``` +/// [MS SQL Server] uses one way of representing these parameters. +/// Syntax +/// ```sql +/// (seed , increment) +/// ``` +/// [MS SQL Server]: https://learn.microsoft.com/en-us/sql/t-sql/statements/create-table-transact-sql-identity-property +/// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum IdentityPropertyFormatKind { + /// A parameters of identity column declared like parameters of function call + /// Example: + /// ```sql + /// (100, 1) + /// ``` + /// [MS SQL Server]: https://learn.microsoft.com/en-us/sql/t-sql/statements/create-table-transact-sql-identity-property + /// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table + FunctionCall(IdentityParameters), + /// A parameters of identity column declared with keywords `START` and `INCREMENT` + /// Example: + /// ```sql + /// START 100 INCREMENT 1 + /// ``` + /// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table + StartAndIncrement(IdentityParameters), +} + +impl fmt::Display for IdentityPropertyFormatKind { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + IdentityPropertyFormatKind::FunctionCall(parameters) => { + write!(f, "({}, {})", parameters.seed, parameters.increment) + } + IdentityPropertyFormatKind::StartAndIncrement(parameters) => { + write!( + f, + " START {} INCREMENT {}", + parameters.seed, parameters.increment + ) + } + } + } +} +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct IdentityParameters { pub seed: Expr, pub increment: Expr, } -impl fmt::Display for IdentityProperty { +/// The identity column option specifies how values are generated for the auto-incremented column, either in increasing or decreasing order. +/// Syntax +/// ```sql +/// ORDER | NOORDER +/// ``` +/// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum IdentityPropertyOrder { + Order, + NoOrder, +} + +impl fmt::Display for IdentityPropertyOrder { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + IdentityPropertyOrder::Order => write!(f, " ORDER"), + IdentityPropertyOrder::NoOrder => write!(f, " NOORDER"), + } + } +} + +/// Column policy that identify a security policy of access to a column. +/// Syntax +/// ```sql +/// [ WITH ] MASKING POLICY [ USING ( , , ... ) ] +/// [ WITH ] PROJECTION POLICY +/// ``` +/// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum ColumnPolicy { + MaskingPolicy(ColumnPolicyProperty), + ProjectionPolicy(ColumnPolicyProperty), +} + +impl fmt::Display for ColumnPolicy { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}, {}", self.seed, self.increment) + let (command, property) = match self { + ColumnPolicy::MaskingPolicy(property) => ("MASKING POLICY", property), + ColumnPolicy::ProjectionPolicy(property) => ("PROJECTION POLICY", property), + }; + if property.with { + write!(f, "WITH ")?; + } + write!(f, "{command} {}", property.policy_name)?; + if let Some(using_columns) = &property.using_columns { + write!(f, " USING ({})", display_comma_separated(using_columns))?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct ColumnPolicyProperty { + /// This flag indicates that the column policy option is declared using the `WITH` prefix. + /// Example + /// ```sql + /// WITH PROJECTION POLICY sample_policy + /// ``` + /// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table + pub with: bool, + pub policy_name: Ident, + pub using_columns: Option>, +} + +/// Tags option of column +/// Syntax +/// ```sql +/// [ WITH ] TAG ( = '' [ , = '' , ... ] ) +/// ``` +/// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct TagsColumnOption { + /// This flag indicates that the tags option is declared using the `WITH` prefix. + /// Example: + /// ```sql + /// WITH TAG (A = 'Tag A') + /// ``` + /// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table + pub with: bool, + pub tags: Vec, +} + +impl fmt::Display for TagsColumnOption { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.with { + write!(f, "WITH ")?; + } + write!(f, "TAG ({})", display_comma_separated(&self.tags))?; + Ok(()) } } @@ -1180,16 +1384,32 @@ pub enum ColumnOption { /// [1]: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#view_column_option_list /// [2]: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#column_option_list Options(Vec), - /// MS SQL Server specific: Creates an identity column in a table. + /// Creates an identity or an autoincrement column in a table. /// Syntax /// ```sql - /// IDENTITY [ (seed , increment) ] + /// { IDENTITY | AUTOINCREMENT } [ (seed , increment) | START num INCREMENT num ] [ ORDER | NOORDER ] /// ``` /// [MS SQL Server]: https://learn.microsoft.com/en-us/sql/t-sql/statements/create-table-transact-sql-identity-property - Identity(Option), + /// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table + Identity(IdentityPropertyKind), /// SQLite specific: ON CONFLICT option on column definition /// OnConflict(Keyword), + /// Snowflake specific: an option of specifying security masking or projection policy to set on a column. + /// Syntax: + /// ```sql + /// [ WITH ] MASKING POLICY [ USING ( , , ... ) ] + /// [ WITH ] PROJECTION POLICY + /// ``` + /// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table + Policy(ColumnPolicy), + /// Snowflake specific: Specifies the tag name and the tag string value. + /// Syntax: + /// ```sql + /// [ WITH ] TAG ( = '' [ , = '' , ... ] ) + /// ``` + /// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table + Tags(TagsColumnOption), } impl fmt::Display for ColumnOption { @@ -1292,16 +1512,18 @@ impl fmt::Display for ColumnOption { write!(f, "OPTIONS({})", display_comma_separated(options)) } Identity(parameters) => { - write!(f, "IDENTITY")?; - if let Some(parameters) = parameters { - write!(f, "({parameters})")?; - } - Ok(()) + write!(f, "{parameters}") } OnConflict(keyword) => { write!(f, "ON CONFLICT {:?}", keyword)?; Ok(()) } + Policy(parameters) => { + write!(f, "{parameters}") + } + Tags(tags) => { + write!(f, "{tags}") + } } } } diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 2fbe91afc..2e46722fa 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -40,11 +40,12 @@ pub use self::data_type::{ pub use self::dcl::{AlterRoleOperation, ResetConfig, RoleOption, SetConfigValue, Use}; pub use self::ddl::{ AlterColumnOperation, AlterIndexOperation, AlterPolicyOperation, AlterTableOperation, - ClusteredBy, ColumnDef, ColumnOption, ColumnOptionDef, ConstraintCharacteristics, Deduplicate, - DeferrableInitial, GeneratedAs, GeneratedExpressionMode, IdentityProperty, IndexOption, - IndexType, KeyOrIndexDisplay, Owner, Partition, ProcedureParam, ReferentialAction, - TableConstraint, UserDefinedTypeCompositeAttributeDef, UserDefinedTypeRepresentation, - ViewColumnDef, + ClusteredBy, ColumnDef, ColumnOption, ColumnOptionDef, ColumnPolicy, ColumnPolicyProperty, + ConstraintCharacteristics, Deduplicate, DeferrableInitial, GeneratedAs, + GeneratedExpressionMode, IdentityParameters, IdentityProperty, IdentityPropertyFormatKind, + IdentityPropertyKind, IdentityPropertyOrder, IndexOption, IndexType, KeyOrIndexDisplay, Owner, + Partition, ProcedureParam, ReferentialAction, TableConstraint, TagsColumnOption, + UserDefinedTypeCompositeAttributeDef, UserDefinedTypeRepresentation, ViewColumnDef, }; pub use self::dml::{CreateIndex, CreateTable, Delete, Insert}; pub use self::operator::{BinaryOperator, UnaryOperator}; diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 871055685..b34e22081 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -49,7 +49,7 @@ pub use self::postgresql::PostgreSqlDialect; pub use self::redshift::RedshiftSqlDialect; pub use self::snowflake::SnowflakeDialect; pub use self::sqlite::SQLiteDialect; -use crate::ast::{Expr, Statement}; +use crate::ast::{ColumnOption, Expr, Statement}; pub use crate::keywords; use crate::keywords::Keyword; use crate::parser::{Parser, ParserError}; @@ -478,6 +478,19 @@ pub trait Dialect: Debug + Any { None } + /// Dialect-specific column option parser override + /// + /// This method is called to parse the next column option. + /// + /// If `None` is returned, falls back to the default behavior. + fn parse_column_option( + &self, + _parser: &mut Parser, + ) -> Option, ParserError>> { + // return None to fall back to the default behavior + None + } + /// Decide the lexical Precedence of operators. /// /// Uses (APPROXIMATELY) as a reference diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index f256c9b53..7c80f0461 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -22,7 +22,11 @@ use crate::ast::helpers::stmt_data_loading::{ DataLoadingOption, DataLoadingOptionType, DataLoadingOptions, StageLoadSelectItem, StageParamsObject, }; -use crate::ast::{Ident, ObjectName, RowAccessPolicy, Statement, Tag, WrappedCollection}; +use crate::ast::{ + ColumnOption, ColumnPolicy, ColumnPolicyProperty, Ident, IdentityParameters, IdentityProperty, + IdentityPropertyFormatKind, IdentityPropertyKind, IdentityPropertyOrder, ObjectName, + RowAccessPolicy, Statement, TagsColumnOption, WrappedCollection, +}; use crate::dialect::{Dialect, Precedence}; use crate::keywords::Keyword; use crate::parser::{Parser, ParserError}; @@ -149,6 +153,36 @@ impl Dialect for SnowflakeDialect { None } + fn parse_column_option( + &self, + parser: &mut Parser, + ) -> Option, ParserError>> { + parser.maybe_parse(|parser| { + let with = parser.parse_keyword(Keyword::WITH); + + if parser.parse_keyword(Keyword::IDENTITY) { + Ok(parse_identity_property(parser) + .map(|p| Some(ColumnOption::Identity(IdentityPropertyKind::Identity(p))))) + } else if parser.parse_keyword(Keyword::AUTOINCREMENT) { + Ok(parse_identity_property(parser).map(|p| { + Some(ColumnOption::Identity(IdentityPropertyKind::Autoincrement( + p, + ))) + })) + } else if parser.parse_keywords(&[Keyword::MASKING, Keyword::POLICY]) { + Ok(parse_column_policy_property(parser, with) + .map(|p| Some(ColumnOption::Policy(ColumnPolicy::MaskingPolicy(p))))) + } else if parser.parse_keywords(&[Keyword::PROJECTION, Keyword::POLICY]) { + Ok(parse_column_policy_property(parser, with) + .map(|p| Some(ColumnOption::Policy(ColumnPolicy::ProjectionPolicy(p))))) + } else if parser.parse_keywords(&[Keyword::TAG]) { + Ok(parse_column_tags(parser, with).map(|p| Some(ColumnOption::Tags(p)))) + } else { + Err(ParserError::ParserError("not found match".to_string())) + } + }) + } + fn get_next_precedence(&self, parser: &Parser) -> Option> { let token = parser.peek_token(); // Snowflake supports the `:` cast operator unlike other dialects @@ -307,16 +341,8 @@ pub fn parse_create_table( builder.with_row_access_policy(Some(RowAccessPolicy::new(policy, columns))) } Keyword::TAG => { - fn parse_tag(parser: &mut Parser) -> Result { - let name = parser.parse_identifier(false)?; - parser.expect_token(&Token::Eq)?; - let value = parser.parse_literal_string()?; - - Ok(Tag::new(name, value)) - } - parser.expect_token(&Token::LParen)?; - let tags = parser.parse_comma_separated(parse_tag)?; + let tags = parser.parse_comma_separated(Parser::parse_tag)?; parser.expect_token(&Token::RParen)?; builder = builder.with_tags(Some(tags)); } @@ -776,3 +802,79 @@ fn parse_parentheses_options(parser: &mut Parser) -> Result Result { + let parameters = if parser.consume_token(&Token::LParen) { + let seed = parser.parse_number()?; + parser.expect_token(&Token::Comma)?; + let increment = parser.parse_number()?; + parser.expect_token(&Token::RParen)?; + + Some(IdentityPropertyFormatKind::FunctionCall( + IdentityParameters { seed, increment }, + )) + } else if parser.parse_keyword(Keyword::START) { + let seed = parser.parse_number()?; + parser.expect_keyword(Keyword::INCREMENT)?; + let increment = parser.parse_number()?; + + Some(IdentityPropertyFormatKind::StartAndIncrement( + IdentityParameters { seed, increment }, + )) + } else { + None + }; + let order = match parser.parse_one_of_keywords(&[Keyword::ORDER, Keyword::NOORDER]) { + Some(Keyword::ORDER) => Some(IdentityPropertyOrder::Order), + Some(Keyword::NOORDER) => Some(IdentityPropertyOrder::NoOrder), + _ => None, + }; + Ok(IdentityProperty { parameters, order }) +} + +/// Parsing a policy property of column option +/// Syntax: +/// ```sql +/// [ USING ( , , ... ) +/// ``` +/// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table +fn parse_column_policy_property( + parser: &mut Parser, + with: bool, +) -> Result { + let policy_name = parser.parse_identifier(false)?; + let using_columns = if parser.parse_keyword(Keyword::USING) { + parser.expect_token(&Token::LParen)?; + let columns = parser.parse_comma_separated(|p| p.parse_identifier(false))?; + parser.expect_token(&Token::RParen)?; + Some(columns) + } else { + None + }; + + Ok(ColumnPolicyProperty { + with, + policy_name, + using_columns, + }) +} + +/// Parsing tags list of column +/// Syntax: +/// ```sql +/// ( = '' [ , = '' , ... ] ) +/// ``` +/// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table +fn parse_column_tags(parser: &mut Parser, with: bool) -> Result { + parser.expect_token(&Token::LParen)?; + let tags = parser.parse_comma_separated(Parser::parse_tag)?; + parser.expect_token(&Token::RParen)?; + + Ok(TagsColumnOption { with, tags }) +} diff --git a/src/keywords.rs b/src/keywords.rs index ecf4bd474..001de7484 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -453,6 +453,7 @@ define_keywords!( MACRO, MANAGEDLOCATION, MAP, + MASKING, MATCH, MATCHED, MATCHES, @@ -504,6 +505,7 @@ define_keywords!( NOINHERIT, NOLOGIN, NONE, + NOORDER, NOREPLICATION, NORMALIZE, NOSCAN, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 842e85c12..5bd64392a 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -6065,7 +6065,7 @@ impl<'a> Parser<'a> { } } else if let Some(option) = self.parse_optional_column_option()? { options.push(ColumnOptionDef { name: None, option }); - } else if dialect_of!(self is MySqlDialect | GenericDialect) + } else if dialect_of!(self is MySqlDialect | SnowflakeDialect | GenericDialect) && self.parse_keyword(Keyword::COLLATE) { collation = Some(self.parse_object_name(false)?); @@ -6105,6 +6105,10 @@ impl<'a> Parser<'a> { } pub fn parse_optional_column_option(&mut self) -> Result, ParserError> { + if let Some(option) = self.dialect.parse_column_option(self) { + return option; + } + if self.parse_keywords(&[Keyword::CHARACTER, Keyword::SET]) { Ok(Some(ColumnOption::CharacterSet( self.parse_object_name(false)?, @@ -6232,17 +6236,24 @@ impl<'a> Parser<'a> { } else if self.parse_keyword(Keyword::IDENTITY) && dialect_of!(self is MsSqlDialect | GenericDialect) { - let property = if self.consume_token(&Token::LParen) { + let parameters = if self.consume_token(&Token::LParen) { let seed = self.parse_number()?; self.expect_token(&Token::Comma)?; let increment = self.parse_number()?; self.expect_token(&Token::RParen)?; - Some(IdentityProperty { seed, increment }) + Some(IdentityPropertyFormatKind::FunctionCall( + IdentityParameters { seed, increment }, + )) } else { None }; - Ok(Some(ColumnOption::Identity(property))) + Ok(Some(ColumnOption::Identity( + IdentityPropertyKind::Identity(IdentityProperty { + parameters, + order: None, + }), + ))) } else if dialect_of!(self is SQLiteDialect | GenericDialect) && self.parse_keywords(&[Keyword::ON, Keyword::CONFLICT]) { @@ -6260,6 +6271,15 @@ impl<'a> Parser<'a> { Ok(None) } } + + pub(crate) fn parse_tag(&mut self) -> Result { + let name = self.parse_identifier(false)?; + self.expect_token(&Token::Eq)?; + let value = self.parse_literal_string()?; + + Ok(Tag::new(name, value)) + } + fn parse_optional_column_option_generated( &mut self, ) -> Result, ParserError> { diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 5a2ef9e87..ef89a4768 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -921,7 +921,12 @@ fn parse_create_table_with_identity_column() { vec![ ColumnOptionDef { name: None, - option: ColumnOption::Identity(None), + option: ColumnOption::Identity(IdentityPropertyKind::Identity( + IdentityProperty { + parameters: None, + order: None, + }, + )), }, ColumnOptionDef { name: None, @@ -934,19 +939,17 @@ fn parse_create_table_with_identity_column() { vec![ ColumnOptionDef { name: None, - #[cfg(not(feature = "bigdecimal"))] - option: ColumnOption::Identity(Some(IdentityProperty { - seed: Expr::Value(Value::Number("1".to_string(), false)), - increment: Expr::Value(Value::Number("1".to_string(), false)), - })), - #[cfg(feature = "bigdecimal")] - option: ColumnOption::Identity(Some(IdentityProperty { - seed: Expr::Value(Value::Number(bigdecimal::BigDecimal::from(1), false)), - increment: Expr::Value(Value::Number( - bigdecimal::BigDecimal::from(1), - false, - )), - })), + option: ColumnOption::Identity(IdentityPropertyKind::Identity( + IdentityProperty { + parameters: Some(IdentityPropertyFormatKind::FunctionCall( + IdentityParameters { + seed: Expr::Value(number("1")), + increment: Expr::Value(number("1")), + }, + )), + order: None, + }, + )), }, ColumnOptionDef { name: None, diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index e73b6999a..d7e967ffe 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -525,6 +525,321 @@ fn test_snowflake_single_line_tokenize() { assert_eq!(expected, tokens); } +#[test] +fn test_snowflake_create_table_with_autoincrement_columns() { + let sql = concat!( + "CREATE TABLE my_table (", + "a INT AUTOINCREMENT ORDER, ", + "b INT AUTOINCREMENT(100, 1) NOORDER, ", + "c INT IDENTITY, ", + "d INT IDENTITY START 100 INCREMENT 1 ORDER", + ")" + ); + // it is a snowflake specific options (AUTOINCREMENT/IDENTITY) + match snowflake().verified_stmt(sql) { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!( + columns, + vec![ + ColumnDef { + name: "a".into(), + data_type: DataType::Int(None), + collation: None, + options: vec![ColumnOptionDef { + name: None, + option: ColumnOption::Identity(IdentityPropertyKind::Autoincrement( + IdentityProperty { + parameters: None, + order: Some(IdentityPropertyOrder::Order), + } + )) + }] + }, + ColumnDef { + name: "b".into(), + data_type: DataType::Int(None), + collation: None, + options: vec![ColumnOptionDef { + name: None, + option: ColumnOption::Identity(IdentityPropertyKind::Autoincrement( + IdentityProperty { + parameters: Some(IdentityPropertyFormatKind::FunctionCall( + IdentityParameters { + seed: Expr::Value(number("100")), + increment: Expr::Value(number("1")), + } + )), + order: Some(IdentityPropertyOrder::NoOrder), + } + )) + }] + }, + ColumnDef { + name: "c".into(), + data_type: DataType::Int(None), + collation: None, + options: vec![ColumnOptionDef { + name: None, + option: ColumnOption::Identity(IdentityPropertyKind::Identity( + IdentityProperty { + parameters: None, + order: None, + } + )) + }] + }, + ColumnDef { + name: "d".into(), + data_type: DataType::Int(None), + collation: None, + options: vec![ColumnOptionDef { + name: None, + option: ColumnOption::Identity(IdentityPropertyKind::Identity( + IdentityProperty { + parameters: Some( + IdentityPropertyFormatKind::StartAndIncrement( + IdentityParameters { + seed: Expr::Value(number("100")), + increment: Expr::Value(number("1")), + } + ) + ), + order: Some(IdentityPropertyOrder::Order), + } + )) + }] + }, + ] + ); + } + _ => unreachable!(), + } +} + +#[test] +fn test_snowflake_create_table_with_collated_column() { + match snowflake_and_generic().verified_stmt("CREATE TABLE my_table (a TEXT COLLATE 'de_DE')") { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!( + columns, + vec![ColumnDef { + name: "a".into(), + data_type: DataType::Text, + collation: Some(ObjectName(vec![Ident::with_quote('\'', "de_DE")])), + options: vec![] + },] + ); + } + _ => unreachable!(), + } +} + +#[test] +fn test_snowflake_create_table_with_columns_masking_policy() { + for (sql, with, using_columns) in [ + ( + "CREATE TABLE my_table (a INT WITH MASKING POLICY p)", + true, + None, + ), + ( + "CREATE TABLE my_table (a INT MASKING POLICY p)", + false, + None, + ), + ( + "CREATE TABLE my_table (a INT WITH MASKING POLICY p USING (a, b))", + true, + Some(vec!["a".into(), "b".into()]), + ), + ( + "CREATE TABLE my_table (a INT MASKING POLICY p USING (a, b))", + false, + Some(vec!["a".into(), "b".into()]), + ), + ] { + match snowflake().verified_stmt(sql) { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!( + columns, + vec![ColumnDef { + name: "a".into(), + data_type: DataType::Int(None), + collation: None, + options: vec![ColumnOptionDef { + name: None, + option: ColumnOption::Policy(ColumnPolicy::MaskingPolicy( + ColumnPolicyProperty { + with, + policy_name: "p".into(), + using_columns, + } + )) + }], + },] + ); + } + _ => unreachable!(), + } + } +} + +#[test] +fn test_snowflake_create_table_with_columns_projection_policy() { + for (sql, with) in [ + ( + "CREATE TABLE my_table (a INT WITH PROJECTION POLICY p)", + true, + ), + ("CREATE TABLE my_table (a INT PROJECTION POLICY p)", false), + ] { + match snowflake().verified_stmt(sql) { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!( + columns, + vec![ColumnDef { + name: "a".into(), + data_type: DataType::Int(None), + collation: None, + options: vec![ColumnOptionDef { + name: None, + option: ColumnOption::Policy(ColumnPolicy::ProjectionPolicy( + ColumnPolicyProperty { + with, + policy_name: "p".into(), + using_columns: None, + } + )) + }], + },] + ); + } + _ => unreachable!(), + } + } +} + +#[test] +fn test_snowflake_create_table_with_columns_tags() { + for (sql, with) in [ + ( + "CREATE TABLE my_table (a INT WITH TAG (A='TAG A', B='TAG B'))", + true, + ), + ( + "CREATE TABLE my_table (a INT TAG (A='TAG A', B='TAG B'))", + false, + ), + ] { + match snowflake().verified_stmt(sql) { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!( + columns, + vec![ColumnDef { + name: "a".into(), + data_type: DataType::Int(None), + collation: None, + options: vec![ColumnOptionDef { + name: None, + option: ColumnOption::Tags(TagsColumnOption { + with, + tags: vec![ + Tag::new("A".into(), "TAG A".into()), + Tag::new("B".into(), "TAG B".into()), + ] + }), + }], + },] + ); + } + _ => unreachable!(), + } + } +} + +#[test] +fn test_snowflake_create_table_with_several_column_options() { + let sql = concat!( + "CREATE TABLE my_table (", + "a INT IDENTITY WITH MASKING POLICY p1 USING (a, b) WITH TAG (A='TAG A', B='TAG B'), ", + "b TEXT COLLATE 'de_DE' PROJECTION POLICY p2 TAG (C='TAG C', D='TAG D')", + ")" + ); + match snowflake().verified_stmt(sql) { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!( + columns, + vec![ + ColumnDef { + name: "a".into(), + data_type: DataType::Int(None), + collation: None, + options: vec![ + ColumnOptionDef { + name: None, + option: ColumnOption::Identity(IdentityPropertyKind::Identity( + IdentityProperty { + parameters: None, + order: None + } + )), + }, + ColumnOptionDef { + name: None, + option: ColumnOption::Policy(ColumnPolicy::MaskingPolicy( + ColumnPolicyProperty { + with: true, + policy_name: "p1".into(), + using_columns: Some(vec!["a".into(), "b".into()]), + } + )), + }, + ColumnOptionDef { + name: None, + option: ColumnOption::Tags(TagsColumnOption { + with: true, + tags: vec![ + Tag::new("A".into(), "TAG A".into()), + Tag::new("B".into(), "TAG B".into()), + ] + }), + } + ], + }, + ColumnDef { + name: "b".into(), + data_type: DataType::Text, + collation: Some(ObjectName(vec![Ident::with_quote('\'', "de_DE")])), + options: vec![ + ColumnOptionDef { + name: None, + option: ColumnOption::Policy(ColumnPolicy::ProjectionPolicy( + ColumnPolicyProperty { + with: false, + policy_name: "p2".into(), + using_columns: None, + } + )), + }, + ColumnOptionDef { + name: None, + option: ColumnOption::Tags(TagsColumnOption { + with: false, + tags: vec![ + Tag::new("C".into(), "TAG C".into()), + Tag::new("D".into(), "TAG D".into()), + ] + }), + } + ], + }, + ] + ); + } + _ => unreachable!(), + } +} + #[test] fn parse_sf_create_or_replace_view_with_comment_missing_equal() { assert!(snowflake_and_generic() From 45c5d69b2298bf54db29810149587a28691063ed Mon Sep 17 00:00:00 2001 From: Yoav Cohen <59807311+yoavcloud@users.noreply.github.com> Date: Sun, 20 Oct 2024 22:29:55 +0200 Subject: [PATCH 30/80] MsSQL TRY_CONVERT (#1477) --- src/ast/mod.rs | 6 +++++- src/dialect/generic.rs | 4 ++++ src/dialect/mod.rs | 5 +++++ src/dialect/mssql.rs | 4 ++++ src/keywords.rs | 1 + src/parser/mod.rs | 12 ++++++++---- tests/sqlparser_common.rs | 11 +++++++++++ tests/sqlparser_mssql.rs | 2 ++ 8 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 2e46722fa..4900307e5 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -669,6 +669,9 @@ pub enum Expr { }, /// CONVERT a value to a different data type or character encoding. e.g. `CONVERT(foo USING utf8mb4)` Convert { + /// CONVERT (false) or TRY_CONVERT (true) + /// + is_try: bool, /// The expression to convert expr: Box, /// The target data type @@ -1371,13 +1374,14 @@ impl fmt::Display for Expr { } } Expr::Convert { + is_try, expr, target_before_value, data_type, charset, styles, } => { - write!(f, "CONVERT(")?; + write!(f, "{}CONVERT(", if *is_try { "TRY_" } else { "" })?; if let Some(data_type) = data_type { if let Some(charset) = charset { write!(f, "{expr}, {data_type} CHARACTER SET {charset}") diff --git a/src/dialect/generic.rs b/src/dialect/generic.rs index 92d720a0b..0a5464c9c 100644 --- a/src/dialect/generic.rs +++ b/src/dialect/generic.rs @@ -107,4 +107,8 @@ impl Dialect for GenericDialect { fn supports_asc_desc_in_column_definition(&self) -> bool { true } + + fn supports_try_convert(&self) -> bool { + true + } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index b34e22081..be97f929b 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -585,6 +585,11 @@ pub trait Dialect: Debug + Any { fn supports_eq_alias_assigment(&self) -> bool { false } + + /// Returns true if this dialect supports the `TRY_CONVERT` function + fn supports_try_convert(&self) -> bool { + false + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/mssql.rs b/src/dialect/mssql.rs index cace372c0..78ec621ed 100644 --- a/src/dialect/mssql.rs +++ b/src/dialect/mssql.rs @@ -53,4 +53,8 @@ impl Dialect for MsSqlDialect { fn supports_eq_alias_assigment(&self) -> bool { true } + + fn supports_try_convert(&self) -> bool { + true + } } diff --git a/src/keywords.rs b/src/keywords.rs index 001de7484..6182ae176 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -771,6 +771,7 @@ define_keywords!( TRUE, TRUNCATE, TRY_CAST, + TRY_CONVERT, TUPLE, TYPE, UESCAPE, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 5bd64392a..a1079f6f7 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1023,7 +1023,8 @@ impl<'a> Parser<'a> { self.parse_time_functions(ObjectName(vec![w.to_ident()])) } Keyword::CASE => self.parse_case_expr(), - Keyword::CONVERT => self.parse_convert_expr(), + Keyword::CONVERT => self.parse_convert_expr(false), + Keyword::TRY_CONVERT if self.dialect.supports_try_convert() => self.parse_convert_expr(true), Keyword::CAST => self.parse_cast_expr(CastKind::Cast), Keyword::TRY_CAST => self.parse_cast_expr(CastKind::TryCast), Keyword::SAFE_CAST => self.parse_cast_expr(CastKind::SafeCast), @@ -1614,7 +1615,7 @@ impl<'a> Parser<'a> { } /// mssql-like convert function - fn parse_mssql_convert(&mut self) -> Result { + fn parse_mssql_convert(&mut self, is_try: bool) -> Result { self.expect_token(&Token::LParen)?; let data_type = self.parse_data_type()?; self.expect_token(&Token::Comma)?; @@ -1626,6 +1627,7 @@ impl<'a> Parser<'a> { }; self.expect_token(&Token::RParen)?; Ok(Expr::Convert { + is_try, expr: Box::new(expr), data_type: Some(data_type), charset: None, @@ -1638,9 +1640,9 @@ impl<'a> Parser<'a> { /// - `CONVERT('héhé' USING utf8mb4)` (MySQL) /// - `CONVERT('héhé', CHAR CHARACTER SET utf8mb4)` (MySQL) /// - `CONVERT(DECIMAL(10, 5), 42)` (MSSQL) - the type comes first - pub fn parse_convert_expr(&mut self) -> Result { + pub fn parse_convert_expr(&mut self, is_try: bool) -> Result { if self.dialect.convert_type_before_value() { - return self.parse_mssql_convert(); + return self.parse_mssql_convert(is_try); } self.expect_token(&Token::LParen)?; let expr = self.parse_expr()?; @@ -1648,6 +1650,7 @@ impl<'a> Parser<'a> { let charset = self.parse_object_name(false)?; self.expect_token(&Token::RParen)?; return Ok(Expr::Convert { + is_try, expr: Box::new(expr), data_type: None, charset: Some(charset), @@ -1664,6 +1667,7 @@ impl<'a> Parser<'a> { }; self.expect_token(&Token::RParen)?; Ok(Expr::Convert { + is_try, expr: Box::new(expr), data_type: Some(data_type), charset, diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 55725eeca..5683bcf91 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11449,3 +11449,14 @@ fn test_alias_equal_expr() { let expected = r#"SELECT x = (a * b) FROM some_table"#; let _ = dialects.one_statement_parses_to(sql, expected); } + +#[test] +fn test_try_convert() { + let dialects = + all_dialects_where(|d| d.supports_try_convert() && d.convert_type_before_value()); + dialects.verified_expr("TRY_CONVERT(VARCHAR(MAX), 'foo')"); + + let dialects = + all_dialects_where(|d| d.supports_try_convert() && !d.convert_type_before_value()); + dialects.verified_expr("TRY_CONVERT('foo', VARCHAR(MAX))"); +} diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index ef89a4768..58765f6c0 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -464,6 +464,7 @@ fn parse_cast_varchar_max() { fn parse_convert() { let sql = "CONVERT(INT, 1, 2, 3, NULL)"; let Expr::Convert { + is_try, expr, data_type, charset, @@ -473,6 +474,7 @@ fn parse_convert() { else { unreachable!() }; + assert!(!is_try); assert_eq!(Expr::Value(number("1")), *expr); assert_eq!(Some(DataType::Int(None)), data_type); assert!(charset.is_none()); From a8432b57db6b8efc635e4903d712e76f7adb6e6d Mon Sep 17 00:00:00 2001 From: David Caldwell Date: Mon, 21 Oct 2024 11:44:38 -0700 Subject: [PATCH 31/80] Add PostgreSQL specfic "CREATE TYPE t AS ENUM (...)" support. (#1460) --- src/ast/ddl.rs | 5 +++++ src/dialect/postgresql.rs | 35 ++++++++++++++++++++++++++++++++++- tests/sqlparser_postgres.rs | 29 +++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 49f6ae4bd..21a716d25 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -1706,6 +1706,8 @@ pub enum UserDefinedTypeRepresentation { Composite { attributes: Vec, }, + /// Note: this is PostgreSQL-specific. See + Enum { labels: Vec }, } impl fmt::Display for UserDefinedTypeRepresentation { @@ -1714,6 +1716,9 @@ impl fmt::Display for UserDefinedTypeRepresentation { UserDefinedTypeRepresentation::Composite { attributes } => { write!(f, "({})", display_comma_separated(attributes)) } + UserDefinedTypeRepresentation::Enum { labels } => { + write!(f, "ENUM ({})", display_comma_separated(labels)) + } } } } diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index 2a66705bb..51dc49849 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -28,7 +28,7 @@ // limitations under the License. use log::debug; -use crate::ast::{CommentObject, Statement}; +use crate::ast::{CommentObject, ObjectName, Statement, UserDefinedTypeRepresentation}; use crate::dialect::{Dialect, Precedence}; use crate::keywords::Keyword; use crate::parser::{Parser, ParserError}; @@ -138,6 +138,9 @@ impl Dialect for PostgreSqlDialect { fn parse_statement(&self, parser: &mut Parser) -> Option> { if parser.parse_keyword(Keyword::COMMENT) { Some(parse_comment(parser)) + } else if parser.parse_keyword(Keyword::CREATE) { + parser.prev_token(); // unconsume the CREATE in case we don't end up parsing anything + parse_create(parser) } else { None } @@ -225,3 +228,33 @@ pub fn parse_comment(parser: &mut Parser) -> Result { if_exists, }) } + +pub fn parse_create(parser: &mut Parser) -> Option> { + let name = parser.maybe_parse(|parser| -> Result { + parser.expect_keyword(Keyword::CREATE)?; + parser.expect_keyword(Keyword::TYPE)?; + let name = parser.parse_object_name(false)?; + parser.expect_keyword(Keyword::AS)?; + parser.expect_keyword(Keyword::ENUM)?; + Ok(name) + }); + name.map(|name| parse_create_type_as_enum(parser, name)) +} + +// https://www.postgresql.org/docs/current/sql-createtype.html +pub fn parse_create_type_as_enum( + parser: &mut Parser, + name: ObjectName, +) -> Result { + if !parser.consume_token(&Token::LParen) { + return parser.expected("'(' after CREATE TYPE AS ENUM", parser.peek_token()); + } + + let labels = parser.parse_comma_separated0(|p| p.parse_identifier(false), Token::RParen)?; + parser.expect_token(&Token::RParen)?; + + Ok(Statement::CreateType { + name, + representation: UserDefinedTypeRepresentation::Enum { labels }, + }) +} diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 735b87b5d..bd37214ce 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -5128,3 +5128,32 @@ fn arrow_cast_precedence() { } ) } + +#[test] +fn parse_create_type_as_enum() { + let statement = pg().one_statement_parses_to( + r#"CREATE TYPE public.my_type AS ENUM ( + 'label1', + 'label2', + 'label3', + 'label4' + );"#, + "CREATE TYPE public.my_type AS ENUM ('label1', 'label2', 'label3', 'label4')", + ); + match statement { + Statement::CreateType { + name, + representation: UserDefinedTypeRepresentation::Enum { labels }, + } => { + assert_eq!("public.my_type", name.to_string()); + assert_eq!( + vec!["label1", "label2", "label3", "label4"] + .into_iter() + .map(|l| Ident::with_quote('\'', l)) + .collect::>(), + labels + ); + } + _ => unreachable!(), + } +} From 38f1e573fe9e176582f0b03867439548e5fd9922 Mon Sep 17 00:00:00 2001 From: Seve Martinez <20816697+seve-martinez@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:52:17 -0700 Subject: [PATCH 32/80] feat: adding Display implementation to DELETE and INSERT (#1427) Co-authored-by: Andrew Lamb --- src/ast/dml.rs | 115 +++++++++++++++++++++++++++++++++++++++- src/ast/mod.rs | 140 +++++-------------------------------------------- 2 files changed, 128 insertions(+), 127 deletions(-) diff --git a/src/ast/dml.rs b/src/ast/dml.rs index 8121f2c5b..2932fafb5 100644 --- a/src/ast/dml.rs +++ b/src/ast/dml.rs @@ -16,7 +16,12 @@ // under the License. #[cfg(not(feature = "std"))] -use alloc::{boxed::Box, string::String, vec::Vec}; +use alloc::{ + boxed::Box, + format, + string::{String, ToString}, + vec::Vec, +}; use core::fmt::{self, Display}; #[cfg(feature = "serde")] @@ -492,6 +497,81 @@ pub struct Insert { pub insert_alias: Option, } +impl Display for Insert { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let table_name = if let Some(alias) = &self.table_alias { + format!("{0} AS {alias}", self.table_name) + } else { + self.table_name.to_string() + }; + + if let Some(action) = self.or { + write!(f, "INSERT OR {action} INTO {table_name} ")?; + } else { + write!( + f, + "{start}", + start = if self.replace_into { + "REPLACE" + } else { + "INSERT" + }, + )?; + if let Some(priority) = self.priority { + write!(f, " {priority}",)?; + } + + write!( + f, + "{ignore}{over}{int}{tbl} {table_name} ", + table_name = table_name, + ignore = if self.ignore { " IGNORE" } else { "" }, + over = if self.overwrite { " OVERWRITE" } else { "" }, + int = if self.into { " INTO" } else { "" }, + tbl = if self.table { " TABLE" } else { "" }, + )?; + } + if !self.columns.is_empty() { + write!(f, "({}) ", display_comma_separated(&self.columns))?; + } + if let Some(ref parts) = self.partitioned { + if !parts.is_empty() { + write!(f, "PARTITION ({}) ", display_comma_separated(parts))?; + } + } + if !self.after_columns.is_empty() { + write!(f, "({}) ", display_comma_separated(&self.after_columns))?; + } + + if let Some(source) = &self.source { + write!(f, "{source}")?; + } + + if self.source.is_none() && self.columns.is_empty() { + write!(f, "DEFAULT VALUES")?; + } + + if let Some(insert_alias) = &self.insert_alias { + write!(f, " AS {0}", insert_alias.row_alias)?; + + if let Some(col_aliases) = &insert_alias.col_aliases { + if !col_aliases.is_empty() { + write!(f, " ({})", display_comma_separated(col_aliases))?; + } + } + } + + if let Some(on) = &self.on { + write!(f, "{on}")?; + } + + if let Some(returning) = &self.returning { + write!(f, " RETURNING {}", display_comma_separated(returning))?; + } + Ok(()) + } +} + /// DELETE statement. #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] @@ -512,3 +592,36 @@ pub struct Delete { /// LIMIT (MySQL) pub limit: Option, } + +impl Display for Delete { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "DELETE ")?; + if !self.tables.is_empty() { + write!(f, "{} ", display_comma_separated(&self.tables))?; + } + match &self.from { + FromTable::WithFromKeyword(from) => { + write!(f, "FROM {}", display_comma_separated(from))?; + } + FromTable::WithoutKeyword(from) => { + write!(f, "{}", display_comma_separated(from))?; + } + } + if let Some(using) = &self.using { + write!(f, " USING {}", display_comma_separated(using))?; + } + if let Some(selection) = &self.selection { + write!(f, " WHERE {selection}")?; + } + if let Some(returning) = &self.returning { + write!(f, " RETURNING {}", display_comma_separated(returning))?; + } + if !self.order_by.is_empty() { + write!(f, " ORDER BY {}", display_comma_separated(&self.order_by))?; + } + if let Some(limit) = &self.limit { + write!(f, " LIMIT {limit}")?; + } + Ok(()) + } +} diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 4900307e5..790a39bdb 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2176,6 +2176,18 @@ pub enum FromTable { /// WithoutKeyword(Vec), } +impl Display for FromTable { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FromTable::WithFromKeyword(tables) => { + write!(f, "FROM {}", display_comma_separated(tables)) + } + FromTable::WithoutKeyword(tables) => { + write!(f, "{}", display_comma_separated(tables)) + } + } + } +} /// Policy type for a `CREATE POLICY` statement. /// ```sql @@ -3533,93 +3545,7 @@ impl fmt::Display for Statement { } Ok(()) } - Statement::Insert(insert) => { - let Insert { - or, - ignore, - into, - table_name, - table_alias, - overwrite, - partitioned, - columns, - after_columns, - source, - table, - on, - returning, - replace_into, - priority, - insert_alias, - } = insert; - let table_name = if let Some(alias) = table_alias { - format!("{table_name} AS {alias}") - } else { - table_name.to_string() - }; - - if let Some(action) = or { - write!(f, "INSERT OR {action} INTO {table_name} ")?; - } else { - write!( - f, - "{start}", - start = if *replace_into { "REPLACE" } else { "INSERT" }, - )?; - if let Some(priority) = priority { - write!(f, " {priority}",)?; - } - - write!( - f, - "{ignore}{over}{int}{tbl} {table_name} ", - table_name = table_name, - ignore = if *ignore { " IGNORE" } else { "" }, - over = if *overwrite { " OVERWRITE" } else { "" }, - int = if *into { " INTO" } else { "" }, - tbl = if *table { " TABLE" } else { "" }, - )?; - } - if !columns.is_empty() { - write!(f, "({}) ", display_comma_separated(columns))?; - } - if let Some(ref parts) = partitioned { - if !parts.is_empty() { - write!(f, "PARTITION ({}) ", display_comma_separated(parts))?; - } - } - if !after_columns.is_empty() { - write!(f, "({}) ", display_comma_separated(after_columns))?; - } - - if let Some(source) = source { - write!(f, "{source}")?; - } - - if source.is_none() && columns.is_empty() { - write!(f, "DEFAULT VALUES")?; - } - - if let Some(insert_alias) = insert_alias { - write!(f, " AS {0}", insert_alias.row_alias)?; - - if let Some(col_aliases) = &insert_alias.col_aliases { - if !col_aliases.is_empty() { - write!(f, " ({})", display_comma_separated(col_aliases))?; - } - } - } - - if let Some(on) = on { - write!(f, "{on}")?; - } - - if let Some(returning) = returning { - write!(f, " RETURNING {}", display_comma_separated(returning))?; - } - - Ok(()) - } + Statement::Insert(insert) => write!(f, "{insert}"), Statement::Install { extension_name: name, } => write!(f, "INSTALL {name}"), @@ -3696,45 +3622,7 @@ impl fmt::Display for Statement { } Ok(()) } - Statement::Delete(delete) => { - let Delete { - tables, - from, - using, - selection, - returning, - order_by, - limit, - } = delete; - write!(f, "DELETE ")?; - if !tables.is_empty() { - write!(f, "{} ", display_comma_separated(tables))?; - } - match from { - FromTable::WithFromKeyword(from) => { - write!(f, "FROM {}", display_comma_separated(from))?; - } - FromTable::WithoutKeyword(from) => { - write!(f, "{}", display_comma_separated(from))?; - } - } - if let Some(using) = using { - write!(f, " USING {}", display_comma_separated(using))?; - } - if let Some(selection) = selection { - write!(f, " WHERE {selection}")?; - } - if let Some(returning) = returning { - write!(f, " RETURNING {}", display_comma_separated(returning))?; - } - if !order_by.is_empty() { - write!(f, " ORDER BY {}", display_comma_separated(order_by))?; - } - if let Some(limit) = limit { - write!(f, " LIMIT {limit}")?; - } - Ok(()) - } + Statement::Delete(delete) => write!(f, "{delete}"), Statement::Close { cursor } => { write!(f, "CLOSE {cursor}")?; From 8e0d26abb3c4dda2fa091dfd9611be1dbdc14755 Mon Sep 17 00:00:00 2001 From: tomershaniii <65544633+tomershaniii@users.noreply.github.com> Date: Mon, 21 Oct 2024 22:41:34 +0300 Subject: [PATCH 33/80] fix for maybe_parse preventing parser from erroring on recursion limit (#1464) --- src/ast/mod.rs | 4 +- src/ast/query.rs | 2 +- src/dialect/mod.rs | 4 +- src/dialect/snowflake.rs | 4 +- src/parser/alter.rs | 2 +- src/parser/mod.rs | 184 ++++++-------- src/test_utils.rs | 27 +- tests/sqlparser_bigquery.rs | 21 +- tests/sqlparser_clickhouse.rs | 13 +- tests/sqlparser_common.rs | 466 ++++++++++++++-------------------- tests/sqlparser_databricks.rs | 13 +- tests/sqlparser_duckdb.rs | 15 +- tests/sqlparser_hive.rs | 15 +- tests/sqlparser_mssql.rs | 10 +- tests/sqlparser_mysql.rs | 109 +++----- tests/sqlparser_postgres.rs | 13 +- tests/sqlparser_redshift.rs | 13 +- tests/sqlparser_snowflake.rs | 54 ++-- tests/sqlparser_sqlite.rs | 25 +- 19 files changed, 423 insertions(+), 571 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 790a39bdb..0573240a2 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -3208,7 +3208,7 @@ pub enum Statement { /// Table confs options: Vec, /// Cache table as a Query - query: Option, + query: Option>, }, /// ```sql /// UNCACHE TABLE [ IF EXISTS ] @@ -6883,7 +6883,7 @@ impl fmt::Display for MacroArg { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum MacroDefinition { Expr(Expr), - Table(Query), + Table(Box), } impl fmt::Display for MacroDefinition { diff --git a/src/ast/query.rs b/src/ast/query.rs index ec0198674..dc5966e5e 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -1103,7 +1103,7 @@ pub enum PivotValueSource { /// Pivot on all values returned by a subquery. /// /// See . - Subquery(Query), + Subquery(Box), } impl fmt::Display for PivotValueSource { diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index be97f929b..28e7ac7d1 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -486,9 +486,9 @@ pub trait Dialect: Debug + Any { fn parse_column_option( &self, _parser: &mut Parser, - ) -> Option, ParserError>> { + ) -> Result, ParserError>>, ParserError> { // return None to fall back to the default behavior - None + Ok(None) } /// Decide the lexical Precedence of operators. diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index 7c80f0461..d9331d952 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -156,7 +156,7 @@ impl Dialect for SnowflakeDialect { fn parse_column_option( &self, parser: &mut Parser, - ) -> Option, ParserError>> { + ) -> Result, ParserError>>, ParserError> { parser.maybe_parse(|parser| { let with = parser.parse_keyword(Keyword::WITH); @@ -247,7 +247,7 @@ pub fn parse_create_table( builder = builder.comment(parser.parse_optional_inline_comment()?); } Keyword::AS => { - let query = parser.parse_boxed_query()?; + let query = parser.parse_query()?; builder = builder.query(Some(query)); break; } diff --git a/src/parser/alter.rs b/src/parser/alter.rs index 28fdaf764..534105790 100644 --- a/src/parser/alter.rs +++ b/src/parser/alter.rs @@ -192,7 +192,7 @@ impl<'a> Parser<'a> { let _ = self.parse_keyword(Keyword::WITH); // option let mut options = vec![]; - while let Some(opt) = self.maybe_parse(|parser| parser.parse_pg_role_option()) { + while let Some(opt) = self.maybe_parse(|parser| parser.parse_pg_role_option())? { options.push(opt); } // check option diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a1079f6f7..a9a5b1df4 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -478,7 +478,7 @@ impl<'a> Parser<'a> { Keyword::ANALYZE => self.parse_analyze(), Keyword::SELECT | Keyword::WITH | Keyword::VALUES => { self.prev_token(); - self.parse_boxed_query().map(Statement::Query) + self.parse_query().map(Statement::Query) } Keyword::TRUNCATE => self.parse_truncate(), Keyword::ATTACH => { @@ -551,7 +551,7 @@ impl<'a> Parser<'a> { }, Token::LParen => { self.prev_token(); - self.parse_boxed_query().map(Statement::Query) + self.parse_query().map(Statement::Query) } _ => self.expected("an SQL statement", next_token), } @@ -662,7 +662,7 @@ impl<'a> Parser<'a> { }; parser.expect_keyword(Keyword::PARTITIONS)?; Ok(pa) - }) + })? .unwrap_or_default(); Ok(Statement::Msck { repair, @@ -829,7 +829,7 @@ impl<'a> Parser<'a> { columns = self .maybe_parse(|parser| { parser.parse_comma_separated(|p| p.parse_identifier(false)) - }) + })? .unwrap_or_default(); for_columns = true } @@ -986,7 +986,7 @@ impl<'a> Parser<'a> { value: parser.parse_literal_string()?, }), } - }); + })?; if let Some(expr) = opt_expr { return Ok(expr); @@ -1061,7 +1061,7 @@ impl<'a> Parser<'a> { && !dialect_of!(self is ClickHouseDialect | DatabricksDialect) => { self.expect_token(&Token::LParen)?; - let query = self.parse_boxed_query()?; + let query = self.parse_query()?; self.expect_token(&Token::RParen)?; Ok(Expr::Function(Function { name: ObjectName(vec![w.to_ident()]), @@ -1228,7 +1228,7 @@ impl<'a> Parser<'a> { Token::LParen => { let expr = if let Some(expr) = self.try_parse_expr_sub_query()? { expr - } else if let Some(lambda) = self.try_parse_lambda() { + } else if let Some(lambda) = self.try_parse_lambda()? { return Ok(lambda); } else { let exprs = self.parse_comma_separated(Parser::parse_expr)?; @@ -1307,12 +1307,12 @@ impl<'a> Parser<'a> { return Ok(None); } - Ok(Some(Expr::Subquery(self.parse_boxed_query()?))) + Ok(Some(Expr::Subquery(self.parse_query()?))) } - fn try_parse_lambda(&mut self) -> Option { + fn try_parse_lambda(&mut self) -> Result, ParserError> { if !self.dialect.supports_lambda_functions() { - return None; + return Ok(None); } self.maybe_parse(|p| { let params = p.parse_comma_separated(|p| p.parse_identifier(false))?; @@ -1332,7 +1332,7 @@ impl<'a> Parser<'a> { // Snowflake permits a subquery to be passed as an argument without // an enclosing set of parens if it's the only argument. if dialect_of!(self is SnowflakeDialect) && self.peek_sub_query() { - let subquery = self.parse_boxed_query()?; + let subquery = self.parse_query()?; self.expect_token(&Token::RParen)?; return Ok(Expr::Function(Function { name, @@ -1697,7 +1697,7 @@ impl<'a> Parser<'a> { self.expect_token(&Token::LParen)?; let exists_node = Expr::Exists { negated, - subquery: self.parse_boxed_query()?, + subquery: self.parse_query()?, }; self.expect_token(&Token::RParen)?; Ok(exists_node) @@ -1777,7 +1777,7 @@ impl<'a> Parser<'a> { expr: Box::new(expr), r#in: Box::new(from), }) - }); + })?; match position_expr { Some(expr) => Ok(expr), // Snowflake supports `position` as an ordinary function call @@ -3032,7 +3032,7 @@ impl<'a> Parser<'a> { self.prev_token(); Expr::InSubquery { expr: Box::new(expr), - subquery: self.parse_boxed_query()?, + subquery: self.parse_query()?, negated, } } else { @@ -3513,17 +3513,19 @@ impl<'a> Parser<'a> { } /// Run a parser method `f`, reverting back to the current position if unsuccessful. - #[must_use] - pub fn maybe_parse(&mut self, mut f: F) -> Option + pub fn maybe_parse(&mut self, mut f: F) -> Result, ParserError> where F: FnMut(&mut Parser) -> Result, { let index = self.index; - if let Ok(t) = f(self) { - Some(t) - } else { - self.index = index; - None + match f(self) { + Ok(t) => Ok(Some(t)), + // Unwind stack if limit exceeded + Err(ParserError::RecursionLimitExceeded) => Err(ParserError::RecursionLimitExceeded), + Err(_) => { + self.index = index; + Ok(None) + } } } @@ -3759,7 +3761,7 @@ impl<'a> Parser<'a> { } /// Parse 'AS' before as query,such as `WITH XXX AS SELECT XXX` oer `CACHE TABLE AS SELECT XXX` - pub fn parse_as_query(&mut self) -> Result<(bool, Query), ParserError> { + pub fn parse_as_query(&mut self) -> Result<(bool, Box), ParserError> { match self.peek_token().token { Token::Word(word) => match word.keyword { Keyword::AS => { @@ -4523,7 +4525,7 @@ impl<'a> Parser<'a> { }; self.expect_keyword(Keyword::AS)?; - let query = self.parse_boxed_query()?; + let query = self.parse_query()?; // Optional `WITH [ CASCADED | LOCAL ] CHECK OPTION` is widely supported here. let with_no_schema_binding = dialect_of!(self is RedshiftSqlDialect | GenericDialect) @@ -5102,7 +5104,7 @@ impl<'a> Parser<'a> { self.expect_keyword(Keyword::FOR)?; - let query = Some(self.parse_boxed_query()?); + let query = Some(self.parse_query()?); Ok(Statement::Declare { stmts: vec![Declare { @@ -5196,7 +5198,7 @@ impl<'a> Parser<'a> { match self.peek_token().token { Token::Word(w) if w.keyword == Keyword::SELECT => ( Some(DeclareType::Cursor), - Some(self.parse_boxed_query()?), + Some(self.parse_query()?), None, None, ), @@ -5889,7 +5891,7 @@ impl<'a> Parser<'a> { // Parse optional `AS ( query )` let query = if self.parse_keyword(Keyword::AS) { - Some(self.parse_boxed_query()?) + Some(self.parse_query()?) } else { None }; @@ -6109,7 +6111,7 @@ impl<'a> Parser<'a> { } pub fn parse_optional_column_option(&mut self) -> Result, ParserError> { - if let Some(option) = self.dialect.parse_column_option(self) { + if let Some(option) = self.dialect.parse_column_option(self)? { return option; } @@ -6483,7 +6485,7 @@ impl<'a> Parser<'a> { } // optional index name - let index_name = self.parse_optional_indent(); + let index_name = self.parse_optional_indent()?; let index_type = self.parse_optional_using_then_index_type()?; let columns = self.parse_parenthesized_column_list(Mandatory, false)?; @@ -6504,7 +6506,7 @@ impl<'a> Parser<'a> { self.expect_keyword(Keyword::KEY)?; // optional index name - let index_name = self.parse_optional_indent(); + let index_name = self.parse_optional_indent()?; let index_type = self.parse_optional_using_then_index_type()?; let columns = self.parse_parenthesized_column_list(Mandatory, false)?; @@ -6566,7 +6568,7 @@ impl<'a> Parser<'a> { let name = match self.peek_token().token { Token::Word(word) if word.keyword == Keyword::USING => None, - _ => self.parse_optional_indent(), + _ => self.parse_optional_indent()?, }; let index_type = self.parse_optional_using_then_index_type()?; @@ -6597,7 +6599,7 @@ impl<'a> Parser<'a> { let index_type_display = self.parse_index_type_display(); - let opt_index_name = self.parse_optional_indent(); + let opt_index_name = self.parse_optional_indent()?; let columns = self.parse_parenthesized_column_list(Mandatory, false)?; @@ -6679,7 +6681,7 @@ impl<'a> Parser<'a> { /// Parse `[ident]`, mostly `ident` is name, like: /// `window_name`, `index_name`, ... - pub fn parse_optional_indent(&mut self) -> Option { + pub fn parse_optional_indent(&mut self) -> Result, ParserError> { self.maybe_parse(|parser| parser.parse_identifier(false)) } @@ -7278,7 +7280,7 @@ impl<'a> Parser<'a> { let with_options = self.parse_options(Keyword::WITH)?; self.expect_keyword(Keyword::AS)?; - let query = self.parse_boxed_query()?; + let query = self.parse_query()?; Ok(Statement::AlterView { name, @@ -7317,7 +7319,7 @@ impl<'a> Parser<'a> { pub fn parse_copy(&mut self) -> Result { let source; if self.consume_token(&Token::LParen) { - source = CopySource::Query(self.parse_boxed_query()?); + source = CopySource::Query(self.parse_query()?); self.expect_token(&Token::RParen)?; } else { let table_name = self.parse_object_name(false)?; @@ -7361,7 +7363,7 @@ impl<'a> Parser<'a> { self.expect_token(&Token::RParen)?; } let mut legacy_options = vec![]; - while let Some(opt) = self.maybe_parse(|parser| parser.parse_copy_legacy_option()) { + while let Some(opt) = self.maybe_parse(|parser| parser.parse_copy_legacy_option())? { legacy_options.push(opt); } let values = if let CopyTarget::Stdin = target { @@ -7453,7 +7455,7 @@ impl<'a> Parser<'a> { Some(Keyword::CSV) => CopyLegacyOption::Csv({ let mut opts = vec![]; while let Some(opt) = - self.maybe_parse(|parser| parser.parse_copy_legacy_csv_option()) + self.maybe_parse(|parser| parser.parse_copy_legacy_csv_option())? { opts.push(opt); } @@ -8035,7 +8037,7 @@ impl<'a> Parser<'a> { // Keyword::ARRAY syntax from above while self.consume_token(&Token::LBracket) { let size = if dialect_of!(self is GenericDialect | DuckDbDialect | PostgreSqlDialect) { - self.maybe_parse(|p| p.parse_literal_uint()) + self.maybe_parse(|p| p.parse_literal_uint())? } else { None }; @@ -8712,7 +8714,7 @@ impl<'a> Parser<'a> { } } - match self.maybe_parse(|parser| parser.parse_statement()) { + match self.maybe_parse(|parser| parser.parse_statement())? { Some(Statement::Explain { .. }) | Some(Statement::ExplainTable { .. }) => Err( ParserError::ParserError("Explain must be root of the plan".to_string()), ), @@ -8751,20 +8753,11 @@ impl<'a> Parser<'a> { } } - /// Call's [`Self::parse_query`] returning a `Box`'ed result. - /// - /// This function can be used to reduce the stack size required in debug - /// builds. Instead of `sizeof(Query)` only a pointer (`Box`) - /// is used. - pub fn parse_boxed_query(&mut self) -> Result, ParserError> { - self.parse_query().map(Box::new) - } - /// Parse a query expression, i.e. a `SELECT` statement optionally /// preceded with some `WITH` CTE declarations and optionally followed /// by `ORDER BY`. Unlike some other parse_... methods, this one doesn't /// expect the initial keyword to be already consumed - pub fn parse_query(&mut self) -> Result { + pub fn parse_query(&mut self) -> Result, ParserError> { let _guard = self.recursion_counter.try_decrease()?; let with = if self.parse_keyword(Keyword::WITH) { Some(With { @@ -8787,7 +8780,8 @@ impl<'a> Parser<'a> { for_clause: None, settings: None, format_clause: None, - }) + } + .into()) } else if self.parse_keyword(Keyword::UPDATE) { Ok(Query { with, @@ -8801,9 +8795,10 @@ impl<'a> Parser<'a> { for_clause: None, settings: None, format_clause: None, - }) + } + .into()) } else { - let body = self.parse_boxed_query_body(self.dialect.prec_unknown())?; + let body = self.parse_query_body(self.dialect.prec_unknown())?; let order_by = self.parse_optional_order_by()?; @@ -8885,7 +8880,8 @@ impl<'a> Parser<'a> { for_clause, settings, format_clause, - }) + } + .into()) } } @@ -9022,7 +9018,7 @@ impl<'a> Parser<'a> { } } self.expect_token(&Token::LParen)?; - let query = self.parse_boxed_query()?; + let query = self.parse_query()?; self.expect_token(&Token::RParen)?; let alias = TableAlias { name, @@ -9046,7 +9042,7 @@ impl<'a> Parser<'a> { } } self.expect_token(&Token::LParen)?; - let query = self.parse_boxed_query()?; + let query = self.parse_query()?; self.expect_token(&Token::RParen)?; let alias = TableAlias { name, columns }; Cte { @@ -9062,15 +9058,6 @@ impl<'a> Parser<'a> { Ok(cte) } - /// Call's [`Self::parse_query_body`] returning a `Box`'ed result. - /// - /// This function can be used to reduce the stack size required in debug - /// builds. Instead of `sizeof(QueryBody)` only a pointer (`Box`) - /// is used. - fn parse_boxed_query_body(&mut self, precedence: u8) -> Result, ParserError> { - self.parse_query_body(precedence).map(Box::new) - } - /// Parse a "query body", which is an expression with roughly the /// following grammar: /// ```sql @@ -9079,17 +9066,14 @@ impl<'a> Parser<'a> { /// subquery ::= query_body [ order_by_limit ] /// set_operation ::= query_body { 'UNION' | 'EXCEPT' | 'INTERSECT' } [ 'ALL' ] query_body /// ``` - /// - /// If you need `Box` then maybe there is sense to use `parse_boxed_query_body` - /// due to prevent stack overflow in debug building(to reserve less memory on stack). - pub fn parse_query_body(&mut self, precedence: u8) -> Result { + pub fn parse_query_body(&mut self, precedence: u8) -> Result, ParserError> { // We parse the expression using a Pratt parser, as in `parse_expr()`. // Start by parsing a restricted SELECT or a `(subquery)`: let expr = if self.parse_keyword(Keyword::SELECT) { SetExpr::Select(self.parse_select().map(Box::new)?) } else if self.consume_token(&Token::LParen) { // CTEs are not allowed here, but the parser currently accepts them - let subquery = self.parse_boxed_query()?; + let subquery = self.parse_query()?; self.expect_token(&Token::RParen)?; SetExpr::Query(subquery) } else if self.parse_keyword(Keyword::VALUES) { @@ -9114,7 +9098,7 @@ impl<'a> Parser<'a> { &mut self, mut expr: SetExpr, precedence: u8, - ) -> Result { + ) -> Result, ParserError> { loop { // The query can be optionally followed by a set operator: let op = self.parse_set_operator(&self.peek_token().token); @@ -9135,11 +9119,11 @@ impl<'a> Parser<'a> { left: Box::new(expr), op: op.unwrap(), set_quantifier, - right: self.parse_boxed_query_body(next_precedence)?, + right: self.parse_query_body(next_precedence)?, }; } - Ok(expr) + Ok(expr.into()) } pub fn parse_set_operator(&mut self, token: &Token) -> Option { @@ -9466,7 +9450,7 @@ impl<'a> Parser<'a> { if let Some(Keyword::HIVEVAR) = modifier { self.expect_token(&Token::Colon)?; } else if let Some(set_role_stmt) = - self.maybe_parse(|parser| parser.parse_set_role(modifier)) + self.maybe_parse(|parser| parser.parse_set_role(modifier))? { return Ok(set_role_stmt); } @@ -9932,7 +9916,7 @@ impl<'a> Parser<'a> { // subquery, followed by the closing ')', and the alias of the derived table. // In the example above this is case (3). if let Some(mut table) = - self.maybe_parse(|parser| parser.parse_derived_table_factor(NotLateral)) + self.maybe_parse(|parser| parser.parse_derived_table_factor(NotLateral))? { while let Some(kw) = self.parse_one_of_keywords(&[Keyword::PIVOT, Keyword::UNPIVOT]) { @@ -10462,7 +10446,7 @@ impl<'a> Parser<'a> { &mut self, lateral: IsLateral, ) -> Result { - let subquery = self.parse_boxed_query()?; + let subquery = self.parse_query()?; self.expect_token(&Token::RParen)?; let alias = self.parse_optional_table_alias(keywords::RESERVED_FOR_TABLE_ALIAS)?; Ok(TableFactor::Derived { @@ -10836,7 +10820,7 @@ impl<'a> Parser<'a> { } else { None }; - let source = self.parse_boxed_query()?; + let source = self.parse_query()?; Ok(Statement::Directory { local, path, @@ -10872,7 +10856,7 @@ impl<'a> Parser<'a> { vec![] }; - let source = Some(self.parse_boxed_query()?); + let source = Some(self.parse_query()?); (columns, partitioned, after_columns, source) }; @@ -11786,7 +11770,7 @@ impl<'a> Parser<'a> { pub fn parse_unload(&mut self) -> Result { self.expect_token(&Token::LParen)?; - let query = self.parse_boxed_query()?; + let query = self.parse_query()?; self.expect_token(&Token::RParen)?; self.expect_keyword(Keyword::TO)?; @@ -12130,7 +12114,9 @@ impl<'a> Parser<'a> { pub fn parse_window_spec(&mut self) -> Result { let window_name = match self.peek_token().token { - Token::Word(word) if word.keyword == Keyword::NoKeyword => self.parse_optional_indent(), + Token::Word(word) if word.keyword == Keyword::NoKeyword => { + self.parse_optional_indent()? + } _ => None, }; @@ -12342,10 +12328,8 @@ mod tests { #[test] fn test_ansii_character_string_types() { // Character string types: - let dialect = TestedDialects { - dialects: vec![Box::new(GenericDialect {}), Box::new(AnsiDialect {})], - options: None, - }; + let dialect = + TestedDialects::new(vec![Box::new(GenericDialect {}), Box::new(AnsiDialect {})]); test_parse_data_type!(dialect, "CHARACTER", DataType::Character(None)); @@ -12472,10 +12456,8 @@ mod tests { #[test] fn test_ansii_character_large_object_types() { // Character large object types: - let dialect = TestedDialects { - dialects: vec![Box::new(GenericDialect {}), Box::new(AnsiDialect {})], - options: None, - }; + let dialect = + TestedDialects::new(vec![Box::new(GenericDialect {}), Box::new(AnsiDialect {})]); test_parse_data_type!( dialect, @@ -12505,10 +12487,9 @@ mod tests { #[test] fn test_parse_custom_types() { - let dialect = TestedDialects { - dialects: vec![Box::new(GenericDialect {}), Box::new(AnsiDialect {})], - options: None, - }; + let dialect = + TestedDialects::new(vec![Box::new(GenericDialect {}), Box::new(AnsiDialect {})]); + test_parse_data_type!( dialect, "GEOMETRY", @@ -12537,10 +12518,8 @@ mod tests { #[test] fn test_ansii_exact_numeric_types() { // Exact numeric types: - let dialect = TestedDialects { - dialects: vec![Box::new(GenericDialect {}), Box::new(AnsiDialect {})], - options: None, - }; + let dialect = + TestedDialects::new(vec![Box::new(GenericDialect {}), Box::new(AnsiDialect {})]); test_parse_data_type!(dialect, "NUMERIC", DataType::Numeric(ExactNumberInfo::None)); @@ -12588,10 +12567,8 @@ mod tests { #[test] fn test_ansii_date_type() { // Datetime types: - let dialect = TestedDialects { - dialects: vec![Box::new(GenericDialect {}), Box::new(AnsiDialect {})], - options: None, - }; + let dialect = + TestedDialects::new(vec![Box::new(GenericDialect {}), Box::new(AnsiDialect {})]); test_parse_data_type!(dialect, "DATE", DataType::Date); @@ -12700,10 +12677,8 @@ mod tests { }}; } - let dialect = TestedDialects { - dialects: vec![Box::new(GenericDialect {}), Box::new(MySqlDialect {})], - options: None, - }; + let dialect = + TestedDialects::new(vec![Box::new(GenericDialect {}), Box::new(MySqlDialect {})]); test_parse_table_constraint!( dialect, @@ -12822,10 +12797,7 @@ mod tests { #[test] fn test_parse_multipart_identifier_positive() { - let dialect = TestedDialects { - dialects: vec![Box::new(GenericDialect {})], - options: None, - }; + let dialect = TestedDialects::new(vec![Box::new(GenericDialect {})]); // parse multipart with quotes let expected = vec![ diff --git a/src/test_utils.rs b/src/test_utils.rs index e588b3506..b35fc45c2 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -44,6 +44,7 @@ use pretty_assertions::assert_eq; pub struct TestedDialects { pub dialects: Vec>, pub options: Option, + pub recursion_limit: Option, } impl TestedDialects { @@ -52,16 +53,38 @@ impl TestedDialects { Self { dialects, options: None, + recursion_limit: None, } } + pub fn new_with_options(dialects: Vec>, options: ParserOptions) -> Self { + Self { + dialects, + options: Some(options), + recursion_limit: None, + } + } + + pub fn with_recursion_limit(mut self, recursion_limit: usize) -> Self { + self.recursion_limit = Some(recursion_limit); + self + } + fn new_parser<'a>(&self, dialect: &'a dyn Dialect) -> Parser<'a> { let parser = Parser::new(dialect); - if let Some(options) = &self.options { + let parser = if let Some(options) = &self.options { parser.with_options(options.clone()) } else { parser - } + }; + + let parser = if let Some(recursion_limit) = &self.recursion_limit { + parser.with_recursion_limit(*recursion_limit) + } else { + parser + }; + + parser } /// Run the given function for all of `self.dialects`, assert that they diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index 63517fe57..2bf470f71 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -40,10 +40,10 @@ fn parse_literal_string() { r#""""triple-double\"escaped""", "#, r#""""triple-double"unescaped""""#, ); - let dialect = TestedDialects { - dialects: vec![Box::new(BigQueryDialect {})], - options: Some(ParserOptions::new().with_unescape(false)), - }; + let dialect = TestedDialects::new_with_options( + vec![Box::new(BigQueryDialect {})], + ParserOptions::new().with_unescape(false), + ); let select = dialect.verified_only_select(sql); assert_eq!(10, select.projection.len()); assert_eq!( @@ -1936,17 +1936,14 @@ fn parse_big_query_declare() { } fn bigquery() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(BigQueryDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(BigQueryDialect {})]) } fn bigquery_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(BigQueryDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![ + Box::new(BigQueryDialect {}), + Box::new(GenericDialect {}), + ]) } #[test] diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index e30c33678..f8c349a37 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -1613,15 +1613,12 @@ fn parse_explain_table() { } fn clickhouse() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(ClickHouseDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(ClickHouseDialect {})]) } fn clickhouse_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(ClickHouseDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![ + Box::new(ClickHouseDialect {}), + Box::new(GenericDialect {}), + ]) } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 5683bcf91..a2eb5070d 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -341,19 +341,16 @@ fn parse_update() { #[test] fn parse_update_set_from() { let sql = "UPDATE t1 SET name = t2.name FROM (SELECT name, id FROM t1 GROUP BY id) AS t2 WHERE t1.id = t2.id"; - let dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(DuckDbDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(BigQueryDialect {}), - Box::new(SnowflakeDialect {}), - Box::new(RedshiftSqlDialect {}), - Box::new(MsSqlDialect {}), - Box::new(SQLiteDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(DuckDbDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(BigQueryDialect {}), + Box::new(SnowflakeDialect {}), + Box::new(RedshiftSqlDialect {}), + Box::new(MsSqlDialect {}), + Box::new(SQLiteDialect {}), + ]); let stmt = dialects.verified_stmt(sql); assert_eq!( stmt, @@ -1051,10 +1048,7 @@ fn test_eof_after_as() { #[test] fn test_no_infix_error() { - let dialects = TestedDialects { - dialects: vec![Box::new(ClickHouseDialect {})], - options: None, - }; + let dialects = TestedDialects::new(vec![Box::new(ClickHouseDialect {})]); let res = dialects.parse_sql_statements("ASSERT-URA<<"); assert_eq!( @@ -1182,23 +1176,20 @@ fn parse_null_in_select() { #[test] fn parse_exponent_in_select() -> Result<(), ParserError> { // all except Hive, as it allows numbers to start an identifier - let dialects = TestedDialects { - dialects: vec![ - Box::new(AnsiDialect {}), - Box::new(BigQueryDialect {}), - Box::new(ClickHouseDialect {}), - Box::new(DuckDbDialect {}), - Box::new(GenericDialect {}), - // Box::new(HiveDialect {}), - Box::new(MsSqlDialect {}), - Box::new(MySqlDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(RedshiftSqlDialect {}), - Box::new(SnowflakeDialect {}), - Box::new(SQLiteDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(AnsiDialect {}), + Box::new(BigQueryDialect {}), + Box::new(ClickHouseDialect {}), + Box::new(DuckDbDialect {}), + Box::new(GenericDialect {}), + // Box::new(HiveDialect {}), + Box::new(MsSqlDialect {}), + Box::new(MySqlDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(RedshiftSqlDialect {}), + Box::new(SnowflakeDialect {}), + Box::new(SQLiteDialect {}), + ]); let sql = "SELECT 10e-20, 1e3, 1e+3, 1e3a, 1e, 0.5e2"; let mut select = dialects.parse_sql_statements(sql)?; @@ -1271,14 +1262,12 @@ fn parse_escaped_single_quote_string_predicate_with_no_escape() { let sql = "SELECT id, fname, lname FROM customer \ WHERE salary <> 'Jim''s salary'"; - let ast = TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: Some( - ParserOptions::new() - .with_trailing_commas(true) - .with_unescape(false), - ), - } + let ast = TestedDialects::new_with_options( + vec![Box::new(MySqlDialect {})], + ParserOptions::new() + .with_trailing_commas(true) + .with_unescape(false), + ) .verified_only_select(sql); assert_eq!( @@ -1400,10 +1389,10 @@ fn parse_mod() { } fn pg_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(PostgreSqlDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![ + Box::new(PostgreSqlDialect {}), + Box::new(GenericDialect {}), + ]) } #[test] @@ -1868,14 +1857,13 @@ fn parse_string_agg() { /// selects all dialects but PostgreSQL pub fn all_dialects_but_pg() -> TestedDialects { - TestedDialects { - dialects: all_dialects() + TestedDialects::new( + all_dialects() .dialects .into_iter() .filter(|x| !x.is::()) .collect(), - options: None, - } + ) } #[test] @@ -2691,17 +2679,14 @@ fn parse_listagg() { #[test] fn parse_array_agg_func() { - let supported_dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(DuckDbDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(MsSqlDialect {}), - Box::new(AnsiDialect {}), - Box::new(HiveDialect {}), - ], - options: None, - }; + let supported_dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(DuckDbDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(MsSqlDialect {}), + Box::new(AnsiDialect {}), + Box::new(HiveDialect {}), + ]); for sql in [ "SELECT ARRAY_AGG(x ORDER BY x) AS a FROM T", @@ -2716,16 +2701,13 @@ fn parse_array_agg_func() { #[test] fn parse_agg_with_order_by() { - let supported_dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(MsSqlDialect {}), - Box::new(AnsiDialect {}), - Box::new(HiveDialect {}), - ], - options: None, - }; + let supported_dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(MsSqlDialect {}), + Box::new(AnsiDialect {}), + Box::new(HiveDialect {}), + ]); for sql in [ "SELECT FIRST_VALUE(x ORDER BY x) AS a FROM T", @@ -2739,17 +2721,14 @@ fn parse_agg_with_order_by() { #[test] fn parse_window_rank_function() { - let supported_dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(MsSqlDialect {}), - Box::new(AnsiDialect {}), - Box::new(HiveDialect {}), - Box::new(SnowflakeDialect {}), - ], - options: None, - }; + let supported_dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(MsSqlDialect {}), + Box::new(AnsiDialect {}), + Box::new(HiveDialect {}), + Box::new(SnowflakeDialect {}), + ]); for sql in [ "SELECT column1, column2, FIRST_VALUE(column2) OVER (PARTITION BY column1 ORDER BY column2 NULLS LAST) AS column2_first FROM t1", @@ -2761,10 +2740,10 @@ fn parse_window_rank_function() { supported_dialects.verified_stmt(sql); } - let supported_dialects_nulls = TestedDialects { - dialects: vec![Box::new(MsSqlDialect {}), Box::new(SnowflakeDialect {})], - options: None, - }; + let supported_dialects_nulls = TestedDialects::new(vec![ + Box::new(MsSqlDialect {}), + Box::new(SnowflakeDialect {}), + ]); for sql in [ "SELECT column1, column2, FIRST_VALUE(column2) IGNORE NULLS OVER (PARTITION BY column1 ORDER BY column2 NULLS LAST) AS column2_first FROM t1", @@ -3321,10 +3300,7 @@ fn parse_create_table_hive_array() { true, ), ] { - let dialects = TestedDialects { - dialects, - options: None, - }; + let dialects = TestedDialects::new(dialects); let sql = format!( "CREATE TABLE IF NOT EXISTS something (name INT, val {})", @@ -3374,14 +3350,11 @@ fn parse_create_table_hive_array() { } // SnowflakeDialect using array different - let dialects = TestedDialects { - dialects: vec![ - Box::new(PostgreSqlDialect {}), - Box::new(HiveDialect {}), - Box::new(MySqlDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(PostgreSqlDialect {}), + Box::new(HiveDialect {}), + Box::new(MySqlDialect {}), + ]); let sql = "CREATE TABLE IF NOT EXISTS something (name int, val array Result<(), Par #[test] fn parse_create_table_with_options() { - let generic = TestedDialects { - dialects: vec![Box::new(GenericDialect {})], - options: None, - }; + let generic = TestedDialects::new(vec![Box::new(GenericDialect {})]); let sql = "CREATE TABLE t (c INT) WITH (foo = 'bar', a = 123)"; match generic.verified_stmt(sql) { @@ -3695,10 +3662,7 @@ fn parse_create_table_clone() { #[test] fn parse_create_table_trailing_comma() { - let dialect = TestedDialects { - dialects: vec![Box::new(DuckDbDialect {})], - options: None, - }; + let dialect = TestedDialects::new(vec![Box::new(DuckDbDialect {})]); let sql = "CREATE TABLE foo (bar int,);"; dialect.one_statement_parses_to(sql, "CREATE TABLE foo (bar INT)"); @@ -4040,15 +4004,12 @@ fn parse_alter_table_add_column() { #[test] fn parse_alter_table_add_column_if_not_exists() { - let dialects = TestedDialects { - dialects: vec![ - Box::new(PostgreSqlDialect {}), - Box::new(BigQueryDialect {}), - Box::new(GenericDialect {}), - Box::new(DuckDbDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(PostgreSqlDialect {}), + Box::new(BigQueryDialect {}), + Box::new(GenericDialect {}), + Box::new(DuckDbDialect {}), + ]); match alter_table_op(dialects.verified_stmt("ALTER TABLE tab ADD IF NOT EXISTS foo TEXT")) { AlterTableOperation::AddColumn { if_not_exists, .. } => { @@ -4191,10 +4152,7 @@ fn parse_alter_table_alter_column_type() { _ => unreachable!(), } - let dialect = TestedDialects { - dialects: vec![Box::new(GenericDialect {})], - options: None, - }; + let dialect = TestedDialects::new(vec![Box::new(GenericDialect {})]); let res = dialect.parse_sql_statements(&format!("{alter_stmt} ALTER COLUMN is_active TYPE TEXT")); @@ -4611,15 +4569,12 @@ fn parse_window_functions() { #[test] fn parse_named_window_functions() { - let supported_dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(MySqlDialect {}), - Box::new(BigQueryDialect {}), - ], - options: None, - }; + let supported_dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(MySqlDialect {}), + Box::new(BigQueryDialect {}), + ]); let sql = "SELECT row_number() OVER (w ORDER BY dt DESC), \ sum(foo) OVER (win PARTITION BY a, b ORDER BY c, d \ @@ -5684,10 +5639,10 @@ fn parse_unnest_in_from_clause() { let select = dialects.verified_only_select(sql); assert_eq!(select.from, want); } - let dialects = TestedDialects { - dialects: vec![Box::new(BigQueryDialect {}), Box::new(GenericDialect {})], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(BigQueryDialect {}), + Box::new(GenericDialect {}), + ]); // 1. both Alias and WITH OFFSET clauses. chk( "expr", @@ -6670,22 +6625,20 @@ fn parse_trim() { ); //keep Snowflake/BigQuery TRIM syntax failing - let all_expected_snowflake = TestedDialects { - dialects: vec![ - //Box::new(GenericDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(MsSqlDialect {}), - Box::new(AnsiDialect {}), - //Box::new(SnowflakeDialect {}), - Box::new(HiveDialect {}), - Box::new(RedshiftSqlDialect {}), - Box::new(MySqlDialect {}), - //Box::new(BigQueryDialect {}), - Box::new(SQLiteDialect {}), - Box::new(DuckDbDialect {}), - ], - options: None, - }; + let all_expected_snowflake = TestedDialects::new(vec![ + //Box::new(GenericDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(MsSqlDialect {}), + Box::new(AnsiDialect {}), + //Box::new(SnowflakeDialect {}), + Box::new(HiveDialect {}), + Box::new(RedshiftSqlDialect {}), + Box::new(MySqlDialect {}), + //Box::new(BigQueryDialect {}), + Box::new(SQLiteDialect {}), + Box::new(DuckDbDialect {}), + ]); + assert_eq!( ParserError::ParserError("Expected: ), found: 'a'".to_owned()), all_expected_snowflake @@ -8582,20 +8535,17 @@ fn test_lock_nonblock() { #[test] fn test_placeholder() { - let dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(DuckDbDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(MsSqlDialect {}), - Box::new(AnsiDialect {}), - Box::new(BigQueryDialect {}), - Box::new(SnowflakeDialect {}), - // Note: `$` is the starting word for the HiveDialect identifier - // Box::new(sqlparser::dialect::HiveDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(DuckDbDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(MsSqlDialect {}), + Box::new(AnsiDialect {}), + Box::new(BigQueryDialect {}), + Box::new(SnowflakeDialect {}), + // Note: `$` is the starting word for the HiveDialect identifier + // Box::new(sqlparser::dialect::HiveDialect {}), + ]); let sql = "SELECT * FROM student WHERE id = $Id1"; let ast = dialects.verified_only_select(sql); assert_eq!( @@ -8621,21 +8571,18 @@ fn test_placeholder() { }), ); - let dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(DuckDbDialect {}), - // Note: `?` is for jsonb operators in PostgreSqlDialect - // Box::new(PostgreSqlDialect {}), - Box::new(MsSqlDialect {}), - Box::new(AnsiDialect {}), - Box::new(BigQueryDialect {}), - Box::new(SnowflakeDialect {}), - // Note: `$` is the starting word for the HiveDialect identifier - // Box::new(sqlparser::dialect::HiveDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(DuckDbDialect {}), + // Note: `?` is for jsonb operators in PostgreSqlDialect + // Box::new(PostgreSqlDialect {}), + Box::new(MsSqlDialect {}), + Box::new(AnsiDialect {}), + Box::new(BigQueryDialect {}), + Box::new(SnowflakeDialect {}), + // Note: `$` is the starting word for the HiveDialect identifier + // Box::new(sqlparser::dialect::HiveDialect {}), + ]); let sql = "SELECT * FROM student WHERE id = ?"; let ast = dialects.verified_only_select(sql); assert_eq!( @@ -9023,7 +8970,7 @@ fn parse_cache_table() { value: Expr::Value(number("0.88")), }, ], - query: Some(query.clone()), + query: Some(query.clone().into()), } ); @@ -9048,7 +8995,7 @@ fn parse_cache_table() { value: Expr::Value(number("0.88")), }, ], - query: Some(query.clone()), + query: Some(query.clone().into()), } ); @@ -9059,7 +9006,7 @@ fn parse_cache_table() { table_name: ObjectName(vec![Ident::with_quote('\'', cache_table_name)]), has_as: false, options: vec![], - query: Some(query.clone()), + query: Some(query.clone().into()), } ); @@ -9070,7 +9017,7 @@ fn parse_cache_table() { table_name: ObjectName(vec![Ident::with_quote('\'', cache_table_name)]), has_as: true, options: vec![], - query: Some(query), + query: Some(query.into()), } ); @@ -9243,14 +9190,11 @@ fn parse_with_recursion_limit() { #[test] fn parse_escaped_string_with_unescape() { fn assert_mysql_query_value(sql: &str, quoted: &str) { - let stmt = TestedDialects { - dialects: vec![ - Box::new(MySqlDialect {}), - Box::new(BigQueryDialect {}), - Box::new(SnowflakeDialect {}), - ], - options: None, - } + let stmt = TestedDialects::new(vec![ + Box::new(MySqlDialect {}), + Box::new(BigQueryDialect {}), + Box::new(SnowflakeDialect {}), + ]) .one_statement_parses_to(sql, ""); match stmt { @@ -9283,14 +9227,14 @@ fn parse_escaped_string_with_unescape() { #[test] fn parse_escaped_string_without_unescape() { fn assert_mysql_query_value(sql: &str, quoted: &str) { - let stmt = TestedDialects { - dialects: vec![ + let stmt = TestedDialects::new_with_options( + vec![ Box::new(MySqlDialect {}), Box::new(BigQueryDialect {}), Box::new(SnowflakeDialect {}), ], - options: Some(ParserOptions::new().with_unescape(false)), - } + ParserOptions::new().with_unescape(false), + ) .one_statement_parses_to(sql, ""); match stmt { @@ -9558,17 +9502,14 @@ fn make_where_clause(num: usize) -> String { #[test] fn parse_non_latin_identifiers() { - let supported_dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(DuckDbDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(MsSqlDialect {}), - Box::new(RedshiftSqlDialect {}), - Box::new(MySqlDialect {}), - ], - options: None, - }; + let supported_dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(DuckDbDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(MsSqlDialect {}), + Box::new(RedshiftSqlDialect {}), + Box::new(MySqlDialect {}), + ]); supported_dialects.verified_stmt("SELECT a.説明 FROM test.public.inter01 AS a"); supported_dialects.verified_stmt("SELECT a.説明 FROM inter01 AS a, inter01_transactions AS b WHERE a.説明 = b.取引 GROUP BY a.説明"); @@ -9582,10 +9523,7 @@ fn parse_non_latin_identifiers() { fn parse_trailing_comma() { // At the moment, DuckDB is the only dialect that allows // trailing commas anywhere in the query - let trailing_commas = TestedDialects { - dialects: vec![Box::new(DuckDbDialect {})], - options: None, - }; + let trailing_commas = TestedDialects::new(vec![Box::new(DuckDbDialect {})]); trailing_commas.one_statement_parses_to( "SELECT album_id, name, FROM track", @@ -9624,10 +9562,7 @@ fn parse_trailing_comma() { trailing_commas.verified_stmt(r#"SELECT "from" FROM "from""#); // doesn't allow any trailing commas - let trailing_commas = TestedDialects { - dialects: vec![Box::new(GenericDialect {})], - options: None, - }; + let trailing_commas = TestedDialects::new(vec![Box::new(GenericDialect {})]); assert_eq!( trailing_commas @@ -9656,10 +9591,10 @@ fn parse_trailing_comma() { #[test] fn parse_projection_trailing_comma() { // Some dialects allow trailing commas only in the projection - let trailing_commas = TestedDialects { - dialects: vec![Box::new(SnowflakeDialect {}), Box::new(BigQueryDialect {})], - options: None, - }; + let trailing_commas = TestedDialects::new(vec![ + Box::new(SnowflakeDialect {}), + Box::new(BigQueryDialect {}), + ]); trailing_commas.one_statement_parses_to( "SELECT album_id, name, FROM track", @@ -9946,14 +9881,11 @@ fn test_release_savepoint() { #[test] fn test_comment_hash_syntax() { - let dialects = TestedDialects { - dialects: vec![ - Box::new(BigQueryDialect {}), - Box::new(SnowflakeDialect {}), - Box::new(MySqlDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(BigQueryDialect {}), + Box::new(SnowflakeDialect {}), + Box::new(MySqlDialect {}), + ]); let sql = r#" # comment SELECT a, b, c # , d, e @@ -10013,10 +9945,10 @@ fn test_buffer_reuse() { #[test] fn parse_map_access_expr() { let sql = "users[-1][safe_offset(2)]"; - let dialects = TestedDialects { - dialects: vec![Box::new(BigQueryDialect {}), Box::new(ClickHouseDialect {})], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(BigQueryDialect {}), + Box::new(ClickHouseDialect {}), + ]); let expr = dialects.verified_expr(sql); let expected = Expr::MapAccess { column: Expr::Identifier(Ident::new("users")).into(), @@ -10591,16 +10523,13 @@ fn test_match_recognize_patterns() { #[test] fn test_select_wildcard_with_replace() { let sql = r#"SELECT * REPLACE (lower(city) AS city) FROM addresses"#; - let dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(BigQueryDialect {}), - Box::new(ClickHouseDialect {}), - Box::new(SnowflakeDialect {}), - Box::new(DuckDbDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(BigQueryDialect {}), + Box::new(ClickHouseDialect {}), + Box::new(SnowflakeDialect {}), + Box::new(DuckDbDialect {}), + ]); let select = dialects.verified_only_select(sql); let expected = SelectItem::Wildcard(WildcardAdditionalOptions { opt_replace: Some(ReplaceSelectItem { @@ -10657,14 +10586,11 @@ fn test_select_wildcard_with_replace() { #[test] fn parse_sized_list() { - let dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(PostgreSqlDialect {}), - Box::new(DuckDbDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(PostgreSqlDialect {}), + Box::new(DuckDbDialect {}), + ]); let sql = r#"CREATE TABLE embeddings (data FLOAT[1536])"#; dialects.verified_stmt(sql); let sql = r#"CREATE TABLE embeddings (data FLOAT[1536][3])"#; @@ -10675,14 +10601,11 @@ fn parse_sized_list() { #[test] fn insert_into_with_parentheses() { - let dialects = TestedDialects { - dialects: vec![ - Box::new(SnowflakeDialect {}), - Box::new(RedshiftSqlDialect {}), - Box::new(GenericDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(SnowflakeDialect {}), + Box::new(RedshiftSqlDialect {}), + Box::new(GenericDialect {}), + ]); dialects.verified_stmt("INSERT INTO t1 (id, name) (SELECT t2.id, t2.name FROM t2)"); } @@ -10850,14 +10773,11 @@ fn parse_within_group() { #[test] fn tests_select_values_without_parens() { - let dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(SnowflakeDialect {}), - Box::new(DatabricksDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(SnowflakeDialect {}), + Box::new(DatabricksDialect {}), + ]); let sql = "SELECT * FROM VALUES (1, 2), (2,3) AS tbl (id, val)"; let canonical = "SELECT * FROM (VALUES (1, 2), (2, 3)) AS tbl (id, val)"; dialects.verified_only_select_with_canonical(sql, canonical); @@ -10865,14 +10785,12 @@ fn tests_select_values_without_parens() { #[test] fn tests_select_values_without_parens_and_set_op() { - let dialects = TestedDialects { - dialects: vec![ - Box::new(GenericDialect {}), - Box::new(SnowflakeDialect {}), - Box::new(DatabricksDialect {}), - ], - options: None, - }; + let dialects = TestedDialects::new(vec![ + Box::new(GenericDialect {}), + Box::new(SnowflakeDialect {}), + Box::new(DatabricksDialect {}), + ]); + let sql = "SELECT id + 1, name FROM VALUES (1, 'Apple'), (2, 'Banana'), (3, 'Orange') AS fruits (id, name) UNION ALL SELECT 5, 'Strawberry'"; let canonical = "SELECT id + 1, name FROM (VALUES (1, 'Apple'), (2, 'Banana'), (3, 'Orange')) AS fruits (id, name) UNION ALL SELECT 5, 'Strawberry'"; let query = dialects.verified_query_with_canonical(sql, canonical); diff --git a/tests/sqlparser_databricks.rs b/tests/sqlparser_databricks.rs index 7dcfee68a..7b917bd06 100644 --- a/tests/sqlparser_databricks.rs +++ b/tests/sqlparser_databricks.rs @@ -24,17 +24,14 @@ use test_utils::*; mod test_utils; fn databricks() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(DatabricksDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(DatabricksDialect {})]) } fn databricks_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(DatabricksDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![ + Box::new(DatabricksDialect {}), + Box::new(GenericDialect {}), + ]) } #[test] diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index 4703f4b60..a4109b0a3 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -24,17 +24,14 @@ use sqlparser::ast::*; use sqlparser::dialect::{DuckDbDialect, GenericDialect}; fn duckdb() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(DuckDbDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(DuckDbDialect {})]) } fn duckdb_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(DuckDbDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![ + Box::new(DuckDbDialect {}), + Box::new(GenericDialect {}), + ]) } #[test] @@ -242,7 +239,7 @@ fn test_create_table_macro() { MacroArg::new("col1_value"), MacroArg::new("col2_value"), ]), - definition: MacroDefinition::Table(duckdb().verified_query(query)), + definition: MacroDefinition::Table(duckdb().verified_query(query).into()), }; assert_eq!(expected, macro_); } diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 069500bf6..10bd374c0 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -418,10 +418,7 @@ fn parse_create_function() { } // Test error in dialect that doesn't support parsing CREATE FUNCTION - let unsupported_dialects = TestedDialects { - dialects: vec![Box::new(MsSqlDialect {})], - options: None, - }; + let unsupported_dialects = TestedDialects::new(vec![Box::new(MsSqlDialect {})]); assert_eq!( unsupported_dialects.parse_sql_statements(sql).unwrap_err(), @@ -538,15 +535,9 @@ fn parse_use() { } fn hive() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(HiveDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(HiveDialect {})]) } fn hive_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(HiveDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(HiveDialect {}), Box::new(GenericDialect {})]) } diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 58765f6c0..0223e2915 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -1030,14 +1030,8 @@ fn parse_create_table_with_identity_column() { } fn ms() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(MsSqlDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(MsSqlDialect {})]) } fn ms_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(MsSqlDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(MsSqlDialect {}), Box::new(GenericDialect {})]) } diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 19dbda21f..db5b9ec8d 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -944,11 +944,7 @@ fn parse_quote_identifiers() { fn parse_escaped_quote_identifiers_with_escape() { let sql = "SELECT `quoted `` identifier`"; assert_eq!( - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: None, - } - .verified_stmt(sql), + TestedDialects::new(vec![Box::new(MySqlDialect {})]).verified_stmt(sql), Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { @@ -991,13 +987,13 @@ fn parse_escaped_quote_identifiers_with_escape() { fn parse_escaped_quote_identifiers_with_no_escape() { let sql = "SELECT `quoted `` identifier`"; assert_eq!( - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: Some(ParserOptions { + TestedDialects::new_with_options( + vec![Box::new(MySqlDialect {})], + ParserOptions { trailing_commas: false, unescape: false, - }), - } + } + ) .verified_stmt(sql), Statement::Query(Box::new(Query { with: None, @@ -1041,11 +1037,7 @@ fn parse_escaped_quote_identifiers_with_no_escape() { fn parse_escaped_backticks_with_escape() { let sql = "SELECT ```quoted identifier```"; assert_eq!( - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: None, - } - .verified_stmt(sql), + TestedDialects::new(vec![Box::new(MySqlDialect {})]).verified_stmt(sql), Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { @@ -1088,10 +1080,10 @@ fn parse_escaped_backticks_with_escape() { fn parse_escaped_backticks_with_no_escape() { let sql = "SELECT ```quoted identifier```"; assert_eq!( - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: Some(ParserOptions::new().with_unescape(false)), - } + TestedDialects::new_with_options( + vec![Box::new(MySqlDialect {})], + ParserOptions::new().with_unescape(false) + ) .verified_stmt(sql), Statement::Query(Box::new(Query { with: None, @@ -1144,55 +1136,26 @@ fn parse_unterminated_escape() { #[test] fn check_roundtrip_of_escaped_string() { - let options = Some(ParserOptions::new().with_unescape(false)); - - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: options.clone(), - } - .verified_stmt(r"SELECT 'I\'m fine'"); - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: options.clone(), - } - .verified_stmt(r#"SELECT 'I''m fine'"#); - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: options.clone(), - } - .verified_stmt(r"SELECT 'I\\\'m fine'"); - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: options.clone(), - } - .verified_stmt(r"SELECT 'I\\\'m fine'"); - - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: options.clone(), - } - .verified_stmt(r#"SELECT "I\"m fine""#); - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: options.clone(), - } - .verified_stmt(r#"SELECT "I""m fine""#); - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: options.clone(), - } - .verified_stmt(r#"SELECT "I\\\"m fine""#); - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: options.clone(), - } - .verified_stmt(r#"SELECT "I\\\"m fine""#); - - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options, - } - .verified_stmt(r#"SELECT "I'm ''fine''""#); + let options = ParserOptions::new().with_unescape(false); + + TestedDialects::new_with_options(vec![Box::new(MySqlDialect {})], options.clone()) + .verified_stmt(r"SELECT 'I\'m fine'"); + TestedDialects::new_with_options(vec![Box::new(MySqlDialect {})], options.clone()) + .verified_stmt(r#"SELECT 'I''m fine'"#); + TestedDialects::new_with_options(vec![Box::new(MySqlDialect {})], options.clone()) + .verified_stmt(r"SELECT 'I\\\'m fine'"); + TestedDialects::new_with_options(vec![Box::new(MySqlDialect {})], options.clone()) + .verified_stmt(r"SELECT 'I\\\'m fine'"); + TestedDialects::new_with_options(vec![Box::new(MySqlDialect {})], options.clone()) + .verified_stmt(r#"SELECT "I\"m fine""#); + TestedDialects::new_with_options(vec![Box::new(MySqlDialect {})], options.clone()) + .verified_stmt(r#"SELECT "I""m fine""#); + TestedDialects::new_with_options(vec![Box::new(MySqlDialect {})], options.clone()) + .verified_stmt(r#"SELECT "I\\\"m fine""#); + TestedDialects::new_with_options(vec![Box::new(MySqlDialect {})], options.clone()) + .verified_stmt(r#"SELECT "I\\\"m fine""#); + TestedDialects::new_with_options(vec![Box::new(MySqlDialect {})], options.clone()) + .verified_stmt(r#"SELECT "I'm ''fine''""#); } #[test] @@ -2624,17 +2587,11 @@ fn parse_create_table_with_fulltext_definition_should_not_accept_constraint_name } fn mysql() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(MySqlDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(MySqlDialect {})]) } fn mysql_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(MySqlDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(MySqlDialect {}), Box::new(GenericDialect {})]) } #[test] diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index bd37214ce..b9b3811ba 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -2973,17 +2973,14 @@ fn parse_on_commit() { } fn pg() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(PostgreSqlDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(PostgreSqlDialect {})]) } fn pg_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(PostgreSqlDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![ + Box::new(PostgreSqlDialect {}), + Box::new(GenericDialect {}), + ]) } #[test] diff --git a/tests/sqlparser_redshift.rs b/tests/sqlparser_redshift.rs index eeba37957..a25d50605 100644 --- a/tests/sqlparser_redshift.rs +++ b/tests/sqlparser_redshift.rs @@ -171,17 +171,14 @@ fn parse_delimited_identifiers() { } fn redshift() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(RedshiftSqlDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(RedshiftSqlDialect {})]) } fn redshift_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(RedshiftSqlDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![ + Box::new(RedshiftSqlDialect {}), + Box::new(GenericDialect {}), + ]) } #[test] diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index d7e967ffe..c17c7b958 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -854,10 +854,8 @@ fn parse_sf_create_or_replace_view_with_comment_missing_equal() { #[test] fn parse_sf_create_or_replace_with_comment_for_snowflake() { let sql = "CREATE OR REPLACE VIEW v COMMENT = 'hello, world' AS SELECT 1"; - let dialect = test_utils::TestedDialects { - dialects: vec![Box::new(SnowflakeDialect {}) as Box], - options: None, - }; + let dialect = + test_utils::TestedDialects::new(vec![Box::new(SnowflakeDialect {}) as Box]); match dialect.verified_stmt(sql) { Statement::CreateView { @@ -1250,24 +1248,25 @@ fn test_array_agg_func() { } fn snowflake() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(SnowflakeDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(SnowflakeDialect {})]) +} + +fn snowflake_with_recursion_limit(recursion_limit: usize) -> TestedDialects { + TestedDialects::new(vec![Box::new(SnowflakeDialect {})]).with_recursion_limit(recursion_limit) } fn snowflake_without_unescape() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(SnowflakeDialect {})], - options: Some(ParserOptions::new().with_unescape(false)), - } + TestedDialects::new_with_options( + vec![Box::new(SnowflakeDialect {})], + ParserOptions::new().with_unescape(false), + ) } fn snowflake_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(SnowflakeDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![ + Box::new(SnowflakeDialect {}), + Box::new(GenericDialect {}), + ]) } #[test] @@ -2759,3 +2758,26 @@ fn parse_view_column_descriptions() { _ => unreachable!(), }; } + +#[test] +fn test_parentheses_overflow() { + let max_nesting_level: usize = 30; + + // Verify the recursion check is not too wasteful... (num of parentheses - 2 is acceptable) + let slack = 2; + let l_parens = "(".repeat(max_nesting_level - slack); + let r_parens = ")".repeat(max_nesting_level - slack); + let sql = format!("SELECT * FROM {l_parens}a.b.c{r_parens}"); + let parsed = + snowflake_with_recursion_limit(max_nesting_level).parse_sql_statements(sql.as_str()); + assert_eq!(parsed.err(), None); + + // Verify the recursion check triggers... (num of parentheses - 1 is acceptable) + let slack = 1; + let l_parens = "(".repeat(max_nesting_level - slack); + let r_parens = ")".repeat(max_nesting_level - slack); + let sql = format!("SELECT * FROM {l_parens}a.b.c{r_parens}"); + let parsed = + snowflake_with_recursion_limit(max_nesting_level).parse_sql_statements(sql.as_str()); + assert_eq!(parsed.err(), Some(ParserError::RecursionLimitExceeded)); +} diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index d3e670e32..6f8bbb2d8 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -529,14 +529,13 @@ fn parse_start_transaction_with_modifier() { sqlite_and_generic().one_statement_parses_to("BEGIN IMMEDIATE", "BEGIN IMMEDIATE TRANSACTION"); sqlite_and_generic().one_statement_parses_to("BEGIN EXCLUSIVE", "BEGIN EXCLUSIVE TRANSACTION"); - let unsupported_dialects = TestedDialects { - dialects: all_dialects() + let unsupported_dialects = TestedDialects::new( + all_dialects() .dialects .into_iter() .filter(|x| !(x.is::() || x.is::())) .collect(), - options: None, - }; + ); let res = unsupported_dialects.parse_sql_statements("BEGIN DEFERRED"); assert_eq!( ParserError::ParserError("Expected: end of statement, found: DEFERRED".to_string()), @@ -571,22 +570,16 @@ fn test_dollar_identifier_as_placeholder() { } fn sqlite() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(SQLiteDialect {})], - options: None, - } + TestedDialects::new(vec![Box::new(SQLiteDialect {})]) } fn sqlite_with_options(options: ParserOptions) -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(SQLiteDialect {})], - options: Some(options), - } + TestedDialects::new_with_options(vec![Box::new(SQLiteDialect {})], options) } fn sqlite_and_generic() -> TestedDialects { - TestedDialects { - dialects: vec![Box::new(SQLiteDialect {}), Box::new(GenericDialect {})], - options: None, - } + TestedDialects::new(vec![ + Box::new(SQLiteDialect {}), + Box::new(GenericDialect {}), + ]) } From ee90373d35e47823c7c0b3afc39beafd1d29f9ca Mon Sep 17 00:00:00 2001 From: Yoav Cohen <59807311+yoavcloud@users.noreply.github.com> Date: Tue, 29 Oct 2024 19:36:47 +0100 Subject: [PATCH 34/80] Fix build (#1483) --- src/dialect/postgresql.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index 51dc49849..361cc74be 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -230,15 +230,17 @@ pub fn parse_comment(parser: &mut Parser) -> Result { } pub fn parse_create(parser: &mut Parser) -> Option> { - let name = parser.maybe_parse(|parser| -> Result { + match parser.maybe_parse(|parser| -> Result { parser.expect_keyword(Keyword::CREATE)?; parser.expect_keyword(Keyword::TYPE)?; let name = parser.parse_object_name(false)?; parser.expect_keyword(Keyword::AS)?; parser.expect_keyword(Keyword::ENUM)?; Ok(name) - }); - name.map(|name| parse_create_type_as_enum(parser, name)) + }) { + Ok(name) => name.map(|name| parse_create_type_as_enum(parser, name)), + Err(e) => Some(Err(e)), + } } // https://www.postgresql.org/docs/current/sql-createtype.html From 8de3cb00742242974ead0632ecfdcd136cd6abd7 Mon Sep 17 00:00:00 2001 From: hulk Date: Fri, 1 Nov 2024 23:20:19 +0800 Subject: [PATCH 35/80] Fix complex blocks warning when running clippy (#1488) --- src/dialect/postgresql.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index 361cc74be..dc458ec5d 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -230,14 +230,16 @@ pub fn parse_comment(parser: &mut Parser) -> Result { } pub fn parse_create(parser: &mut Parser) -> Option> { - match parser.maybe_parse(|parser| -> Result { + let name = parser.maybe_parse(|parser| -> Result { parser.expect_keyword(Keyword::CREATE)?; parser.expect_keyword(Keyword::TYPE)?; let name = parser.parse_object_name(false)?; parser.expect_keyword(Keyword::AS)?; parser.expect_keyword(Keyword::ENUM)?; Ok(name) - }) { + }); + + match name { Ok(name) => name.map(|name| parse_create_type_as_enum(parser, name)), Err(e) => Some(Err(e)), } From e2197eeca9ef2d51a26f29ac23c15515aa668a0f Mon Sep 17 00:00:00 2001 From: Yoav Cohen <59807311+yoavcloud@users.noreply.github.com> Date: Sun, 3 Nov 2024 14:07:06 +0100 Subject: [PATCH 36/80] Add support for SHOW DATABASES/SCHEMAS/TABLES/VIEWS in Hive (#1487) --- src/ast/mod.rs | 81 ++++++++++++++++++++++++++++++++++++++- src/keywords.rs | 3 ++ src/parser/mod.rs | 50 ++++++++++++++++++++++-- tests/sqlparser_common.rs | 21 ++++++++++ tests/sqlparser_mysql.rs | 9 ++++- 5 files changed, 157 insertions(+), 7 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 0573240a2..b2672552e 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2782,12 +2782,29 @@ pub enum Statement { filter: Option, }, /// ```sql + /// SHOW DATABASES [LIKE 'pattern'] + /// ``` + ShowDatabases { filter: Option }, + /// ```sql + /// SHOW SCHEMAS [LIKE 'pattern'] + /// ``` + ShowSchemas { filter: Option }, + /// ```sql /// SHOW TABLES /// ``` - /// Note: this is a MySQL-specific statement. ShowTables { extended: bool, full: bool, + clause: Option, + db_name: Option, + filter: Option, + }, + /// ```sql + /// SHOW VIEWS + /// ``` + ShowViews { + materialized: bool, + clause: Option, db_name: Option, filter: Option, }, @@ -4363,9 +4380,24 @@ impl fmt::Display for Statement { } Ok(()) } + Statement::ShowDatabases { filter } => { + write!(f, "SHOW DATABASES")?; + if let Some(filter) = filter { + write!(f, " {filter}")?; + } + Ok(()) + } + Statement::ShowSchemas { filter } => { + write!(f, "SHOW SCHEMAS")?; + if let Some(filter) = filter { + write!(f, " {filter}")?; + } + Ok(()) + } Statement::ShowTables { extended, full, + clause: show_clause, db_name, filter, } => { @@ -4375,8 +4407,33 @@ impl fmt::Display for Statement { extended = if *extended { "EXTENDED " } else { "" }, full = if *full { "FULL " } else { "" }, )?; + if let Some(show_clause) = show_clause { + write!(f, " {show_clause}")?; + } + if let Some(db_name) = db_name { + write!(f, " {db_name}")?; + } + if let Some(filter) = filter { + write!(f, " {filter}")?; + } + Ok(()) + } + Statement::ShowViews { + materialized, + clause: show_clause, + db_name, + filter, + } => { + write!( + f, + "SHOW {}VIEWS", + if *materialized { "MATERIALIZED " } else { "" } + )?; + if let Some(show_clause) = show_clause { + write!(f, " {show_clause}")?; + } if let Some(db_name) = db_name { - write!(f, " FROM {db_name}")?; + write!(f, " {db_name}")?; } if let Some(filter) = filter { write!(f, " {filter}")?; @@ -6057,6 +6114,7 @@ pub enum ShowStatementFilter { Like(String), ILike(String), Where(Expr), + NoKeyword(String), } impl fmt::Display for ShowStatementFilter { @@ -6066,6 +6124,25 @@ impl fmt::Display for ShowStatementFilter { Like(pattern) => write!(f, "LIKE '{}'", value::escape_single_quote_string(pattern)), ILike(pattern) => write!(f, "ILIKE {}", value::escape_single_quote_string(pattern)), Where(expr) => write!(f, "WHERE {expr}"), + NoKeyword(pattern) => write!(f, "'{}'", value::escape_single_quote_string(pattern)), + } + } +} + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum ShowClause { + IN, + FROM, +} + +impl fmt::Display for ShowClause { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use ShowClause::*; + match self { + FROM => write!(f, "FROM"), + IN => write!(f, "IN"), } } } diff --git a/src/keywords.rs b/src/keywords.rs index 6182ae176..e98309681 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -217,6 +217,7 @@ define_keywords!( CYCLE, DATA, DATABASE, + DATABASES, DATA_RETENTION_TIME_IN_DAYS, DATE, DATE32, @@ -662,6 +663,7 @@ define_keywords!( SAFE_CAST, SAVEPOINT, SCHEMA, + SCHEMAS, SCOPE, SCROLL, SEARCH, @@ -822,6 +824,7 @@ define_keywords!( VERSION, VERSIONING, VIEW, + VIEWS, VIRTUAL, VOLATILE, WAREHOUSE, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a9a5b1df4..c4b92ba4e 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -9579,6 +9579,10 @@ impl<'a> Parser<'a> { Ok(self.parse_show_columns(extended, full)?) } else if self.parse_keyword(Keyword::TABLES) { Ok(self.parse_show_tables(extended, full)?) + } else if self.parse_keywords(&[Keyword::MATERIALIZED, Keyword::VIEWS]) { + Ok(self.parse_show_views(true)?) + } else if self.parse_keyword(Keyword::VIEWS) { + Ok(self.parse_show_views(false)?) } else if self.parse_keyword(Keyword::FUNCTIONS) { Ok(self.parse_show_functions()?) } else if extended || full { @@ -9605,6 +9609,10 @@ impl<'a> Parser<'a> { session, global, }) + } else if self.parse_keyword(Keyword::DATABASES) { + self.parse_show_databases() + } else if self.parse_keyword(Keyword::SCHEMAS) { + self.parse_show_schemas() } else { Ok(Statement::ShowVariable { variable: self.parse_identifiers()?, @@ -9612,6 +9620,18 @@ impl<'a> Parser<'a> { } } + fn parse_show_databases(&mut self) -> Result { + Ok(Statement::ShowDatabases { + filter: self.parse_show_statement_filter()?, + }) + } + + fn parse_show_schemas(&mut self) -> Result { + Ok(Statement::ShowSchemas { + filter: self.parse_show_statement_filter()?, + }) + } + pub fn parse_show_create(&mut self) -> Result { let obj_type = match self.expect_one_of_keywords(&[ Keyword::TABLE, @@ -9667,14 +9687,31 @@ impl<'a> Parser<'a> { extended: bool, full: bool, ) -> Result { - let db_name = match self.parse_one_of_keywords(&[Keyword::FROM, Keyword::IN]) { - Some(_) => Some(self.parse_identifier(false)?), - None => None, + let (clause, db_name) = match self.parse_one_of_keywords(&[Keyword::FROM, Keyword::IN]) { + Some(Keyword::FROM) => (Some(ShowClause::FROM), Some(self.parse_identifier(false)?)), + Some(Keyword::IN) => (Some(ShowClause::IN), Some(self.parse_identifier(false)?)), + _ => (None, None), }; let filter = self.parse_show_statement_filter()?; Ok(Statement::ShowTables { extended, full, + clause, + db_name, + filter, + }) + } + + fn parse_show_views(&mut self, materialized: bool) -> Result { + let (clause, db_name) = match self.parse_one_of_keywords(&[Keyword::FROM, Keyword::IN]) { + Some(Keyword::FROM) => (Some(ShowClause::FROM), Some(self.parse_identifier(false)?)), + Some(Keyword::IN) => (Some(ShowClause::IN), Some(self.parse_identifier(false)?)), + _ => (None, None), + }; + let filter = self.parse_show_statement_filter()?; + Ok(Statement::ShowViews { + materialized, + clause, db_name, filter, }) @@ -9704,7 +9741,12 @@ impl<'a> Parser<'a> { } else if self.parse_keyword(Keyword::WHERE) { Ok(Some(ShowStatementFilter::Where(self.parse_expr()?))) } else { - Ok(None) + self.maybe_parse(|parser| -> Result { + parser.parse_literal_string() + })? + .map_or(Ok(None), |filter| { + Ok(Some(ShowStatementFilter::NoKeyword(filter))) + }) } } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index a2eb5070d..4016e5a69 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11378,3 +11378,24 @@ fn test_try_convert() { all_dialects_where(|d| d.supports_try_convert() && !d.convert_type_before_value()); dialects.verified_expr("TRY_CONVERT('foo', VARCHAR(MAX))"); } + +#[test] +fn test_show_dbs_schemas_tables_views() { + verified_stmt("SHOW DATABASES"); + verified_stmt("SHOW DATABASES LIKE '%abc'"); + verified_stmt("SHOW SCHEMAS"); + verified_stmt("SHOW SCHEMAS LIKE '%abc'"); + verified_stmt("SHOW TABLES"); + verified_stmt("SHOW TABLES IN db1"); + verified_stmt("SHOW TABLES IN db1 'abc'"); + verified_stmt("SHOW VIEWS"); + verified_stmt("SHOW VIEWS IN db1"); + verified_stmt("SHOW VIEWS IN db1 'abc'"); + verified_stmt("SHOW VIEWS FROM db1"); + verified_stmt("SHOW VIEWS FROM db1 'abc'"); + verified_stmt("SHOW MATERIALIZED VIEWS"); + verified_stmt("SHOW MATERIALIZED VIEWS IN db1"); + verified_stmt("SHOW MATERIALIZED VIEWS IN db1 'abc'"); + verified_stmt("SHOW MATERIALIZED VIEWS FROM db1"); + verified_stmt("SHOW MATERIALIZED VIEWS FROM db1 'abc'"); +} diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index db5b9ec8d..4b9354e85 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -329,6 +329,7 @@ fn parse_show_tables() { Statement::ShowTables { extended: false, full: false, + clause: None, db_name: None, filter: None, } @@ -338,6 +339,7 @@ fn parse_show_tables() { Statement::ShowTables { extended: false, full: false, + clause: Some(ShowClause::FROM), db_name: Some(Ident::new("mydb")), filter: None, } @@ -347,6 +349,7 @@ fn parse_show_tables() { Statement::ShowTables { extended: true, full: false, + clause: None, db_name: None, filter: None, } @@ -356,6 +359,7 @@ fn parse_show_tables() { Statement::ShowTables { extended: false, full: true, + clause: None, db_name: None, filter: None, } @@ -365,6 +369,7 @@ fn parse_show_tables() { Statement::ShowTables { extended: false, full: false, + clause: None, db_name: None, filter: Some(ShowStatementFilter::Like("pattern".into())), } @@ -374,13 +379,15 @@ fn parse_show_tables() { Statement::ShowTables { extended: false, full: false, + clause: None, db_name: None, filter: Some(ShowStatementFilter::Where( mysql_and_generic().verified_expr("1 = 2") )), } ); - mysql_and_generic().one_statement_parses_to("SHOW TABLES IN mydb", "SHOW TABLES FROM mydb"); + mysql_and_generic().verified_stmt("SHOW TABLES IN mydb"); + mysql_and_generic().verified_stmt("SHOW TABLES FROM mydb"); } #[test] From a9a9d58c389a77f4ea3fc4ee2395b34739b64c62 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 6 Nov 2024 10:47:01 -0500 Subject: [PATCH 37/80] Fix typo in `Dialect::supports_eq_alias_assigment` (#1478) --- src/dialect/mod.rs | 2 +- src/dialect/mssql.rs | 2 +- src/parser/mod.rs | 2 +- tests/sqlparser_common.rs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 28e7ac7d1..7e43439a4 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -582,7 +582,7 @@ pub trait Dialect: Debug + Any { /// SELECT col_alias = col FROM tbl; /// SELECT col_alias AS col FROM tbl; /// ``` - fn supports_eq_alias_assigment(&self) -> bool { + fn supports_eq_alias_assignment(&self) -> bool { false } diff --git a/src/dialect/mssql.rs b/src/dialect/mssql.rs index 78ec621ed..a5ee0bf75 100644 --- a/src/dialect/mssql.rs +++ b/src/dialect/mssql.rs @@ -50,7 +50,7 @@ impl Dialect for MsSqlDialect { true } - fn supports_eq_alias_assigment(&self) -> bool { + fn supports_eq_alias_assignment(&self) -> bool { true } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c4b92ba4e..1ad637f90 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -11235,7 +11235,7 @@ impl<'a> Parser<'a> { left, op: BinaryOperator::Eq, right, - } if self.dialect.supports_eq_alias_assigment() + } if self.dialect.supports_eq_alias_assignment() && matches!(left.as_ref(), Expr::Identifier(_)) => { let Expr::Identifier(alias) = *left else { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 4016e5a69..94dfcfec1 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11353,7 +11353,7 @@ fn test_any_some_all_comparison() { #[test] fn test_alias_equal_expr() { - let dialects = all_dialects_where(|d| d.supports_eq_alias_assigment()); + let dialects = all_dialects_where(|d| d.supports_eq_alias_assignment()); let sql = r#"SELECT some_alias = some_column FROM some_table"#; let expected = r#"SELECT some_column AS some_alias FROM some_table"#; let _ = dialects.one_statement_parses_to(sql, expected); @@ -11362,7 +11362,7 @@ fn test_alias_equal_expr() { let expected = r#"SELECT (a * b) AS some_alias FROM some_table"#; let _ = dialects.one_statement_parses_to(sql, expected); - let dialects = all_dialects_where(|d| !d.supports_eq_alias_assigment()); + let dialects = all_dialects_where(|d| !d.supports_eq_alias_assignment()); let sql = r#"SELECT x = (a * b) FROM some_table"#; let expected = r#"SELECT x = (a * b) FROM some_table"#; let _ = dialects.one_statement_parses_to(sql, expected); From 05821cc7db7b499528ac47a93d8d86a43c889933 Mon Sep 17 00:00:00 2001 From: wugeer <1284057728@qq.com> Date: Wed, 6 Nov 2024 23:51:08 +0800 Subject: [PATCH 38/80] Add support for PostgreSQL `LISTEN/NOTIFY` syntax (#1485) --- src/ast/mod.rs | 28 +++++++++++++++ src/dialect/mod.rs | 10 ++++++ src/dialect/postgresql.rs | 10 ++++++ src/keywords.rs | 2 ++ src/parser/mod.rs | 19 ++++++++++ tests/sqlparser_common.rs | 76 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 145 insertions(+) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index b2672552e..2ef3a460b 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -3295,6 +3295,23 @@ pub enum Statement { include_final: bool, deduplicate: Option, }, + /// ```sql + /// LISTEN + /// ``` + /// listen for a notification channel + /// + /// See Postgres + LISTEN { channel: Ident }, + /// ```sql + /// NOTIFY channel [ , payload ] + /// ``` + /// send a notification event together with an optional “payload” string to channel + /// + /// See Postgres + NOTIFY { + channel: Ident, + payload: Option, + }, } impl fmt::Display for Statement { @@ -4839,6 +4856,17 @@ impl fmt::Display for Statement { } Ok(()) } + Statement::LISTEN { channel } => { + write!(f, "LISTEN {channel}")?; + Ok(()) + } + Statement::NOTIFY { channel, payload } => { + write!(f, "NOTIFY {channel}")?; + if let Some(payload) = payload { + write!(f, ", '{payload}'")?; + } + Ok(()) + } } } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 7e43439a4..5abddba38 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -590,6 +590,16 @@ pub trait Dialect: Debug + Any { fn supports_try_convert(&self) -> bool { false } + + /// Returns true if the dialect supports the `LISTEN` statement + fn supports_listen(&self) -> bool { + false + } + + /// Returns true if the dialect supports the `NOTIFY` statement + fn supports_notify(&self) -> bool { + false + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index dc458ec5d..c40c826c4 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -191,6 +191,16 @@ impl Dialect for PostgreSqlDialect { fn supports_explain_with_utility_options(&self) -> bool { true } + + /// see + fn supports_listen(&self) -> bool { + true + } + + /// see + fn supports_notify(&self) -> bool { + true + } } pub fn parse_comment(parser: &mut Parser) -> Result { diff --git a/src/keywords.rs b/src/keywords.rs index e98309681..d60227c99 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -438,6 +438,7 @@ define_keywords!( LIKE_REGEX, LIMIT, LINES, + LISTEN, LN, LOAD, LOCAL, @@ -513,6 +514,7 @@ define_keywords!( NOSUPERUSER, NOT, NOTHING, + NOTIFY, NOWAIT, NO_WRITE_TO_BINLOG, NTH_VALUE, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 1ad637f90..fd7d1c578 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -532,6 +532,10 @@ impl<'a> Parser<'a> { Keyword::EXECUTE => self.parse_execute(), Keyword::PREPARE => self.parse_prepare(), Keyword::MERGE => self.parse_merge(), + // `LISTEN` and `NOTIFY` are Postgres-specific + // syntaxes. They are used for Postgres statement. + Keyword::LISTEN if self.dialect.supports_listen() => self.parse_listen(), + Keyword::NOTIFY if self.dialect.supports_notify() => self.parse_notify(), // `PRAGMA` is sqlite specific https://www.sqlite.org/pragma.html Keyword::PRAGMA => self.parse_pragma(), Keyword::UNLOAD => self.parse_unload(), @@ -946,6 +950,21 @@ impl<'a> Parser<'a> { Ok(Statement::ReleaseSavepoint { name }) } + pub fn parse_listen(&mut self) -> Result { + let channel = self.parse_identifier(false)?; + Ok(Statement::LISTEN { channel }) + } + + pub fn parse_notify(&mut self) -> Result { + let channel = self.parse_identifier(false)?; + let payload = if self.consume_token(&Token::Comma) { + Some(self.parse_literal_string()?) + } else { + None + }; + Ok(Statement::NOTIFY { channel, payload }) + } + /// Parse an expression prefix. pub fn parse_prefix(&mut self) -> Result { // allow the dialect to override prefix parsing diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 94dfcfec1..334dae2b3 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11399,3 +11399,79 @@ fn test_show_dbs_schemas_tables_views() { verified_stmt("SHOW MATERIALIZED VIEWS FROM db1"); verified_stmt("SHOW MATERIALIZED VIEWS FROM db1 'abc'"); } + +#[test] +fn parse_listen_channel() { + let dialects = all_dialects_where(|d| d.supports_listen()); + + match dialects.verified_stmt("LISTEN test1") { + Statement::LISTEN { channel } => { + assert_eq!(Ident::new("test1"), channel); + } + _ => unreachable!(), + }; + + assert_eq!( + dialects.parse_sql_statements("LISTEN *").unwrap_err(), + ParserError::ParserError("Expected: identifier, found: *".to_string()) + ); + + let dialects = all_dialects_where(|d| !d.supports_listen()); + + assert_eq!( + dialects.parse_sql_statements("LISTEN test1").unwrap_err(), + ParserError::ParserError("Expected: an SQL statement, found: LISTEN".to_string()) + ); +} + +#[test] +fn parse_notify_channel() { + let dialects = all_dialects_where(|d| d.supports_notify()); + + match dialects.verified_stmt("NOTIFY test1") { + Statement::NOTIFY { channel, payload } => { + assert_eq!(Ident::new("test1"), channel); + assert_eq!(payload, None); + } + _ => unreachable!(), + }; + + match dialects.verified_stmt("NOTIFY test1, 'this is a test notification'") { + Statement::NOTIFY { + channel, + payload: Some(payload), + } => { + assert_eq!(Ident::new("test1"), channel); + assert_eq!("this is a test notification", payload); + } + _ => unreachable!(), + }; + + assert_eq!( + dialects.parse_sql_statements("NOTIFY *").unwrap_err(), + ParserError::ParserError("Expected: identifier, found: *".to_string()) + ); + assert_eq!( + dialects + .parse_sql_statements("NOTIFY test1, *") + .unwrap_err(), + ParserError::ParserError("Expected: literal string, found: *".to_string()) + ); + + let sql_statements = [ + "NOTIFY test1", + "NOTIFY test1, 'this is a test notification'", + ]; + let dialects = all_dialects_where(|d| !d.supports_notify()); + + for &sql in &sql_statements { + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError("Expected: an SQL statement, found: NOTIFY".to_string()) + ); + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError("Expected: an SQL statement, found: NOTIFY".to_string()) + ); + } +} From a5b0092506afd871c42a7ff2421d42091600f0d3 Mon Sep 17 00:00:00 2001 From: Yoav Cohen <59807311+yoavcloud@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:09:55 +0100 Subject: [PATCH 39/80] Add support for TOP before ALL/DISTINCT (#1495) --- src/ast/query.rs | 12 +++++++++++- src/dialect/mod.rs | 6 ++++++ src/dialect/redshift.rs | 6 ++++++ src/parser/mod.rs | 16 ++++++++++------ tests/sqlparser_clickhouse.rs | 1 + tests/sqlparser_common.rs | 18 ++++++++++++++++++ tests/sqlparser_duckdb.rs | 2 ++ tests/sqlparser_mssql.rs | 2 ++ tests/sqlparser_mysql.rs | 8 ++++++++ tests/sqlparser_postgres.rs | 3 +++ 10 files changed, 67 insertions(+), 7 deletions(-) diff --git a/src/ast/query.rs b/src/ast/query.rs index dc5966e5e..6767662d5 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -279,6 +279,8 @@ pub struct Select { pub distinct: Option, /// MSSQL syntax: `TOP () [ PERCENT ] [ WITH TIES ]` pub top: Option, + /// Whether the top was located before `ALL`/`DISTINCT` + pub top_before_distinct: bool, /// projection expressions pub projection: Vec, /// INTO @@ -327,12 +329,20 @@ impl fmt::Display for Select { write!(f, " {value_table_mode}")?; } + if let Some(ref top) = self.top { + if self.top_before_distinct { + write!(f, " {top}")?; + } + } if let Some(ref distinct) = self.distinct { write!(f, " {distinct}")?; } if let Some(ref top) = self.top { - write!(f, " {top}")?; + if !self.top_before_distinct { + write!(f, " {top}")?; + } } + write!(f, " {}", display_comma_separated(&self.projection))?; if let Some(ref into) = self.into { diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 5abddba38..453fee3de 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -600,6 +600,12 @@ pub trait Dialect: Debug + Any { fn supports_notify(&self) -> bool { false } + + /// Returns true if this dialect expects the the `TOP` option + /// before the `ALL`/`DISTINCT` options in a `SELECT` statement. + fn supports_top_before_distinct(&self) -> bool { + false + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/redshift.rs b/src/dialect/redshift.rs index 3bfdec3b0..4d0773843 100644 --- a/src/dialect/redshift.rs +++ b/src/dialect/redshift.rs @@ -68,4 +68,10 @@ impl Dialect for RedshiftSqlDialect { fn supports_connect_by(&self) -> bool { true } + + /// Redshift expects the `TOP` option before the `ALL/DISTINCT` option: + /// + fn supports_top_before_distinct(&self) -> bool { + true + } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index fd7d1c578..de11ba7c9 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -9193,13 +9193,16 @@ impl<'a> Parser<'a> { None }; + let mut top_before_distinct = false; + let mut top = None; + if self.dialect.supports_top_before_distinct() && self.parse_keyword(Keyword::TOP) { + top = Some(self.parse_top()?); + top_before_distinct = true; + } let distinct = self.parse_all_or_distinct()?; - - let top = if self.parse_keyword(Keyword::TOP) { - Some(self.parse_top()?) - } else { - None - }; + if !self.dialect.supports_top_before_distinct() && self.parse_keyword(Keyword::TOP) { + top = Some(self.parse_top()?); + } let projection = self.parse_projection()?; @@ -9342,6 +9345,7 @@ impl<'a> Parser<'a> { Ok(Select { distinct, top, + top_before_distinct, projection, into, from, diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index f8c349a37..a71871115 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -40,6 +40,7 @@ fn parse_map_access_expr() { Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![UnnamedExpr(MapAccess { column: Box::new(Identifier(Ident { value: "string_values".to_string(), diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 334dae2b3..49753a1f4 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -379,6 +379,7 @@ fn parse_update_set_from() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![ SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("name"))), SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("id"))), @@ -4649,6 +4650,7 @@ fn test_parse_named_window() { let expected = Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![ SelectItem::ExprWithAlias { expr: Expr::Function(Function { @@ -5289,6 +5291,7 @@ fn parse_interval_and_or_xor() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![UnnamedExpr(Expr::Identifier(Ident { value: "col".to_string(), quote_style: None, @@ -7367,6 +7370,7 @@ fn lateral_function() { let expected = Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions { opt_ilike: None, opt_exclude: None, @@ -8215,6 +8219,7 @@ fn parse_merge() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::Wildcard( WildcardAdditionalOptions::default() )], @@ -9803,6 +9808,7 @@ fn parse_unload() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![UnnamedExpr(Expr::Identifier(Ident::new("cola"))),], into: None, from: vec![TableWithJoins { @@ -9978,6 +9984,7 @@ fn parse_connect_by() { let expect_query = Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![ SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("employee_id"))), SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("manager_id"))), @@ -10064,6 +10071,7 @@ fn parse_connect_by() { Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![ SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("employee_id"))), SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("manager_id"))), @@ -11475,3 +11483,13 @@ fn parse_notify_channel() { ); } } + +#[test] +fn test_select_top() { + let dialects = all_dialects_where(|d| d.supports_top_before_distinct()); + dialects.one_statement_parses_to("SELECT ALL * FROM tbl", "SELECT * FROM tbl"); + dialects.verified_stmt("SELECT TOP 3 * FROM tbl"); + dialects.one_statement_parses_to("SELECT TOP 3 ALL * FROM tbl", "SELECT TOP 3 * FROM tbl"); + dialects.verified_stmt("SELECT TOP 3 DISTINCT * FROM tbl"); + dialects.verified_stmt("SELECT TOP 3 DISTINCT a, b, c FROM tbl"); +} diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index a4109b0a3..d68f37713 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -261,6 +261,7 @@ fn test_select_union_by_name() { left: Box::::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions { opt_ilike: None, opt_exclude: None, @@ -301,6 +302,7 @@ fn test_select_union_by_name() { right: Box::::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions { opt_ilike: None, opt_exclude: None, diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 0223e2915..c5f43b072 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -114,6 +114,7 @@ fn parse_create_procedure() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Value(number("1")))], into: None, from: vec![], @@ -514,6 +515,7 @@ fn parse_substring_in_select() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: Some(Distinct::Distinct), top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Substring { expr: Box::new(Expr::Identifier(Ident { value: "description".to_string(), diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 4b9354e85..6cd08df18 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -957,6 +957,7 @@ fn parse_escaped_quote_identifiers_with_escape() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "quoted ` identifier".into(), quote_style: Some('`'), @@ -1007,6 +1008,7 @@ fn parse_escaped_quote_identifiers_with_no_escape() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "quoted `` identifier".into(), quote_style: Some('`'), @@ -1050,6 +1052,7 @@ fn parse_escaped_backticks_with_escape() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "`quoted identifier`".into(), quote_style: Some('`'), @@ -1097,6 +1100,7 @@ fn parse_escaped_backticks_with_no_escape() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "``quoted identifier``".into(), quote_style: Some('`'), @@ -1741,6 +1745,7 @@ fn parse_select_with_numeric_prefix_column_name() { Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident::new( "123col_$@123abc" )))], @@ -1795,6 +1800,7 @@ fn parse_select_with_concatenation_of_exp_number_and_numeric_prefix_column() { Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![ SelectItem::UnnamedExpr(Expr::Value(number("123e4"))), SelectItem::UnnamedExpr(Expr::Identifier(Ident::new("123col_$@123abc"))) @@ -2295,6 +2301,7 @@ fn parse_substring_in_select() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: Some(Distinct::Distinct), top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Substring { expr: Box::new(Expr::Identifier(Ident { value: "description".to_string(), @@ -2616,6 +2623,7 @@ fn parse_hex_string_introducer() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::IntroducedString { introducer: "_latin1".to_string(), value: Value::HexStringLiteral("4D7953514C".to_string()) diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index b9b3811ba..c30603baa 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -1165,6 +1165,7 @@ fn parse_copy_to() { body: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![ SelectItem::ExprWithAlias { expr: Expr::Value(number("42")), @@ -2505,6 +2506,7 @@ fn parse_array_subquery_expr() { left: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Value(number("1")))], into: None, from: vec![], @@ -2525,6 +2527,7 @@ fn parse_array_subquery_expr() { right: Box::new(SetExpr::Select(Box::new(Select { distinct: None, top: None, + top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Value(number("2")))], into: None, from: vec![], From fc0e13b80ea76274891e05290be34b0478075245 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Wed, 6 Nov 2024 22:04:13 +0100 Subject: [PATCH 40/80] add support for `FOR ORDINALITY` and `NESTED` in JSON_TABLE (#1493) --- src/ast/mod.rs | 15 +++++---- src/ast/query.rs | 71 ++++++++++++++++++++++++++++++++++++++-- src/parser/mod.rs | 20 +++++++++-- tests/sqlparser_mysql.rs | 10 ++++-- 4 files changed, 102 insertions(+), 14 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 2ef3a460b..a24739a60 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -54,13 +54,14 @@ pub use self::query::{ ExceptSelectItem, ExcludeSelectItem, ExprWithAlias, Fetch, ForClause, ForJson, ForXml, FormatClause, GroupByExpr, GroupByWithModifier, IdentWithAlias, IlikeSelectItem, Interpolate, InterpolateExpr, Join, JoinConstraint, JoinOperator, JsonTableColumn, - JsonTableColumnErrorHandling, LateralView, LockClause, LockType, MatchRecognizePattern, - MatchRecognizeSymbol, Measure, NamedWindowDefinition, NamedWindowExpr, NonBlock, Offset, - OffsetRows, OrderBy, OrderByExpr, PivotValueSource, ProjectionSelect, Query, RenameSelectItem, - RepetitionQuantifier, ReplaceSelectElement, ReplaceSelectItem, RowsPerMatch, Select, - SelectInto, SelectItem, SetExpr, SetOperator, SetQuantifier, Setting, SymbolDefinition, Table, - TableAlias, TableFactor, TableFunctionArgs, TableVersion, TableWithJoins, Top, TopQuantity, - ValueTableMode, Values, WildcardAdditionalOptions, With, WithFill, + JsonTableColumnErrorHandling, JsonTableNamedColumn, JsonTableNestedColumn, LateralView, + LockClause, LockType, MatchRecognizePattern, MatchRecognizeSymbol, Measure, + NamedWindowDefinition, NamedWindowExpr, NonBlock, Offset, OffsetRows, OrderBy, OrderByExpr, + PivotValueSource, ProjectionSelect, Query, RenameSelectItem, RepetitionQuantifier, + ReplaceSelectElement, ReplaceSelectItem, RowsPerMatch, Select, SelectInto, SelectItem, SetExpr, + SetOperator, SetQuantifier, Setting, SymbolDefinition, Table, TableAlias, TableFactor, + TableFunctionArgs, TableVersion, TableWithJoins, Top, TopQuantity, ValueTableMode, Values, + WildcardAdditionalOptions, With, WithFill, }; pub use self::trigger::{ diff --git a/src/ast/query.rs b/src/ast/query.rs index 6767662d5..7af472430 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -2286,19 +2286,84 @@ impl fmt::Display for ForJson { } /// A single column definition in MySQL's `JSON_TABLE` table valued function. +/// +/// See +/// - [MySQL's JSON_TABLE documentation](https://dev.mysql.com/doc/refman/8.0/en/json-table-functions.html#function_json-table) +/// - [Oracle's JSON_TABLE documentation](https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/JSON_TABLE.html) +/// - [MariaDB's JSON_TABLE documentation](https://mariadb.com/kb/en/json_table/) +/// /// ```sql /// SELECT * /// FROM JSON_TABLE( /// '["a", "b"]', /// '$[*]' COLUMNS ( -/// value VARCHAR(20) PATH '$' +/// name FOR ORDINALITY, +/// value VARCHAR(20) PATH '$', +/// NESTED PATH '$[*]' COLUMNS ( +/// value VARCHAR(20) PATH '$' +/// ) /// ) /// ) AS jt; /// ``` #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -pub struct JsonTableColumn { +pub enum JsonTableColumn { + /// A named column with a JSON path + Named(JsonTableNamedColumn), + /// The FOR ORDINALITY column, which is a special column that returns the index of the current row in a JSON array. + ForOrdinality(Ident), + /// A set of nested columns, which extracts data from a nested JSON array. + Nested(JsonTableNestedColumn), +} + +impl fmt::Display for JsonTableColumn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JsonTableColumn::Named(json_table_named_column) => { + write!(f, "{json_table_named_column}") + } + JsonTableColumn::ForOrdinality(ident) => write!(f, "{} FOR ORDINALITY", ident), + JsonTableColumn::Nested(json_table_nested_column) => { + write!(f, "{json_table_nested_column}") + } + } + } +} + +/// A nested column in a JSON_TABLE column list +/// +/// See +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct JsonTableNestedColumn { + pub path: Value, + pub columns: Vec, +} + +impl fmt::Display for JsonTableNestedColumn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "NESTED PATH {} COLUMNS ({})", + self.path, + display_comma_separated(&self.columns) + ) + } +} + +/// A single column definition in MySQL's `JSON_TABLE` table valued function. +/// +/// See +/// +/// ```sql +/// value VARCHAR(20) PATH '$' +/// ``` +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct JsonTableNamedColumn { /// The name of the column to be extracted. pub name: Ident, /// The type of the column to be extracted. @@ -2313,7 +2378,7 @@ pub struct JsonTableColumn { pub on_error: Option, } -impl fmt::Display for JsonTableColumn { +impl fmt::Display for JsonTableNamedColumn { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index de11ba7c9..2bd454369 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -10466,7 +10466,23 @@ impl<'a> Parser<'a> { /// Parses MySQL's JSON_TABLE column definition. /// For example: `id INT EXISTS PATH '$' DEFAULT '0' ON EMPTY ERROR ON ERROR` pub fn parse_json_table_column_def(&mut self) -> Result { + if self.parse_keyword(Keyword::NESTED) { + let _has_path_keyword = self.parse_keyword(Keyword::PATH); + let path = self.parse_value()?; + self.expect_keyword(Keyword::COLUMNS)?; + let columns = self.parse_parenthesized(|p| { + p.parse_comma_separated(Self::parse_json_table_column_def) + })?; + return Ok(JsonTableColumn::Nested(JsonTableNestedColumn { + path, + columns, + })); + } let name = self.parse_identifier(false)?; + if self.parse_keyword(Keyword::FOR) { + self.expect_keyword(Keyword::ORDINALITY)?; + return Ok(JsonTableColumn::ForOrdinality(name)); + } let r#type = self.parse_data_type()?; let exists = self.parse_keyword(Keyword::EXISTS); self.expect_keyword(Keyword::PATH)?; @@ -10481,14 +10497,14 @@ impl<'a> Parser<'a> { on_error = Some(error_handling); } } - Ok(JsonTableColumn { + Ok(JsonTableColumn::Named(JsonTableNamedColumn { name, r#type, path, exists, on_empty, on_error, - }) + })) } fn parse_json_table_column_error_handling( diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 6cd08df18..47f7f5b4b 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -2773,6 +2773,12 @@ fn parse_json_table() { r#"SELECT * FROM JSON_TABLE('[1,2]', '$[*]' COLUMNS(x INT PATH '$' ERROR ON EMPTY)) AS t"#, ); mysql().verified_only_select(r#"SELECT * FROM JSON_TABLE('[1,2]', '$[*]' COLUMNS(x INT PATH '$' ERROR ON EMPTY DEFAULT '0' ON ERROR)) AS t"#); + mysql().verified_only_select( + r#"SELECT jt.* FROM JSON_TABLE('["Alice", "Bob", "Charlie"]', '$[*]' COLUMNS(row_num FOR ORDINALITY, name VARCHAR(50) PATH '$')) AS jt"#, + ); + mysql().verified_only_select( + r#"SELECT * FROM JSON_TABLE('[ {"a": 1, "b": [11,111]}, {"a": 2, "b": [22,222]}, {"a":3}]', '$[*]' COLUMNS(a INT PATH '$.a', NESTED PATH '$.b[*]' COLUMNS (b INT PATH '$'))) AS jt"#, + ); assert_eq!( mysql() .verified_only_select( @@ -2784,14 +2790,14 @@ fn parse_json_table() { json_expr: Expr::Value(Value::SingleQuotedString("[1,2]".to_string())), json_path: Value::SingleQuotedString("$[*]".to_string()), columns: vec![ - JsonTableColumn { + JsonTableColumn::Named(JsonTableNamedColumn { name: Ident::new("x"), r#type: DataType::Int(None), path: Value::SingleQuotedString("$".to_string()), exists: false, on_empty: Some(JsonTableColumnErrorHandling::Default(Value::SingleQuotedString("0".to_string()))), on_error: Some(JsonTableColumnErrorHandling::Null), - }, + }), ], alias: Some(TableAlias { name: Ident::new("t"), From 9394ad4c0cfa7f9a264ba2109764f3309c39c41d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 6 Nov 2024 16:48:49 -0500 Subject: [PATCH 41/80] Add Apache License to additional files (#1502) --- .github/dependabot.yml | 17 +++++++++++++++++ .github/workflows/rust.yml | 17 +++++++++++++++++ README.md | 19 +++++++++++++++++++ tests/queries/tpch/1.sql | 18 ++++++++++++++++++ tests/queries/tpch/10.sql | 17 +++++++++++++++++ tests/queries/tpch/11.sql | 17 +++++++++++++++++ tests/queries/tpch/12.sql | 17 +++++++++++++++++ tests/queries/tpch/13.sql | 17 +++++++++++++++++ tests/queries/tpch/14.sql | 17 +++++++++++++++++ tests/queries/tpch/15.sql | 17 +++++++++++++++++ tests/queries/tpch/16.sql | 17 +++++++++++++++++ tests/queries/tpch/17.sql | 17 +++++++++++++++++ tests/queries/tpch/18.sql | 17 +++++++++++++++++ tests/queries/tpch/19.sql | 17 +++++++++++++++++ tests/queries/tpch/2.sql | 18 +++++++++++++++++- tests/queries/tpch/20.sql | 17 +++++++++++++++++ tests/queries/tpch/21.sql | 17 +++++++++++++++++ tests/queries/tpch/22.sql | 17 +++++++++++++++++ tests/queries/tpch/3.sql | 17 +++++++++++++++++ tests/queries/tpch/4.sql | 17 +++++++++++++++++ tests/queries/tpch/5.sql | 17 +++++++++++++++++ tests/queries/tpch/6.sql | 19 ++++++++++++++++++- tests/queries/tpch/7.sql | 17 +++++++++++++++++ tests/queries/tpch/8.sql | 17 +++++++++++++++++ tests/queries/tpch/9.sql | 17 +++++++++++++++++ 25 files changed, 429 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a73d1dc4e..7d2903d29 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + version: 2 updates: - package-ecosystem: cargo diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4aab9cee7..253d8ab27 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + name: Rust on: [push, pull_request] diff --git a/README.md b/README.md index 3226b9549..934d9d06d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,22 @@ + + # Extensible SQL Lexer and Parser for Rust [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) diff --git a/tests/queries/tpch/1.sql b/tests/queries/tpch/1.sql index ae44c94d2..89232181e 100644 --- a/tests/queries/tpch/1.sql +++ b/tests/queries/tpch/1.sql @@ -1,3 +1,21 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + + select l_returnflag, l_linestatus, diff --git a/tests/queries/tpch/10.sql b/tests/queries/tpch/10.sql index a8de12995..06f6256b4 100644 --- a/tests/queries/tpch/10.sql +++ b/tests/queries/tpch/10.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/11.sql b/tests/queries/tpch/11.sql index f9cf254b5..6115eb59d 100644 --- a/tests/queries/tpch/11.sql +++ b/tests/queries/tpch/11.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/12.sql b/tests/queries/tpch/12.sql index ca9c494e7..0ffdb6668 100644 --- a/tests/queries/tpch/12.sql +++ b/tests/queries/tpch/12.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/13.sql b/tests/queries/tpch/13.sql index 32b0ebeb9..64572aba8 100644 --- a/tests/queries/tpch/13.sql +++ b/tests/queries/tpch/13.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/14.sql b/tests/queries/tpch/14.sql index 74f9643ef..abe7f2ea5 100644 --- a/tests/queries/tpch/14.sql +++ b/tests/queries/tpch/14.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/15.sql b/tests/queries/tpch/15.sql index 8b3b8c1ef..16646fb60 100644 --- a/tests/queries/tpch/15.sql +++ b/tests/queries/tpch/15.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions create view revenue0 (supplier_no, total_revenue) as diff --git a/tests/queries/tpch/16.sql b/tests/queries/tpch/16.sql index a0412fcbb..e7e489796 100644 --- a/tests/queries/tpch/16.sql +++ b/tests/queries/tpch/16.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/17.sql b/tests/queries/tpch/17.sql index d59bc18ab..d895fb27f 100644 --- a/tests/queries/tpch/17.sql +++ b/tests/queries/tpch/17.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/18.sql b/tests/queries/tpch/18.sql index e07956fed..54b8b85d3 100644 --- a/tests/queries/tpch/18.sql +++ b/tests/queries/tpch/18.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/19.sql b/tests/queries/tpch/19.sql index 908e08297..1aca74329 100644 --- a/tests/queries/tpch/19.sql +++ b/tests/queries/tpch/19.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/2.sql b/tests/queries/tpch/2.sql index f04c1d497..9426b91d6 100644 --- a/tests/queries/tpch/2.sql +++ b/tests/queries/tpch/2.sql @@ -1,5 +1,21 @@ --- using default substitutions +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- using default substitutions select s_acctbal, diff --git a/tests/queries/tpch/20.sql b/tests/queries/tpch/20.sql index 7aaabc2d5..6f06dcb7a 100644 --- a/tests/queries/tpch/20.sql +++ b/tests/queries/tpch/20.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/21.sql b/tests/queries/tpch/21.sql index 5a287f9a3..58879495f 100644 --- a/tests/queries/tpch/21.sql +++ b/tests/queries/tpch/21.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/22.sql b/tests/queries/tpch/22.sql index 1fc8523ad..2a7f45e95 100644 --- a/tests/queries/tpch/22.sql +++ b/tests/queries/tpch/22.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/3.sql b/tests/queries/tpch/3.sql index 710aac520..df50cf760 100644 --- a/tests/queries/tpch/3.sql +++ b/tests/queries/tpch/3.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/4.sql b/tests/queries/tpch/4.sql index e5adb9349..e4a84f79c 100644 --- a/tests/queries/tpch/4.sql +++ b/tests/queries/tpch/4.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/5.sql b/tests/queries/tpch/5.sql index ea376576f..9b92b30ed 100644 --- a/tests/queries/tpch/5.sql +++ b/tests/queries/tpch/5.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/6.sql b/tests/queries/tpch/6.sql index 949b7b16a..709fd201f 100644 --- a/tests/queries/tpch/6.sql +++ b/tests/queries/tpch/6.sql @@ -1,5 +1,22 @@ --- using default substitutions +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- using default substitutions select sum(l_extendedprice * l_discount) as revenue diff --git a/tests/queries/tpch/7.sql b/tests/queries/tpch/7.sql index 85dd8f9c8..1379dbb47 100644 --- a/tests/queries/tpch/7.sql +++ b/tests/queries/tpch/7.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/8.sql b/tests/queries/tpch/8.sql index e6e4d30b8..22ae8b906 100644 --- a/tests/queries/tpch/8.sql +++ b/tests/queries/tpch/8.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions diff --git a/tests/queries/tpch/9.sql b/tests/queries/tpch/9.sql index f9eaf65ee..fa957b052 100644 --- a/tests/queries/tpch/9.sql +++ b/tests/queries/tpch/9.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- using default substitutions From 543ec6c584b9dbb5855f7364cd5d80b0b4ed563e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 6 Nov 2024 16:50:27 -0500 Subject: [PATCH 42/80] Move CHANGELOG content (#1503) --- CHANGELOG.md | 1180 +------------------------------------- changelog/0.51.0-pre.md | 1188 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 1192 insertions(+), 1176 deletions(-) create mode 100644 changelog/0.51.0-pre.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 07142602d..e047515ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,1181 +20,9 @@ # Changelog All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project aims to adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). Given that the parser produces a typed AST, any changes to the AST will -technically be breaking and thus will result in a `0.(N+1)` version. We document -changes that break via addition as "Added". - -## [Unreleased] -Check https://github.com/sqlparser-rs/sqlparser-rs/commits/main for undocumented changes. - - -## [0.51.0] 2024-09-11 -As always, huge props to @iffyio @jmhain and @lovasoa for their help reviewing and merging PRs 🙏. -Without them this project would not be possible. - -Reminder: we are in the final phases of moving sqlparser-rs into the Apache -DataFusion project: https://github.com/sqlparser-rs/sqlparser-rs/issues/1294 - -### Fixed -* Fix Hive table comment should be after table column definitions (#1413) - Thanks @git-hulk -* Fix stack overflow in `parse_subexpr` (#1410) - Thanks @eejbyfeldt -* Fix `INTERVAL` parsing to support expressions and units via dialect (#1398) - Thanks @samuelcolvin -* Fix identifiers starting with `$` should be regarded as a placeholder in SQLite (#1402) - Thanks @git-hulk - -### Added -* Support for MSSQL table options (#1414) - Thanks @bombsimon -* Test showing how negative constants are parsed (#1421) - Thanks @alamb -* Support databricks dialect to dialect_from_str (#1416) - Thanks @milenkovicmalamb -* Support `DROP|CLEAR|MATERIALIZE PROJECTION` syntax for ClickHouse (#1417) - Thanks @git-hulk -* Support postgres `TRUNCATE` syntax (#1406) - Thanks @tobyhede -* Support `CREATE INDEX` with clause (#1389) - Thanks @lewiszlw -* Support parsing `CLUSTERED BY` clause for Hive (#1397) - Thanks @git-hulk -* Support different `USE` statement syntaxes (#1387) - Thanks @kacpermuda -* Support `ADD PROJECTION` syntax for ClickHouse (#1390) - Thanks @git-hulk - -### Changed -* Implement common traits for OneOrManyWithParens (#1368) - Thanks @gstvg -* Cleanup parse_statement (#1407) - Thanks @samuelcolvin -* Allow `DateTimeField::Custom` with `EXTRACT` in Postgres (#1394) - Thanks @samuelcolvin - - -## [0.50.0] 2024-08-15 -Again, huge props to @iffyio @jmhain and @lovasoa for their help reviewing and merging PRs 🙏. -Without them this project would not be possible. - -Reminder: are in the process of moving sqlparser to governed as part of the Apache -DataFusion project: https://github.com/sqlparser-rs/sqlparser-rs/issues/1294 - -### Fixed -* Clippy 1.80 warnings (#1357) - Thanks @lovasoa - -### Added -* Support `STRUCT` and list of structs for DuckDB dialect (#1372) - Thanks @jayzhan211 -* Support custom lexical precedence in PostgreSQL dialect (#1379) - Thanks @samuelcolvin -* Support `FREEZE|UNFREEZE PARTITION` syntax for ClickHouse (#1380) - Thanks @git-hulk -* Support scale in `CEIL` and `FLOOR` functions (#1377) - Thanks @seve-martinez -* Support `CREATE TRIGGER` and `DROP TRIGGER` statements (#1352) - Thanks @LucaCappelletti94 -* Support `EXTRACT` syntax for snowflake (#1374) - Thanks @seve-martinez -* Support `ATTACH` / `DETACH PARTITION` for ClickHouse (#1362) - Thanks @git-hulk -* Support Dialect level precedence, update Postgres Dialect to match Postgres (#1360) - Thanks @samuelcolvin -* Support parsing empty map literal syntax for DuckDB and Generic dialects (#1361) - Thanks @goldmedal -* Support `SETTINGS` clause for ClickHouse table-valued functions (#1358) - Thanks @Jesse-Bakker -* Support `OPTIMIZE TABLE` statement for ClickHouse (#1359) - Thanks @git-hulk -* Support `ON CLUSTER` in `ALTER TABLE` for ClickHouse (#1342) - Thanks @git-hulk -* Support `GLOBAL` keyword before the join operator (#1353) - Thanks @git-hulk -* Support postgres String Constants with Unicode Escapes (#1355) - Thanks @lovasoa -* Support position with normal function call syntax for Snowflake (#1341) - Thanks @jmhain -* Support `TABLE` keyword in `DESC|DESCRIBE|EXPLAIN TABLE` statement (#1351) - Thanks @git-hulk - -### Changed -* Only require `DESCRIBE TABLE` for Snowflake and ClickHouse dialect (#1386) - Thanks @ alamb -* Rename (unreleased) `get_next_precedence_full` to `get_next_precedence_default` (#1378) - Thanks @samuelcolvin -* Use local GitHub Action to replace setup-rust-action (#1371) - Thanks @git-hulk -* Simplify arrow_cast tests (#1367) - Thanks @alamb -* Update version of GitHub Actions (#1363) - Thanks @git-hulk -* Make `Parser::maybe_parse` pub (#1364) - Thanks @Jesse-Bakker -* Improve comments on 1Dialect` (#1366) - Thanks @alamb - - -## [0.49.0] 2024-07-23 -As always, huge props to @iffyio @jmhain and @lovasoa for their help reviewing and merging PRs! - -We are in the process of moving sqlparser to governed as part of the Apache -DataFusion project: https://github.com/sqlparser-rs/sqlparser-rs/issues/1294 - -### Fixed -* Fix quoted identifier regression edge-case with "from" in SELECT (#1346) - Thanks @alexander-beedie -* Fix `AS` query clause should be after the create table options (#1339) - Thanks @git-hulk - -### Added - -* Support `MATERIALIZED`/`ALIAS`/`EPHERMERAL` default column options for ClickHouse (#1348) - Thanks @git-hulk -* Support `()` as the `GROUP BY` nothing (#1347) - Thanks @git-hulk -* Support Map literal syntax for DuckDB and Generic (#1344) - Thanks @goldmedal -* Support subquery expression in `SET` expressions (#1343) - Thanks @iffyio -* Support `WITH FILL` for ClickHouse (#1330) - Thanks @nickpresta -* Support `PARTITION BY` for PostgreSQL in `CREATE TABLE` statement (#1338) - Thanks @git-hulk -* Support of table function `WITH ORDINALITY` modifier for Postgres (#1337) - Thanks @git-hulk - - -## [0.48.0] 2024-07-09 - -Huge shout out to @iffyio @jmhain and @lovasoa for their help reviewing and merging PRs! - -### Fixed -* Fix CI error message in CI (#1333) - Thanks @alamb -* Fix typo in sqlparser-derive README (#1310) - Thanks @leoyvens -* Re-enable trailing commas in DCL (#1318) - Thanks @MohamedAbdeen21 -* Fix a few typos in comment lines (#1316) - Thanks @git-hulk -* Fix Snowflake `SELECT * wildcard REPLACE ... RENAME` order (#1321) - Thanks @alexander-beedie -* Allow semi-colon at the end of UNCACHE statement (#1320) - Thanks @LorrensP-2158466 -* Return errors, not panic, when integers fail to parse in `AUTO_INCREMENT` and `TOP` (#1305) - Thanks @eejbyfeldt - -### Added -* Support `OWNER TO` clause in Postgres (#1314) - Thanks @gainings -* Support `FORMAT` clause for ClickHouse (#1335) - Thanks @git-hulk -* Support `DROP PROCEDURE` statement (#1324) - Thanks @LorrensP-2158466 -* Support `PREWHERE` condition for ClickHouse dialect (#1328) - Thanks @git-hulk -* Support `SETTINGS` pairs for ClickHouse dialect (#1327) - Thanks @git-hulk -* Support `GROUP BY WITH MODIFIER` for ClickHouse dialect (#1323) - Thanks @git-hulk -* Support DuckDB Union datatype (#1322) - Thanks @gstvg -* Support parametric arguments to `FUNCTION` for ClickHouse dialect (#1315) - Thanks @git-hulk -* Support `TO` in `CREATE VIEW` clause for Clickhouse (#1313) - Thanks @Bidaya0 -* Support `UPDATE` statements that contain tuple assignments (#1317) - Thanks @lovasoa -* Support `BY NAME quantifier across all set ops (#1309) - Thanks @alexander-beedie -* Support SnowFlake exclusive `CREATE TABLE` options (#1233) - Thanks @balliegojr -* Support ClickHouse `CREATE TABLE` with primary key and parametrised table engine (#1289) - Thanks @7phs -* Support custom operators in Postgres (#1302) - Thanks @lovasoa -* Support ClickHouse data types (#1285) - Thanks @7phs - -### Changed -* Add stale PR github workflow (#1331) - Thanks @alamb -* Refine docs (#1326) - Thanks @emilsivervik -* Improve error messages with additional colons (#1319) - Thanks @LorrensP-2158466 -* Move Display fmt to struct for `CreateIndex` (#1307) - Thanks @philipcristiano -* Enhancing Trailing Comma Option (#1212) - Thanks @MohamedAbdeen21 -* Encapsulate `CreateTable`, `CreateIndex` into specific structs (#1291) - Thanks @philipcristiano - -## [0.47.0] 2024-06-01 - -### Fixed -* Re-support Postgres array slice syntax (#1290) - Thanks @jmhain -* Fix DoubleColon cast skipping AT TIME ZONE #1266 (#1267) - Thanks @dmitrybugakov -* Fix for values as table name in Databricks and generic (#1278) - Thanks @jmhain - -### Added -* Support `ASOF` joins in Snowflake (#1288) - Thanks @jmhain -* Support `CREATE VIEW` with fields and data types ClickHouse (#1292) - Thanks @7phs -* Support view comments for Snowflake (#1287) - Thanks @bombsimon -* Support dynamic pivot in Snowflake (#1280) - Thanks @jmhain -* Support `CREATE FUNCTION` for BigQuery, generalize AST (#1253) - Thanks @iffyio -* Support expression in `AT TIME ZONE` and fix precedence (#1272) - Thanks @jmhain -* Support `IGNORE/RESPECT NULLS` inside function argument list for Databricks (#1263) - Thanks @jmhain -* Support `SELECT * EXCEPT` Databricks (#1261) - Thanks @jmhain -* Support triple quoted strings (#1262) - Thanks @iffyio -* Support array indexing for duckdb (#1265) - Thanks @JichaoS -* Support multiple SET variables (#1252) - Thanks @iffyio -* Support `ANY_VALUE` `HAVING` clause (#1258) in BigQuery - Thanks @jmhain -* Support keywords as field names in BigQuery struct syntax (#1254) - Thanks @iffyio -* Support `GROUP_CONCAT()` in MySQL (#1256) - Thanks @jmhain -* Support lambda functions in Databricks (#1257) - Thanks @jmhain -* Add const generic peek_tokens method to parser (#1255) - Thanks @jmhain - - -## [0.46.0] 2024-05-03 - -### Changed -* Consolidate representation of function calls, remove `AggregateExpressionWithFilter`, `ArraySubquery`, `ListAgg` and `ArrayAgg` (#1247) - Thanks jmhain -* Extended dialect trait to support numeric prefixed identifiers (#1188) - Thanks @groobyming -* Update simple_logger requirement from 4.0 to 5.0 (#1246) - Thanks @dependabot -* Improve parsing of JSON accesses on Postgres and Snowflake (#1215) - Thanks @jmhain -* Encapsulate Insert and Delete into specific structs (#1224) - Thanks @tisonkun -* Preserve double colon casts (and simplify cast representations) (#1221) - Thanks @jmhain - -### Fixed -* Fix redundant brackets in Hive/Snowflake/Redshift (#1229) - Thanks @yuval-illumex - -### Added -* Support values without parens in Snowflake and DataBricks (#1249) - Thanks @HiranmayaGundu -* Support WINDOW clause after QUALIFY when parsing (#1248) - Thanks @iffyio -* Support `DECLARE` parsing for mssql (#1235) - Thanks @devanbenz -* Support `?`-based jsonb operators in Postgres (#1242) - THanks @ReppCodes -* Support Struct datatype parsing for GenericDialect (#1241) - Thanks @duongcongtoai -* Support BigQuery window function null treatment (#1239) - Thanks @iffyio -* Support extend pivot operator - Thanks @iffyio -* Support Databricks SQL dialect (#1220) - Thanks @jmhain -* Support for MSSQL CONVERT styles (#1219) - Thanks @iffyio -* Support window clause using named window in BigQuery (#1237) - Thanks @iffyio -* Support for CONNECT BY (#1138) - Thanks @jmhain -* Support object constants in Snowflake (#1223) - Thanks @jmhain -* Support BigQuery MERGE syntax (#1217) - Thanks @iffyio -* Support for MAX for NVARCHAR (#1232) - Thanks @ bombsimon -* Support fixed size list types (#1231) - @@universalmind303 -* Support Snowflake MATCH_RECOGNIZE syntax (#1222) - Thanks @jmhain -* Support quoted string backslash escaping (#1177) - Thanks @iffyio -* Support Modify Column for MySQL dialect (#1216) - Thanks @KKould -* Support `select * ilike` for snowflake (#1228) - Thanks @HiranmayaGundu -* Support wildcard replace in duckdb and snowflake syntax (#1226) - Thanks @HiranmayaGundu - - - -## [0.45.0] 2024-04-12 - -### Added -* Support `DateTimeField` variants: `CUSTOM` and `WEEK(MONDAY)` (#1191) - Thanks @iffyio -* Support for arbitrary expr in `MapAccessSyntax` (#1179) - Thanks @iffyio -* Support unquoted hyphen in table/view declaration for BigQuery (#1178) - Thanks @iffyio -* Support `CREATE/DROP SECRET` for duckdb dialect (#1208) - Thanks @JichaoS -* Support MySQL `UNIQUE` table constraint (#1164) - Thanks @Nikita-str -* Support tailing commas on Snowflake. (#1205) - Thanks @yassun7010 -* Support `[FIRST | AFTER column_name]` in `ALTER TABLE` for MySQL (#1180) - Thanks @xring -* Support inline comment with hash syntax for BigQuery (#1192) - Thanks @iffyio -* Support named windows in OVER (window_definition) clause (#1166) - Thanks @Nikita-str -* Support PARALLEL ... and for ..ON NULL INPUT ... to CREATE FUNCTION` (#1202) - Thanks @dimfeld -* Support DuckDB functions named arguments with assignment operator (#1195) - Thanks @alamb -* Support DuckDB struct literal syntax (#1194) - Thanks @gstvg -* Support `$$` in generic dialect ... (#1185)- Thanks @milenkovicm -* Support row_alias and col_aliases in `INSERT` statement for MySQL and Generic dialects (#1136) - Thanks @emin100 - -### Fixed -* Fix dollar quoted string tokenizer (#1193) - Thanks @ZacJW -* Do not allocate in `impl Display` for `DateTimeField` (#1209) - Thanks @alamb -* Fix parse `COPY INTO` stage names without parens for SnowFlake (#1187) - Thanks @mobuchowski -* Solve stack overflow on RecursionLimitExceeded on debug builds (#1171) - Thanks @Nikita-str -* Fix parsing of equality binary operator in function argument (#1182) - Thanks @jmhain -* Fix some comments (#1184) - Thanks @sunxunle - -### Changed -* Cleanup `CREATE FUNCTION` tests (#1203) - Thanks @alamb -* Parse `SUBSTRING FROM` syntax in all dialects, reflect change in the AST (#1173) - Thanks @lovasoa -* Add identifier quote style to Dialect trait (#1170) - Thanks @backkem - -## [0.44.0] 2024-03-02 - -### Added -* Support EXPLAIN / DESCR / DESCRIBE [FORMATTED | EXTENDED] (#1156) - Thanks @jonathanlehtoalamb -* Support ALTER TABLE ... SET LOCATION (#1154) - Thanks @jonathanlehto -* Support `ROW FORMAT DELIMITED` in Hive (#1155) - Thanks @jonathanlehto -* Support `SERDEPROPERTIES` for `CREATE TABLE` with Hive (#1152) - Thanks @jonathanlehto -* Support `EXECUTE ... USING` for Postgres (#1153) - Thanks @jonathanlehto -* Support Postgres style `CREATE FUNCTION` in GenericDialect (#1159) - Thanks @alamb -* Support `SET TBLPROPERTIES` (#1151) - Thanks @jonathanlehto -* Support `UNLOAD` statement (#1150) - Thanks @jonathanlehto -* Support `MATERIALIZED CTEs` (#1148) - Thanks @ReppCodes -* Support `DECLARE` syntax for snowflake and bigquery (#1122) - Thanks @iffyio -* Support `SELECT AS VALUE` and `SELECT AS STRUCT` for BigQuery (#1135) - Thanks @lustefaniak -* Support `(+)` outer join syntax (#1145) - Thanks @jmhain -* Support `INSERT INTO ... SELECT ... RETURNING`(#1132) - Thanks @lovasoa -* Support DuckDB `INSTALL` and `LOAD` (#1127) - Thanks @universalmind303 -* Support `=` operator in function args (#1128) - Thanks @universalmind303 -* Support `CREATE VIEW IF NOT EXISTS` (#1118) - Thanks @7phs -* Support `UPDATE FROM` for SQLite (further to #694) (#1117) - Thanks @ggaughan -* Support optional `DELETE FROM` statement (#1120) - Thanks @iffyio -* Support MySQL `SHOW STATUS` statement (#1119) - Thanks invm - -### Fixed -* Clean up nightly clippy lints (#1158) - Thanks @alamb -* Handle escape, unicode, and hex in tokenize_escaped_single_quoted_string (#1146) - Thanks @JasonLi-cn -* Fix panic while parsing `REPLACE` (#1140) - THanks @jjbayer -* Fix clippy warning from rust 1.76 (#1130) - Thanks @alamb -* Fix release instructions (#1115) - Thanks @alamb - -### Changed -* Add `parse_keyword_with_tokens` for paring keyword and tokens combination (#1141) - Thanks @viirya -* Add ParadeDB to list of known users (#1142) - Thanks @philippemnoel -* Accept JSON_TABLE both as an unquoted table name and a table-valued function (#1134) - Thanks @lovasoa - - -## [0.43.1] 2024-01-22 -### Changes -* Fixed CHANGELOG - - -## [0.43.0] 2024-01-22 -* NO CHANGES - -## [0.42.0] 2024-01-22 - -### Added -* Support for constraint `CHARACTERISTICS` clause (#1099) - Thanks @dimfeld -* Support for unquoted hyphenated identifiers on bigquery (#1109) - Thanks @jmhain -* Support `BigQuery` table and view options (#1061) - Thanks @iffyio -* Support Postgres operators for the LIKE expression variants (#1096) - Thanks @gruuya -* Support "timezone_region" and "timezone_abbr" for `EXTRACT` (and `DATE_PART`) (#1090) - Thanks @alexander-beedie -* Support `JSONB` datatype (#1089) - Thanks @alexander-beedie -* Support PostgreSQL `^@` starts-with operator (#1091) - Thanks @alexander-beedie -* Support PostgreSQL Insert table aliases (#1069) (#1084) - Thanks @boydjohnson -* Support PostgreSQL `CREATE EXTENSION` (#1078) - Thanks @tobyhede -* Support PostgreSQL `ADD GENERATED` in `ALTER COLUMN` statements (#1079) - Thanks @tobyhede -* Support SQLite column definitions with no type (#1075) - Thanks @takluyver -* Support PostgreSQL `ENABLE` and `DISABLE` on `ALTER TABLE` (#1077) - Thanks @tobyhede -* Support MySQL `FLUSH` statement (#1076) - Thanks @emin100 -* Support Mysql `REPLACE` statement and `PRIORITY` clause of `INSERT` (#1072) - Thanks @emin100 - -### Fixed -* Fix `:start` and `:end` json accesses on SnowFlake (#1110) - Thanks @jmhain -* Fix array_agg wildcard behavior (#1093) - Thanks @ReppCodes -* Error on dangling `NO` in `CREATE SEQUENCE` options (#1104) - Thanks @PartiallyTyped -* Allow string values in `PRAGMA` commands (#1101) - Thanks @invm - -### Changed -* Use `Option` for Min and Max vals in Seq Opts, fix alter col seq display (#1106) - Thanks @PartiallyTyped -* Replace `AtomicUsize` with Cell in the recursion counter (#1098) - Thanks @wzzzzd -* Add Qrlew as a user in README.md (#1107) - Thanks @ngrislain -* Add APIs to reuse token buffers in `Tokenizer` (#1094) - Thanks @0rphon -* Bump version of `sqlparser-derive` to 0.2.2 (#1083) - Thanks @alamb - -## [0.41.0] 2023-12-22 - -### Added -* Support `DEFERRED`, `IMMEDIATE`, and `EXCLUSIVE` in SQLite's `BEGIN TRANSACTION` command (#1067) - Thanks @takaebato -* Support generated columns skipping `GENERATED ALWAYS` keywords (#1058) - Thanks @takluyver -* Support `LOCK/UNLOCK TABLES` for MySQL (#1059) - Thanks @zzzdong -* Support `JSON_TABLE` (#1062) - Thanks @lovasoa -* Support `CALL` statements (#1063) - Thanks @lovasoa - -### Fixed -* fix rendering of SELECT TOP (#1070) for Snowflake - Thanks jmhain - -### Changed -* Improve documentation formatting (#1068) - Thanks @alamb -* Replace type_id() by trait method to allow wrapping dialects (#1065) - Thanks @jjbayer -* Document that comments aren't preserved for round trip (#1060) - Thanks @takluyver -* Update sqlparser-derive to use `syn 2.0` (#1040) - Thanks @serprex - -## [0.40.0] 2023-11-27 - -### Added -* Add `{pre,post}_visit_query` to `Visitor` (#1044) - Thanks @jmhain -* Support generated virtual columns with expression (#1051) - Thanks @takluyver -* Support PostgreSQL `END` (#1035) - Thanks @tobyhede -* Support `INSERT INTO ... DEFAULT VALUES ...` (#1036) - Thanks @CDThomas -* Support `RELEASE` and `ROLLBACK TO SAVEPOINT` (#1045) - Thanks @CDThomas -* Support `CONVERT` expressions (#1048) - Thanks @lovasoa -* Support `GLOBAL` and `SESSION` parts in `SHOW VARIABLES` for mysql and generic - Thanks @emin100 -* Support snowflake `PIVOT` on derived table factors (#1027) - Thanks @lustefaniak -* Support mssql json and xml extensions (#1043) - Thanks @lovasoa -* Support for `MAX` as a character length (#1038) - Thanks @lovasoa -* Support `IN ()` syntax of SQLite (#1028) - Thanks @alamb - -### Fixed -* Fix extra whitespace printed before `ON CONFLICT` (#1037) - Thanks @CDThomas - -### Changed -* Document round trip ability (#1052) - Thanks @alamb -* Add PRQL to list of users (#1031) - Thanks @vanillajonathan - -## [0.39.0] 2023-10-27 - -### Added -* Support for `LATERAL FLATTEN` and similar (#1026) - Thanks @lustefaniak -* Support BigQuery struct, array and bytes , int64, `float64` datatypes (#1003) - Thanks @iffyio -* Support numbers as placeholders in Snowflake (e.g. `:1)` (#1001) - Thanks @yuval-illumex -* Support date 'key' when using semi structured data (#1023) @yuval-illumex -* Support IGNORE|RESPECT NULLs clause in window functions (#998) - Thanks @yuval-illumex -* Support for single-quoted identifiers (#1021) - Thanks @lovasoa -* Support multiple PARTITION statements in ALTER TABLE ADD statement (#1011) - Thanks @bitemyapp -* Support "with" identifiers surrounded by backticks in GenericDialect (#1010) - Thanks @bitemyapp -* Support INSERT IGNORE in MySql and GenericDialect (#1004) - Thanks @emin100 -* Support SQLite `pragma` statement (#969) - Thanks @marhoily -* Support `position` as a column name (#1022) - Thanks @lustefaniak -* Support `FILTER` in Functions (for `OVER`) clause (#1007) - Thanks @lovasoa -* Support `SELECT * EXCEPT/REPLACE` syntax from ClickHouse (#1013) - Thanks @lustefaniak -* Support subquery as function arg w/o parens in Snowflake dialect (#996) - Thanks @jmhain -* Support `UNION DISTINCT BY NAME` syntax (#997) - Thanks @alexander-beedie -* Support mysql `RLIKE` and `REGEXP` binary operators (#1017) - Thanks @lovasoa -* Support bigquery `CAST AS x [STRING|DATE] FORMAT` syntax (#978) - Thanks @lustefaniak -* Support Snowflake/BigQuery `TRIM`. (#975) - Thanks @zdenal -* Support `CREATE [TEMPORARY|TEMP] VIEW [IF NOT EXISTS] `(#993) - Thanks @gabivlj -* Support for `CREATE VIEW … WITH NO SCHEMA BINDING` Redshift (#979) - Thanks @lustefaniak -* Support `UNPIVOT` and a fix for chained PIVOTs (#983) - @jmhain -* Support for `LIMIT BY` (#977) - Thanks @lustefaniak -* Support for mixed BigQuery table name quoting (#971) - Thanks @iffyio -* Support `DELETE` with `ORDER BY` and `LIMIT` (MySQL) (#992) - Thanks @ulrichsg -* Support `EXTRACT` for `DAYOFWEEK`, `DAYOFYEAR`, `ISOWEEK`, `TIME` (#980) - Thanks @lustefaniak -* Support `ATTACH DATABASE` (#989) - Thanks @lovasoa - -### Fixed -* Fix handling of `/~%` in Snowflake stage name (#1009) - Thanks @lustefaniak -* Fix column `COLLATE` not displayed (#1012) - Thanks @lustefaniak -* Fix for clippy 1.73 (#995) - Thanks @alamb - -### Changed -* Test to ensure `+ - * / %` binary operators work the same in all dialects (#1025) - Thanks @lustefaniak -* Improve documentation on Parser::consume_token and friends (#994) - Thanks @alamb -* Test that regexp can be used as an identifier in postgres (#1018) - Thanks @lovasoa -* Add docstrings for Dialects, update README (#1016) - Thanks @alamb -* Add JumpWire to users in README (#990) - Thanks @hexedpackets -* Add tests for clickhouse: `tokenize == as Token::DoubleEq` (#981)- Thanks @lustefaniak - -## [0.38.0] 2023-09-21 - -### Added - -* Support `==`operator for Sqlite (#970) - Thanks @marhoily -* Support mysql `PARTITION` to table selection (#959) - Thanks @chunshao90 -* Support `UNNEST` as a table factor for PostgreSQL (#968) @hexedpackets -* Support MySQL `UNIQUE KEY` syntax (#962) - Thanks @artorias1024 -* Support` `GROUP BY ALL` (#964) - @berkaysynnada -* Support multiple actions in one ALTER TABLE statement (#960) - Thanks @ForbesLindesay -* Add `--sqlite param` to CLI (#956) - Thanks @ddol - -### Fixed -* Fix Rust 1.72 clippy lints (#957) - Thanks @alamb - -### Changed -* Add missing token loc in parse err msg (#965) - Thanks @ding-young -* Change how `ANY` and `ALL` expressions are represented in AST (#963) - Thanks @SeanTroyUWO -* Show location info in parse errors (#958) - Thanks @MartinNowak -* Update release documentation (#954) - Thanks @alamb -* Break test and coverage test into separate jobs (#949) - Thanks @alamb - - -## [0.37.0] 2023-08-22 - -### Added -* Support `FOR SYSTEM_TIME AS OF` table time travel clause support, `visit_table_factor` to Visitor (#951) - Thanks @gruuya -* Support MySQL `auto_increment` offset in table definition (#950) - Thanks @ehoeve -* Test for mssql table name in square brackets (#952) - Thanks @lovasoa -* Support additional Postgres `CREATE INDEX` syntax (#943) - Thanks @ForbesLindesay -* Support `ALTER ROLE` syntax of PostgreSQL and MS SQL Server (#942) - Thanks @r4ntix -* Support table-level comments (#946) - Thanks @ehoeve -* Support `DROP TEMPORARY TABLE`, MySQL syntax (#916) - Thanks @liadgiladi -* Support posgres type alias (#933) - Thanks @Kikkon - -### Fixed -* Clarify the value of the special flag (#948) - Thanks @alamb -* Fix `SUBSTRING` from/to argument construction for mssql (#947) - Thanks @jmaness -* Fix: use Rust idiomatic capitalization for newly added DataType enums (#939) - Thanks @Kikkon -* Fix `BEGIN TRANSACTION` being serialized as `START TRANSACTION` (#935) - Thanks @lovasoa -* Fix parsing of datetime functions without parenthesis (#930) - Thanks @lovasoa - -## [0.36.1] 2023-07-19 - -### Fixed -* Fix parsing of identifiers after '%' symbol (#927) - Thanks @alamb - -## [0.36.0] 2023-07-19 - -### Added -* Support toggling "unescape" mode to retain original escaping (#870) - Thanks @canalun -* Support UNION (ALL) BY NAME syntax (#915) - Thanks @parkma99 -* Add doc comment for all operators (#917) - Thanks @izveigor -* Support `PGOverlap` operator (#912) - Thanks @izveigor -* Support multi args for unnest (#909) - Thanks @jayzhan211 -* Support `ALTER VIEW`, MySQL syntax (#907) - Thanks @liadgiladi -* Add DeltaLake keywords (#906) - Thanks @roeap - -### Fixed -* Parse JsonOperators correctly (#913) - Thanks @izveigor -* Fix dependabot by removing rust-toolchain toml (#922) - Thanks @alamb - -### Changed -* Clean up JSON operator tokenizing code (#923) - Thanks @alamb -* Upgrade bigdecimal to 0.4.1 (#921) - Thanks @jinlee0 -* Remove most instances of #[cfg(feature(bigdecimal))] in tests (#910) - Thanks @alamb - -## [0.35.0] 2023-06-23 - -### Added -* Support `CREATE PROCEDURE` of MSSQL (#900) - Thanks @delsehi -* Support DuckDB's `CREATE MACRO` statements (#897) - Thanks @MartinNowak -* Support for `CREATE TYPE (AS)` statements (#888) - Thanks @srijs -* Support `STRICT` tables of sqlite (#903) - Thanks @parkma99 - -### Fixed -* Fixed precedence of unary negation operator with operators: Mul, Div and Mod (#902) - Thanks @izveigor - -### Changed -* Add `support_group_by_expr` to `Dialect` trait (#896) - Thanks @jdye64 -* Update criterion requirement from `0.4` to `0.5` in `/sqlparser_bench` (#890) - Thanks @dependabot (!!) - -## [0.34.0] 2023-05-19 - -### Added - -* Support named window frames (#881) - Thanks @berkaysynnada, @mustafasrepo, and @ozankabak -* Support for `ORDER BY` clauses in aggregate functions (#882) - Thanks @mustafasrepo -* Support `DuckDB` dialect (#878) - Thanks @eitsupi -* Support optional `TABLE` keyword for `TRUNCATE TABLE` (#883) - Thanks @mobuchowski -* Support MySQL's `DIV` operator (#876) - Thanks @eitsupi -* Support Custom operators (#868) - Thanks @max-sixty -* Add `Parser::parse_multipart_identifier` (#860) - Thanks @Jefffrey -* Support for multiple expressions, order by in `ARRAY_AGG` (#879) - Thanks @mustafasrepo -* Support for query source in `COPY .. TO` statement (#858) - Thanks @aprimadi -* Support `DISTINCT ON (...)` (#852) - Thanks @aljazerzen -* Support multiple-table `DELETE` syntax (#855) - Thanks @AviRaboah -* Support `COPY INTO` in `SnowflakeDialect` (#841) - Thanks @pawel-big-lebowski -* Support identifiers beginning with digits in MySQL (#856) - Thanks @AviRaboah - -### Changed -* Include license file in published crate (#871) - Thanks @ankane -* Make `Expr::Interval` its own struct (#872) - Thanks @aprimadi -* Add dialect_from_str and improve Dialect documentation (#848) - Thanks @alamb -* Add clickhouse to example (#849) - Thanks @anglinb - -### Fixed -* Fix merge conflict (#885) - Thanks @alamb -* Fix tiny typo in custom_sql_parser.md (#864) - Thanks @okue -* Fix logical merge conflict (#865) - Thanks @alamb -* Test trailing commas (#859) - Thanks @aljazerzen - - -## [0.33.0] 2023-04-10 - -### Added -* Support for Mysql Backslash escapes (enabled by default) (#844) - Thanks @cobyge -* Support "UPDATE" statement in "WITH" subquery (#842) - Thanks @nicksrandall -* Support PIVOT table syntax (#836) - Thanks @pawel-big-lebowski -* Support CREATE/DROP STAGE for Snowflake (#833) - Thanks @pawel-big-lebowski -* Support Non-Latin characters (#840) - Thanks @mskrzypkows -* Support PostgreSQL: GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY and GENERATED - Thanks @sam-mmm -* Support IF EXISTS in COMMENT statements (#831) - Thanks @pawel-big-lebowski -* Support snowflake alter table swap with (#825) - Thanks @pawel-big-lebowski - -### Changed -* Move tests from parser.rs to appropriate parse_XX tests (#845) - Thanks @alamb -* Correct typos in parser.rs (#838) - Thanks @felixonmars -* Improve documentation on verified_* methods (#828) - Thanks @alamb - -## [0.32.0] 2023-03-6 - -### Added -* Support ClickHouse `CREATE TABLE` with `ORDER BY` (#824) - Thanks @ankrgyl -* Support PostgreSQL exponentiation `^` operator (#813) - Thanks @michael-2956 -* Support `BIGNUMERIC` type in BigQuery (#811) - Thanks @togami2864 -* Support for optional trailing commas (#810) - Thanks @ankrgyl - -### Fixed -* Fix table alias parsing regression by backing out redshift column definition list (#827) - Thanks @alamb -* Fix typo in `ReplaceSelectElement` `colum_name` --> `column_name` (#822) - Thanks @togami2864 - -## [0.31.0] 2023-03-1 - -### Added -* Support raw string literals for BigQuery dialect (#812) - Thanks @togami2864 -* Support `SELECT * REPLACE AS ` in BigQuery dialect (#798) - Thanks @togami2864 -* Support byte string literals for BigQuery dialect (#802) - Thanks @togami2864 -* Support columns definition list for system information functions in RedShift dialect (#769) - Thanks @mskrzypkows -* Support `TRANSIENT` keyword in Snowflake dialect (#807) - Thanks @mobuchowski -* Support `JSON` keyword (#799) - Thanks @togami2864 -* Support MySQL Character Set Introducers (#788) - Thanks @mskrzypkows - -### Fixed -* Fix clippy error in ci (#803) - Thanks @togami2864 -* Handle offset in map key in BigQuery dialect (#797) - Thanks @Ziinc -* Fix a typo (precendence -> precedence) (#794) - Thanks @SARDONYX-sard -* use post_* visitors for mutable visits (#789) - Thanks @lovasoa - -### Changed -* Add another known user (#787) - Thanks @joocer - -## [0.30.0] 2023-01-02 - -### Added -* Support `RENAME` for wildcard `SELECTs` (#784) - Thanks @Jefffrey -* Add a mutable visitor (#782) - Thanks @lovasoa - -### Changed -* Allow parsing of mysql empty row inserts (#783) - Thanks @Jefffrey - -### Fixed -* Fix logical conflict (#785) - Thanks @alamb - -## [0.29.0] 2022-12-29 - -### Highlights -* Partial source location tracking: see #710 -* Recursion limit to prevent stack overflows: #764 -* AST visitor: #765 - -### Added -feat: dollar-quoted strings support (#772) - Thanks @vasilev-alex -* Add derive based AST visitor (#765) - Thanks @tustvold -* Support `ALTER INDEX {INDEX_NAME} RENAME TO {NEW_INDEX_NAME}` (#767) - Thanks @devgony -* Support `CREATE TABLE ON UPDATE ` Function (#685) - Thanks @CEOJINSUNG -* Support `CREATE FUNCTION` definition with `$$` (#755)- Thanks @zidaye -* Add location tracking in the tokenizer and parser (#710) - Thanks @ankrgyl -* Add configurable recursion limit to parser, to protect against stackoverflows (#764) - Thanks @alamb -* Support parsing scientific notation (such as `10e5`) (#768) - Thanks @Jefffrey -* Support `DROP FUNCTION` syntax (#752) - Thanks @zidaye -* Support json operators `@>` `<@`, `@?` and `@@` - Thanks @audunska -* Support the type key (#750)- Thanks @yuval-illumex - -### Changed -* Improve docs and add examples for Visitor (#778) - Thanks @alamb -* Add a backlink from sqlparse_derive to sqlparser and publishing instructions (#779) - Thanks @alamb -* Document new features, update authors (#776) - Thanks @alamb -* Improve Readme (#774) - Thanks @alamb -* Standardize comments on parsing optional keywords (#773) - Thanks @alamb -* Enable grouping sets parsing for `GenericDialect` (#771) - Thanks @Jefffrey -* Generalize conflict target (#762) - Thanks @audunska -* Generalize locking clause (#759) - Thanks @audunska -* Add negative test for except clause on wildcards (#746)- Thanks @alamb -* Add `NANOSECOND` keyword (#749)- Thanks @waitingkuo - -### Fixed -* ParserError if nested explain (#781) - Thanks @Jefffrey -* Fix cargo docs / warnings and add CI check (#777) - Thanks @alamb -* unnest join constraint with alias parsing for BigQuery dialect (#732)- Thanks @Ziinc - -## [0.28.0] 2022-12-05 - -### Added -* Support for `EXCEPT` clause on wildcards (#745) - Thanks @AugustoFKL -* Support `CREATE FUNCTION` Postgres options (#722) - Thanks @wangrunji0408 -* Support `CREATE TABLE x AS TABLE y` (#704) - Thanks @sarahyurick -* Support MySQL `ROWS` syntax for `VALUES` (#737) - Thanks @aljazerzen -* Support `WHERE` condition for `UPDATE ON CONFLICT` (#735) - Thanks @zidaye -* Support `CLUSTER BY` when creating Materialized View (#736) - Thanks @yuval-illumex -* Support nested comments (#726) - Thanks @yang-han -* Support `USING` method when creating indexes. (#731) - Thanks @step-baby and @yangjiaxin01 -* Support `SEMI`/`ANTI` `JOIN` syntax (#723) - Thanks @mingmwang -* Support `EXCLUDE` support for snowflake and generic dialect (#721) - Thanks @AugustoFKL -* Support `MATCH AGAINST` (#708) - Thanks @AugustoFKL -* Support `IF NOT EXISTS` in `ALTER TABLE ADD COLUMN` (#707) - Thanks @AugustoFKL -* Support `SET TIME ZONE ` (#727) - Thanks @waitingkuo -* Support `UPDATE ... FROM ( subquery )` (#694) - Thanks @unvalley - -### Changed -* Add `Parser::index()` method to get current parsing index (#728) - Thanks @neverchanje -* Add `COMPRESSION` as keyword (#720)- Thanks @AugustoFKL -* Derive `PartialOrd`, `Ord`, and `Copy` whenever possible (#717) - Thanks @AugustoFKL -* Fixed `INTERVAL` parsing logic and precedence (#705) - Thanks @sarahyurick -* Support updating multiple column names whose names are the same as(#725) - Thanks @step-baby - -### Fixed -* Clean up some redundant code in parser (#741) - Thanks @alamb -* Fix logical conflict - Thanks @alamb -* Cleanup to avoid is_ok() (#740) - Thanks @alamb -* Cleanup to avoid using unreachable! when parsing semi/anti join (#738) - Thanks @alamb -* Add an example to docs to clarify semantic analysis (#739) - Thanks @alamb -* Add information about parting semantic logic to README.md (#724) - Thanks @AugustoFKL -* Logical conflicts - Thanks @alamb -* Tiny typo in docs (#709) - Thanks @pmcgee69 - - -## [0.27.0] 2022-11-11 - -### Added -* Support `ON CONFLICT` and `RETURNING` in `UPDATE` statement (#666) - Thanks @main and @gamife -* Support `FULLTEXT` option on create table for MySQL and Generic dialects (#702) - Thanks @AugustoFKL -* Support `ARRAY_AGG` for Bigquery and Snowflake (#662) - Thanks @SuperBo -* Support DISTINCT for SetOperator (#689) - Thanks @unvalley -* Support the ARRAY type of Snowflake (#699) - Thanks @yuval-illumex -* Support create sequence with options INCREMENT, MINVALUE, MAXVALUE, START etc. (#681) - Thanks @sam-mmm -* Support `:` operator for semi-structured data in Snowflake(#693) - Thanks @yuval-illumex -* Support ALTER TABLE DROP PRIMARY KEY (#682) - Thanks @ding-young -* Support `NUMERIC` and `DEC` ANSI data types (#695) - Thanks @AugustoFKL -* Support modifiers for Custom Datatype (#680) - Thanks @sunng87 - -### Changed -* Add precision for TIME, DATETIME, and TIMESTAMP data types (#701) - Thanks @AugustoFKL -* add Date keyword (#691) - Thanks @sarahyurick -* Update simple_logger requirement from 2.1 to 4.0 - Thanks @dependabot - -### Fixed -* Fix broken DataFusion link (#703) - Thanks @jmg-duarte -* Add MySql, BigQuery to all dialects tests, fixed bugs (#697) - Thanks @omer-shtivi - - -## [0.26.0] 2022-10-19 - -### Added -* Support MySQL table option `{INDEX | KEY}` in CREATE TABLE definiton (#665) - Thanks @AugustoFKL -* Support `CREATE [ { TEMPORARY | TEMP } ] SEQUENCE [ IF NOT EXISTS ] ` (#678) - Thanks @sam-mmm -* Support `DROP SEQUENCE` statement (#673) - Thanks @sam-mmm -* Support for ANSI types `CHARACTER LARGE OBJECT[(p)]` and `CHAR LARGE OBJECT[(p)]` (#671) - Thanks @AugustoFKL -* Support `[CACHE|UNCACHE] TABLE` (#670) - Thanks @francis-du -* Support `CEIL(expr TO DateTimeField)` and `FLOOR(expr TO DateTimeField)` - Thanks @sarahyurick -* Support all ansii character string types, (#648) - Thanks @AugustoFKL - -### Changed -* Support expressions inside window frames (#655) - Thanks @mustafasrepo and @ozankabak -* Support unit on char length units for small character strings (#663) - Thanks @AugustoFKL -* Replace booleans on `SET ROLE` with a single enum. (#664) - Thanks @AugustoFKL -* Replace `Option`s with enum for `DECIMAL` precision (#654) - Thanks @AugustoFKL - -## [0.25.0] 2022-10-03 - -### Added - -* Support `AUTHORIZATION` clause in `CREATE SCHEMA` statements (#641) - Thanks @AugustoFKL -* Support optional precision for `CLOB` and `BLOB` (#639) - Thanks @AugustoFKL -* Support optional precision in `VARBINARY` and `BINARY` (#637) - Thanks @AugustoFKL - - -### Changed -* `TIMESTAMP` and `TIME` parsing preserve zone information (#646) - Thanks @AugustoFKL - -### Fixed - -* Correct order of arguments when parsing `LIMIT x,y` , restrict to `MySql` and `Generic` dialects - Thanks @AugustoFKL - - -## [0.24.0] 2022-09-29 - -### Added - -* Support `MILLENNIUM` (2 Ns) (#633) - Thanks @sarahyurick -* Support `MEDIUMINT` (#630) - Thanks @AugustoFKL -* Support `DOUBLE PRECISION` (#629) - Thanks @AugustoFKL -* Support precision in `CLOB`, `BINARY`, `VARBINARY`, `BLOB` data type (#618) - Thanks @ding-young -* Support `CREATE ROLE` and `DROP ROLE` (#598) - Thanks @blx -* Support full range of sqlite prepared statement placeholders (#604) - Thanks @lovasoa -* Support National string literal with lower case `n` (#612) - Thanks @mskrzypkows -* Support SHOW FUNCTIONS (#620) - Thanks @joocer -* Support `set time zone to 'some-timezone'` (#617) - Thanks @waitingkuo - -### Changed -* Move `Value::Interval` to `Expr::Interval` (#609) - Thanks @ding-young -* Update `criterion` dev-requirement from 0.3 to 0.4 in /sqlparser_bench (#611) - Thanks @dependabot -* Box `Query` in `Cte` (#572) - Thanks @MazterQyou - -### Other -* Disambiguate CREATE ROLE ... USER and GROUP (#628) - Thanks @alamb -* Add test for optional WITH in CREATE ROLE (#627) - Thanks @alamb - -## [0.23.0] 2022-09-08 - -### Added -* Add support for aggregate expressions with filters (#585) - Thanks @andygrove -* Support `LOCALTIME` and `LOCALTIMESTAMP` time functions (#592) - Thanks @MazterQyou - -## [0.22.0] 2022-08-26 - -### Added -* Support `OVERLAY` expressions (#594) - Thanks @ayushg -* Support `WITH TIMEZONE` and `WITHOUT TIMEZONE` when parsing `TIMESTAMP` expressions (#589) - Thanks @waitingkuo -* Add ability for dialects to override prefix, infix, and statement parsing (#581) - Thanks @andygrove - -## [0.21.0] 2022-08-18 - -### Added -* Support `IS [NOT] TRUE`, `IS [NOT] FALSE`, and `IS [NOT] UNKNOWN` - Thanks (#583) @sarahyurick -* Support `SIMILAR TO` syntax (#569) - Thanks @ayushdg -* Support `SHOW COLLATION` (#564) - Thanks @MazterQyou -* Support `SHOW TABLES` (#563) - Thanks @MazterQyou -* Support `SET NAMES literal [COLLATE literal]` (#558) - Thanks @ovr -* Support trailing commas (#557) in `BigQuery` dialect - Thanks @komukomo -* Support `USE ` (#565) - Thanks @MazterQyou -* Support `SHOW COLUMNS FROM tbl FROM db` (#562) - Thanks @MazterQyou -* Support `SHOW VARIABLES` for `MySQL` dialect (#559) - Thanks @ovr and @vasilev-alex - -### Changed -* Support arbitrary expression in `SET` statement (#574) - Thanks @ovr and @vasilev-alex -* Parse LIKE patterns as Expr not Value (#579) - Thanks @andygrove -* Update Ballista link in README (#576) - Thanks @sanxiyn -* Parse `TRIM` from with optional expr and `FROM` expr (#573) - Thanks @ayushdg -* Support PostgreSQL array subquery constructor (#566) - Thanks @MazterQyou -* Clarify contribution licensing (#570) - Thanks @alamb -* Update for new clippy ints (#571) - Thanks @alamb -* Change `Like` and `ILike` to `Expr` variants, allow escape char (#569) - Thanks @ayushdg -* Parse special keywords as functions (`current_user`, `user`, etc) (#561) - Thanks @ovr -* Support expressions in `LIMIT`/`OFFSET` (#567) - Thanks @MazterQyou - -## [0.20.0] 2022-08-05 - -### Added -* Support custom `OPERATOR` postgres syntax (#548) - Thanks @iskakaushik -* Support `SAFE_CAST` for BigQuery (#552) - Thanks @togami2864 - -### Changed -* Added SECURITY.md (#546) - Thanks @JamieSlome -* Allow `>>` and `<<` binary operators in Generic dialect (#553) - Thanks @ovr -* Allow `NestedJoin` with an alias (#551) - Thanks @waitingkuo - -## [0.19.0] 2022-07-28 - -### Added - -* Support `ON CLUSTER` for `CREATE TABLE` statement (ClickHouse DDL) (#527) - Thanks @andyrichardson -* Support empty `ARRAY` literals (#532) - Thanks @bitemyapp -* Support `AT TIME ZONE` clause (#539) - Thanks @bitemyapp -* Support `USING` clause and table aliases in `DELETE` (#541) - Thanks @mobuchowski -* Support `SHOW CREATE VIEW` statement (#536) - Thanks @mrob95 -* Support `CLONE` clause in `CREATE TABLE` statements (#542) - Thanks @mobuchowski -* Support `WITH OFFSET Alias` in table references (#528) - Thanks @sivchari -* Support double quoted (`"`) literal strings: (#530) - Thanks @komukomo -* Support `ON UPDATE` clause on column definitions in `CREATE TABLE` statements (#522) - Thanks @frolovdev - - -### Changed: - -* `Box`ed `Query` body to save stack space (#540) - Thanks @5tan -* Distinguish between `INT` and `INTEGER` types (#525) - Thanks @frolovdev -* Parse `WHERE NOT EXISTS` as `Expr::Exists` rather than `Expr::UnaryOp` for consistency (#523) - Thanks @frolovdev -* Support `Expr` instead of `String` for argument to `INTERVAL` (#517) - Thanks @togami2864 - -### Fixed: - -* Report characters instead of bytes in error messages (#529) - Thanks @michael-2956 - - -## [0.18.0] 2022-06-06 - -### Added - -* Support `CLOSE` (cursors) (#515) - Thanks @ovr -* Support `DECLARE` (cursors) (#509) - Thanks @ovr -* Support `FETCH` (cursors) (#510) - Thanks @ovr -* Support `DATETIME` keyword (#512) - Thanks @komukomo -* Support `UNNEST` as a table factor (#493) - Thanks @sivchari -* Support `CREATE FUNCTION` (hive flavor) (#496) - Thanks @mobuchowski -* Support placeholders (`$` or `?`) in `LIMIT` clause (#494) - Thanks @step-baby -* Support escaped string literals (PostgreSQL) (#502) - Thanks @ovr -* Support `IS TRUE` and `IS FALSE` (#499) - Thanks @ovr -* Support `DISCARD [ALL | PLANS | SEQUENCES | TEMPORARY | TEMP]` (#500) - Thanks @gandronchik -* Support `array<..>` HIVE data types (#491) - Thanks @mobuchowski -* Support `SET` values that begin with `-` #495 - Thanks @mobuchowski -* Support unicode whitespace (#482) - Thanks @alexsatori -* Support `BigQuery` dialect (#490) - Thanks @komukomo - -### Changed: -* Add docs for MapAccess (#489) - Thanks @alamb -* Rename `ArrayIndex::indexs` to `ArrayIndex::indexes` (#492) - Thanks @alamb - -### Fixed: -* Fix escaping of trailing quote in quoted identifiers (#505) - Thanks @razzolini-qpq -* Fix parsing of `COLLATE` after parentheses in expressions (#507) - Thanks @razzolini-qpq -* Distinguish tables and nullary functions in `FROM` (#506) - Thanks @razzolini-qpq -* Fix `MERGE INTO` semicolon handling (#508) - Thanks @mskrzypkows - -## [0.17.0] 2022-05-09 - -### Added - -* Support `#` as first character in field name for `RedShift` dialect (#485) - Thanks @yuval-illumex -* Support for postgres composite types (#466) - Thanks @poonai -* Support `TABLE` keyword with SELECT INTO (#487) - Thanks @MazterQyou -* Support `ANY`/`ALL` operators (#477) - Thanks @ovr -* Support `ArrayIndex` in `GenericDialect` (#480) - Thanks @ovr -* Support `Redshift` dialect, handle square brackets properly (#471) - Thanks @mskrzypkows -* Support `KILL` statement (#479) - Thanks @ovr -* Support `QUALIFY` clause on `SELECT` for `Snowflake` dialect (#465) - Thanks @mobuchowski -* Support `POSITION(x IN y)` function syntax (#463) @yuval-illumex -* Support `global`,`local`, `on commit` for `create temporary table` (#456) - Thanks @gandronchik -* Support `NVARCHAR` data type (#462) - Thanks @yuval-illumex -* Support for postgres json operators `->`, `->>`, `#>`, and `#>>` (#458) - Thanks @poonai -* Support `SET ROLE` statement (#455) - Thanks @slhmy - -### Changed: -* Improve docstrings for `KILL` statement (#481) - Thanks @alamb -* Add negative tests for `POSITION` (#469) - Thanks @alamb -* Add negative tests for `IN` parsing (#468) - Thanks @alamb -* Suppport table names (as well as subqueries) as source in `MERGE` statements (#483) - Thanks @mskrzypkows - - -### Fixed: -* `INTO` keyword is optional for `INSERT`, `MERGE` (#473) - Thanks @mobuchowski -* Support `IS TRUE` and `IS FALSE` expressions in boolean filter (#474) - Thanks @yuval-illumex -* Support fully qualified object names in `SET VARIABLE` (#484) - Thanks mobuchowski - -## [0.16.0] 2022-04-03 - -### Added - -* Support `WEEK` keyword in `EXTRACT` (#436) - Thanks @Ted-Jiang -* Support `MERGE` statement (#430) - Thanks @mobuchowski -* Support `SAVEPOINT` statement (#438) - Thanks @poonai -* Support `TO` clause in `COPY` (#441) - Thanks @matthewmturner -* Support `CREATE DATABASE` statement (#451) - Thanks @matthewmturner -* Support `FROM` clause in `UPDATE` statement (#450) - Thanks @slhmy -* Support additional `COPY` options (#446) - Thanks @wangrunji0408 - -### Fixed: -* Bug in array / map access parsing (#433) - Thanks @monadbobo - -## [0.15.0] 2022-03-07 - -### Added - -* Support for ClickHouse array types (e.g. [1,2,3]) (#429) - Thanks @monadbobo -* Support for `unsigned tinyint`, `unsigned int`, `unsigned smallint` and `unsigned bigint` datatypes (#428) - Thanks @watarukura -* Support additional keywords for `EXTRACT` (#427) - Thanks @mobuchowski -* Support IN UNNEST(expression) (#426) - Thanks @komukomo -* Support COLLATION keywork on CREATE TABLE (#424) - Thanks @watarukura -* Support FOR UPDATE/FOR SHARE clause (#418) - Thanks @gamife -* Support prepared statement placeholder arg `?` and `$` (#420) - Thanks @gamife -* Support array expressions such as `ARRAY[1,2]` , `foo[1]` and `INT[][]` (#419) - Thanks @gamife - -### Changed: -* remove Travis CI (#421) - Thanks @efx - -### Fixed: -* Allow `array` to be used as a function name again (#432) - @alamb -* Update docstring reference to `Query` (#423) - Thanks @max-sixty - -## [0.14.0] 2022-02-09 - -### Added -* Support `CURRENT_TIMESTAMP`, `CURRENT_TIME`, and `CURRENT_DATE` (#391) - Thanks @yuval-illumex -* SUPPORT `SUPER` keyword (#387) - Thanks @flaneur2020 -* Support differing orders of `OFFSET` `LIMIT` as well as `LIMIT` `OFFSET` (#413) - Thanks @yuval-illumex -* Support for `FROM `, `DELIMITER`, and `CSV HEADER` options for `COPY` command (#409) - Thanks @poonai -* Support `CHARSET` and `ENGINE` clauses on `CREATE TABLE` for mysql (#392) - Thanks @antialize -* Support `DROP CONSTRAINT [ IF EXISTS ] [ CASCADE ]` (#396) - Thanks @tvallotton -* Support parsing tuples and add `Expr::Tuple` (#414) - @alamb -* Support MySQL style `LIMIT X, Y` (#415) - @alamb -* Support `SESSION TRANSACTION` and `TRANSACTION SNAPSHOT`. (#379) - Thanks @poonai -* Support `ALTER COLUMN` and `RENAME CONSTRAINT` (#381) - Thanks @zhamlin -* Support for Map access, add ClickHouse dialect (#382) - Thanks @monadbobo - -### Changed -* Restrict where wildcard (`*`) can appear, add to `FunctionArgExpr` remove `Expr::[Qualified]Wildcard`, (#378) - Thanks @panarch -* Update simple_logger requirement from 1.9 to 2.1 (#403) -* export all methods of parser (#397) - Thanks @neverchanje! -* Clarify maintenance status on README (#416) - @alamb - -### Fixed -* Fix new clippy errors (#412) - @alamb -* Fix panic with `GRANT/REVOKE` in `CONNECT`, `CREATE`, `EXECUTE` or `TEMPORARY` - Thanks @evgenyx00 -* Handle double quotes inside quoted identifiers correctly (#411) - Thanks @Marwes -* Handle mysql backslash escaping (#373) - Thanks @vasilev-alex - -## [0.13.0] 2021-12-10 - -### Added -* Add ALTER TABLE CHANGE COLUMN, extend the UPDATE statement with ON clause (#375) - Thanks @0xA537FD! -* Add support for GROUPIING SETS, ROLLUP and CUBE - Thanks @Jimexist! -* Add basic support for GRANT and REVOKE (#365) - Thanks @blx! - -### Changed -* Use Rust 2021 edition (#368) - Thanks @Jimexist! - -### Fixed -* Fix clippy errors (#367, #374) - Thanks @Jimexist! - - -## [0.12.0] 2021-10-14 - -### Added -* Add support for [NOT] IS DISTINCT FROM (#306) - @Dandandan - -### Changed -* Move the keywords module - Thanks @koushiro! - - -## [0.11.0] 2021-09-24 - -### Added -* Support minimum display width for integer data types (#337) Thanks @vasilev-alex! -* Add logical XOR operator (#357) - Thanks @xzmrdltl! -* Support DESCRIBE table_name (#340) - Thanks @ovr! -* Support SHOW CREATE TABLE|EVENT|FUNCTION (#338) - Thanks @ovr! -* Add referential actions to TableConstraint foreign key (#306) - Thanks @joshwd36! - -### Changed -* Enable map access for numbers, multiple nesting levels (#356) - Thanks @Igosuki! -* Rename Token::Mult to Token::Mul (#353) - Thanks @koushiro! -* Use derive(Default) for HiveFormat (#348) - Thanks @koushiro! -* Improve tokenizer error (#347) - Thanks @koushiro! -* Eliminate redundant string copy in Tokenizer (#343) - Thanks @koushiro! -* Update bigdecimal requirement from 0.2 to 0.3 dependencies (#341) -* Support parsing hexadecimal literals that start with `0x` (#324) - Thanks @TheSchemm! - - -## [0.10.0] 2021-08-23 - -### Added -* Support for `no_std` (#332) - Thanks @koushiro! -* Postgres regular expression operators (`~`, `~*`, `!~`, `!~*`) (#328) - Thanks @b41sh! -* tinyint (#320) - Thanks @sundy-li -* ILIKE (#300) - Thanks @maxcountryman! -* TRIM syntax (#331, #334) - Thanks ever0de - - -### Fixed -* Return error instead of panic (#316) - Thanks @BohuTANG! - -### Changed -- Rename `Modulus` to `Modulo` (#335) - Thanks @RGRAVITY817! -- Update links to reflect repository move to `sqlparser-rs` GitHub org (#333) - Thanks @andygrove -- Add default value for `WindowFrame` (#313) - Thanks @Jimexist! - -## [0.9.0] 2021-03-21 - -### Added -* Add support for `TRY_CAST` syntax (#299) - Thanks @seddonm1! - -## [0.8.0] 2021-02-20 - -### Added -* Introduce Hive QL dialect `HiveDialect` and syntax (#235) - Thanks @hntd187! -* Add `SUBSTRING(col [FROM ] [FOR ])` syntax (#293) -* Support parsing floats without leading digits `.01` (#294) -* Support parsing multiple show variables (#290) - Thanks @francis-du! -* Support SQLite `INSERT OR [..]` syntax (#281) - Thanks @zhangli-pear! - -## [0.7.0] 2020-12-28 - -### Changed -- Change the MySQL dialect to support `` `identifiers` `` quoted with backticks instead of the standard `"double-quoted"` identifiers (#247) - thanks @mashuai! -- Update bigdecimal requirement from 0.1 to 0.2 (#268) - -### Added -- Enable dialect-specific behaviours in the parser (`dialect_of!()`) (#254) - thanks @eyalleshem! -- Support named arguments in function invocations (`ARG_NAME => val`) (#250) - thanks @eyalleshem! -- Support `TABLE()` functions in `FROM` (#253) - thanks @eyalleshem! -- Support Snowflake's single-line comments starting with '#' or '//' (#264) - thanks @eyalleshem! -- Support PostgreSQL `PREPARE`, `EXECUTE`, and `DEALLOCATE` (#243) - thanks @silathdiir! -- Support PostgreSQL math operators (#267) - thanks @alex-dukhno! -- Add SQLite dialect (#248) - thanks @mashuai! -- Add Snowflake dialect (#259) - thanks @eyalleshem! -- Support for Recursive CTEs - thanks @rhanqtl! -- Support `FROM (table_name) alias` syntax - thanks @eyalleshem! -- Support for `EXPLAIN [ANALYZE] VERBOSE` - thanks @ovr! -- Support `ANALYZE TABLE` -- DDL: - - Support `OR REPLACE` in `CREATE VIEW`/`TABLE` (#239) - thanks @Dandandan! - - Support specifying `ASC`/`DESC` in index columns (#249) - thanks @mashuai! - - Support SQLite `AUTOINCREMENT` and MySQL `AUTO_INCREMENT` column option in `CREATE TABLE` (#234) - thanks @mashuai! - - Support PostgreSQL `IF NOT EXISTS` for `CREATE SCHEMA` (#276) - thanks @alex-dukhno! - -### Fixed -- Fix a typo in `JSONFILE` serialization, introduced in 0.3.1 (#237) -- Change `CREATE INDEX` serialization to not end with a semicolon, introduced in 0.5.1 (#245) -- Don't fail parsing `ALTER TABLE ADD COLUMN` ending with a semicolon, introduced in 0.5.1 (#246) - thanks @mashuai - -## [0.6.1] - 2020-07-20 - -### Added -- Support BigQuery `ASSERT` statement (#226) - -## [0.6.0] - 2020-07-20 - -### Added -- Support SQLite's `CREATE TABLE (...) WITHOUT ROWID` (#208) - thanks @mashuai! -- Support SQLite's `CREATE VIRTUAL TABLE` (#209) - thanks @mashuai! - -## [0.5.1] - 2020-06-26 -This release should have been called `0.6`, as it introduces multiple incompatible changes to the API. If you don't want to upgrade yet, you can revert to the previous version by changing your `Cargo.toml` to: - - sqlparser = "= 0.5.0" - - -### Changed -- **`Parser::parse_sql` now accepts a `&str` instead of `String` (#182)** - thanks @Dandandan! -- Change `Ident` (previously a simple `String`) to store the parsed (unquoted) `value` of the identifier and the `quote_style` separately (#143) - thanks @apparebit! -- Support Snowflake's `FROM (table_name)` (#155) - thanks @eyalleshem! -- Add line and column number to TokenizerError (#194) - thanks @Dandandan! -- Use Token::EOF instead of Option (#195) -- Make the units keyword following `INTERVAL '...'` optional (#184) - thanks @maxcountryman! -- Generalize `DATE`/`TIME`/`TIMESTAMP` literals representation in the AST (`TypedString { data_type, value }`) and allow `DATE` and other keywords to be used as identifiers when not followed by a string (#187) - thanks @maxcountryman! -- Output DataType capitalized (`fmt::Display`) (#202) - thanks @Dandandan! - -### Added -- Support MSSQL `TOP () [ PERCENT ] [ WITH TIES ]` (#150) - thanks @alexkyllo! -- Support MySQL `LIMIT row_count OFFSET offset` (not followed by `ROW` or `ROWS`) and remember which variant was parsed (#158) - thanks @mjibson! -- Support PostgreSQL `CREATE TABLE IF NOT EXISTS table_name` (#163) - thanks @alex-dukhno! -- Support basic forms of `CREATE INDEX` and `DROP INDEX` (#167) - thanks @mashuai! -- Support `ON { UPDATE | DELETE } { RESTRICT | CASCADE | SET NULL | NO ACTION | SET DEFAULT }` in `FOREIGN KEY` constraints (#170) - thanks @c7hm4r! -- Support basic forms of `CREATE SCHEMA` and `DROP SCHEMA` (#173) - thanks @alex-dukhno! -- Support `NULLS FIRST`/`LAST` in `ORDER BY` expressions (#176) - thanks @houqp! -- Support `LISTAGG()` (#174) - thanks @maxcountryman! -- Support the string concatentation operator `||` (#178) - thanks @Dandandan! -- Support bitwise AND (`&`), OR (`|`), XOR (`^`) (#181) - thanks @Dandandan! -- Add serde support to AST structs and enums (#196) - thanks @panarch! -- Support `ALTER TABLE ADD COLUMN`, `RENAME COLUMN`, and `RENAME TO` (#203) - thanks @mashuai! -- Support `ALTER TABLE DROP COLUMN` (#148) - thanks @ivanceras! -- Support `CREATE TABLE ... AS ...` (#206) - thanks @Dandandan! - -### Fixed -- Report an error for unterminated string literals (#165) -- Make file format (`STORED AS`) case insensitive (#200) and don't allow quoting it (#201) - thanks @Dandandan! - -## [0.5.0] - 2019-10-10 - -### Changed -- Replace the `Value::Long(u64)` and `Value::Double(f64)` variants with `Value::Number(String)` to avoid losing precision when parsing decimal literals (#130) - thanks @benesch! -- `--features bigdecimal` can be enabled to work with `Value::Number(BigDecimal)` instead, at the cost of an additional dependency. - -### Added -- Support MySQL `SHOW COLUMNS`, `SET =`, and `SHOW ` statements (#135) - thanks @quodlibetor and @benesch! - -### Fixed -- Don't fail to parse `START TRANSACTION` followed by a semicolon (#139) - thanks @gaffneyk! - - -## [0.4.0] - 2019-07-02 -This release brings us closer to SQL-92 support, mainly thanks to the improvements contributed back from @MaterializeInc's fork and other work by @benesch. - -### Changed -- Remove "SQL" from type and enum variant names, `SQLType` -> `DataType`, remove "sql" prefix from module names (#105, #122) -- Rename `ASTNode` -> `Expr` (#119) -- Improve consistency of binary/unary op nodes (#112): - - `ASTNode::SQLBinaryExpr` is now `Expr::BinaryOp` and `ASTNode::SQLUnary` is `Expr::UnaryOp`; - - The `op: SQLOperator` field is now either a `BinaryOperator` or an `UnaryOperator`. -- Change the representation of JOINs to match the standard (#109): `SQLSelect`'s `relation` and `joins` are replaced with `from: Vec`. Before this change `FROM foo NATURAL JOIN bar, baz` was represented as "foo" as the `relation` followed by two joins (`Inner(Natural)` and `Implicit`); now it's two `TableWithJoins` (`foo NATURAL JOIN bar` and `baz`). -- Extract a `SQLFunction` struct (#89) -- Replace `Option>` with `Vec` in the AST structs (#73) -- Change `Value::Long()` to be unsigned, use u64 consistently (#65) - -### Added -- Infra: - - Implement `fmt::Display` on AST nodes (#124) - thanks @vemoo! - - Implement `Hash` (#88) and `Eq` (#123) on all AST nodes - - Implement `std::error::Error` for `ParserError` (#72) - - Handle Windows line-breaks (#54) -- Expressions: - - Support `INTERVAL` literals (#103) - - Support `DATE` / `TIME` / `TIMESTAMP` literals (#99) - - Support `EXTRACT` (#96) - - Support `X'hex value'` literals (#95) - - Support `EXISTS` subqueries (#90) - - Support nested expressions in `BETWEEN` (#80) - - Support `COUNT(DISTINCT x)` and similar (#77) - - Support `CASE operand WHEN expected_value THEN ..` and table-valued functions (#59) - - Support analytic (window) functions (`OVER` clause) (#50) -- Queries / DML: - - Support nested joins (#100) and derived tables with set operations (#111) - - Support `UPDATE` statements (#97) - - Support `INSERT INTO foo SELECT * FROM bar` and `FROM VALUES (...)` (#91) - - Support `SELECT ALL` (#76) - - Add `FETCH` and `OFFSET` support, and `LATERAL` (#69) - thanks @thomas-jeepe! - - Support `COLLATE`, optional column list in CTEs (#64) -- DDL/TCL: - - Support `START/SET/COMMIT/ROLLBACK TRANSACTION` (#106) - thanks @SamuelMarks! - - Parse column constraints in any order (#93) - - Parse `DECIMAL` and `DEC` aliases for `NUMERIC` type (#92) - - Support `DROP [TABLE|VIEW]` (#75) - - Support arbitrary `WITH` options for `CREATE [TABLE|VIEW]` (#74) - - Support constraints in `CREATE TABLE` (#65) -- Add basic MSSQL dialect (#61) and some MSSQL-specific features: - - `CROSS`/`OUTER APPLY` (#120) - - MSSQL identifier and alias parsing rules (#66) - - `WITH` hints (#59) - -### Fixed -- Report an error for `SELECT * FROM a OUTER JOIN b` instead of parsing `OUTER` as an alias (#118) -- Fix the precedence of `NOT LIKE` (#82) and unary `NOT` (#107) -- Do not panic when `NOT` is not followed by an expected keyword (#71) -successfully instead of returning a parse error - thanks @ivanceras! (#67) - and similar fixes for queries with no `FROM` (#116) -- Fix issues with `ALTER TABLE ADD CONSTRAINT` parsing (#65) -- Serialize the "not equals" operator as `<>` instead of `!=` (#64) -- Remove dependencies on `uuid` (#59) and `chrono` (#61) -- Make `SELECT` query with `LIMIT` clause but no `WHERE` parse - Fix incorrect behavior of `ASTNode::SQLQualifiedWildcard::to_string()` (returned `foo*` instead of `foo.*`) - thanks @thomas-jeepe! (#52) - -## [0.3.1] - 2019-04-20 -### Added -- Extended `SQLStatement::SQLCreateTable` to support Hive's EXTERNAL TABLES (`CREATE EXTERNAL TABLE .. STORED AS .. LOCATION '..'`) - thanks @zhzy0077! (#46) -- Parse `SELECT DISTINCT` to `SQLSelect::distinct` (#49) - -## [0.3.0] - 2019-04-03 -### Changed -This release includes major changes to the AST structs to add a number of features, as described in #37 and #43. In particular: -- `ASTNode` variants that represent statements were extracted from `ASTNode` into a separate `SQLStatement` enum; - - `Parser::parse_sql` now returns a `Vec` of parsed statements. - - `ASTNode` now represents an expression (renamed to `Expr` in 0.4.0) -- The query representation (formerly `ASTNode::SQLSelect`) became more complicated to support: - - `WITH` and `UNION`/`EXCEPT`/`INTERSECT` (via `SQLQuery`, `Cte`, and `SQLSetExpr`), - - aliases and qualified wildcards in `SELECT` (via `SQLSelectItem`), - - and aliases in `FROM`/`JOIN` (via `TableFactor`). -- A new `SQLObjectName` struct is used instead of `String` or `ASTNode::SQLCompoundIdentifier` - for objects like tables, custom types, etc. -- Added support for "delimited identifiers" and made keywords context-specific (thus accepting them as valid identifiers in most contexts) - **this caused a regression in parsing `SELECT .. FROM .. LIMIT ..` (#67), fixed in 0.4.0** - -### Added -Other than the changes listed above, some less intrusive additions include: -- Support `CREATE [MATERIALIZED] VIEW` statement -- Support `IN`, `BETWEEN`, unary +/- in epressions -- Support `CHAR` data type and `NUMERIC` not followed by `(p,s)`. -- Support national string literals (`N'...'`) - -## [0.2.4] - 2019-03-08 -Same as 0.2.2. - -## [0.2.3] - 2019-03-08 [YANKED] - -## [0.2.2] - 2019-03-08 -### Changed -- Removed `Value::String`, `Value::DoubleQuotedString`, and `Token::String`, making - - `'...'` parse as a string literal (`Value::SingleQuotedString`), and - - `"..."` fail to parse until version 0.3.0 (#36) - -## [0.2.1] - 2019-01-13 -We don't have a changelog for the changes made in 2018, but thanks to @crw5996, @cswinter, @fredrikroos, @ivanceras, @nickolay, @virattara for their contributions in the early stages of the project! +technically be breaking and thus will result in a `0.(N+1)` version. -## [0.1.0] - 2018-09-03 -Initial release +- Unreleased: Check https://github.com/sqlparser-rs/sqlparser-rs/commits/main for undocumented changes. +- `0.51.0` and earlier: [changelog/0.51.0-pre.md](changelog/0.51.0-pre.md) \ No newline at end of file diff --git a/changelog/0.51.0-pre.md b/changelog/0.51.0-pre.md new file mode 100644 index 000000000..18c7aefb6 --- /dev/null +++ b/changelog/0.51.0-pre.md @@ -0,0 +1,1188 @@ + + + +## [0.51.0] 2024-09-11 +As always, huge props to @iffyio @jmhain and @lovasoa for their help reviewing and merging PRs 🙏. +Without them this project would not be possible. + +Reminder: we are in the final phases of moving sqlparser-rs into the Apache +DataFusion project: https://github.com/sqlparser-rs/sqlparser-rs/issues/1294 + +### Fixed +* Fix Hive table comment should be after table column definitions (#1413) - Thanks @git-hulk +* Fix stack overflow in `parse_subexpr` (#1410) - Thanks @eejbyfeldt +* Fix `INTERVAL` parsing to support expressions and units via dialect (#1398) - Thanks @samuelcolvin +* Fix identifiers starting with `$` should be regarded as a placeholder in SQLite (#1402) - Thanks @git-hulk + +### Added +* Support for MSSQL table options (#1414) - Thanks @bombsimon +* Test showing how negative constants are parsed (#1421) - Thanks @alamb +* Support databricks dialect to dialect_from_str (#1416) - Thanks @milenkovicmalamb +* Support `DROP|CLEAR|MATERIALIZE PROJECTION` syntax for ClickHouse (#1417) - Thanks @git-hulk +* Support postgres `TRUNCATE` syntax (#1406) - Thanks @tobyhede +* Support `CREATE INDEX` with clause (#1389) - Thanks @lewiszlw +* Support parsing `CLUSTERED BY` clause for Hive (#1397) - Thanks @git-hulk +* Support different `USE` statement syntaxes (#1387) - Thanks @kacpermuda +* Support `ADD PROJECTION` syntax for ClickHouse (#1390) - Thanks @git-hulk + +### Changed +* Implement common traits for OneOrManyWithParens (#1368) - Thanks @gstvg +* Cleanup parse_statement (#1407) - Thanks @samuelcolvin +* Allow `DateTimeField::Custom` with `EXTRACT` in Postgres (#1394) - Thanks @samuelcolvin + + +## [0.50.0] 2024-08-15 +Again, huge props to @iffyio @jmhain and @lovasoa for their help reviewing and merging PRs 🙏. +Without them this project would not be possible. + +Reminder: are in the process of moving sqlparser to governed as part of the Apache +DataFusion project: https://github.com/sqlparser-rs/sqlparser-rs/issues/1294 + +### Fixed +* Clippy 1.80 warnings (#1357) - Thanks @lovasoa + +### Added +* Support `STRUCT` and list of structs for DuckDB dialect (#1372) - Thanks @jayzhan211 +* Support custom lexical precedence in PostgreSQL dialect (#1379) - Thanks @samuelcolvin +* Support `FREEZE|UNFREEZE PARTITION` syntax for ClickHouse (#1380) - Thanks @git-hulk +* Support scale in `CEIL` and `FLOOR` functions (#1377) - Thanks @seve-martinez +* Support `CREATE TRIGGER` and `DROP TRIGGER` statements (#1352) - Thanks @LucaCappelletti94 +* Support `EXTRACT` syntax for snowflake (#1374) - Thanks @seve-martinez +* Support `ATTACH` / `DETACH PARTITION` for ClickHouse (#1362) - Thanks @git-hulk +* Support Dialect level precedence, update Postgres Dialect to match Postgres (#1360) - Thanks @samuelcolvin +* Support parsing empty map literal syntax for DuckDB and Generic dialects (#1361) - Thanks @goldmedal +* Support `SETTINGS` clause for ClickHouse table-valued functions (#1358) - Thanks @Jesse-Bakker +* Support `OPTIMIZE TABLE` statement for ClickHouse (#1359) - Thanks @git-hulk +* Support `ON CLUSTER` in `ALTER TABLE` for ClickHouse (#1342) - Thanks @git-hulk +* Support `GLOBAL` keyword before the join operator (#1353) - Thanks @git-hulk +* Support postgres String Constants with Unicode Escapes (#1355) - Thanks @lovasoa +* Support position with normal function call syntax for Snowflake (#1341) - Thanks @jmhain +* Support `TABLE` keyword in `DESC|DESCRIBE|EXPLAIN TABLE` statement (#1351) - Thanks @git-hulk + +### Changed +* Only require `DESCRIBE TABLE` for Snowflake and ClickHouse dialect (#1386) - Thanks @ alamb +* Rename (unreleased) `get_next_precedence_full` to `get_next_precedence_default` (#1378) - Thanks @samuelcolvin +* Use local GitHub Action to replace setup-rust-action (#1371) - Thanks @git-hulk +* Simplify arrow_cast tests (#1367) - Thanks @alamb +* Update version of GitHub Actions (#1363) - Thanks @git-hulk +* Make `Parser::maybe_parse` pub (#1364) - Thanks @Jesse-Bakker +* Improve comments on 1Dialect` (#1366) - Thanks @alamb + + +## [0.49.0] 2024-07-23 +As always, huge props to @iffyio @jmhain and @lovasoa for their help reviewing and merging PRs! + +We are in the process of moving sqlparser to governed as part of the Apache +DataFusion project: https://github.com/sqlparser-rs/sqlparser-rs/issues/1294 + +### Fixed +* Fix quoted identifier regression edge-case with "from" in SELECT (#1346) - Thanks @alexander-beedie +* Fix `AS` query clause should be after the create table options (#1339) - Thanks @git-hulk + +### Added + +* Support `MATERIALIZED`/`ALIAS`/`EPHERMERAL` default column options for ClickHouse (#1348) - Thanks @git-hulk +* Support `()` as the `GROUP BY` nothing (#1347) - Thanks @git-hulk +* Support Map literal syntax for DuckDB and Generic (#1344) - Thanks @goldmedal +* Support subquery expression in `SET` expressions (#1343) - Thanks @iffyio +* Support `WITH FILL` for ClickHouse (#1330) - Thanks @nickpresta +* Support `PARTITION BY` for PostgreSQL in `CREATE TABLE` statement (#1338) - Thanks @git-hulk +* Support of table function `WITH ORDINALITY` modifier for Postgres (#1337) - Thanks @git-hulk + + +## [0.48.0] 2024-07-09 + +Huge shout out to @iffyio @jmhain and @lovasoa for their help reviewing and merging PRs! + +### Fixed +* Fix CI error message in CI (#1333) - Thanks @alamb +* Fix typo in sqlparser-derive README (#1310) - Thanks @leoyvens +* Re-enable trailing commas in DCL (#1318) - Thanks @MohamedAbdeen21 +* Fix a few typos in comment lines (#1316) - Thanks @git-hulk +* Fix Snowflake `SELECT * wildcard REPLACE ... RENAME` order (#1321) - Thanks @alexander-beedie +* Allow semi-colon at the end of UNCACHE statement (#1320) - Thanks @LorrensP-2158466 +* Return errors, not panic, when integers fail to parse in `AUTO_INCREMENT` and `TOP` (#1305) - Thanks @eejbyfeldt + +### Added +* Support `OWNER TO` clause in Postgres (#1314) - Thanks @gainings +* Support `FORMAT` clause for ClickHouse (#1335) - Thanks @git-hulk +* Support `DROP PROCEDURE` statement (#1324) - Thanks @LorrensP-2158466 +* Support `PREWHERE` condition for ClickHouse dialect (#1328) - Thanks @git-hulk +* Support `SETTINGS` pairs for ClickHouse dialect (#1327) - Thanks @git-hulk +* Support `GROUP BY WITH MODIFIER` for ClickHouse dialect (#1323) - Thanks @git-hulk +* Support DuckDB Union datatype (#1322) - Thanks @gstvg +* Support parametric arguments to `FUNCTION` for ClickHouse dialect (#1315) - Thanks @git-hulk +* Support `TO` in `CREATE VIEW` clause for Clickhouse (#1313) - Thanks @Bidaya0 +* Support `UPDATE` statements that contain tuple assignments (#1317) - Thanks @lovasoa +* Support `BY NAME quantifier across all set ops (#1309) - Thanks @alexander-beedie +* Support SnowFlake exclusive `CREATE TABLE` options (#1233) - Thanks @balliegojr +* Support ClickHouse `CREATE TABLE` with primary key and parametrised table engine (#1289) - Thanks @7phs +* Support custom operators in Postgres (#1302) - Thanks @lovasoa +* Support ClickHouse data types (#1285) - Thanks @7phs + +### Changed +* Add stale PR github workflow (#1331) - Thanks @alamb +* Refine docs (#1326) - Thanks @emilsivervik +* Improve error messages with additional colons (#1319) - Thanks @LorrensP-2158466 +* Move Display fmt to struct for `CreateIndex` (#1307) - Thanks @philipcristiano +* Enhancing Trailing Comma Option (#1212) - Thanks @MohamedAbdeen21 +* Encapsulate `CreateTable`, `CreateIndex` into specific structs (#1291) - Thanks @philipcristiano + +## [0.47.0] 2024-06-01 + +### Fixed +* Re-support Postgres array slice syntax (#1290) - Thanks @jmhain +* Fix DoubleColon cast skipping AT TIME ZONE #1266 (#1267) - Thanks @dmitrybugakov +* Fix for values as table name in Databricks and generic (#1278) - Thanks @jmhain + +### Added +* Support `ASOF` joins in Snowflake (#1288) - Thanks @jmhain +* Support `CREATE VIEW` with fields and data types ClickHouse (#1292) - Thanks @7phs +* Support view comments for Snowflake (#1287) - Thanks @bombsimon +* Support dynamic pivot in Snowflake (#1280) - Thanks @jmhain +* Support `CREATE FUNCTION` for BigQuery, generalize AST (#1253) - Thanks @iffyio +* Support expression in `AT TIME ZONE` and fix precedence (#1272) - Thanks @jmhain +* Support `IGNORE/RESPECT NULLS` inside function argument list for Databricks (#1263) - Thanks @jmhain +* Support `SELECT * EXCEPT` Databricks (#1261) - Thanks @jmhain +* Support triple quoted strings (#1262) - Thanks @iffyio +* Support array indexing for duckdb (#1265) - Thanks @JichaoS +* Support multiple SET variables (#1252) - Thanks @iffyio +* Support `ANY_VALUE` `HAVING` clause (#1258) in BigQuery - Thanks @jmhain +* Support keywords as field names in BigQuery struct syntax (#1254) - Thanks @iffyio +* Support `GROUP_CONCAT()` in MySQL (#1256) - Thanks @jmhain +* Support lambda functions in Databricks (#1257) - Thanks @jmhain +* Add const generic peek_tokens method to parser (#1255) - Thanks @jmhain + + +## [0.46.0] 2024-05-03 + +### Changed +* Consolidate representation of function calls, remove `AggregateExpressionWithFilter`, `ArraySubquery`, `ListAgg` and `ArrayAgg` (#1247) - Thanks jmhain +* Extended dialect trait to support numeric prefixed identifiers (#1188) - Thanks @groobyming +* Update simple_logger requirement from 4.0 to 5.0 (#1246) - Thanks @dependabot +* Improve parsing of JSON accesses on Postgres and Snowflake (#1215) - Thanks @jmhain +* Encapsulate Insert and Delete into specific structs (#1224) - Thanks @tisonkun +* Preserve double colon casts (and simplify cast representations) (#1221) - Thanks @jmhain + +### Fixed +* Fix redundant brackets in Hive/Snowflake/Redshift (#1229) - Thanks @yuval-illumex + +### Added +* Support values without parens in Snowflake and DataBricks (#1249) - Thanks @HiranmayaGundu +* Support WINDOW clause after QUALIFY when parsing (#1248) - Thanks @iffyio +* Support `DECLARE` parsing for mssql (#1235) - Thanks @devanbenz +* Support `?`-based jsonb operators in Postgres (#1242) - THanks @ReppCodes +* Support Struct datatype parsing for GenericDialect (#1241) - Thanks @duongcongtoai +* Support BigQuery window function null treatment (#1239) - Thanks @iffyio +* Support extend pivot operator - Thanks @iffyio +* Support Databricks SQL dialect (#1220) - Thanks @jmhain +* Support for MSSQL CONVERT styles (#1219) - Thanks @iffyio +* Support window clause using named window in BigQuery (#1237) - Thanks @iffyio +* Support for CONNECT BY (#1138) - Thanks @jmhain +* Support object constants in Snowflake (#1223) - Thanks @jmhain +* Support BigQuery MERGE syntax (#1217) - Thanks @iffyio +* Support for MAX for NVARCHAR (#1232) - Thanks @ bombsimon +* Support fixed size list types (#1231) - @@universalmind303 +* Support Snowflake MATCH_RECOGNIZE syntax (#1222) - Thanks @jmhain +* Support quoted string backslash escaping (#1177) - Thanks @iffyio +* Support Modify Column for MySQL dialect (#1216) - Thanks @KKould +* Support `select * ilike` for snowflake (#1228) - Thanks @HiranmayaGundu +* Support wildcard replace in duckdb and snowflake syntax (#1226) - Thanks @HiranmayaGundu + + + +## [0.45.0] 2024-04-12 + +### Added +* Support `DateTimeField` variants: `CUSTOM` and `WEEK(MONDAY)` (#1191) - Thanks @iffyio +* Support for arbitrary expr in `MapAccessSyntax` (#1179) - Thanks @iffyio +* Support unquoted hyphen in table/view declaration for BigQuery (#1178) - Thanks @iffyio +* Support `CREATE/DROP SECRET` for duckdb dialect (#1208) - Thanks @JichaoS +* Support MySQL `UNIQUE` table constraint (#1164) - Thanks @Nikita-str +* Support tailing commas on Snowflake. (#1205) - Thanks @yassun7010 +* Support `[FIRST | AFTER column_name]` in `ALTER TABLE` for MySQL (#1180) - Thanks @xring +* Support inline comment with hash syntax for BigQuery (#1192) - Thanks @iffyio +* Support named windows in OVER (window_definition) clause (#1166) - Thanks @Nikita-str +* Support PARALLEL ... and for ..ON NULL INPUT ... to CREATE FUNCTION` (#1202) - Thanks @dimfeld +* Support DuckDB functions named arguments with assignment operator (#1195) - Thanks @alamb +* Support DuckDB struct literal syntax (#1194) - Thanks @gstvg +* Support `$$` in generic dialect ... (#1185)- Thanks @milenkovicm +* Support row_alias and col_aliases in `INSERT` statement for MySQL and Generic dialects (#1136) - Thanks @emin100 + +### Fixed +* Fix dollar quoted string tokenizer (#1193) - Thanks @ZacJW +* Do not allocate in `impl Display` for `DateTimeField` (#1209) - Thanks @alamb +* Fix parse `COPY INTO` stage names without parens for SnowFlake (#1187) - Thanks @mobuchowski +* Solve stack overflow on RecursionLimitExceeded on debug builds (#1171) - Thanks @Nikita-str +* Fix parsing of equality binary operator in function argument (#1182) - Thanks @jmhain +* Fix some comments (#1184) - Thanks @sunxunle + +### Changed +* Cleanup `CREATE FUNCTION` tests (#1203) - Thanks @alamb +* Parse `SUBSTRING FROM` syntax in all dialects, reflect change in the AST (#1173) - Thanks @lovasoa +* Add identifier quote style to Dialect trait (#1170) - Thanks @backkem + +## [0.44.0] 2024-03-02 + +### Added +* Support EXPLAIN / DESCR / DESCRIBE [FORMATTED | EXTENDED] (#1156) - Thanks @jonathanlehtoalamb +* Support ALTER TABLE ... SET LOCATION (#1154) - Thanks @jonathanlehto +* Support `ROW FORMAT DELIMITED` in Hive (#1155) - Thanks @jonathanlehto +* Support `SERDEPROPERTIES` for `CREATE TABLE` with Hive (#1152) - Thanks @jonathanlehto +* Support `EXECUTE ... USING` for Postgres (#1153) - Thanks @jonathanlehto +* Support Postgres style `CREATE FUNCTION` in GenericDialect (#1159) - Thanks @alamb +* Support `SET TBLPROPERTIES` (#1151) - Thanks @jonathanlehto +* Support `UNLOAD` statement (#1150) - Thanks @jonathanlehto +* Support `MATERIALIZED CTEs` (#1148) - Thanks @ReppCodes +* Support `DECLARE` syntax for snowflake and bigquery (#1122) - Thanks @iffyio +* Support `SELECT AS VALUE` and `SELECT AS STRUCT` for BigQuery (#1135) - Thanks @lustefaniak +* Support `(+)` outer join syntax (#1145) - Thanks @jmhain +* Support `INSERT INTO ... SELECT ... RETURNING`(#1132) - Thanks @lovasoa +* Support DuckDB `INSTALL` and `LOAD` (#1127) - Thanks @universalmind303 +* Support `=` operator in function args (#1128) - Thanks @universalmind303 +* Support `CREATE VIEW IF NOT EXISTS` (#1118) - Thanks @7phs +* Support `UPDATE FROM` for SQLite (further to #694) (#1117) - Thanks @ggaughan +* Support optional `DELETE FROM` statement (#1120) - Thanks @iffyio +* Support MySQL `SHOW STATUS` statement (#1119) - Thanks invm + +### Fixed +* Clean up nightly clippy lints (#1158) - Thanks @alamb +* Handle escape, unicode, and hex in tokenize_escaped_single_quoted_string (#1146) - Thanks @JasonLi-cn +* Fix panic while parsing `REPLACE` (#1140) - THanks @jjbayer +* Fix clippy warning from rust 1.76 (#1130) - Thanks @alamb +* Fix release instructions (#1115) - Thanks @alamb + +### Changed +* Add `parse_keyword_with_tokens` for paring keyword and tokens combination (#1141) - Thanks @viirya +* Add ParadeDB to list of known users (#1142) - Thanks @philippemnoel +* Accept JSON_TABLE both as an unquoted table name and a table-valued function (#1134) - Thanks @lovasoa + + +## [0.43.1] 2024-01-22 +### Changes +* Fixed CHANGELOG + + +## [0.43.0] 2024-01-22 +* NO CHANGES + +## [0.42.0] 2024-01-22 + +### Added +* Support for constraint `CHARACTERISTICS` clause (#1099) - Thanks @dimfeld +* Support for unquoted hyphenated identifiers on bigquery (#1109) - Thanks @jmhain +* Support `BigQuery` table and view options (#1061) - Thanks @iffyio +* Support Postgres operators for the LIKE expression variants (#1096) - Thanks @gruuya +* Support "timezone_region" and "timezone_abbr" for `EXTRACT` (and `DATE_PART`) (#1090) - Thanks @alexander-beedie +* Support `JSONB` datatype (#1089) - Thanks @alexander-beedie +* Support PostgreSQL `^@` starts-with operator (#1091) - Thanks @alexander-beedie +* Support PostgreSQL Insert table aliases (#1069) (#1084) - Thanks @boydjohnson +* Support PostgreSQL `CREATE EXTENSION` (#1078) - Thanks @tobyhede +* Support PostgreSQL `ADD GENERATED` in `ALTER COLUMN` statements (#1079) - Thanks @tobyhede +* Support SQLite column definitions with no type (#1075) - Thanks @takluyver +* Support PostgreSQL `ENABLE` and `DISABLE` on `ALTER TABLE` (#1077) - Thanks @tobyhede +* Support MySQL `FLUSH` statement (#1076) - Thanks @emin100 +* Support Mysql `REPLACE` statement and `PRIORITY` clause of `INSERT` (#1072) - Thanks @emin100 + +### Fixed +* Fix `:start` and `:end` json accesses on SnowFlake (#1110) - Thanks @jmhain +* Fix array_agg wildcard behavior (#1093) - Thanks @ReppCodes +* Error on dangling `NO` in `CREATE SEQUENCE` options (#1104) - Thanks @PartiallyTyped +* Allow string values in `PRAGMA` commands (#1101) - Thanks @invm + +### Changed +* Use `Option` for Min and Max vals in Seq Opts, fix alter col seq display (#1106) - Thanks @PartiallyTyped +* Replace `AtomicUsize` with Cell in the recursion counter (#1098) - Thanks @wzzzzd +* Add Qrlew as a user in README.md (#1107) - Thanks @ngrislain +* Add APIs to reuse token buffers in `Tokenizer` (#1094) - Thanks @0rphon +* Bump version of `sqlparser-derive` to 0.2.2 (#1083) - Thanks @alamb + +## [0.41.0] 2023-12-22 + +### Added +* Support `DEFERRED`, `IMMEDIATE`, and `EXCLUSIVE` in SQLite's `BEGIN TRANSACTION` command (#1067) - Thanks @takaebato +* Support generated columns skipping `GENERATED ALWAYS` keywords (#1058) - Thanks @takluyver +* Support `LOCK/UNLOCK TABLES` for MySQL (#1059) - Thanks @zzzdong +* Support `JSON_TABLE` (#1062) - Thanks @lovasoa +* Support `CALL` statements (#1063) - Thanks @lovasoa + +### Fixed +* fix rendering of SELECT TOP (#1070) for Snowflake - Thanks jmhain + +### Changed +* Improve documentation formatting (#1068) - Thanks @alamb +* Replace type_id() by trait method to allow wrapping dialects (#1065) - Thanks @jjbayer +* Document that comments aren't preserved for round trip (#1060) - Thanks @takluyver +* Update sqlparser-derive to use `syn 2.0` (#1040) - Thanks @serprex + +## [0.40.0] 2023-11-27 + +### Added +* Add `{pre,post}_visit_query` to `Visitor` (#1044) - Thanks @jmhain +* Support generated virtual columns with expression (#1051) - Thanks @takluyver +* Support PostgreSQL `END` (#1035) - Thanks @tobyhede +* Support `INSERT INTO ... DEFAULT VALUES ...` (#1036) - Thanks @CDThomas +* Support `RELEASE` and `ROLLBACK TO SAVEPOINT` (#1045) - Thanks @CDThomas +* Support `CONVERT` expressions (#1048) - Thanks @lovasoa +* Support `GLOBAL` and `SESSION` parts in `SHOW VARIABLES` for mysql and generic - Thanks @emin100 +* Support snowflake `PIVOT` on derived table factors (#1027) - Thanks @lustefaniak +* Support mssql json and xml extensions (#1043) - Thanks @lovasoa +* Support for `MAX` as a character length (#1038) - Thanks @lovasoa +* Support `IN ()` syntax of SQLite (#1028) - Thanks @alamb + +### Fixed +* Fix extra whitespace printed before `ON CONFLICT` (#1037) - Thanks @CDThomas + +### Changed +* Document round trip ability (#1052) - Thanks @alamb +* Add PRQL to list of users (#1031) - Thanks @vanillajonathan + +## [0.39.0] 2023-10-27 + +### Added +* Support for `LATERAL FLATTEN` and similar (#1026) - Thanks @lustefaniak +* Support BigQuery struct, array and bytes , int64, `float64` datatypes (#1003) - Thanks @iffyio +* Support numbers as placeholders in Snowflake (e.g. `:1)` (#1001) - Thanks @yuval-illumex +* Support date 'key' when using semi structured data (#1023) @yuval-illumex +* Support IGNORE|RESPECT NULLs clause in window functions (#998) - Thanks @yuval-illumex +* Support for single-quoted identifiers (#1021) - Thanks @lovasoa +* Support multiple PARTITION statements in ALTER TABLE ADD statement (#1011) - Thanks @bitemyapp +* Support "with" identifiers surrounded by backticks in GenericDialect (#1010) - Thanks @bitemyapp +* Support INSERT IGNORE in MySql and GenericDialect (#1004) - Thanks @emin100 +* Support SQLite `pragma` statement (#969) - Thanks @marhoily +* Support `position` as a column name (#1022) - Thanks @lustefaniak +* Support `FILTER` in Functions (for `OVER`) clause (#1007) - Thanks @lovasoa +* Support `SELECT * EXCEPT/REPLACE` syntax from ClickHouse (#1013) - Thanks @lustefaniak +* Support subquery as function arg w/o parens in Snowflake dialect (#996) - Thanks @jmhain +* Support `UNION DISTINCT BY NAME` syntax (#997) - Thanks @alexander-beedie +* Support mysql `RLIKE` and `REGEXP` binary operators (#1017) - Thanks @lovasoa +* Support bigquery `CAST AS x [STRING|DATE] FORMAT` syntax (#978) - Thanks @lustefaniak +* Support Snowflake/BigQuery `TRIM`. (#975) - Thanks @zdenal +* Support `CREATE [TEMPORARY|TEMP] VIEW [IF NOT EXISTS] `(#993) - Thanks @gabivlj +* Support for `CREATE VIEW … WITH NO SCHEMA BINDING` Redshift (#979) - Thanks @lustefaniak +* Support `UNPIVOT` and a fix for chained PIVOTs (#983) - @jmhain +* Support for `LIMIT BY` (#977) - Thanks @lustefaniak +* Support for mixed BigQuery table name quoting (#971) - Thanks @iffyio +* Support `DELETE` with `ORDER BY` and `LIMIT` (MySQL) (#992) - Thanks @ulrichsg +* Support `EXTRACT` for `DAYOFWEEK`, `DAYOFYEAR`, `ISOWEEK`, `TIME` (#980) - Thanks @lustefaniak +* Support `ATTACH DATABASE` (#989) - Thanks @lovasoa + +### Fixed +* Fix handling of `/~%` in Snowflake stage name (#1009) - Thanks @lustefaniak +* Fix column `COLLATE` not displayed (#1012) - Thanks @lustefaniak +* Fix for clippy 1.73 (#995) - Thanks @alamb + +### Changed +* Test to ensure `+ - * / %` binary operators work the same in all dialects (#1025) - Thanks @lustefaniak +* Improve documentation on Parser::consume_token and friends (#994) - Thanks @alamb +* Test that regexp can be used as an identifier in postgres (#1018) - Thanks @lovasoa +* Add docstrings for Dialects, update README (#1016) - Thanks @alamb +* Add JumpWire to users in README (#990) - Thanks @hexedpackets +* Add tests for clickhouse: `tokenize == as Token::DoubleEq` (#981)- Thanks @lustefaniak + +## [0.38.0] 2023-09-21 + +### Added + +* Support `==`operator for Sqlite (#970) - Thanks @marhoily +* Support mysql `PARTITION` to table selection (#959) - Thanks @chunshao90 +* Support `UNNEST` as a table factor for PostgreSQL (#968) @hexedpackets +* Support MySQL `UNIQUE KEY` syntax (#962) - Thanks @artorias1024 +* Support` `GROUP BY ALL` (#964) - @berkaysynnada +* Support multiple actions in one ALTER TABLE statement (#960) - Thanks @ForbesLindesay +* Add `--sqlite param` to CLI (#956) - Thanks @ddol + +### Fixed +* Fix Rust 1.72 clippy lints (#957) - Thanks @alamb + +### Changed +* Add missing token loc in parse err msg (#965) - Thanks @ding-young +* Change how `ANY` and `ALL` expressions are represented in AST (#963) - Thanks @SeanTroyUWO +* Show location info in parse errors (#958) - Thanks @MartinNowak +* Update release documentation (#954) - Thanks @alamb +* Break test and coverage test into separate jobs (#949) - Thanks @alamb + + +## [0.37.0] 2023-08-22 + +### Added +* Support `FOR SYSTEM_TIME AS OF` table time travel clause support, `visit_table_factor` to Visitor (#951) - Thanks @gruuya +* Support MySQL `auto_increment` offset in table definition (#950) - Thanks @ehoeve +* Test for mssql table name in square brackets (#952) - Thanks @lovasoa +* Support additional Postgres `CREATE INDEX` syntax (#943) - Thanks @ForbesLindesay +* Support `ALTER ROLE` syntax of PostgreSQL and MS SQL Server (#942) - Thanks @r4ntix +* Support table-level comments (#946) - Thanks @ehoeve +* Support `DROP TEMPORARY TABLE`, MySQL syntax (#916) - Thanks @liadgiladi +* Support posgres type alias (#933) - Thanks @Kikkon + +### Fixed +* Clarify the value of the special flag (#948) - Thanks @alamb +* Fix `SUBSTRING` from/to argument construction for mssql (#947) - Thanks @jmaness +* Fix: use Rust idiomatic capitalization for newly added DataType enums (#939) - Thanks @Kikkon +* Fix `BEGIN TRANSACTION` being serialized as `START TRANSACTION` (#935) - Thanks @lovasoa +* Fix parsing of datetime functions without parenthesis (#930) - Thanks @lovasoa + +## [0.36.1] 2023-07-19 + +### Fixed +* Fix parsing of identifiers after '%' symbol (#927) - Thanks @alamb + +## [0.36.0] 2023-07-19 + +### Added +* Support toggling "unescape" mode to retain original escaping (#870) - Thanks @canalun +* Support UNION (ALL) BY NAME syntax (#915) - Thanks @parkma99 +* Add doc comment for all operators (#917) - Thanks @izveigor +* Support `PGOverlap` operator (#912) - Thanks @izveigor +* Support multi args for unnest (#909) - Thanks @jayzhan211 +* Support `ALTER VIEW`, MySQL syntax (#907) - Thanks @liadgiladi +* Add DeltaLake keywords (#906) - Thanks @roeap + +### Fixed +* Parse JsonOperators correctly (#913) - Thanks @izveigor +* Fix dependabot by removing rust-toolchain toml (#922) - Thanks @alamb + +### Changed +* Clean up JSON operator tokenizing code (#923) - Thanks @alamb +* Upgrade bigdecimal to 0.4.1 (#921) - Thanks @jinlee0 +* Remove most instances of #[cfg(feature(bigdecimal))] in tests (#910) - Thanks @alamb + +## [0.35.0] 2023-06-23 + +### Added +* Support `CREATE PROCEDURE` of MSSQL (#900) - Thanks @delsehi +* Support DuckDB's `CREATE MACRO` statements (#897) - Thanks @MartinNowak +* Support for `CREATE TYPE (AS)` statements (#888) - Thanks @srijs +* Support `STRICT` tables of sqlite (#903) - Thanks @parkma99 + +### Fixed +* Fixed precedence of unary negation operator with operators: Mul, Div and Mod (#902) - Thanks @izveigor + +### Changed +* Add `support_group_by_expr` to `Dialect` trait (#896) - Thanks @jdye64 +* Update criterion requirement from `0.4` to `0.5` in `/sqlparser_bench` (#890) - Thanks @dependabot (!!) + +## [0.34.0] 2023-05-19 + +### Added + +* Support named window frames (#881) - Thanks @berkaysynnada, @mustafasrepo, and @ozankabak +* Support for `ORDER BY` clauses in aggregate functions (#882) - Thanks @mustafasrepo +* Support `DuckDB` dialect (#878) - Thanks @eitsupi +* Support optional `TABLE` keyword for `TRUNCATE TABLE` (#883) - Thanks @mobuchowski +* Support MySQL's `DIV` operator (#876) - Thanks @eitsupi +* Support Custom operators (#868) - Thanks @max-sixty +* Add `Parser::parse_multipart_identifier` (#860) - Thanks @Jefffrey +* Support for multiple expressions, order by in `ARRAY_AGG` (#879) - Thanks @mustafasrepo +* Support for query source in `COPY .. TO` statement (#858) - Thanks @aprimadi +* Support `DISTINCT ON (...)` (#852) - Thanks @aljazerzen +* Support multiple-table `DELETE` syntax (#855) - Thanks @AviRaboah +* Support `COPY INTO` in `SnowflakeDialect` (#841) - Thanks @pawel-big-lebowski +* Support identifiers beginning with digits in MySQL (#856) - Thanks @AviRaboah + +### Changed +* Include license file in published crate (#871) - Thanks @ankane +* Make `Expr::Interval` its own struct (#872) - Thanks @aprimadi +* Add dialect_from_str and improve Dialect documentation (#848) - Thanks @alamb +* Add clickhouse to example (#849) - Thanks @anglinb + +### Fixed +* Fix merge conflict (#885) - Thanks @alamb +* Fix tiny typo in custom_sql_parser.md (#864) - Thanks @okue +* Fix logical merge conflict (#865) - Thanks @alamb +* Test trailing commas (#859) - Thanks @aljazerzen + + +## [0.33.0] 2023-04-10 + +### Added +* Support for Mysql Backslash escapes (enabled by default) (#844) - Thanks @cobyge +* Support "UPDATE" statement in "WITH" subquery (#842) - Thanks @nicksrandall +* Support PIVOT table syntax (#836) - Thanks @pawel-big-lebowski +* Support CREATE/DROP STAGE for Snowflake (#833) - Thanks @pawel-big-lebowski +* Support Non-Latin characters (#840) - Thanks @mskrzypkows +* Support PostgreSQL: GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY and GENERATED - Thanks @sam-mmm +* Support IF EXISTS in COMMENT statements (#831) - Thanks @pawel-big-lebowski +* Support snowflake alter table swap with (#825) - Thanks @pawel-big-lebowski + +### Changed +* Move tests from parser.rs to appropriate parse_XX tests (#845) - Thanks @alamb +* Correct typos in parser.rs (#838) - Thanks @felixonmars +* Improve documentation on verified_* methods (#828) - Thanks @alamb + +## [0.32.0] 2023-03-6 + +### Added +* Support ClickHouse `CREATE TABLE` with `ORDER BY` (#824) - Thanks @ankrgyl +* Support PostgreSQL exponentiation `^` operator (#813) - Thanks @michael-2956 +* Support `BIGNUMERIC` type in BigQuery (#811) - Thanks @togami2864 +* Support for optional trailing commas (#810) - Thanks @ankrgyl + +### Fixed +* Fix table alias parsing regression by backing out redshift column definition list (#827) - Thanks @alamb +* Fix typo in `ReplaceSelectElement` `colum_name` --> `column_name` (#822) - Thanks @togami2864 + +## [0.31.0] 2023-03-1 + +### Added +* Support raw string literals for BigQuery dialect (#812) - Thanks @togami2864 +* Support `SELECT * REPLACE AS ` in BigQuery dialect (#798) - Thanks @togami2864 +* Support byte string literals for BigQuery dialect (#802) - Thanks @togami2864 +* Support columns definition list for system information functions in RedShift dialect (#769) - Thanks @mskrzypkows +* Support `TRANSIENT` keyword in Snowflake dialect (#807) - Thanks @mobuchowski +* Support `JSON` keyword (#799) - Thanks @togami2864 +* Support MySQL Character Set Introducers (#788) - Thanks @mskrzypkows + +### Fixed +* Fix clippy error in ci (#803) - Thanks @togami2864 +* Handle offset in map key in BigQuery dialect (#797) - Thanks @Ziinc +* Fix a typo (precendence -> precedence) (#794) - Thanks @SARDONYX-sard +* use post_* visitors for mutable visits (#789) - Thanks @lovasoa + +### Changed +* Add another known user (#787) - Thanks @joocer + +## [0.30.0] 2023-01-02 + +### Added +* Support `RENAME` for wildcard `SELECTs` (#784) - Thanks @Jefffrey +* Add a mutable visitor (#782) - Thanks @lovasoa + +### Changed +* Allow parsing of mysql empty row inserts (#783) - Thanks @Jefffrey + +### Fixed +* Fix logical conflict (#785) - Thanks @alamb + +## [0.29.0] 2022-12-29 + +### Highlights +* Partial source location tracking: see #710 +* Recursion limit to prevent stack overflows: #764 +* AST visitor: #765 + +### Added +feat: dollar-quoted strings support (#772) - Thanks @vasilev-alex +* Add derive based AST visitor (#765) - Thanks @tustvold +* Support `ALTER INDEX {INDEX_NAME} RENAME TO {NEW_INDEX_NAME}` (#767) - Thanks @devgony +* Support `CREATE TABLE ON UPDATE ` Function (#685) - Thanks @CEOJINSUNG +* Support `CREATE FUNCTION` definition with `$$` (#755)- Thanks @zidaye +* Add location tracking in the tokenizer and parser (#710) - Thanks @ankrgyl +* Add configurable recursion limit to parser, to protect against stackoverflows (#764) - Thanks @alamb +* Support parsing scientific notation (such as `10e5`) (#768) - Thanks @Jefffrey +* Support `DROP FUNCTION` syntax (#752) - Thanks @zidaye +* Support json operators `@>` `<@`, `@?` and `@@` - Thanks @audunska +* Support the type key (#750)- Thanks @yuval-illumex + +### Changed +* Improve docs and add examples for Visitor (#778) - Thanks @alamb +* Add a backlink from sqlparse_derive to sqlparser and publishing instructions (#779) - Thanks @alamb +* Document new features, update authors (#776) - Thanks @alamb +* Improve Readme (#774) - Thanks @alamb +* Standardize comments on parsing optional keywords (#773) - Thanks @alamb +* Enable grouping sets parsing for `GenericDialect` (#771) - Thanks @Jefffrey +* Generalize conflict target (#762) - Thanks @audunska +* Generalize locking clause (#759) - Thanks @audunska +* Add negative test for except clause on wildcards (#746)- Thanks @alamb +* Add `NANOSECOND` keyword (#749)- Thanks @waitingkuo + +### Fixed +* ParserError if nested explain (#781) - Thanks @Jefffrey +* Fix cargo docs / warnings and add CI check (#777) - Thanks @alamb +* unnest join constraint with alias parsing for BigQuery dialect (#732)- Thanks @Ziinc + +## [0.28.0] 2022-12-05 + +### Added +* Support for `EXCEPT` clause on wildcards (#745) - Thanks @AugustoFKL +* Support `CREATE FUNCTION` Postgres options (#722) - Thanks @wangrunji0408 +* Support `CREATE TABLE x AS TABLE y` (#704) - Thanks @sarahyurick +* Support MySQL `ROWS` syntax for `VALUES` (#737) - Thanks @aljazerzen +* Support `WHERE` condition for `UPDATE ON CONFLICT` (#735) - Thanks @zidaye +* Support `CLUSTER BY` when creating Materialized View (#736) - Thanks @yuval-illumex +* Support nested comments (#726) - Thanks @yang-han +* Support `USING` method when creating indexes. (#731) - Thanks @step-baby and @yangjiaxin01 +* Support `SEMI`/`ANTI` `JOIN` syntax (#723) - Thanks @mingmwang +* Support `EXCLUDE` support for snowflake and generic dialect (#721) - Thanks @AugustoFKL +* Support `MATCH AGAINST` (#708) - Thanks @AugustoFKL +* Support `IF NOT EXISTS` in `ALTER TABLE ADD COLUMN` (#707) - Thanks @AugustoFKL +* Support `SET TIME ZONE ` (#727) - Thanks @waitingkuo +* Support `UPDATE ... FROM ( subquery )` (#694) - Thanks @unvalley + +### Changed +* Add `Parser::index()` method to get current parsing index (#728) - Thanks @neverchanje +* Add `COMPRESSION` as keyword (#720)- Thanks @AugustoFKL +* Derive `PartialOrd`, `Ord`, and `Copy` whenever possible (#717) - Thanks @AugustoFKL +* Fixed `INTERVAL` parsing logic and precedence (#705) - Thanks @sarahyurick +* Support updating multiple column names whose names are the same as(#725) - Thanks @step-baby + +### Fixed +* Clean up some redundant code in parser (#741) - Thanks @alamb +* Fix logical conflict - Thanks @alamb +* Cleanup to avoid is_ok() (#740) - Thanks @alamb +* Cleanup to avoid using unreachable! when parsing semi/anti join (#738) - Thanks @alamb +* Add an example to docs to clarify semantic analysis (#739) - Thanks @alamb +* Add information about parting semantic logic to README.md (#724) - Thanks @AugustoFKL +* Logical conflicts - Thanks @alamb +* Tiny typo in docs (#709) - Thanks @pmcgee69 + + +## [0.27.0] 2022-11-11 + +### Added +* Support `ON CONFLICT` and `RETURNING` in `UPDATE` statement (#666) - Thanks @main and @gamife +* Support `FULLTEXT` option on create table for MySQL and Generic dialects (#702) - Thanks @AugustoFKL +* Support `ARRAY_AGG` for Bigquery and Snowflake (#662) - Thanks @SuperBo +* Support DISTINCT for SetOperator (#689) - Thanks @unvalley +* Support the ARRAY type of Snowflake (#699) - Thanks @yuval-illumex +* Support create sequence with options INCREMENT, MINVALUE, MAXVALUE, START etc. (#681) - Thanks @sam-mmm +* Support `:` operator for semi-structured data in Snowflake(#693) - Thanks @yuval-illumex +* Support ALTER TABLE DROP PRIMARY KEY (#682) - Thanks @ding-young +* Support `NUMERIC` and `DEC` ANSI data types (#695) - Thanks @AugustoFKL +* Support modifiers for Custom Datatype (#680) - Thanks @sunng87 + +### Changed +* Add precision for TIME, DATETIME, and TIMESTAMP data types (#701) - Thanks @AugustoFKL +* add Date keyword (#691) - Thanks @sarahyurick +* Update simple_logger requirement from 2.1 to 4.0 - Thanks @dependabot + +### Fixed +* Fix broken DataFusion link (#703) - Thanks @jmg-duarte +* Add MySql, BigQuery to all dialects tests, fixed bugs (#697) - Thanks @omer-shtivi + + +## [0.26.0] 2022-10-19 + +### Added +* Support MySQL table option `{INDEX | KEY}` in CREATE TABLE definiton (#665) - Thanks @AugustoFKL +* Support `CREATE [ { TEMPORARY | TEMP } ] SEQUENCE [ IF NOT EXISTS ] ` (#678) - Thanks @sam-mmm +* Support `DROP SEQUENCE` statement (#673) - Thanks @sam-mmm +* Support for ANSI types `CHARACTER LARGE OBJECT[(p)]` and `CHAR LARGE OBJECT[(p)]` (#671) - Thanks @AugustoFKL +* Support `[CACHE|UNCACHE] TABLE` (#670) - Thanks @francis-du +* Support `CEIL(expr TO DateTimeField)` and `FLOOR(expr TO DateTimeField)` - Thanks @sarahyurick +* Support all ansii character string types, (#648) - Thanks @AugustoFKL + +### Changed +* Support expressions inside window frames (#655) - Thanks @mustafasrepo and @ozankabak +* Support unit on char length units for small character strings (#663) - Thanks @AugustoFKL +* Replace booleans on `SET ROLE` with a single enum. (#664) - Thanks @AugustoFKL +* Replace `Option`s with enum for `DECIMAL` precision (#654) - Thanks @AugustoFKL + +## [0.25.0] 2022-10-03 + +### Added + +* Support `AUTHORIZATION` clause in `CREATE SCHEMA` statements (#641) - Thanks @AugustoFKL +* Support optional precision for `CLOB` and `BLOB` (#639) - Thanks @AugustoFKL +* Support optional precision in `VARBINARY` and `BINARY` (#637) - Thanks @AugustoFKL + + +### Changed +* `TIMESTAMP` and `TIME` parsing preserve zone information (#646) - Thanks @AugustoFKL + +### Fixed + +* Correct order of arguments when parsing `LIMIT x,y` , restrict to `MySql` and `Generic` dialects - Thanks @AugustoFKL + + +## [0.24.0] 2022-09-29 + +### Added + +* Support `MILLENNIUM` (2 Ns) (#633) - Thanks @sarahyurick +* Support `MEDIUMINT` (#630) - Thanks @AugustoFKL +* Support `DOUBLE PRECISION` (#629) - Thanks @AugustoFKL +* Support precision in `CLOB`, `BINARY`, `VARBINARY`, `BLOB` data type (#618) - Thanks @ding-young +* Support `CREATE ROLE` and `DROP ROLE` (#598) - Thanks @blx +* Support full range of sqlite prepared statement placeholders (#604) - Thanks @lovasoa +* Support National string literal with lower case `n` (#612) - Thanks @mskrzypkows +* Support SHOW FUNCTIONS (#620) - Thanks @joocer +* Support `set time zone to 'some-timezone'` (#617) - Thanks @waitingkuo + +### Changed +* Move `Value::Interval` to `Expr::Interval` (#609) - Thanks @ding-young +* Update `criterion` dev-requirement from 0.3 to 0.4 in /sqlparser_bench (#611) - Thanks @dependabot +* Box `Query` in `Cte` (#572) - Thanks @MazterQyou + +### Other +* Disambiguate CREATE ROLE ... USER and GROUP (#628) - Thanks @alamb +* Add test for optional WITH in CREATE ROLE (#627) - Thanks @alamb + +## [0.23.0] 2022-09-08 + +### Added +* Add support for aggregate expressions with filters (#585) - Thanks @andygrove +* Support `LOCALTIME` and `LOCALTIMESTAMP` time functions (#592) - Thanks @MazterQyou + +## [0.22.0] 2022-08-26 + +### Added +* Support `OVERLAY` expressions (#594) - Thanks @ayushg +* Support `WITH TIMEZONE` and `WITHOUT TIMEZONE` when parsing `TIMESTAMP` expressions (#589) - Thanks @waitingkuo +* Add ability for dialects to override prefix, infix, and statement parsing (#581) - Thanks @andygrove + +## [0.21.0] 2022-08-18 + +### Added +* Support `IS [NOT] TRUE`, `IS [NOT] FALSE`, and `IS [NOT] UNKNOWN` - Thanks (#583) @sarahyurick +* Support `SIMILAR TO` syntax (#569) - Thanks @ayushdg +* Support `SHOW COLLATION` (#564) - Thanks @MazterQyou +* Support `SHOW TABLES` (#563) - Thanks @MazterQyou +* Support `SET NAMES literal [COLLATE literal]` (#558) - Thanks @ovr +* Support trailing commas (#557) in `BigQuery` dialect - Thanks @komukomo +* Support `USE ` (#565) - Thanks @MazterQyou +* Support `SHOW COLUMNS FROM tbl FROM db` (#562) - Thanks @MazterQyou +* Support `SHOW VARIABLES` for `MySQL` dialect (#559) - Thanks @ovr and @vasilev-alex + +### Changed +* Support arbitrary expression in `SET` statement (#574) - Thanks @ovr and @vasilev-alex +* Parse LIKE patterns as Expr not Value (#579) - Thanks @andygrove +* Update Ballista link in README (#576) - Thanks @sanxiyn +* Parse `TRIM` from with optional expr and `FROM` expr (#573) - Thanks @ayushdg +* Support PostgreSQL array subquery constructor (#566) - Thanks @MazterQyou +* Clarify contribution licensing (#570) - Thanks @alamb +* Update for new clippy ints (#571) - Thanks @alamb +* Change `Like` and `ILike` to `Expr` variants, allow escape char (#569) - Thanks @ayushdg +* Parse special keywords as functions (`current_user`, `user`, etc) (#561) - Thanks @ovr +* Support expressions in `LIMIT`/`OFFSET` (#567) - Thanks @MazterQyou + +## [0.20.0] 2022-08-05 + +### Added +* Support custom `OPERATOR` postgres syntax (#548) - Thanks @iskakaushik +* Support `SAFE_CAST` for BigQuery (#552) - Thanks @togami2864 + +### Changed +* Added SECURITY.md (#546) - Thanks @JamieSlome +* Allow `>>` and `<<` binary operators in Generic dialect (#553) - Thanks @ovr +* Allow `NestedJoin` with an alias (#551) - Thanks @waitingkuo + +## [0.19.0] 2022-07-28 + +### Added + +* Support `ON CLUSTER` for `CREATE TABLE` statement (ClickHouse DDL) (#527) - Thanks @andyrichardson +* Support empty `ARRAY` literals (#532) - Thanks @bitemyapp +* Support `AT TIME ZONE` clause (#539) - Thanks @bitemyapp +* Support `USING` clause and table aliases in `DELETE` (#541) - Thanks @mobuchowski +* Support `SHOW CREATE VIEW` statement (#536) - Thanks @mrob95 +* Support `CLONE` clause in `CREATE TABLE` statements (#542) - Thanks @mobuchowski +* Support `WITH OFFSET Alias` in table references (#528) - Thanks @sivchari +* Support double quoted (`"`) literal strings: (#530) - Thanks @komukomo +* Support `ON UPDATE` clause on column definitions in `CREATE TABLE` statements (#522) - Thanks @frolovdev + + +### Changed: + +* `Box`ed `Query` body to save stack space (#540) - Thanks @5tan +* Distinguish between `INT` and `INTEGER` types (#525) - Thanks @frolovdev +* Parse `WHERE NOT EXISTS` as `Expr::Exists` rather than `Expr::UnaryOp` for consistency (#523) - Thanks @frolovdev +* Support `Expr` instead of `String` for argument to `INTERVAL` (#517) - Thanks @togami2864 + +### Fixed: + +* Report characters instead of bytes in error messages (#529) - Thanks @michael-2956 + + +## [0.18.0] 2022-06-06 + +### Added + +* Support `CLOSE` (cursors) (#515) - Thanks @ovr +* Support `DECLARE` (cursors) (#509) - Thanks @ovr +* Support `FETCH` (cursors) (#510) - Thanks @ovr +* Support `DATETIME` keyword (#512) - Thanks @komukomo +* Support `UNNEST` as a table factor (#493) - Thanks @sivchari +* Support `CREATE FUNCTION` (hive flavor) (#496) - Thanks @mobuchowski +* Support placeholders (`$` or `?`) in `LIMIT` clause (#494) - Thanks @step-baby +* Support escaped string literals (PostgreSQL) (#502) - Thanks @ovr +* Support `IS TRUE` and `IS FALSE` (#499) - Thanks @ovr +* Support `DISCARD [ALL | PLANS | SEQUENCES | TEMPORARY | TEMP]` (#500) - Thanks @gandronchik +* Support `array<..>` HIVE data types (#491) - Thanks @mobuchowski +* Support `SET` values that begin with `-` #495 - Thanks @mobuchowski +* Support unicode whitespace (#482) - Thanks @alexsatori +* Support `BigQuery` dialect (#490) - Thanks @komukomo + +### Changed: +* Add docs for MapAccess (#489) - Thanks @alamb +* Rename `ArrayIndex::indexs` to `ArrayIndex::indexes` (#492) - Thanks @alamb + +### Fixed: +* Fix escaping of trailing quote in quoted identifiers (#505) - Thanks @razzolini-qpq +* Fix parsing of `COLLATE` after parentheses in expressions (#507) - Thanks @razzolini-qpq +* Distinguish tables and nullary functions in `FROM` (#506) - Thanks @razzolini-qpq +* Fix `MERGE INTO` semicolon handling (#508) - Thanks @mskrzypkows + +## [0.17.0] 2022-05-09 + +### Added + +* Support `#` as first character in field name for `RedShift` dialect (#485) - Thanks @yuval-illumex +* Support for postgres composite types (#466) - Thanks @poonai +* Support `TABLE` keyword with SELECT INTO (#487) - Thanks @MazterQyou +* Support `ANY`/`ALL` operators (#477) - Thanks @ovr +* Support `ArrayIndex` in `GenericDialect` (#480) - Thanks @ovr +* Support `Redshift` dialect, handle square brackets properly (#471) - Thanks @mskrzypkows +* Support `KILL` statement (#479) - Thanks @ovr +* Support `QUALIFY` clause on `SELECT` for `Snowflake` dialect (#465) - Thanks @mobuchowski +* Support `POSITION(x IN y)` function syntax (#463) @yuval-illumex +* Support `global`,`local`, `on commit` for `create temporary table` (#456) - Thanks @gandronchik +* Support `NVARCHAR` data type (#462) - Thanks @yuval-illumex +* Support for postgres json operators `->`, `->>`, `#>`, and `#>>` (#458) - Thanks @poonai +* Support `SET ROLE` statement (#455) - Thanks @slhmy + +### Changed: +* Improve docstrings for `KILL` statement (#481) - Thanks @alamb +* Add negative tests for `POSITION` (#469) - Thanks @alamb +* Add negative tests for `IN` parsing (#468) - Thanks @alamb +* Suppport table names (as well as subqueries) as source in `MERGE` statements (#483) - Thanks @mskrzypkows + + +### Fixed: +* `INTO` keyword is optional for `INSERT`, `MERGE` (#473) - Thanks @mobuchowski +* Support `IS TRUE` and `IS FALSE` expressions in boolean filter (#474) - Thanks @yuval-illumex +* Support fully qualified object names in `SET VARIABLE` (#484) - Thanks mobuchowski + +## [0.16.0] 2022-04-03 + +### Added + +* Support `WEEK` keyword in `EXTRACT` (#436) - Thanks @Ted-Jiang +* Support `MERGE` statement (#430) - Thanks @mobuchowski +* Support `SAVEPOINT` statement (#438) - Thanks @poonai +* Support `TO` clause in `COPY` (#441) - Thanks @matthewmturner +* Support `CREATE DATABASE` statement (#451) - Thanks @matthewmturner +* Support `FROM` clause in `UPDATE` statement (#450) - Thanks @slhmy +* Support additional `COPY` options (#446) - Thanks @wangrunji0408 + +### Fixed: +* Bug in array / map access parsing (#433) - Thanks @monadbobo + +## [0.15.0] 2022-03-07 + +### Added + +* Support for ClickHouse array types (e.g. [1,2,3]) (#429) - Thanks @monadbobo +* Support for `unsigned tinyint`, `unsigned int`, `unsigned smallint` and `unsigned bigint` datatypes (#428) - Thanks @watarukura +* Support additional keywords for `EXTRACT` (#427) - Thanks @mobuchowski +* Support IN UNNEST(expression) (#426) - Thanks @komukomo +* Support COLLATION keywork on CREATE TABLE (#424) - Thanks @watarukura +* Support FOR UPDATE/FOR SHARE clause (#418) - Thanks @gamife +* Support prepared statement placeholder arg `?` and `$` (#420) - Thanks @gamife +* Support array expressions such as `ARRAY[1,2]` , `foo[1]` and `INT[][]` (#419) - Thanks @gamife + +### Changed: +* remove Travis CI (#421) - Thanks @efx + +### Fixed: +* Allow `array` to be used as a function name again (#432) - @alamb +* Update docstring reference to `Query` (#423) - Thanks @max-sixty + +## [0.14.0] 2022-02-09 + +### Added +* Support `CURRENT_TIMESTAMP`, `CURRENT_TIME`, and `CURRENT_DATE` (#391) - Thanks @yuval-illumex +* SUPPORT `SUPER` keyword (#387) - Thanks @flaneur2020 +* Support differing orders of `OFFSET` `LIMIT` as well as `LIMIT` `OFFSET` (#413) - Thanks @yuval-illumex +* Support for `FROM `, `DELIMITER`, and `CSV HEADER` options for `COPY` command (#409) - Thanks @poonai +* Support `CHARSET` and `ENGINE` clauses on `CREATE TABLE` for mysql (#392) - Thanks @antialize +* Support `DROP CONSTRAINT [ IF EXISTS ] [ CASCADE ]` (#396) - Thanks @tvallotton +* Support parsing tuples and add `Expr::Tuple` (#414) - @alamb +* Support MySQL style `LIMIT X, Y` (#415) - @alamb +* Support `SESSION TRANSACTION` and `TRANSACTION SNAPSHOT`. (#379) - Thanks @poonai +* Support `ALTER COLUMN` and `RENAME CONSTRAINT` (#381) - Thanks @zhamlin +* Support for Map access, add ClickHouse dialect (#382) - Thanks @monadbobo + +### Changed +* Restrict where wildcard (`*`) can appear, add to `FunctionArgExpr` remove `Expr::[Qualified]Wildcard`, (#378) - Thanks @panarch +* Update simple_logger requirement from 1.9 to 2.1 (#403) +* export all methods of parser (#397) - Thanks @neverchanje! +* Clarify maintenance status on README (#416) - @alamb + +### Fixed +* Fix new clippy errors (#412) - @alamb +* Fix panic with `GRANT/REVOKE` in `CONNECT`, `CREATE`, `EXECUTE` or `TEMPORARY` - Thanks @evgenyx00 +* Handle double quotes inside quoted identifiers correctly (#411) - Thanks @Marwes +* Handle mysql backslash escaping (#373) - Thanks @vasilev-alex + +## [0.13.0] 2021-12-10 + +### Added +* Add ALTER TABLE CHANGE COLUMN, extend the UPDATE statement with ON clause (#375) - Thanks @0xA537FD! +* Add support for GROUPIING SETS, ROLLUP and CUBE - Thanks @Jimexist! +* Add basic support for GRANT and REVOKE (#365) - Thanks @blx! + +### Changed +* Use Rust 2021 edition (#368) - Thanks @Jimexist! + +### Fixed +* Fix clippy errors (#367, #374) - Thanks @Jimexist! + + +## [0.12.0] 2021-10-14 + +### Added +* Add support for [NOT] IS DISTINCT FROM (#306) - @Dandandan + +### Changed +* Move the keywords module - Thanks @koushiro! + + +## [0.11.0] 2021-09-24 + +### Added +* Support minimum display width for integer data types (#337) Thanks @vasilev-alex! +* Add logical XOR operator (#357) - Thanks @xzmrdltl! +* Support DESCRIBE table_name (#340) - Thanks @ovr! +* Support SHOW CREATE TABLE|EVENT|FUNCTION (#338) - Thanks @ovr! +* Add referential actions to TableConstraint foreign key (#306) - Thanks @joshwd36! + +### Changed +* Enable map access for numbers, multiple nesting levels (#356) - Thanks @Igosuki! +* Rename Token::Mult to Token::Mul (#353) - Thanks @koushiro! +* Use derive(Default) for HiveFormat (#348) - Thanks @koushiro! +* Improve tokenizer error (#347) - Thanks @koushiro! +* Eliminate redundant string copy in Tokenizer (#343) - Thanks @koushiro! +* Update bigdecimal requirement from 0.2 to 0.3 dependencies (#341) +* Support parsing hexadecimal literals that start with `0x` (#324) - Thanks @TheSchemm! + + +## [0.10.0] 2021-08-23 + +### Added +* Support for `no_std` (#332) - Thanks @koushiro! +* Postgres regular expression operators (`~`, `~*`, `!~`, `!~*`) (#328) - Thanks @b41sh! +* tinyint (#320) - Thanks @sundy-li +* ILIKE (#300) - Thanks @maxcountryman! +* TRIM syntax (#331, #334) - Thanks ever0de + + +### Fixed +* Return error instead of panic (#316) - Thanks @BohuTANG! + +### Changed +- Rename `Modulus` to `Modulo` (#335) - Thanks @RGRAVITY817! +- Update links to reflect repository move to `sqlparser-rs` GitHub org (#333) - Thanks @andygrove +- Add default value for `WindowFrame` (#313) - Thanks @Jimexist! + +## [0.9.0] 2021-03-21 + +### Added +* Add support for `TRY_CAST` syntax (#299) - Thanks @seddonm1! + +## [0.8.0] 2021-02-20 + +### Added +* Introduce Hive QL dialect `HiveDialect` and syntax (#235) - Thanks @hntd187! +* Add `SUBSTRING(col [FROM ] [FOR ])` syntax (#293) +* Support parsing floats without leading digits `.01` (#294) +* Support parsing multiple show variables (#290) - Thanks @francis-du! +* Support SQLite `INSERT OR [..]` syntax (#281) - Thanks @zhangli-pear! + +## [0.7.0] 2020-12-28 + +### Changed +- Change the MySQL dialect to support `` `identifiers` `` quoted with backticks instead of the standard `"double-quoted"` identifiers (#247) - thanks @mashuai! +- Update bigdecimal requirement from 0.1 to 0.2 (#268) + +### Added +- Enable dialect-specific behaviours in the parser (`dialect_of!()`) (#254) - thanks @eyalleshem! +- Support named arguments in function invocations (`ARG_NAME => val`) (#250) - thanks @eyalleshem! +- Support `TABLE()` functions in `FROM` (#253) - thanks @eyalleshem! +- Support Snowflake's single-line comments starting with '#' or '//' (#264) - thanks @eyalleshem! +- Support PostgreSQL `PREPARE`, `EXECUTE`, and `DEALLOCATE` (#243) - thanks @silathdiir! +- Support PostgreSQL math operators (#267) - thanks @alex-dukhno! +- Add SQLite dialect (#248) - thanks @mashuai! +- Add Snowflake dialect (#259) - thanks @eyalleshem! +- Support for Recursive CTEs - thanks @rhanqtl! +- Support `FROM (table_name) alias` syntax - thanks @eyalleshem! +- Support for `EXPLAIN [ANALYZE] VERBOSE` - thanks @ovr! +- Support `ANALYZE TABLE` +- DDL: + - Support `OR REPLACE` in `CREATE VIEW`/`TABLE` (#239) - thanks @Dandandan! + - Support specifying `ASC`/`DESC` in index columns (#249) - thanks @mashuai! + - Support SQLite `AUTOINCREMENT` and MySQL `AUTO_INCREMENT` column option in `CREATE TABLE` (#234) - thanks @mashuai! + - Support PostgreSQL `IF NOT EXISTS` for `CREATE SCHEMA` (#276) - thanks @alex-dukhno! + +### Fixed +- Fix a typo in `JSONFILE` serialization, introduced in 0.3.1 (#237) +- Change `CREATE INDEX` serialization to not end with a semicolon, introduced in 0.5.1 (#245) +- Don't fail parsing `ALTER TABLE ADD COLUMN` ending with a semicolon, introduced in 0.5.1 (#246) - thanks @mashuai + +## [0.6.1] - 2020-07-20 + +### Added +- Support BigQuery `ASSERT` statement (#226) + +## [0.6.0] - 2020-07-20 + +### Added +- Support SQLite's `CREATE TABLE (...) WITHOUT ROWID` (#208) - thanks @mashuai! +- Support SQLite's `CREATE VIRTUAL TABLE` (#209) - thanks @mashuai! + +## [0.5.1] - 2020-06-26 +This release should have been called `0.6`, as it introduces multiple incompatible changes to the API. If you don't want to upgrade yet, you can revert to the previous version by changing your `Cargo.toml` to: + + sqlparser = "= 0.5.0" + + +### Changed +- **`Parser::parse_sql` now accepts a `&str` instead of `String` (#182)** - thanks @Dandandan! +- Change `Ident` (previously a simple `String`) to store the parsed (unquoted) `value` of the identifier and the `quote_style` separately (#143) - thanks @apparebit! +- Support Snowflake's `FROM (table_name)` (#155) - thanks @eyalleshem! +- Add line and column number to TokenizerError (#194) - thanks @Dandandan! +- Use Token::EOF instead of Option (#195) +- Make the units keyword following `INTERVAL '...'` optional (#184) - thanks @maxcountryman! +- Generalize `DATE`/`TIME`/`TIMESTAMP` literals representation in the AST (`TypedString { data_type, value }`) and allow `DATE` and other keywords to be used as identifiers when not followed by a string (#187) - thanks @maxcountryman! +- Output DataType capitalized (`fmt::Display`) (#202) - thanks @Dandandan! + +### Added +- Support MSSQL `TOP () [ PERCENT ] [ WITH TIES ]` (#150) - thanks @alexkyllo! +- Support MySQL `LIMIT row_count OFFSET offset` (not followed by `ROW` or `ROWS`) and remember which variant was parsed (#158) - thanks @mjibson! +- Support PostgreSQL `CREATE TABLE IF NOT EXISTS table_name` (#163) - thanks @alex-dukhno! +- Support basic forms of `CREATE INDEX` and `DROP INDEX` (#167) - thanks @mashuai! +- Support `ON { UPDATE | DELETE } { RESTRICT | CASCADE | SET NULL | NO ACTION | SET DEFAULT }` in `FOREIGN KEY` constraints (#170) - thanks @c7hm4r! +- Support basic forms of `CREATE SCHEMA` and `DROP SCHEMA` (#173) - thanks @alex-dukhno! +- Support `NULLS FIRST`/`LAST` in `ORDER BY` expressions (#176) - thanks @houqp! +- Support `LISTAGG()` (#174) - thanks @maxcountryman! +- Support the string concatentation operator `||` (#178) - thanks @Dandandan! +- Support bitwise AND (`&`), OR (`|`), XOR (`^`) (#181) - thanks @Dandandan! +- Add serde support to AST structs and enums (#196) - thanks @panarch! +- Support `ALTER TABLE ADD COLUMN`, `RENAME COLUMN`, and `RENAME TO` (#203) - thanks @mashuai! +- Support `ALTER TABLE DROP COLUMN` (#148) - thanks @ivanceras! +- Support `CREATE TABLE ... AS ...` (#206) - thanks @Dandandan! + +### Fixed +- Report an error for unterminated string literals (#165) +- Make file format (`STORED AS`) case insensitive (#200) and don't allow quoting it (#201) - thanks @Dandandan! + +## [0.5.0] - 2019-10-10 + +### Changed +- Replace the `Value::Long(u64)` and `Value::Double(f64)` variants with `Value::Number(String)` to avoid losing precision when parsing decimal literals (#130) - thanks @benesch! +- `--features bigdecimal` can be enabled to work with `Value::Number(BigDecimal)` instead, at the cost of an additional dependency. + +### Added +- Support MySQL `SHOW COLUMNS`, `SET =`, and `SHOW ` statements (#135) - thanks @quodlibetor and @benesch! + +### Fixed +- Don't fail to parse `START TRANSACTION` followed by a semicolon (#139) - thanks @gaffneyk! + + +## [0.4.0] - 2019-07-02 +This release brings us closer to SQL-92 support, mainly thanks to the improvements contributed back from @MaterializeInc's fork and other work by @benesch. + +### Changed +- Remove "SQL" from type and enum variant names, `SQLType` -> `DataType`, remove "sql" prefix from module names (#105, #122) +- Rename `ASTNode` -> `Expr` (#119) +- Improve consistency of binary/unary op nodes (#112): + - `ASTNode::SQLBinaryExpr` is now `Expr::BinaryOp` and `ASTNode::SQLUnary` is `Expr::UnaryOp`; + - The `op: SQLOperator` field is now either a `BinaryOperator` or an `UnaryOperator`. +- Change the representation of JOINs to match the standard (#109): `SQLSelect`'s `relation` and `joins` are replaced with `from: Vec`. Before this change `FROM foo NATURAL JOIN bar, baz` was represented as "foo" as the `relation` followed by two joins (`Inner(Natural)` and `Implicit`); now it's two `TableWithJoins` (`foo NATURAL JOIN bar` and `baz`). +- Extract a `SQLFunction` struct (#89) +- Replace `Option>` with `Vec` in the AST structs (#73) +- Change `Value::Long()` to be unsigned, use u64 consistently (#65) + +### Added +- Infra: + - Implement `fmt::Display` on AST nodes (#124) - thanks @vemoo! + - Implement `Hash` (#88) and `Eq` (#123) on all AST nodes + - Implement `std::error::Error` for `ParserError` (#72) + - Handle Windows line-breaks (#54) +- Expressions: + - Support `INTERVAL` literals (#103) + - Support `DATE` / `TIME` / `TIMESTAMP` literals (#99) + - Support `EXTRACT` (#96) + - Support `X'hex value'` literals (#95) + - Support `EXISTS` subqueries (#90) + - Support nested expressions in `BETWEEN` (#80) + - Support `COUNT(DISTINCT x)` and similar (#77) + - Support `CASE operand WHEN expected_value THEN ..` and table-valued functions (#59) + - Support analytic (window) functions (`OVER` clause) (#50) +- Queries / DML: + - Support nested joins (#100) and derived tables with set operations (#111) + - Support `UPDATE` statements (#97) + - Support `INSERT INTO foo SELECT * FROM bar` and `FROM VALUES (...)` (#91) + - Support `SELECT ALL` (#76) + - Add `FETCH` and `OFFSET` support, and `LATERAL` (#69) - thanks @thomas-jeepe! + - Support `COLLATE`, optional column list in CTEs (#64) +- DDL/TCL: + - Support `START/SET/COMMIT/ROLLBACK TRANSACTION` (#106) - thanks @SamuelMarks! + - Parse column constraints in any order (#93) + - Parse `DECIMAL` and `DEC` aliases for `NUMERIC` type (#92) + - Support `DROP [TABLE|VIEW]` (#75) + - Support arbitrary `WITH` options for `CREATE [TABLE|VIEW]` (#74) + - Support constraints in `CREATE TABLE` (#65) +- Add basic MSSQL dialect (#61) and some MSSQL-specific features: + - `CROSS`/`OUTER APPLY` (#120) + - MSSQL identifier and alias parsing rules (#66) + - `WITH` hints (#59) + +### Fixed +- Report an error for `SELECT * FROM a OUTER JOIN b` instead of parsing `OUTER` as an alias (#118) +- Fix the precedence of `NOT LIKE` (#82) and unary `NOT` (#107) +- Do not panic when `NOT` is not followed by an expected keyword (#71) + successfully instead of returning a parse error - thanks @ivanceras! (#67) - and similar fixes for queries with no `FROM` (#116) +- Fix issues with `ALTER TABLE ADD CONSTRAINT` parsing (#65) +- Serialize the "not equals" operator as `<>` instead of `!=` (#64) +- Remove dependencies on `uuid` (#59) and `chrono` (#61) +- Make `SELECT` query with `LIMIT` clause but no `WHERE` parse - Fix incorrect behavior of `ASTNode::SQLQualifiedWildcard::to_string()` (returned `foo*` instead of `foo.*`) - thanks @thomas-jeepe! (#52) + +## [0.3.1] - 2019-04-20 +### Added +- Extended `SQLStatement::SQLCreateTable` to support Hive's EXTERNAL TABLES (`CREATE EXTERNAL TABLE .. STORED AS .. LOCATION '..'`) - thanks @zhzy0077! (#46) +- Parse `SELECT DISTINCT` to `SQLSelect::distinct` (#49) + +## [0.3.0] - 2019-04-03 +### Changed +This release includes major changes to the AST structs to add a number of features, as described in #37 and #43. In particular: +- `ASTNode` variants that represent statements were extracted from `ASTNode` into a separate `SQLStatement` enum; + - `Parser::parse_sql` now returns a `Vec` of parsed statements. + - `ASTNode` now represents an expression (renamed to `Expr` in 0.4.0) +- The query representation (formerly `ASTNode::SQLSelect`) became more complicated to support: + - `WITH` and `UNION`/`EXCEPT`/`INTERSECT` (via `SQLQuery`, `Cte`, and `SQLSetExpr`), + - aliases and qualified wildcards in `SELECT` (via `SQLSelectItem`), + - and aliases in `FROM`/`JOIN` (via `TableFactor`). +- A new `SQLObjectName` struct is used instead of `String` or `ASTNode::SQLCompoundIdentifier` - for objects like tables, custom types, etc. +- Added support for "delimited identifiers" and made keywords context-specific (thus accepting them as valid identifiers in most contexts) - **this caused a regression in parsing `SELECT .. FROM .. LIMIT ..` (#67), fixed in 0.4.0** + +### Added +Other than the changes listed above, some less intrusive additions include: +- Support `CREATE [MATERIALIZED] VIEW` statement +- Support `IN`, `BETWEEN`, unary +/- in epressions +- Support `CHAR` data type and `NUMERIC` not followed by `(p,s)`. +- Support national string literals (`N'...'`) + +## [0.2.4] - 2019-03-08 +Same as 0.2.2. + +## [0.2.3] - 2019-03-08 [YANKED] + +## [0.2.2] - 2019-03-08 +### Changed +- Removed `Value::String`, `Value::DoubleQuotedString`, and `Token::String`, making + - `'...'` parse as a string literal (`Value::SingleQuotedString`), and + - `"..."` fail to parse until version 0.3.0 (#36) + +## [0.2.1] - 2019-01-13 +We don't have a changelog for the changes made in 2018, but thanks to @crw5996, @cswinter, @fredrikroos, @ivanceras, @nickolay, @virattara for their contributions in the early stages of the project! + +## [0.1.0] - 2018-09-03 +Initial release From d853c35391a3e53a5d1e03b51383de54055afae6 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 7 Nov 2024 16:59:14 +0100 Subject: [PATCH 43/80] improve support for T-SQL EXECUTE statements (#1490) --- src/ast/mod.rs | 23 ++++++++++++++++------ src/parser/mod.rs | 20 ++++++++++++++----- tests/sqlparser_common.rs | 39 +++++++++++++++++++++++++++++++++++++ tests/sqlparser_postgres.rs | 9 ++++++--- 4 files changed, 77 insertions(+), 14 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index a24739a60..31d7af1ba 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -3113,10 +3113,14 @@ pub enum Statement { /// EXECUTE name [ ( parameter [, ...] ) ] [USING ] /// ``` /// - /// Note: this is a PostgreSQL-specific statement. + /// Note: this statement is supported by Postgres and MSSQL, with slight differences in syntax. + /// + /// Postgres: + /// MSSQL: Execute { - name: Ident, + name: ObjectName, parameters: Vec, + has_parentheses: bool, using: Vec, }, /// ```sql @@ -4585,12 +4589,19 @@ impl fmt::Display for Statement { Statement::Execute { name, parameters, + has_parentheses, using, } => { - write!(f, "EXECUTE {name}")?; - if !parameters.is_empty() { - write!(f, "({})", display_comma_separated(parameters))?; - } + let (open, close) = if *has_parentheses { + ("(", ")") + } else { + (if parameters.is_empty() { "" } else { " " }, "") + }; + write!( + f, + "EXECUTE {name}{open}{}{close}", + display_comma_separated(parameters), + )?; if !using.is_empty() { write!(f, " USING {}", display_comma_separated(using))?; }; diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 2bd454369..942ff19fd 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -529,7 +529,7 @@ impl<'a> Parser<'a> { // `PREPARE`, `EXECUTE` and `DEALLOCATE` are Postgres-specific // syntaxes. They are used for Postgres prepared statement. Keyword::DEALLOCATE => self.parse_deallocate(), - Keyword::EXECUTE => self.parse_execute(), + Keyword::EXECUTE | Keyword::EXEC => self.parse_execute(), Keyword::PREPARE => self.parse_prepare(), Keyword::MERGE => self.parse_merge(), // `LISTEN` and `NOTIFY` are Postgres-specific @@ -11807,11 +11807,20 @@ impl<'a> Parser<'a> { } pub fn parse_execute(&mut self) -> Result { - let name = self.parse_identifier(false)?; + let name = self.parse_object_name(false)?; - let mut parameters = vec![]; - if self.consume_token(&Token::LParen) { - parameters = self.parse_comma_separated(Parser::parse_expr)?; + let has_parentheses = self.consume_token(&Token::LParen); + + let end_token = match (has_parentheses, self.peek_token().token) { + (true, _) => Token::RParen, + (false, Token::EOF) => Token::EOF, + (false, Token::Word(w)) if w.keyword == Keyword::USING => Token::Word(w), + (false, _) => Token::SemiColon, + }; + + let parameters = self.parse_comma_separated0(Parser::parse_expr, end_token)?; + + if has_parentheses { self.expect_token(&Token::RParen)?; } @@ -11827,6 +11836,7 @@ impl<'a> Parser<'a> { Ok(Statement::Execute { name, parameters, + has_parentheses, using, }) } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 49753a1f4..e37280636 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -1396,6 +1396,10 @@ fn pg_and_generic() -> TestedDialects { ]) } +fn ms_and_generic() -> TestedDialects { + TestedDialects::new(vec![Box::new(MsSqlDialect {}), Box::new(GenericDialect {})]) +} + #[test] fn parse_json_ops_without_colon() { use self::BinaryOperator::*; @@ -9735,6 +9739,41 @@ fn parse_call() { ); } +#[test] +fn parse_execute_stored_procedure() { + let expected = Statement::Execute { + name: ObjectName(vec![ + Ident { + value: "my_schema".to_string(), + quote_style: None, + }, + Ident { + value: "my_stored_procedure".to_string(), + quote_style: None, + }, + ]), + parameters: vec![ + Expr::Value(Value::NationalStringLiteral("param1".to_string())), + Expr::Value(Value::NationalStringLiteral("param2".to_string())), + ], + has_parentheses: false, + using: vec![], + }; + assert_eq!( + // Microsoft SQL Server does not use parentheses around arguments for EXECUTE + ms_and_generic() + .verified_stmt("EXECUTE my_schema.my_stored_procedure N'param1', N'param2'"), + expected + ); + assert_eq!( + ms_and_generic().one_statement_parses_to( + "EXEC my_schema.my_stored_procedure N'param1', N'param2';", + "EXECUTE my_schema.my_stored_procedure N'param1', N'param2'", + ), + expected + ); +} + #[test] fn parse_create_table_collate() { pg_and_generic().verified_stmt("CREATE TABLE tbl (foo INT, bar TEXT COLLATE \"de_DE\")"); diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index c30603baa..100c8eeb2 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -1539,8 +1539,9 @@ fn parse_execute() { assert_eq!( stmt, Statement::Execute { - name: "a".into(), + name: ObjectName(vec!["a".into()]), parameters: vec![], + has_parentheses: false, using: vec![] } ); @@ -1549,11 +1550,12 @@ fn parse_execute() { assert_eq!( stmt, Statement::Execute { - name: "a".into(), + name: ObjectName(vec!["a".into()]), parameters: vec![ Expr::Value(number("1")), Expr::Value(Value::SingleQuotedString("t".to_string())) ], + has_parentheses: true, using: vec![] } ); @@ -1563,8 +1565,9 @@ fn parse_execute() { assert_eq!( stmt, Statement::Execute { - name: "a".into(), + name: ObjectName(vec!["a".into()]), parameters: vec![], + has_parentheses: false, using: vec![ Expr::Cast { kind: CastKind::Cast, From 334a5bf354ac964a14f2e2dc3851ed4d853b28d7 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 7 Nov 2024 11:26:57 -0500 Subject: [PATCH 44/80] Update CHANGELOG.md for `0.52.0` release, add scripts/ instructions for ASF releases (#1479) --- .github/workflows/rust.yml | 35 ----- .gitignore | 1 + CHANGELOG.md | 4 +- Cargo.toml | 14 +- changelog/0.52.0.md | 104 ++++++++++++++ dev/release/README.md | 181 ++++++++++++++++++++++++ dev/release/check-rat-report.py | 59 ++++++++ dev/release/create-tarball.sh | 135 ++++++++++++++++++ dev/release/generate-changelog.py | 164 +++++++++++++++++++++ dev/release/rat_exclude_files.txt | 6 + dev/release/release-tarball.sh | 74 ++++++++++ dev/release/run-rat.sh | 43 ++++++ dev/release/verify-release-candidate.sh | 152 ++++++++++++++++++++ docs/releasing.md | 81 ----------- 14 files changed, 926 insertions(+), 127 deletions(-) create mode 100644 changelog/0.52.0.md create mode 100644 dev/release/README.md create mode 100644 dev/release/check-rat-report.py create mode 100755 dev/release/create-tarball.sh create mode 100755 dev/release/generate-changelog.py create mode 100644 dev/release/rat_exclude_files.txt create mode 100755 dev/release/release-tarball.sh create mode 100755 dev/release/run-rat.sh create mode 100755 dev/release/verify-release-candidate.sh delete mode 100644 docs/releasing.md diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 253d8ab27..2502abe9d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -85,38 +85,3 @@ jobs: use-tool-cache: true - name: Test run: cargo test --all-features - - test-coverage: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Rust Toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Install Tarpaulin - uses: actions-rs/install@v0.1 - with: - crate: cargo-tarpaulin - version: 0.14.2 - use-tool-cache: true - - name: Coverage - run: cargo tarpaulin -o Lcov --output-dir ./coverage - - name: Coveralls - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - publish-crate: - if: startsWith(github.ref, 'refs/tags/v0') - runs-on: ubuntu-latest - needs: [test] - steps: - - uses: actions/checkout@v4 - - name: Setup Rust Toolchain - uses: ./.github/actions/setup-builder - - name: Publish - shell: bash - run: | - cargo publish --token ${{ secrets.CRATES_TOKEN }} diff --git a/.gitignore b/.gitignore index 4c6821d47..f705d0b0d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /target/ /sqlparser_bench/target/ /derive/target/ +dev/dist # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock diff --git a/CHANGELOG.md b/CHANGELOG.md index e047515ad..ec74bf633 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,5 +24,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm Given that the parser produces a typed AST, any changes to the AST will technically be breaking and thus will result in a `0.(N+1)` version. + - Unreleased: Check https://github.com/sqlparser-rs/sqlparser-rs/commits/main for undocumented changes. -- `0.51.0` and earlier: [changelog/0.51.0-pre.md](changelog/0.51.0-pre.md) \ No newline at end of file +- `0.52.0`: [changelog/0.52.0.md](changelog/0.52.0.md) +- `0.51.0` and earlier: [changelog/0.51.0-pre.md](changelog/0.51.0-pre.md) diff --git a/Cargo.toml b/Cargo.toml index 99546465b..18b246e04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,12 +18,12 @@ [package] name = "sqlparser" description = "Extensible SQL Lexer and Parser with support for ANSI SQL:2011" -version = "0.51.0" -authors = ["Andy Grove "] -homepage = "https://github.com/sqlparser-rs/sqlparser-rs" +version = "0.52.0" +authors = ["Apache DataFusion "] +homepage = "https://github.com/apache/datafusion-sqlparser-rs" documentation = "https://docs.rs/sqlparser/" keywords = ["ansi", "sql", "lexer", "parser"] -repository = "https://github.com/sqlparser-rs/sqlparser-rs" +repository = "https://github.com/apache/datafusion-sqlparser-rs" license = "Apache-2.0" include = [ "src/**/*.rs", @@ -58,12 +58,6 @@ simple_logger = "5.0" matches = "0.1" pretty_assertions = "1" -[package.metadata.release] -# Instruct `cargo release` to not run `cargo publish` locally: -# https://github.com/sunng87/cargo-release/blob/master/docs/reference.md#config-fields -# See docs/releasing.md for details. -publish = false - [package.metadata.docs.rs] # Document these features on docs.rs features = ["serde", "visitor"] diff --git a/changelog/0.52.0.md b/changelog/0.52.0.md new file mode 100644 index 000000000..9d5b16c7c --- /dev/null +++ b/changelog/0.52.0.md @@ -0,0 +1,104 @@ + + +# sqlparser-rs 0.52.0 Changelog + +This release consists of 45 commits from 20 contributors. See credits at the end of this changelog for more information. + +**Implemented enhancements:** + +- feat: support explain options [#1426](https://github.com/apache/datafusion-sqlparser-rs/pull/1426) (kysshsy) +- feat: adding Display implementation to DELETE and INSERT [#1427](https://github.com/apache/datafusion-sqlparser-rs/pull/1427) (seve-martinez) + +**Fixed bugs:** + +- fix: `maybe_parse` preventing parser from erroring on recursion limit [#1464](https://github.com/apache/datafusion-sqlparser-rs/pull/1464) (tomershaniii) + +**Other:** + +- Fix parsing of negative values [#1419](https://github.com/apache/datafusion-sqlparser-rs/pull/1419) (agscpp) +- Allow to use ON CLUSTER cluster_name in TRUNCATE syntax [#1428](https://github.com/apache/datafusion-sqlparser-rs/pull/1428) (git-hulk) +- chore: remove redundant punctuation [#1434](https://github.com/apache/datafusion-sqlparser-rs/pull/1434) (Fischer0522) +- MS SQL Server: add support for IDENTITY column option [#1432](https://github.com/apache/datafusion-sqlparser-rs/pull/1432) (7phs) +- Update to ASF header / add when missing [#1437](https://github.com/apache/datafusion-sqlparser-rs/pull/1437) (alamb) +- Some small optimizations [#1424](https://github.com/apache/datafusion-sqlparser-rs/pull/1424) (exrok) +- Fix `codestyle` CI check [#1438](https://github.com/apache/datafusion-sqlparser-rs/pull/1438) (alamb) +- Implements CREATE POLICY syntax for PostgreSQL [#1440](https://github.com/apache/datafusion-sqlparser-rs/pull/1440) (git-hulk) +- make `parse_expr_with_alias` public [#1444](https://github.com/apache/datafusion-sqlparser-rs/pull/1444) (Eason0729) +- Implements DROP POLICY syntax for PostgreSQL [#1445](https://github.com/apache/datafusion-sqlparser-rs/pull/1445) (git-hulk) +- Support `DROP DATABASE` [#1443](https://github.com/apache/datafusion-sqlparser-rs/pull/1443) (linhr) +- Implements ALTER POLICY syntax for PostgreSQL [#1446](https://github.com/apache/datafusion-sqlparser-rs/pull/1446) (git-hulk) +- Add a note discouraging new use of `dialect_of` macro [#1448](https://github.com/apache/datafusion-sqlparser-rs/pull/1448) (alamb) +- Expand handling of `LIMIT 1, 2` handling to include sqlite [#1447](https://github.com/apache/datafusion-sqlparser-rs/pull/1447) (joshuawarner32) +- Fix always uses CommentDef::WithoutEq while parsing the inline comment [#1453](https://github.com/apache/datafusion-sqlparser-rs/pull/1453) (git-hulk) +- Add support for the LIKE ANY and ILIKE ANY pattern-matching condition [#1456](https://github.com/apache/datafusion-sqlparser-rs/pull/1456) (yoavcloud) +- added ability to parse extension to parse_comment inside postgres dialect [#1451](https://github.com/apache/datafusion-sqlparser-rs/pull/1451) (MaxwellKnight) +- Snowflake: support of views column comment [#1441](https://github.com/apache/datafusion-sqlparser-rs/pull/1441) (7phs) +- Add SQLite "ON CONFLICT" column option in CREATE TABLE statements [#1442](https://github.com/apache/datafusion-sqlparser-rs/pull/1442) (nucccc) +- Add support for ASC and DESC in CREATE TABLE column constraints for SQLite. [#1462](https://github.com/apache/datafusion-sqlparser-rs/pull/1462) (caldwell) +- Add support of `EXPLAIN QUERY PLAN` syntax for SQLite dialect [#1458](https://github.com/apache/datafusion-sqlparser-rs/pull/1458) (git-hulk) +- Add "DROP TYPE" support. [#1461](https://github.com/apache/datafusion-sqlparser-rs/pull/1461) (caldwell) +- chore: Add asf.yaml [#1463](https://github.com/apache/datafusion-sqlparser-rs/pull/1463) (Xuanwo) +- Add support for quantified comparison predicates (ALL/ANY/SOME) [#1459](https://github.com/apache/datafusion-sqlparser-rs/pull/1459) (yoavcloud) +- MySQL dialect: Add support for hash comments [#1466](https://github.com/apache/datafusion-sqlparser-rs/pull/1466) (hansott) +- Fix #1469 (SET ROLE regression) [#1474](https://github.com/apache/datafusion-sqlparser-rs/pull/1474) (lovasoa) +- Add support for parsing MsSql alias with equals [#1467](https://github.com/apache/datafusion-sqlparser-rs/pull/1467) (yoavcloud) +- Snowflake: support for extended column options in `CREATE TABLE` [#1454](https://github.com/apache/datafusion-sqlparser-rs/pull/1454) (7phs) +- MsSQL TRY_CONVERT [#1477](https://github.com/apache/datafusion-sqlparser-rs/pull/1477) (yoavcloud) +- Add PostgreSQL specfic "CREATE TYPE t AS ENUM (...)" support. [#1460](https://github.com/apache/datafusion-sqlparser-rs/pull/1460) (caldwell) +- Fix build [#1483](https://github.com/apache/datafusion-sqlparser-rs/pull/1483) (yoavcloud) +- Fix complex blocks warning when running clippy [#1488](https://github.com/apache/datafusion-sqlparser-rs/pull/1488) (git-hulk) +- Add support for SHOW DATABASES/SCHEMAS/TABLES/VIEWS in Hive [#1487](https://github.com/apache/datafusion-sqlparser-rs/pull/1487) (yoavcloud) +- Fix typo in `Dialect::supports_eq_alias_assigment` [#1478](https://github.com/apache/datafusion-sqlparser-rs/pull/1478) (alamb) +- Add support for PostgreSQL `LISTEN/NOTIFY` syntax [#1485](https://github.com/apache/datafusion-sqlparser-rs/pull/1485) (wugeer) +- Add support for TOP before ALL/DISTINCT [#1495](https://github.com/apache/datafusion-sqlparser-rs/pull/1495) (yoavcloud) +- add support for `FOR ORDINALITY` and `NESTED` in JSON_TABLE [#1493](https://github.com/apache/datafusion-sqlparser-rs/pull/1493) (lovasoa) +- Add Apache License to additional files [#1502](https://github.com/apache/datafusion-sqlparser-rs/pull/1502) (alamb) +- Move CHANGELOG content [#1503](https://github.com/apache/datafusion-sqlparser-rs/pull/1503) (alamb) +- improve support for T-SQL EXECUTE statements [#1490](https://github.com/apache/datafusion-sqlparser-rs/pull/1490) (lovasoa) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 8 Andrew Lamb + 7 Yoav Cohen + 7 hulk + 3 Aleksei Piianin + 3 David Caldwell + 3 Ophir LOJKINE + 1 Agaev Guseyn + 1 Eason + 1 Fischer + 1 Hans Ott + 1 Heran Lin + 1 Joshua Warner + 1 Maxwell Knight + 1 Seve Martinez + 1 Siyuan Huang + 1 Thomas Dagenais + 1 Xuanwo + 1 nucccc + 1 tomershaniii + 1 wugeer +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + diff --git a/dev/release/README.md b/dev/release/README.md new file mode 100644 index 000000000..c440f7387 --- /dev/null +++ b/dev/release/README.md @@ -0,0 +1,181 @@ + + + +## Process Overview + +As part of the Apache governance model, official releases consist of signed +source tarballs approved by the DataFusion PMC. + +We then use the code in the approved artifacts to release to crates.io. + +### Change Log + +We maintain a `CHANGELOG.md` so our users know what has been changed between releases. + +You will need a GitHub Personal Access Token for the following steps. Follow +[these instructions](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) +to generate one if you do not already have one. + +The changelog is generated using a Python script which needs `PyGitHub`, installed using pip: + +```shell +pip3 install PyGitHub +``` + +To generate the changelog, set the `GITHUB_TOKEN` environment variable to a valid token and then run the script +providing two commit ids or tags followed by the version number of the release being created. The following +example generates a change log of all changes between the first commit and the current HEAD revision. + +```shell +export GITHUB_TOKEN= +python ./dev/release/generate-changelog.py v0.51.0 HEAD 0.52.0 > changelog/0.52.0.md +``` + +This script creates a changelog from GitHub PRs based on the labels associated with them as well as looking for +titles starting with `feat:`, `fix:`, or `docs:`. + +Add an entry to CHANGELOG.md for the new version + +## Prepare release commits and PR + +### Update Version + +Checkout the main commit to be released + +```shell +git fetch apache +git checkout apache/main +``` + +Manually update the version in the root `Cargo.toml` to the release version (e.g. `0.52.0`). + +Lastly commit the version change: + +```shell +git commit -a -m 'Update version' +``` + +## Prepare release candidate artifacts + +After the PR gets merged, you are ready to create release artifacts from the +merged commit. + +(Note you need to be a committer to run these scripts as they upload to the apache svn distribution servers) + +### Pick a Release Candidate (RC) number + +Pick numbers in sequential order, with `0` for `rc0`, `1` for `rc1`, etc. + +### Create git tag for the release: + +While the official release artifacts are signed tarballs and zip files, we also +tag the commit it was created for convenience and code archaeology. + +Using a string such as `v0.52.0` as the ``, create and push the tag by running these commands: + +For example to tag version `0.52.0` + +```shell +git fetch apache +git tag v0.52.0-rc1 apache/main +# push tag to Github remote +git push apache v0.52.0-rc1 +``` + +### Create, sign, and upload artifacts + +Run `create-tarball.sh` with the `` tag and `` and you found in previous steps: + +```shell +GITHUB_TOKEN= ./dev/release/create-tarball.sh 0.52.0 1 +``` + +The `create-tarball.sh` script + +1. creates and uploads all release candidate artifacts to the [datafusion + dev](https://dist.apache.org/repos/dist/dev/datafusion) location on the + apache distribution svn server + +2. provide you an email template to + send to dev@datafusion.apache.org for release voting. + +### Vote on Release Candidate artifacts + +Send the email output from the script to dev@datafusion.apache.org. + +For the release to become "official" it needs at least three PMC members to vote +1 on it. + +### Verifying Release Candidates + +The `dev/release/verify-release-candidate.sh` is a script in this repository that can assist in the verification process. Run it like: + +```shell +./dev/release/verify-release-candidate.sh 0.52.0 1 +``` + +#### If the release is not approved + +If the release is not approved, fix whatever the problem is, merge changelog +changes into main if there is any and try again with the next RC number. + +## Finalize the release + +NOTE: steps in this section can only be done by PMC members. + +### After the release is approved + +Move artifacts to the release location in SVN, using the `release-tarball.sh` script: + +```shell +./dev/release/release-tarball.sh 0.52.0 1 +``` + +Congratulations! The release is now official! + +### Publish on Crates.io + +Only approved releases of the tarball should be published to +crates.io, in order to conform to Apache Software Foundation +governance standards. + +A DataFusion committer can publish this crate after an official project release has +been made to crates.io using the following instructions. + +Follow [these +instructions](https://doc.rust-lang.org/cargo/reference/publishing.html) to +create an account and login to crates.io before asking to be added as an owner +to the sqlparser DataFusion crates. + +Download and unpack the official release tarball + +Verify that the Cargo.toml in the tarball contains the correct version +(e.g. `version = "0.52.0"`) and then publish the crates by running the following commands + +```shell +cargo publish +``` + +If necessary, also publish the `sqlparser_derive` crate: + +crates.io homepage: https://crates.io/crates/sqlparser_derive + +```shell +(cd derive && cargo publish +``` diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py new file mode 100644 index 000000000..e30d72bdd --- /dev/null +++ b/dev/release/check-rat-report.py @@ -0,0 +1,59 @@ +#!/usr/bin/python +############################################################################## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +############################################################################## +import fnmatch +import re +import sys +import xml.etree.ElementTree as ET + +if len(sys.argv) != 3: + sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" % + sys.argv[0]) + sys.exit(1) + +exclude_globs_filename = sys.argv[1] +xml_filename = sys.argv[2] + +globs = [line.strip() for line in open(exclude_globs_filename, "r")] + +tree = ET.parse(xml_filename) +root = tree.getroot() +resources = root.findall('resource') + +all_ok = True +for r in resources: + approvals = r.findall('license-approval') + if not approvals or approvals[0].attrib['name'] == 'true': + continue + clean_name = re.sub('^[^/]+/', '', r.attrib['name']) + excluded = False + for g in globs: + if fnmatch.fnmatch(clean_name, g): + excluded = True + break + if not excluded: + sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % ( + clean_name, r.attrib['name'], approvals[0].attrib['name'])) + all_ok = False + +if not all_ok: + sys.exit(1) + +print('OK') +sys.exit(0) diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh new file mode 100755 index 000000000..4cb17cd36 --- /dev/null +++ b/dev/release/create-tarball.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Adapted from https://github.com/apache/datafusion/tree/master/dev/release/create-tarball.sh + +# This script creates a signed tarball in +# dev/dist/apache-datafusion-sqlparser-rs--rc.tar.gz and uploads it to +# the "dev" area of the dist.apache.datafusion repository and prepares an +# email for sending to the dev@datafusion.apache.org list for a formal +# vote. +# +# See release/README.md for full release instructions +# +# Requirements: +# +# 1. gpg setup for signing and have uploaded your public +# signature to https://pgp.mit.edu/ +# +# 2. Logged into the apache svn server with the appropriate +# credentials +# +# 3. Install the requests python package +# +# +# Based in part on 02-source.sh from apache/arrow +# + +set -e + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo "ex. $0 0.52.0 2" + exit +fi + +if [[ -z "${GITHUB_TOKEN}" ]]; then + echo "Please set personal github token through GITHUB_TOKEN environment variable" + exit +fi + +version=$1 +rc=$2 +tag="v${version}-rc${rc}" + +echo "Attempting to create ${tarball} from tag ${tag}" +release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag}) + +release=apache-datafusion-sqlparser-rs-${version} +distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc} +tarname=${release}.tar.gz +tarball=${distdir}/${tarname} +url="https://dist.apache.org/repos/dist/dev/datafusion/${release}-rc${rc}" + +if [ -z "$release_hash" ]; then + echo "Cannot continue: unknown git tag: ${tag}" +fi + +echo "Draft email for dev@datafusion.apache.org mailing list" +echo "" +echo "---------------------------------------------------------" +cat < containing the files in git at $release_hash +# the files in the tarball are prefixed with {version} (e.g. 4.0.1) +mkdir -p ${distdir} +(cd "${SOURCE_TOP_DIR}" && git archive ${release_hash} --prefix ${release}/ | gzip > ${tarball}) + +echo "Running rat license checker on ${tarball}" +${SOURCE_DIR}/run-rat.sh ${tarball} + +echo "Signing tarball and creating checksums" +gpg --armor --output ${tarball}.asc --detach-sig ${tarball} +# create signing with relative path of tarball +# so that they can be verified with a command such as +# shasum --check apache-datafusion-sqlparser-rs-0.52.0-rc1.tar.gz.sha512 +(cd ${distdir} && shasum -a 256 ${tarname}) > ${tarball}.sha256 +(cd ${distdir} && shasum -a 512 ${tarname}) > ${tarball}.sha512 + + +echo "Uploading to sqlparser-rs dist/dev to ${url}" +svn co --depth=empty https://dist.apache.org/repos/dist/dev/datafusion ${SOURCE_TOP_DIR}/dev/dist +svn add ${distdir} +svn ci -m "Apache DataFusion ${version} ${rc}" ${distdir} diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py new file mode 100755 index 000000000..52fd2e548 --- /dev/null +++ b/dev/release/generate-changelog.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys +from github import Github +import os +import re +import subprocess + +def print_pulls(repo_name, title, pulls): + if len(pulls) > 0: + print("**{}:**".format(title)) + print() + for (pull, commit) in pulls: + url = "https://github.com/{}/pull/{}".format(repo_name, pull.number) + print("- {} [#{}]({}) ({})".format(pull.title, pull.number, url, commit.author.login)) + print() + + +def generate_changelog(repo, repo_name, tag1, tag2, version): + + # get a list of commits between two tags + print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr) + comparison = repo.compare(tag1, tag2) + + # get the pull requests for these commits + print("Fetching pull requests", file=sys.stderr) + unique_pulls = [] + all_pulls = [] + for commit in comparison.commits: + pulls = commit.get_pulls() + for pull in pulls: + # there can be multiple commits per PR if squash merge is not being used and + # in this case we should get all the author names, but for now just pick one + if pull.number not in unique_pulls: + unique_pulls.append(pull.number) + all_pulls.append((pull, commit)) + + # we split the pulls into categories + breaking = [] + bugs = [] + docs = [] + enhancements = [] + performance = [] + other = [] + + # categorize the pull requests based on GitHub labels + print("Categorizing pull requests", file=sys.stderr) + for (pull, commit) in all_pulls: + + # see if PR title uses Conventional Commits + cc_type = '' + cc_scope = '' + cc_breaking = '' + parts = re.findall(r'^([a-z]+)(\([a-z]+\))?(!)?:', pull.title) + if len(parts) == 1: + parts_tuple = parts[0] + cc_type = parts_tuple[0] # fix, feat, docs, chore + cc_scope = parts_tuple[1] # component within project + cc_breaking = parts_tuple[2] == '!' + + labels = [label.name for label in pull.labels] + if 'api change' in labels or cc_breaking: + breaking.append((pull, commit)) + elif 'bug' in labels or cc_type == 'fix': + bugs.append((pull, commit)) + elif 'performance' in labels or cc_type == 'perf': + performance.append((pull, commit)) + elif 'enhancement' in labels or cc_type == 'feat': + enhancements.append((pull, commit)) + elif 'documentation' in labels or cc_type == 'docs' or cc_type == 'doc': + docs.append((pull, commit)) + else: + other.append((pull, commit)) + + # produce the changelog content + print("Generating changelog content", file=sys.stderr) + + # ASF header + print("""\n""") + + print(f"# sqlparser-rs {version} Changelog\n") + + # get the number of commits + commit_count = subprocess.check_output(f"git log --pretty=oneline {tag1}..{tag2} | wc -l", shell=True, text=True).strip() + + # get number of contributors + contributor_count = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2} | wc -l", shell=True, text=True).strip() + + print(f"This release consists of {commit_count} commits from {contributor_count} contributors. " + f"See credits at the end of this changelog for more information.\n") + + print_pulls(repo_name, "Breaking changes", breaking) + print_pulls(repo_name, "Performance related", performance) + print_pulls(repo_name, "Implemented enhancements", enhancements) + print_pulls(repo_name, "Fixed bugs", bugs) + print_pulls(repo_name, "Documentation updates", docs) + print_pulls(repo_name, "Other", other) + + # show code contributions + credits = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2}", shell=True, text=True).rstrip() + + print("## Credits\n") + print("Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) " + "per contributor.\n") + print("```") + print(credits) + print("```\n") + + print("Thank you also to everyone who contributed in other ways such as filing issues, reviewing " + "PRs, and providing feedback on this release.\n") + +def cli(args=None): + """Process command line arguments.""" + if not args: + args = sys.argv[1:] + + parser = argparse.ArgumentParser() + parser.add_argument("tag1", help="The previous commit or tag (e.g. 0.1.0)") + parser.add_argument("tag2", help="The current commit or tag (e.g. HEAD)") + parser.add_argument("version", help="The version number to include in the changelog") + args = parser.parse_args() + + token = os.getenv("GITHUB_TOKEN") + project = "apache/datafusion-sqlparser-rs" + + g = Github(token) + repo = g.get_repo(project) + generate_changelog(repo, project, args.tag1, args.tag2, args.version) + +if __name__ == "__main__": + cli() \ No newline at end of file diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt new file mode 100644 index 000000000..a567eda9c --- /dev/null +++ b/dev/release/rat_exclude_files.txt @@ -0,0 +1,6 @@ +# Files to exclude from the Apache Rat (license) check +.gitignore +.tool-versions +dev/release/rat_exclude_files.txt +fuzz/.gitignore + diff --git a/dev/release/release-tarball.sh b/dev/release/release-tarball.sh new file mode 100755 index 000000000..e59b2776c --- /dev/null +++ b/dev/release/release-tarball.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/release-tarball.sh + +# This script copies a tarball from the "dev" area of the +# dist.apache.datafusion repository to the "release" area +# +# This script should only be run after the release has been approved +# by the Apache DataFusion PMC committee. +# +# See release/README.md for full release instructions +# +# Based in part on post-01-upload.sh from apache/arrow + + +set -e +set -u + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo "ex. $0 0.52.0 2" + exit +fi + +version=$1 +rc=$2 + +tmp_dir=tmp-apache-datafusion-dist + +echo "Recreate temporary directory: ${tmp_dir}" +rm -rf ${tmp_dir} +mkdir -p ${tmp_dir} + +echo "Clone dev dist repository" +svn \ + co \ + https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-sqlparser-rs-${version}-rc${rc} \ + ${tmp_dir}/dev + +echo "Clone release dist repository" +svn co https://dist.apache.org/repos/dist/release/datafusion ${tmp_dir}/release + +echo "Copy ${version}-rc${rc} to release working copy" +release_version=datafusion-sqlparser-rs-${version} +mkdir -p ${tmp_dir}/release/${release_version} +cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/ +svn add ${tmp_dir}/release/${release_version} + +echo "Commit release" +svn ci -m "Apache DataFusion sqlparser-rs ${version}" ${tmp_dir}/release + +echo "Clean up" +rm -rf ${tmp_dir} + +echo "Success! The release is available here:" +echo " https://dist.apache.org/repos/dist/release/datafusion/${release_version}" diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh new file mode 100755 index 000000000..94fa55fbe --- /dev/null +++ b/dev/release/run-rat.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +RAT_VERSION=0.13 + +# download apache rat +if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then + curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar > apache-rat-${RAT_VERSION}.jar +fi + +RAT="java -jar apache-rat-${RAT_VERSION}.jar -x " + +RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) + +# generate the rat report +$RAT $1 > rat.txt +python $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt rat.txt > filtered_rat.txt +cat filtered_rat.txt +UNAPPROVED=`cat filtered_rat.txt | grep "NOT APPROVED" | wc -l` + +if [ "0" -eq "${UNAPPROVED}" ]; then + echo "No unapproved licenses" +else + echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt" + exit 1 +fi diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh new file mode 100755 index 000000000..9ff7e17b5 --- /dev/null +++ b/dev/release/verify-release-candidate.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +case $# in + 2) VERSION="$1" + RC_NUMBER="$2" + ;; + *) echo "Usage: $0 X.Y.Z RC_NUMBER" + exit 1 + ;; +esac + +set -e +set -x +set -o pipefail + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +ARROW_DIR="$(dirname $(dirname ${SOURCE_DIR}))" +ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/datafusion' + +download_dist_file() { + curl \ + --silent \ + --show-error \ + --fail \ + --location \ + --remote-name $ARROW_DIST_URL/$1 +} + +download_rc_file() { + download_dist_file apache-datafusion-sqlparser-rs-${VERSION}-rc${RC_NUMBER}/$1 +} + +import_gpg_keys() { + download_dist_file KEYS + gpg --import KEYS +} + +if type shasum >/dev/null 2>&1; then + sha256_verify="shasum -a 256 -c" + sha512_verify="shasum -a 512 -c" +else + sha256_verify="sha256sum -c" + sha512_verify="sha512sum -c" +fi + +fetch_archive() { + local dist_name=$1 + download_rc_file ${dist_name}.tar.gz + download_rc_file ${dist_name}.tar.gz.asc + download_rc_file ${dist_name}.tar.gz.sha256 + download_rc_file ${dist_name}.tar.gz.sha512 + verify_dir_artifact_signatures +} + +verify_dir_artifact_signatures() { + # verify the signature and the checksums of each artifact + find . -name '*.asc' | while read sigfile; do + artifact=${sigfile/.asc/} + gpg --verify $sigfile $artifact || exit 1 + + # go into the directory because the checksum files contain only the + # basename of the artifact + pushd $(dirname $artifact) + base_artifact=$(basename $artifact) + ${sha256_verify} $base_artifact.sha256 || exit 1 + ${sha512_verify} $base_artifact.sha512 || exit 1 + popd + done +} + +setup_tempdir() { + cleanup() { + if [ "${TEST_SUCCESS}" = "yes" ]; then + rm -fr "${ARROW_TMPDIR}" + else + echo "Failed to verify release candidate. See ${ARROW_TMPDIR} for details." + fi + } + + if [ -z "${ARROW_TMPDIR}" ]; then + # clean up automatically if ARROW_TMPDIR is not defined + ARROW_TMPDIR=$(mktemp -d -t "$1.XXXXX") + trap cleanup EXIT + else + # don't clean up automatically + mkdir -p "${ARROW_TMPDIR}" + fi +} + +test_source_distribution() { + # install rust toolchain in a similar fashion like test-miniconda + export RUSTUP_HOME=$PWD/test-rustup + export CARGO_HOME=$PWD/test-rustup + + curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path + + export PATH=$RUSTUP_HOME/bin:$PATH + source $RUSTUP_HOME/env + + # build and test rust + + # raises on any formatting errors + rustup component add rustfmt --toolchain stable + cargo fmt --all -- --check + + cargo build + cargo test --all-features + + if ( find -iname 'Cargo.toml' | xargs grep SNAPSHOT ); then + echo "Cargo.toml version should not contain SNAPSHOT for releases" + exit 1 + fi + + # Check that publish works + cargo publish --dry-run +} + +TEST_SUCCESS=no + +setup_tempdir "datafusion-sqlparser-rs-${VERSION}" +echo "Working in sandbox ${ARROW_TMPDIR}" +cd ${ARROW_TMPDIR} + +dist_name="apache-datafusion-sqlparser-rs-${VERSION}" +import_gpg_keys +fetch_archive ${dist_name} +tar xf ${dist_name}.tar.gz +pushd ${dist_name} + test_source_distribution +popd + +TEST_SUCCESS=yes +echo 'Release candidate looks good!' +exit 0 diff --git a/docs/releasing.md b/docs/releasing.md deleted file mode 100644 index c1b85a20c..000000000 --- a/docs/releasing.md +++ /dev/null @@ -1,81 +0,0 @@ - - -# Releasing - -## Prerequisites -Publishing to crates.io has been automated via GitHub Actions, so you will only -need push access to the [sqlparser-rs GitHub repository](https://github.com/sqlparser-rs/sqlparser-rs) -in order to publish a release. - -We use the [`cargo release`](https://github.com/sunng87/cargo-release) -subcommand to ensure correct versioning. Install via: - -``` -$ cargo install cargo-release -``` - -## Process - -1. **Before releasing** ensure `CHANGELOG.md` is updated appropriately and that - you have a clean checkout of the `main` branch of the sqlparser repository: - ``` - $ git fetch && git status - On branch main - Your branch is up to date with 'origin/main'. - - nothing to commit, working tree clean - ``` - * If you have the time, check that the examples in the README are up to date. - -2. Using `cargo-release` we can publish a new release like so: - - ``` - $ cargo release minor --push-remote origin - ``` - - After verifying, you can rerun with `--execute` if all looks good. - You can add `--no-push` to stop before actually publishing the release. - - `cargo release` will then: - - * Bump the minor part of the version in `Cargo.toml` (e.g. `0.7.1-alpha.0` - -> `0.8.0`. You can use `patch` instead of `minor`, as appropriate). - * Create a new tag (e.g. `v0.8.0`) locally - * Push the new tag to the specified remote (`origin` in the above - example), which will trigger a publishing process to crates.io as part of - the [corresponding GitHub Action](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/.github/workflows/rust.yml). - - Note that credentials for authoring in this way are securely stored in - the (GitHub) repo secrets as `CRATE_TOKEN`. - -4. Check that the new version of the crate is available on crates.io: - https://crates.io/crates/sqlparser - - -## `sqlparser_derive` crate - -Currently this crate is manually published via `cargo publish`. - -crates.io homepage: https://crates.io/crates/sqlparser_derive - -```shell -cd derive -cargo publish -``` From e857787309e1d03189578b393f20c2e4a850af8d Mon Sep 17 00:00:00 2001 From: wugeer <1284057728@qq.com> Date: Wed, 13 Nov 2024 14:36:33 +0800 Subject: [PATCH 45/80] hive: support for special not expression `!a` and raise error for `a!` factorial operator (#1472) Co-authored-by: Ifeanyi Ubah --- src/ast/operator.rs | 3 ++ src/dialect/hive.rs | 5 ++ src/dialect/mod.rs | 10 ++++ src/dialect/postgresql.rs | 5 ++ src/parser/mod.rs | 12 +++-- tests/sqlparser_common.rs | 110 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 142 insertions(+), 3 deletions(-) diff --git a/src/ast/operator.rs b/src/ast/operator.rs index c3bb379d6..e44ea2bf4 100644 --- a/src/ast/operator.rs +++ b/src/ast/operator.rs @@ -51,6 +51,8 @@ pub enum UnaryOperator { PGPrefixFactorial, /// Absolute value, e.g. `@ -9` (PostgreSQL-specific) PGAbs, + /// Unary logical not operator: e.g. `! false` (Hive-specific) + BangNot, } impl fmt::Display for UnaryOperator { @@ -65,6 +67,7 @@ impl fmt::Display for UnaryOperator { UnaryOperator::PGPostfixFactorial => "!", UnaryOperator::PGPrefixFactorial => "!!", UnaryOperator::PGAbs => "@", + UnaryOperator::BangNot => "!", }) } } diff --git a/src/dialect/hive.rs b/src/dialect/hive.rs index 63642b33c..b97bf69be 100644 --- a/src/dialect/hive.rs +++ b/src/dialect/hive.rs @@ -51,4 +51,9 @@ impl Dialect for HiveDialect { fn require_interval_qualifier(&self) -> bool { true } + + /// See Hive + fn supports_bang_not_operator(&self) -> bool { + true + } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 453fee3de..7592740ca 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -575,6 +575,11 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if the dialect supports `a!` expressions + fn supports_factorial_operator(&self) -> bool { + false + } + /// Returns true if this dialect supports treating the equals operator `=` within a `SelectItem` /// as an alias assignment operator, rather than a boolean expression. /// For example: the following statements are equivalent for such a dialect: @@ -591,6 +596,11 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if the dialect supports `!a` syntax for boolean `NOT` expressions. + fn supports_bang_not_operator(&self) -> bool { + false + } + /// Returns true if the dialect supports the `LISTEN` statement fn supports_listen(&self) -> bool { false diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index c40c826c4..72841c604 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -201,6 +201,11 @@ impl Dialect for PostgreSqlDialect { fn supports_notify(&self) -> bool { true } + + /// see + fn supports_factorial_operator(&self) -> bool { + true + } } pub fn parse_comment(parser: &mut Parser) -> Result { diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 942ff19fd..e329c0177 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1194,6 +1194,14 @@ impl<'a> Parser<'a> { ), }) } + Token::ExclamationMark if self.dialect.supports_bang_not_operator() => { + Ok(Expr::UnaryOp { + op: UnaryOperator::BangNot, + expr: Box::new( + self.parse_subexpr(self.dialect.prec_value(Precedence::UnaryNot))?, + ), + }) + } tok @ Token::DoubleExclamationMark | tok @ Token::PGSquareRoot | tok @ Token::PGCubeRoot @@ -1287,7 +1295,6 @@ impl<'a> Parser<'a> { } _ => self.expected("an expression", next_token), }?; - if self.parse_keyword(Keyword::COLLATE) { Ok(Expr::Collate { expr: Box::new(expr), @@ -2818,8 +2825,7 @@ impl<'a> Parser<'a> { data_type: self.parse_data_type()?, format: None, }) - } else if Token::ExclamationMark == tok { - // PostgreSQL factorial operation + } else if Token::ExclamationMark == tok && self.dialect.supports_factorial_operator() { Ok(Expr::UnaryOp { op: UnaryOperator::PGPostfixFactorial, expr: Box::new(expr), diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index e37280636..84f2f718b 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11532,3 +11532,113 @@ fn test_select_top() { dialects.verified_stmt("SELECT TOP 3 DISTINCT * FROM tbl"); dialects.verified_stmt("SELECT TOP 3 DISTINCT a, b, c FROM tbl"); } + +#[test] +fn parse_bang_not() { + let dialects = all_dialects_where(|d| d.supports_bang_not_operator()); + let sql = "SELECT !a, !(b > 3)"; + let Select { projection, .. } = dialects.verified_only_select(sql); + + for (i, expr) in [ + Box::new(Expr::Identifier(Ident::new("a"))), + Box::new(Expr::Nested(Box::new(Expr::BinaryOp { + left: Box::new(Expr::Identifier(Ident::new("b"))), + op: BinaryOperator::Gt, + right: Box::new(Expr::Value(Value::Number("3".parse().unwrap(), false))), + }))), + ] + .into_iter() + .enumerate() + { + assert_eq!( + SelectItem::UnnamedExpr(Expr::UnaryOp { + op: UnaryOperator::BangNot, + expr + }), + projection[i] + ) + } + + let sql_statements = ["SELECT a!", "SELECT a ! b", "SELECT a ! as b"]; + + for &sql in &sql_statements { + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError("No infix parser for token ExclamationMark".to_string()) + ); + } + + let sql_statements = ["SELECT !a", "SELECT !a b", "SELECT !a as b"]; + let dialects = all_dialects_where(|d| !d.supports_bang_not_operator()); + + for &sql in &sql_statements { + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError("Expected: an expression, found: !".to_string()) + ); + } +} + +#[test] +fn parse_factorial_operator() { + let dialects = all_dialects_where(|d| d.supports_factorial_operator()); + let sql = "SELECT a!, (b + c)!"; + let Select { projection, .. } = dialects.verified_only_select(sql); + + for (i, expr) in [ + Box::new(Expr::Identifier(Ident::new("a"))), + Box::new(Expr::Nested(Box::new(Expr::BinaryOp { + left: Box::new(Expr::Identifier(Ident::new("b"))), + op: BinaryOperator::Plus, + right: Box::new(Expr::Identifier(Ident::new("c"))), + }))), + ] + .into_iter() + .enumerate() + { + assert_eq!( + SelectItem::UnnamedExpr(Expr::UnaryOp { + op: UnaryOperator::PGPostfixFactorial, + expr + }), + projection[i] + ) + } + + let sql_statements = ["SELECT !a", "SELECT !a b", "SELECT !a as b"]; + + for &sql in &sql_statements { + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError("Expected: an expression, found: !".to_string()) + ); + } + + let sql_statements = ["SELECT a!", "SELECT a ! b", "SELECT a ! as b"]; + + // Due to the exclamation mark, which is both part of the `bang not` operator + // and the `factorial` operator, additional filtering not supports + // `bang not` operator is required here. + let dialects = + all_dialects_where(|d| !d.supports_factorial_operator() && !d.supports_bang_not_operator()); + + for &sql in &sql_statements { + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError("No infix parser for token ExclamationMark".to_string()) + ); + } + + // Due to the exclamation mark, which is both part of the `bang not` operator + // and the `factorial` operator, additional filtering supports + // `bang not` operator is required here. + let dialects = + all_dialects_where(|d| !d.supports_factorial_operator() && d.supports_bang_not_operator()); + + for &sql in &sql_statements { + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError("No infix parser for token ExclamationMark".to_string()) + ); + } +} From 90824486df096f37b9f9dd8f1d74f8043c7b1f64 Mon Sep 17 00:00:00 2001 From: gaoqiangz <38213294+gaoqiangz@users.noreply.github.com> Date: Wed, 13 Nov 2024 14:41:13 +0800 Subject: [PATCH 46/80] Add support for MSSQL's `OPENJSON WITH` clause (#1498) --- src/ast/mod.rs | 12 +- src/ast/query.rs | 74 +++++++++ src/keywords.rs | 1 + src/parser/mod.rs | 60 +++++++ tests/sqlparser_mssql.rs | 335 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 476 insertions(+), 6 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 31d7af1ba..81bddcd17 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -56,12 +56,12 @@ pub use self::query::{ InterpolateExpr, Join, JoinConstraint, JoinOperator, JsonTableColumn, JsonTableColumnErrorHandling, JsonTableNamedColumn, JsonTableNestedColumn, LateralView, LockClause, LockType, MatchRecognizePattern, MatchRecognizeSymbol, Measure, - NamedWindowDefinition, NamedWindowExpr, NonBlock, Offset, OffsetRows, OrderBy, OrderByExpr, - PivotValueSource, ProjectionSelect, Query, RenameSelectItem, RepetitionQuantifier, - ReplaceSelectElement, ReplaceSelectItem, RowsPerMatch, Select, SelectInto, SelectItem, SetExpr, - SetOperator, SetQuantifier, Setting, SymbolDefinition, Table, TableAlias, TableFactor, - TableFunctionArgs, TableVersion, TableWithJoins, Top, TopQuantity, ValueTableMode, Values, - WildcardAdditionalOptions, With, WithFill, + NamedWindowDefinition, NamedWindowExpr, NonBlock, Offset, OffsetRows, OpenJsonTableColumn, + OrderBy, OrderByExpr, PivotValueSource, ProjectionSelect, Query, RenameSelectItem, + RepetitionQuantifier, ReplaceSelectElement, ReplaceSelectItem, RowsPerMatch, Select, + SelectInto, SelectItem, SetExpr, SetOperator, SetQuantifier, Setting, SymbolDefinition, Table, + TableAlias, TableFactor, TableFunctionArgs, TableVersion, TableWithJoins, Top, TopQuantity, + ValueTableMode, Values, WildcardAdditionalOptions, With, WithFill, }; pub use self::trigger::{ diff --git a/src/ast/query.rs b/src/ast/query.rs index 7af472430..60ebe3765 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -1036,6 +1036,27 @@ pub enum TableFactor { /// The alias for the table. alias: Option, }, + /// The MSSQL's `OPENJSON` table-valued function. + /// + /// ```sql + /// OPENJSON( jsonExpression [ , path ] ) [ ] + /// + /// ::= WITH ( { colName type [ column_path ] [ AS JSON ] } [ ,...n ] ) + /// ```` + /// + /// Reference: + OpenJsonTable { + /// The JSON expression to be evaluated. It must evaluate to a json string + json_expr: Expr, + /// The path to the array or object to be iterated over. + /// It must evaluate to a json array or object. + json_path: Option, + /// The columns to be extracted from each element of the array or object. + /// Each column must have a name and a type. + columns: Vec, + /// The alias for the table. + alias: Option, + }, /// Represents a parenthesized table factor. The SQL spec only allows a /// join expression (`(foo bar [ baz ... ])`) to be nested, /// possibly several times. @@ -1461,6 +1482,25 @@ impl fmt::Display for TableFactor { } Ok(()) } + TableFactor::OpenJsonTable { + json_expr, + json_path, + columns, + alias, + } => { + write!(f, "OPENJSON({json_expr}")?; + if let Some(json_path) = json_path { + write!(f, ", {json_path}")?; + } + write!(f, ")")?; + if !columns.is_empty() { + write!(f, " WITH ({})", display_comma_separated(columns))?; + } + if let Some(alias) = alias { + write!(f, " AS {alias}")?; + } + Ok(()) + } TableFactor::NestedJoin { table_with_joins, alias, @@ -2421,6 +2461,40 @@ impl fmt::Display for JsonTableColumnErrorHandling { } } +/// A single column definition in MSSQL's `OPENJSON WITH` clause. +/// +/// ```sql +/// colName type [ column_path ] [ AS JSON ] +/// ``` +/// +/// Reference: +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct OpenJsonTableColumn { + /// The name of the column to be extracted. + pub name: Ident, + /// The type of the column to be extracted. + pub r#type: DataType, + /// The path to the column to be extracted. Must be a literal string. + pub path: Option, + /// The `AS JSON` option. + pub as_json: bool, +} + +impl fmt::Display for OpenJsonTableColumn { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{} {}", self.name, self.r#type)?; + if let Some(path) = &self.path { + write!(f, " '{}'", value::escape_single_quote_string(path))?; + } + if self.as_json { + write!(f, " AS JSON")?; + } + Ok(()) + } +} + /// BigQuery supports ValueTables which have 2 modes: /// `SELECT AS STRUCT` /// `SELECT AS VALUE` diff --git a/src/keywords.rs b/src/keywords.rs index d60227c99..982cea81b 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -537,6 +537,7 @@ define_keywords!( ONE, ONLY, OPEN, + OPENJSON, OPERATOR, OPTIMIZE, OPTIMIZER_COSTS, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index e329c0177..a69f1db10 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -10049,6 +10049,7 @@ impl<'a> Parser<'a> { | TableFactor::Function { alias, .. } | TableFactor::UNNEST { alias, .. } | TableFactor::JsonTable { alias, .. } + | TableFactor::OpenJsonTable { alias, .. } | TableFactor::TableFunction { alias, .. } | TableFactor::Pivot { alias, .. } | TableFactor::Unpivot { alias, .. } @@ -10162,6 +10163,9 @@ impl<'a> Parser<'a> { columns, alias, }) + } else if self.parse_keyword_with_tokens(Keyword::OPENJSON, &[Token::LParen]) { + self.prev_token(); + self.parse_open_json_table_factor() } else { let name = self.parse_object_name(true)?; @@ -10227,6 +10231,34 @@ impl<'a> Parser<'a> { } } + /// Parses `OPENJSON( jsonExpression [ , path ] ) [ ]` clause, + /// assuming the `OPENJSON` keyword was already consumed. + fn parse_open_json_table_factor(&mut self) -> Result { + self.expect_token(&Token::LParen)?; + let json_expr = self.parse_expr()?; + let json_path = if self.consume_token(&Token::Comma) { + Some(self.parse_value()?) + } else { + None + }; + self.expect_token(&Token::RParen)?; + let columns = if self.parse_keyword(Keyword::WITH) { + self.expect_token(&Token::LParen)?; + let columns = self.parse_comma_separated(Parser::parse_openjson_table_column_def)?; + self.expect_token(&Token::RParen)?; + columns + } else { + Vec::new() + }; + let alias = self.parse_optional_table_alias(keywords::RESERVED_FOR_TABLE_ALIAS)?; + Ok(TableFactor::OpenJsonTable { + json_expr, + json_path, + columns, + alias, + }) + } + fn parse_match_recognize(&mut self, table: TableFactor) -> Result { self.expect_token(&Token::LParen)?; @@ -10513,6 +10545,34 @@ impl<'a> Parser<'a> { })) } + /// Parses MSSQL's `OPENJSON WITH` column definition. + /// + /// ```sql + /// colName type [ column_path ] [ AS JSON ] + /// ``` + /// + /// Reference: + pub fn parse_openjson_table_column_def(&mut self) -> Result { + let name = self.parse_identifier(false)?; + let r#type = self.parse_data_type()?; + let path = if let Token::SingleQuotedString(path) = self.peek_token().token { + self.next_token(); + Some(path) + } else { + None + }; + let as_json = self.parse_keyword(Keyword::AS); + if as_json { + self.expect_keyword(Keyword::JSON)?; + } + Ok(OpenJsonTableColumn { + name, + r#type, + path, + as_json, + }) + } + fn parse_json_table_column_error_handling( &mut self, ) -> Result, ParserError> { diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index c5f43b072..a1ec5e24a 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -193,6 +193,341 @@ fn parse_mssql_apply_join() { ); } +#[test] +fn parse_mssql_openjson() { + let select = ms().verified_only_select( + "SELECT B.kind, B.id_list \ + FROM t_test_table AS A \ + CROSS APPLY OPENJSON(A.param, '$.config') WITH (kind VARCHAR(20) '$.kind', [id_list] NVARCHAR(MAX) '$.id_list' AS JSON) AS B", + ); + assert_eq!( + vec![TableWithJoins { + relation: TableFactor::Table { + name: ObjectName(vec![Ident { + value: "t_test_table".into(), + quote_style: None, + },]), + alias: Some(TableAlias { + name: Ident { + value: "A".into(), + quote_style: None + }, + columns: vec![] + }), + args: None, + with_hints: vec![], + version: None, + with_ordinality: false, + partitions: vec![] + }, + joins: vec![Join { + relation: TableFactor::OpenJsonTable { + json_expr: Expr::CompoundIdentifier(vec![ + Ident { + value: "A".into(), + quote_style: None, + }, + Ident { + value: "param".into(), + quote_style: None, + } + ]), + json_path: Some(Value::SingleQuotedString("$.config".into())), + columns: vec![ + OpenJsonTableColumn { + name: Ident { + value: "kind".into(), + quote_style: None, + }, + r#type: DataType::Varchar(Some(CharacterLength::IntegerLength { + length: 20, + unit: None + })), + path: Some("$.kind".into()), + as_json: false + }, + OpenJsonTableColumn { + name: Ident { + value: "id_list".into(), + quote_style: Some('['), + }, + r#type: DataType::Nvarchar(Some(CharacterLength::Max)), + path: Some("$.id_list".into()), + as_json: true + } + ], + alias: Some(TableAlias { + name: Ident { + value: "B".into(), + quote_style: None + }, + columns: vec![] + }) + }, + global: false, + join_operator: JoinOperator::CrossApply + }] + }], + select.from + ); + let select = ms().verified_only_select( + "SELECT B.kind, B.id_list \ + FROM t_test_table AS A \ + CROSS APPLY OPENJSON(A.param) WITH (kind VARCHAR(20) '$.kind', [id_list] NVARCHAR(MAX) '$.id_list' AS JSON) AS B", + ); + assert_eq!( + vec![TableWithJoins { + relation: TableFactor::Table { + name: ObjectName(vec![Ident { + value: "t_test_table".into(), + quote_style: None, + },]), + alias: Some(TableAlias { + name: Ident { + value: "A".into(), + quote_style: None + }, + columns: vec![] + }), + args: None, + with_hints: vec![], + version: None, + with_ordinality: false, + partitions: vec![] + }, + joins: vec![Join { + relation: TableFactor::OpenJsonTable { + json_expr: Expr::CompoundIdentifier(vec![ + Ident { + value: "A".into(), + quote_style: None, + }, + Ident { + value: "param".into(), + quote_style: None, + } + ]), + json_path: None, + columns: vec![ + OpenJsonTableColumn { + name: Ident { + value: "kind".into(), + quote_style: None, + }, + r#type: DataType::Varchar(Some(CharacterLength::IntegerLength { + length: 20, + unit: None + })), + path: Some("$.kind".into()), + as_json: false + }, + OpenJsonTableColumn { + name: Ident { + value: "id_list".into(), + quote_style: Some('['), + }, + r#type: DataType::Nvarchar(Some(CharacterLength::Max)), + path: Some("$.id_list".into()), + as_json: true + } + ], + alias: Some(TableAlias { + name: Ident { + value: "B".into(), + quote_style: None + }, + columns: vec![] + }) + }, + global: false, + join_operator: JoinOperator::CrossApply + }] + }], + select.from + ); + let select = ms().verified_only_select( + "SELECT B.kind, B.id_list \ + FROM t_test_table AS A \ + CROSS APPLY OPENJSON(A.param) WITH (kind VARCHAR(20), [id_list] NVARCHAR(MAX)) AS B", + ); + assert_eq!( + vec![TableWithJoins { + relation: TableFactor::Table { + name: ObjectName(vec![Ident { + value: "t_test_table".into(), + quote_style: None, + },]), + alias: Some(TableAlias { + name: Ident { + value: "A".into(), + quote_style: None + }, + columns: vec![] + }), + args: None, + with_hints: vec![], + version: None, + with_ordinality: false, + partitions: vec![] + }, + joins: vec![Join { + relation: TableFactor::OpenJsonTable { + json_expr: Expr::CompoundIdentifier(vec![ + Ident { + value: "A".into(), + quote_style: None, + }, + Ident { + value: "param".into(), + quote_style: None, + } + ]), + json_path: None, + columns: vec![ + OpenJsonTableColumn { + name: Ident { + value: "kind".into(), + quote_style: None, + }, + r#type: DataType::Varchar(Some(CharacterLength::IntegerLength { + length: 20, + unit: None + })), + path: None, + as_json: false + }, + OpenJsonTableColumn { + name: Ident { + value: "id_list".into(), + quote_style: Some('['), + }, + r#type: DataType::Nvarchar(Some(CharacterLength::Max)), + path: None, + as_json: false + } + ], + alias: Some(TableAlias { + name: Ident { + value: "B".into(), + quote_style: None + }, + columns: vec![] + }) + }, + global: false, + join_operator: JoinOperator::CrossApply + }] + }], + select.from + ); + let select = ms_and_generic().verified_only_select( + "SELECT B.kind, B.id_list \ + FROM t_test_table AS A \ + CROSS APPLY OPENJSON(A.param, '$.config') AS B", + ); + assert_eq!( + vec![TableWithJoins { + relation: TableFactor::Table { + name: ObjectName(vec![Ident { + value: "t_test_table".into(), + quote_style: None, + },]), + alias: Some(TableAlias { + name: Ident { + value: "A".into(), + quote_style: None + }, + columns: vec![] + }), + args: None, + with_hints: vec![], + version: None, + with_ordinality: false, + partitions: vec![] + }, + joins: vec![Join { + relation: TableFactor::OpenJsonTable { + json_expr: Expr::CompoundIdentifier(vec![ + Ident { + value: "A".into(), + quote_style: None, + }, + Ident { + value: "param".into(), + quote_style: None, + } + ]), + json_path: Some(Value::SingleQuotedString("$.config".into())), + columns: vec![], + alias: Some(TableAlias { + name: Ident { + value: "B".into(), + quote_style: None + }, + columns: vec![] + }) + }, + global: false, + join_operator: JoinOperator::CrossApply + }] + }], + select.from + ); + let select = ms_and_generic().verified_only_select( + "SELECT B.kind, B.id_list \ + FROM t_test_table AS A \ + CROSS APPLY OPENJSON(A.param) AS B", + ); + assert_eq!( + vec![TableWithJoins { + relation: TableFactor::Table { + name: ObjectName(vec![Ident { + value: "t_test_table".into(), + quote_style: None, + },]), + alias: Some(TableAlias { + name: Ident { + value: "A".into(), + quote_style: None + }, + columns: vec![] + }), + args: None, + with_hints: vec![], + version: None, + with_ordinality: false, + partitions: vec![] + }, + joins: vec![Join { + relation: TableFactor::OpenJsonTable { + json_expr: Expr::CompoundIdentifier(vec![ + Ident { + value: "A".into(), + quote_style: None, + }, + Ident { + value: "param".into(), + quote_style: None, + } + ]), + json_path: None, + columns: vec![], + alias: Some(TableAlias { + name: Ident { + value: "B".into(), + quote_style: None + }, + columns: vec![] + }) + }, + global: false, + join_operator: JoinOperator::CrossApply + }] + }], + select.from + ); +} + #[test] fn parse_mssql_top_paren() { let sql = "SELECT TOP (5) * FROM foo"; From 3a8369aaf5eb0d7daebf906326e80ae08db45c90 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Wed, 13 Nov 2024 11:25:26 +0100 Subject: [PATCH 47/80] Parse true and false as identifiers in mssql (#1510) --- src/dialect/mod.rs | 6 +++ src/dialect/mssql.rs | 5 +++ src/parser/mod.rs | 14 +++++-- tests/sqlparser_common.rs | 81 ++++++++++++--------------------------- tests/sqlparser_mssql.rs | 12 ++++++ tests/sqlparser_mysql.rs | 39 +++++++++++++++++++ 6 files changed, 97 insertions(+), 60 deletions(-) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 7592740ca..c8c11bc95 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -616,6 +616,12 @@ pub trait Dialect: Debug + Any { fn supports_top_before_distinct(&self) -> bool { false } + + /// Returns true if the dialect supports boolean literals (`true` and `false`). + /// For example, in MSSQL these are treated as identifiers rather than boolean literals. + fn supports_boolean_literals(&self) -> bool { + true + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/mssql.rs b/src/dialect/mssql.rs index a5ee0bf75..8aab0bc8a 100644 --- a/src/dialect/mssql.rs +++ b/src/dialect/mssql.rs @@ -57,4 +57,9 @@ impl Dialect for MsSqlDialect { fn supports_try_convert(&self) -> bool { true } + + /// In MSSQL, there is no boolean type, and `true` and `false` are valid column names + fn supports_boolean_literals(&self) -> bool { + false + } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a69f1db10..355456d52 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1014,7 +1014,11 @@ impl<'a> Parser<'a> { let next_token = self.next_token(); let expr = match next_token.token { Token::Word(w) => match w.keyword { - Keyword::TRUE | Keyword::FALSE | Keyword::NULL => { + Keyword::TRUE | Keyword::FALSE if self.dialect.supports_boolean_literals() => { + self.prev_token(); + Ok(Expr::Value(self.parse_value()?)) + } + Keyword::NULL => { self.prev_token(); Ok(Expr::Value(self.parse_value()?)) } @@ -7577,8 +7581,12 @@ impl<'a> Parser<'a> { let location = next_token.location; match next_token.token { Token::Word(w) => match w.keyword { - Keyword::TRUE => Ok(Value::Boolean(true)), - Keyword::FALSE => Ok(Value::Boolean(false)), + Keyword::TRUE if self.dialect.supports_boolean_literals() => { + Ok(Value::Boolean(true)) + } + Keyword::FALSE if self.dialect.supports_boolean_literals() => { + Ok(Value::Boolean(false)) + } Keyword::NULL => Ok(Value::Null), Keyword::NoKeyword if w.quote_style.is_some() => match w.quote_style { Some('"') => Ok(Value::DoubleQuotedString(w.value)), diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 84f2f718b..bef0f535c 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -820,7 +820,7 @@ fn parse_top_level() { verified_stmt("(SELECT 1)"); verified_stmt("((SELECT 1))"); verified_stmt("VALUES (1)"); - verified_stmt("VALUES ROW(1, true, 'a'), ROW(2, false, 'b')"); + verified_stmt("VALUES ROW(1, NULL, 'a'), ROW(2, NULL, 'b')"); } #[test] @@ -1499,7 +1499,7 @@ fn parse_is_not_distinct_from() { #[test] fn parse_not_precedence() { // NOT has higher precedence than OR/AND, so the following must parse as (NOT true) OR true - let sql = "NOT true OR true"; + let sql = "NOT 1 OR 1"; assert_matches!( verified_expr(sql), Expr::BinaryOp { @@ -1919,44 +1919,6 @@ fn parse_binary_all() { ); } -#[test] -fn parse_logical_xor() { - let sql = "SELECT true XOR true, false XOR false, true XOR false, false XOR true"; - let select = verified_only_select(sql); - assert_eq!( - SelectItem::UnnamedExpr(Expr::BinaryOp { - left: Box::new(Expr::Value(Value::Boolean(true))), - op: BinaryOperator::Xor, - right: Box::new(Expr::Value(Value::Boolean(true))), - }), - select.projection[0] - ); - assert_eq!( - SelectItem::UnnamedExpr(Expr::BinaryOp { - left: Box::new(Expr::Value(Value::Boolean(false))), - op: BinaryOperator::Xor, - right: Box::new(Expr::Value(Value::Boolean(false))), - }), - select.projection[1] - ); - assert_eq!( - SelectItem::UnnamedExpr(Expr::BinaryOp { - left: Box::new(Expr::Value(Value::Boolean(true))), - op: BinaryOperator::Xor, - right: Box::new(Expr::Value(Value::Boolean(false))), - }), - select.projection[2] - ); - assert_eq!( - SelectItem::UnnamedExpr(Expr::BinaryOp { - left: Box::new(Expr::Value(Value::Boolean(false))), - op: BinaryOperator::Xor, - right: Box::new(Expr::Value(Value::Boolean(true))), - }), - select.projection[3] - ); -} - #[test] fn parse_between() { fn chk(negated: bool) { @@ -4113,14 +4075,14 @@ fn parse_alter_table_alter_column() { ); match alter_table_op(verified_stmt(&format!( - "{alter_stmt} ALTER COLUMN is_active SET DEFAULT false" + "{alter_stmt} ALTER COLUMN is_active SET DEFAULT 0" ))) { AlterTableOperation::AlterColumn { column_name, op } => { assert_eq!("is_active", column_name.to_string()); assert_eq!( op, AlterColumnOperation::SetDefault { - value: Expr::Value(Value::Boolean(false)) + value: Expr::Value(test_utils::number("0")) } ); } @@ -6502,7 +6464,7 @@ fn parse_values() { verified_stmt("SELECT * FROM (VALUES (1), (2), (3))"); verified_stmt("SELECT * FROM (VALUES (1), (2), (3)), (VALUES (1, 2, 3))"); verified_stmt("SELECT * FROM (VALUES (1)) UNION VALUES (1)"); - verified_stmt("SELECT * FROM (VALUES ROW(1, true, 'a'), ROW(2, false, 'b')) AS t (a, b, c)"); + verified_stmt("SELECT * FROM (VALUES ROW(1, NULL, 'a'), ROW(2, NULL, 'b')) AS t (a, b, c)"); } #[test] @@ -7321,7 +7283,7 @@ fn lateral_derived() { let lateral_str = if lateral_in { "LATERAL " } else { "" }; let sql = format!( "SELECT * FROM customer LEFT JOIN {lateral_str}\ - (SELECT * FROM order WHERE order.customer = customer.id LIMIT 3) AS order ON true" + (SELECT * FROM orders WHERE orders.customer = customer.id LIMIT 3) AS orders ON 1" ); let select = verified_only_select(&sql); let from = only(select.from); @@ -7329,7 +7291,7 @@ fn lateral_derived() { let join = &from.joins[0]; assert_eq!( join.join_operator, - JoinOperator::LeftOuter(JoinConstraint::On(Expr::Value(Value::Boolean(true)))) + JoinOperator::LeftOuter(JoinConstraint::On(Expr::Value(test_utils::number("1")))) ); if let TableFactor::Derived { lateral, @@ -7338,10 +7300,10 @@ fn lateral_derived() { } = join.relation { assert_eq!(lateral_in, lateral); - assert_eq!(Ident::new("order"), alias.name); + assert_eq!(Ident::new("orders"), alias.name); assert_eq!( subquery.to_string(), - "SELECT * FROM order WHERE order.customer = customer.id LIMIT 3" + "SELECT * FROM orders WHERE orders.customer = customer.id LIMIT 3" ); } else { unreachable!() @@ -8381,7 +8343,7 @@ fn parse_merge() { _ => unreachable!(), }; - let sql = "MERGE INTO s.bar AS dest USING newArrivals AS S ON false WHEN NOT MATCHED THEN INSERT VALUES (stg.A, stg.B, stg.C)"; + let sql = "MERGE INTO s.bar AS dest USING newArrivals AS S ON (1 > 1) WHEN NOT MATCHED THEN INSERT VALUES (stg.A, stg.B, stg.C)"; verified_stmt(sql); } @@ -11160,13 +11122,11 @@ fn parse_explain_with_option_list() { #[test] fn test_create_policy() { - let sql = concat!( - "CREATE POLICY my_policy ON my_table ", - "AS PERMISSIVE FOR SELECT ", - "TO my_role, CURRENT_USER ", - "USING (c0 = 1) ", - "WITH CHECK (true)" - ); + let sql: &str = "CREATE POLICY my_policy ON my_table \ + AS PERMISSIVE FOR SELECT \ + TO my_role, CURRENT_USER \ + USING (c0 = 1) \ + WITH CHECK (1 = 1)"; match all_dialects().verified_stmt(sql) { Statement::CreatePolicy { @@ -11194,7 +11154,14 @@ fn test_create_policy() { right: Box::new(Expr::Value(Value::Number("1".parse().unwrap(), false))), }) ); - assert_eq!(with_check, Some(Expr::Value(Value::Boolean(true)))); + assert_eq!( + with_check, + Some(Expr::BinaryOp { + left: Box::new(Expr::Value(Value::Number("1".parse().unwrap(), false))), + op: BinaryOperator::Eq, + right: Box::new(Expr::Value(Value::Number("1".parse().unwrap(), false))), + }) + ); } _ => unreachable!(), } @@ -11205,7 +11172,7 @@ fn test_create_policy() { "AS PERMISSIVE FOR SELECT ", "TO my_role, CURRENT_USER ", "USING (c0 IN (SELECT column FROM t0)) ", - "WITH CHECK (true)" + "WITH CHECK (1 = 1)" )); // omit AS / FOR / TO / USING / WITH CHECK clauses is allowed all_dialects().verified_stmt("CREATE POLICY my_policy ON my_table"); diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index a1ec5e24a..4f9f6bb82 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -1366,6 +1366,18 @@ fn parse_create_table_with_identity_column() { } } +#[test] +fn parse_true_false_as_identifiers() { + assert_eq!( + ms().verified_expr("true"), + Expr::Identifier(Ident::new("true")) + ); + assert_eq!( + ms().verified_expr("false"), + Expr::Identifier(Ident::new("false")) + ); +} + fn ms() -> TestedDialects { TestedDialects::new(vec![Box::new(MsSqlDialect {})]) } diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 47f7f5b4b..44b2ac6ba 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -2817,3 +2817,42 @@ fn test_group_concat() { mysql_and_generic() .verified_expr("GROUP_CONCAT(DISTINCT test_score ORDER BY test_score DESC SEPARATOR ' ')"); } + +/// The XOR binary operator is only supported in MySQL +#[test] +fn parse_logical_xor() { + let sql = "SELECT true XOR true, false XOR false, true XOR false, false XOR true"; + let select = mysql_and_generic().verified_only_select(sql); + assert_eq!( + SelectItem::UnnamedExpr(Expr::BinaryOp { + left: Box::new(Expr::Value(Value::Boolean(true))), + op: BinaryOperator::Xor, + right: Box::new(Expr::Value(Value::Boolean(true))), + }), + select.projection[0] + ); + assert_eq!( + SelectItem::UnnamedExpr(Expr::BinaryOp { + left: Box::new(Expr::Value(Value::Boolean(false))), + op: BinaryOperator::Xor, + right: Box::new(Expr::Value(Value::Boolean(false))), + }), + select.projection[1] + ); + assert_eq!( + SelectItem::UnnamedExpr(Expr::BinaryOp { + left: Box::new(Expr::Value(Value::Boolean(true))), + op: BinaryOperator::Xor, + right: Box::new(Expr::Value(Value::Boolean(false))), + }), + select.projection[2] + ); + assert_eq!( + SelectItem::UnnamedExpr(Expr::BinaryOp { + left: Box::new(Expr::Value(Value::Boolean(false))), + op: BinaryOperator::Xor, + right: Box::new(Expr::Value(Value::Boolean(true))), + }), + select.projection[3] + ); +} From 632ba4cf8e6448e67faf6c3d2dd600642dca207c Mon Sep 17 00:00:00 2001 From: wugeer <1284057728@qq.com> Date: Wed, 13 Nov 2024 19:54:57 +0800 Subject: [PATCH 48/80] Fix the parsing error in MSSQL for multiple statements that include `DECLARE` statements (#1497) --- src/parser/mod.rs | 94 +++++++++++++++++++++------------------- tests/sqlparser_mssql.rs | 71 +++++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 46 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 355456d52..d3f432041 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -5321,55 +5321,61 @@ impl<'a> Parser<'a> { /// ``` /// [MsSql]: https://learn.microsoft.com/en-us/sql/t-sql/language-elements/declare-local-variable-transact-sql?view=sql-server-ver16 pub fn parse_mssql_declare(&mut self) -> Result { - let mut stmts = vec![]; - - loop { - let name = { - let ident = self.parse_identifier(false)?; - if !ident.value.starts_with('@') { - Err(ParserError::TokenizerError( - "Invalid MsSql variable declaration.".to_string(), - )) - } else { - Ok(ident) - } - }?; + let stmts = self.parse_comma_separated(Parser::parse_mssql_declare_stmt)?; - let (declare_type, data_type) = match self.peek_token().token { - Token::Word(w) => match w.keyword { - Keyword::CURSOR => { - self.next_token(); - (Some(DeclareType::Cursor), None) - } - Keyword::AS => { - self.next_token(); - (None, Some(self.parse_data_type()?)) - } - _ => (None, Some(self.parse_data_type()?)), - }, - _ => (None, Some(self.parse_data_type()?)), - }; + Ok(Statement::Declare { stmts }) + } - let assignment = self.parse_mssql_variable_declaration_expression()?; + /// Parse the body of a [MsSql] `DECLARE`statement. + /// + /// Syntax: + /// ```text + // { + // { @local_variable [AS] data_type [ = value ] } + // | { @cursor_variable_name CURSOR } + // } [ ,...n ] + /// ``` + /// [MsSql]: https://learn.microsoft.com/en-us/sql/t-sql/language-elements/declare-local-variable-transact-sql?view=sql-server-ver16 + pub fn parse_mssql_declare_stmt(&mut self) -> Result { + let name = { + let ident = self.parse_identifier(false)?; + if !ident.value.starts_with('@') { + Err(ParserError::TokenizerError( + "Invalid MsSql variable declaration.".to_string(), + )) + } else { + Ok(ident) + } + }?; - stmts.push(Declare { - names: vec![name], - data_type, - assignment, - declare_type, - binary: None, - sensitive: None, - scroll: None, - hold: None, - for_query: None, - }); + let (declare_type, data_type) = match self.peek_token().token { + Token::Word(w) => match w.keyword { + Keyword::CURSOR => { + self.next_token(); + (Some(DeclareType::Cursor), None) + } + Keyword::AS => { + self.next_token(); + (None, Some(self.parse_data_type()?)) + } + _ => (None, Some(self.parse_data_type()?)), + }, + _ => (None, Some(self.parse_data_type()?)), + }; - if self.next_token() != Token::Comma { - break; - } - } + let assignment = self.parse_mssql_variable_declaration_expression()?; - Ok(Statement::Declare { stmts }) + Ok(Declare { + names: vec![name], + data_type, + assignment, + declare_type, + binary: None, + sensitive: None, + scroll: None, + hold: None, + for_query: None, + }) } /// Parses the assigned expression in a variable declaration. diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 4f9f6bb82..c28f89e37 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -29,7 +29,7 @@ use sqlparser::ast::DeclareAssignment::MsSqlAssignment; use sqlparser::ast::Value::SingleQuotedString; use sqlparser::ast::*; use sqlparser::dialect::{GenericDialect, MsSqlDialect}; -use sqlparser::parser::{Parser, ParserError}; +use sqlparser::parser::ParserError; #[test] fn parse_mssql_identifiers() { @@ -910,7 +910,7 @@ fn parse_substring_in_select() { #[test] fn parse_mssql_declare() { let sql = "DECLARE @foo CURSOR, @bar INT, @baz AS TEXT = 'foobar';"; - let ast = Parser::parse_sql(&MsSqlDialect {}, sql).unwrap(); + let ast = ms().parse_sql_statements(sql).unwrap(); assert_eq!( vec![Statement::Declare { @@ -963,6 +963,73 @@ fn parse_mssql_declare() { }], ast ); + + let sql = "DECLARE @bar INT;SET @bar = 2;SELECT @bar * 4"; + let ast = ms().parse_sql_statements(sql).unwrap(); + assert_eq!( + vec![ + Statement::Declare { + stmts: vec![Declare { + names: vec![Ident { + value: "@bar".to_string(), + quote_style: None + }], + data_type: Some(Int(None)), + assignment: None, + declare_type: None, + binary: None, + sensitive: None, + scroll: None, + hold: None, + for_query: None + }] + }, + Statement::SetVariable { + local: false, + hivevar: false, + variables: OneOrManyWithParens::One(ObjectName(vec![Ident::new("@bar")])), + value: vec![Expr::Value(Value::Number("2".parse().unwrap(), false))], + }, + Statement::Query(Box::new(Query { + with: None, + limit: None, + limit_by: vec![], + offset: None, + fetch: None, + locks: vec![], + for_clause: None, + order_by: None, + settings: None, + format_clause: None, + body: Box::new(SetExpr::Select(Box::new(Select { + distinct: None, + top: None, + top_before_distinct: false, + projection: vec![SelectItem::UnnamedExpr(Expr::BinaryOp { + left: Box::new(Expr::Identifier(Ident::new("@bar"))), + op: BinaryOperator::Multiply, + right: Box::new(Expr::Value(Value::Number("4".parse().unwrap(), false))), + })], + into: None, + from: vec![], + lateral_views: vec![], + prewhere: None, + selection: None, + group_by: GroupByExpr::Expressions(vec![], vec![]), + cluster_by: vec![], + distribute_by: vec![], + sort_by: vec![], + having: None, + named_window: vec![], + window_before_qualify: false, + qualify: None, + value_table_mode: None, + connect_by: None, + }))) + })) + ], + ast + ); } #[test] From 76322baf2f126faea5ea416be176218fc964699c Mon Sep 17 00:00:00 2001 From: Yoav Cohen <59807311+yoavcloud@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:55:26 +0200 Subject: [PATCH 49/80] Add support for Snowflake SHOW DATABASES/SCHEMAS/TABLES/VIEWS/COLUMNS statements (#1501) --- src/ast/mod.rs | 223 +++++++++++++++++++++++++---------- src/dialect/mod.rs | 6 + src/dialect/snowflake.rs | 6 + src/keywords.rs | 4 + src/parser/mod.rs | 215 ++++++++++++++++++++++++++------- tests/sqlparser_common.rs | 54 ++++++--- tests/sqlparser_mysql.rs | 177 +++++++++++++++++++++------ tests/sqlparser_snowflake.rs | 65 ++++++++++ 8 files changed, 591 insertions(+), 159 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 81bddcd17..848e6bdbd 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2773,41 +2773,45 @@ pub enum Statement { /// ```sql /// SHOW COLUMNS /// ``` - /// - /// Note: this is a MySQL-specific statement. ShowColumns { extended: bool, full: bool, - #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))] - table_name: ObjectName, - filter: Option, + show_options: ShowStatementOptions, }, /// ```sql - /// SHOW DATABASES [LIKE 'pattern'] + /// SHOW DATABASES /// ``` - ShowDatabases { filter: Option }, + ShowDatabases { + terse: bool, + history: bool, + show_options: ShowStatementOptions, + }, /// ```sql - /// SHOW SCHEMAS [LIKE 'pattern'] + /// SHOW SCHEMAS /// ``` - ShowSchemas { filter: Option }, + ShowSchemas { + terse: bool, + history: bool, + show_options: ShowStatementOptions, + }, /// ```sql /// SHOW TABLES /// ``` ShowTables { + terse: bool, + history: bool, extended: bool, full: bool, - clause: Option, - db_name: Option, - filter: Option, + external: bool, + show_options: ShowStatementOptions, }, /// ```sql /// SHOW VIEWS /// ``` ShowViews { + terse: bool, materialized: bool, - clause: Option, - db_name: Option, - filter: Option, + show_options: ShowStatementOptions, }, /// ```sql /// SHOW COLLATION @@ -4387,79 +4391,72 @@ impl fmt::Display for Statement { Statement::ShowColumns { extended, full, - table_name, - filter, + show_options, } => { write!( f, - "SHOW {extended}{full}COLUMNS FROM {table_name}", + "SHOW {extended}{full}COLUMNS{show_options}", extended = if *extended { "EXTENDED " } else { "" }, full = if *full { "FULL " } else { "" }, - table_name = table_name, )?; - if let Some(filter) = filter { - write!(f, " {filter}")?; - } Ok(()) } - Statement::ShowDatabases { filter } => { - write!(f, "SHOW DATABASES")?; - if let Some(filter) = filter { - write!(f, " {filter}")?; - } + Statement::ShowDatabases { + terse, + history, + show_options, + } => { + write!( + f, + "SHOW {terse}DATABASES{history}{show_options}", + terse = if *terse { "TERSE " } else { "" }, + history = if *history { " HISTORY" } else { "" }, + )?; Ok(()) } - Statement::ShowSchemas { filter } => { - write!(f, "SHOW SCHEMAS")?; - if let Some(filter) = filter { - write!(f, " {filter}")?; - } + Statement::ShowSchemas { + terse, + history, + show_options, + } => { + write!( + f, + "SHOW {terse}SCHEMAS{history}{show_options}", + terse = if *terse { "TERSE " } else { "" }, + history = if *history { " HISTORY" } else { "" }, + )?; Ok(()) } Statement::ShowTables { + terse, + history, extended, full, - clause: show_clause, - db_name, - filter, + external, + show_options, } => { write!( f, - "SHOW {extended}{full}TABLES", + "SHOW {terse}{extended}{full}{external}TABLES{history}{show_options}", + terse = if *terse { "TERSE " } else { "" }, extended = if *extended { "EXTENDED " } else { "" }, full = if *full { "FULL " } else { "" }, + external = if *external { "EXTERNAL " } else { "" }, + history = if *history { " HISTORY" } else { "" }, )?; - if let Some(show_clause) = show_clause { - write!(f, " {show_clause}")?; - } - if let Some(db_name) = db_name { - write!(f, " {db_name}")?; - } - if let Some(filter) = filter { - write!(f, " {filter}")?; - } Ok(()) } Statement::ShowViews { + terse, materialized, - clause: show_clause, - db_name, - filter, + show_options, } => { write!( f, - "SHOW {}VIEWS", - if *materialized { "MATERIALIZED " } else { "" } + "SHOW {terse}{materialized}VIEWS{show_options}", + terse = if *terse { "TERSE " } else { "" }, + materialized = if *materialized { "MATERIALIZED " } else { "" } )?; - if let Some(show_clause) = show_clause { - write!(f, " {show_clause}")?; - } - if let Some(db_name) = db_name { - write!(f, " {db_name}")?; - } - if let Some(filter) = filter { - write!(f, " {filter}")?; - } Ok(()) } Statement::ShowFunctions { filter } => { @@ -6172,14 +6169,14 @@ impl fmt::Display for ShowStatementFilter { #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] -pub enum ShowClause { +pub enum ShowStatementInClause { IN, FROM, } -impl fmt::Display for ShowClause { +impl fmt::Display for ShowStatementInClause { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use ShowClause::*; + use ShowStatementInClause::*; match self { FROM => write!(f, "FROM"), IN => write!(f, "IN"), @@ -7357,6 +7354,108 @@ impl Display for UtilityOption { } } +/// Represents the different options available for `SHOW` +/// statements to filter the results. Example from Snowflake: +/// +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct ShowStatementOptions { + pub show_in: Option, + pub starts_with: Option, + pub limit: Option, + pub limit_from: Option, + pub filter_position: Option, +} + +impl Display for ShowStatementOptions { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (like_in_infix, like_in_suffix) = match &self.filter_position { + Some(ShowStatementFilterPosition::Infix(filter)) => { + (format!(" {filter}"), "".to_string()) + } + Some(ShowStatementFilterPosition::Suffix(filter)) => { + ("".to_string(), format!(" {filter}")) + } + None => ("".to_string(), "".to_string()), + }; + write!( + f, + "{like_in_infix}{show_in}{starts_with}{limit}{from}{like_in_suffix}", + show_in = match &self.show_in { + Some(i) => format!(" {i}"), + None => String::new(), + }, + starts_with = match &self.starts_with { + Some(s) => format!(" STARTS WITH {s}"), + None => String::new(), + }, + limit = match &self.limit { + Some(l) => format!(" LIMIT {l}"), + None => String::new(), + }, + from = match &self.limit_from { + Some(f) => format!(" FROM {f}"), + None => String::new(), + } + )?; + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum ShowStatementFilterPosition { + Infix(ShowStatementFilter), // For example: SHOW COLUMNS LIKE '%name%' IN TABLE tbl + Suffix(ShowStatementFilter), // For example: SHOW COLUMNS IN tbl LIKE '%name%' +} + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum ShowStatementInParentType { + Account, + Database, + Schema, + Table, + View, +} + +impl fmt::Display for ShowStatementInParentType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + ShowStatementInParentType::Account => write!(f, "ACCOUNT"), + ShowStatementInParentType::Database => write!(f, "DATABASE"), + ShowStatementInParentType::Schema => write!(f, "SCHEMA"), + ShowStatementInParentType::Table => write!(f, "TABLE"), + ShowStatementInParentType::View => write!(f, "VIEW"), + } + } +} + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct ShowStatementIn { + pub clause: ShowStatementInClause, + pub parent_type: Option, + pub parent_name: Option, +} + +impl fmt::Display for ShowStatementIn { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.clause)?; + if let Some(parent_type) = &self.parent_type { + write!(f, " {}", parent_type)?; + } + if let Some(parent_name) = &self.parent_name { + write!(f, " {}", parent_name)?; + } + Ok(()) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index c8c11bc95..f37c0d85c 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -622,6 +622,12 @@ pub trait Dialect: Debug + Any { fn supports_boolean_literals(&self) -> bool { true } + + /// Returns true if this dialect supports the `LIKE 'pattern'` option in + /// a `SHOW` statement before the `IN` option + fn supports_show_like_before_in(&self) -> bool { + false + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index d9331d952..98e8f5e2f 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -203,6 +203,12 @@ impl Dialect for SnowflakeDialect { fn allow_extract_single_quotes(&self) -> bool { true } + + /// Snowflake expects the `LIKE` option before the `IN` option, + /// for example: + fn supports_show_like_before_in(&self) -> bool { + true + } } /// Parse snowflake create table statement. diff --git a/src/keywords.rs b/src/keywords.rs index 982cea81b..9cdc90ce2 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -76,6 +76,7 @@ define_keywords!( ABS, ABSOLUTE, ACCESS, + ACCOUNT, ACTION, ADD, ADMIN, @@ -91,6 +92,7 @@ define_keywords!( AND, ANTI, ANY, + APPLICATION, APPLY, ARCHIVE, ARE, @@ -710,6 +712,7 @@ define_keywords!( STABLE, STAGE, START, + STARTS, STATEMENT, STATIC, STATISTICS, @@ -746,6 +749,7 @@ define_keywords!( TEMP, TEMPORARY, TERMINATED, + TERSE, TEXT, TEXTFILE, THEN, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index d3f432041..7f6961ae8 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -3205,6 +3205,22 @@ impl<'a> Parser<'a> { }) } + /// Look for all of the expected keywords in sequence, without consuming them + fn peek_keyword(&mut self, expected: Keyword) -> bool { + let index = self.index; + let matched = self.parse_keyword(expected); + self.index = index; + matched + } + + /// Look for all of the expected keywords in sequence, without consuming them + fn peek_keywords(&mut self, expected: &[Keyword]) -> bool { + let index = self.index; + let matched = self.parse_keywords(expected); + self.index = index; + matched + } + /// Return the first non-whitespace token that has not yet been processed /// (or None if reached end-of-file) and mark it as processed. OK to call /// repeatedly after reaching EOF. @@ -9611,21 +9627,23 @@ impl<'a> Parser<'a> { } pub fn parse_show(&mut self) -> Result { + let terse = self.parse_keyword(Keyword::TERSE); let extended = self.parse_keyword(Keyword::EXTENDED); let full = self.parse_keyword(Keyword::FULL); let session = self.parse_keyword(Keyword::SESSION); let global = self.parse_keyword(Keyword::GLOBAL); + let external = self.parse_keyword(Keyword::EXTERNAL); if self .parse_one_of_keywords(&[Keyword::COLUMNS, Keyword::FIELDS]) .is_some() { Ok(self.parse_show_columns(extended, full)?) } else if self.parse_keyword(Keyword::TABLES) { - Ok(self.parse_show_tables(extended, full)?) + Ok(self.parse_show_tables(terse, extended, full, external)?) } else if self.parse_keywords(&[Keyword::MATERIALIZED, Keyword::VIEWS]) { - Ok(self.parse_show_views(true)?) + Ok(self.parse_show_views(terse, true)?) } else if self.parse_keyword(Keyword::VIEWS) { - Ok(self.parse_show_views(false)?) + Ok(self.parse_show_views(terse, false)?) } else if self.parse_keyword(Keyword::FUNCTIONS) { Ok(self.parse_show_functions()?) } else if extended || full { @@ -9653,9 +9671,9 @@ impl<'a> Parser<'a> { global, }) } else if self.parse_keyword(Keyword::DATABASES) { - self.parse_show_databases() + self.parse_show_databases(terse) } else if self.parse_keyword(Keyword::SCHEMAS) { - self.parse_show_schemas() + self.parse_show_schemas(terse) } else { Ok(Statement::ShowVariable { variable: self.parse_identifiers()?, @@ -9663,15 +9681,23 @@ impl<'a> Parser<'a> { } } - fn parse_show_databases(&mut self) -> Result { + fn parse_show_databases(&mut self, terse: bool) -> Result { + let history = self.parse_keyword(Keyword::HISTORY); + let show_options = self.parse_show_stmt_options()?; Ok(Statement::ShowDatabases { - filter: self.parse_show_statement_filter()?, + terse, + history, + show_options, }) } - fn parse_show_schemas(&mut self) -> Result { + fn parse_show_schemas(&mut self, terse: bool) -> Result { + let history = self.parse_keyword(Keyword::HISTORY); + let show_options = self.parse_show_stmt_options()?; Ok(Statement::ShowSchemas { - filter: self.parse_show_statement_filter()?, + terse, + history, + show_options, }) } @@ -9705,58 +9731,43 @@ impl<'a> Parser<'a> { extended: bool, full: bool, ) -> Result { - self.expect_one_of_keywords(&[Keyword::FROM, Keyword::IN])?; - let object_name = self.parse_object_name(false)?; - let table_name = match self.parse_one_of_keywords(&[Keyword::FROM, Keyword::IN]) { - Some(_) => { - let db_name = vec![self.parse_identifier(false)?]; - let ObjectName(table_name) = object_name; - let object_name = db_name.into_iter().chain(table_name).collect(); - ObjectName(object_name) - } - None => object_name, - }; - let filter = self.parse_show_statement_filter()?; + let show_options = self.parse_show_stmt_options()?; Ok(Statement::ShowColumns { extended, full, - table_name, - filter, + show_options, }) } - pub fn parse_show_tables( + fn parse_show_tables( &mut self, + terse: bool, extended: bool, full: bool, + external: bool, ) -> Result { - let (clause, db_name) = match self.parse_one_of_keywords(&[Keyword::FROM, Keyword::IN]) { - Some(Keyword::FROM) => (Some(ShowClause::FROM), Some(self.parse_identifier(false)?)), - Some(Keyword::IN) => (Some(ShowClause::IN), Some(self.parse_identifier(false)?)), - _ => (None, None), - }; - let filter = self.parse_show_statement_filter()?; + let history = !external && self.parse_keyword(Keyword::HISTORY); + let show_options = self.parse_show_stmt_options()?; Ok(Statement::ShowTables { + terse, + history, extended, full, - clause, - db_name, - filter, + external, + show_options, }) } - fn parse_show_views(&mut self, materialized: bool) -> Result { - let (clause, db_name) = match self.parse_one_of_keywords(&[Keyword::FROM, Keyword::IN]) { - Some(Keyword::FROM) => (Some(ShowClause::FROM), Some(self.parse_identifier(false)?)), - Some(Keyword::IN) => (Some(ShowClause::IN), Some(self.parse_identifier(false)?)), - _ => (None, None), - }; - let filter = self.parse_show_statement_filter()?; + fn parse_show_views( + &mut self, + terse: bool, + materialized: bool, + ) -> Result { + let show_options = self.parse_show_stmt_options()?; Ok(Statement::ShowViews { materialized, - clause, - db_name, - filter, + terse, + show_options, }) } @@ -12395,6 +12406,124 @@ impl<'a> Parser<'a> { } false } + + fn parse_show_stmt_options(&mut self) -> Result { + let show_in; + let mut filter_position = None; + if self.dialect.supports_show_like_before_in() { + if let Some(filter) = self.parse_show_statement_filter()? { + filter_position = Some(ShowStatementFilterPosition::Infix(filter)); + } + show_in = self.maybe_parse_show_stmt_in()?; + } else { + show_in = self.maybe_parse_show_stmt_in()?; + if let Some(filter) = self.parse_show_statement_filter()? { + filter_position = Some(ShowStatementFilterPosition::Suffix(filter)); + } + } + let starts_with = self.maybe_parse_show_stmt_starts_with()?; + let limit = self.maybe_parse_show_stmt_limit()?; + let from = self.maybe_parse_show_stmt_from()?; + Ok(ShowStatementOptions { + filter_position, + show_in, + starts_with, + limit, + limit_from: from, + }) + } + + fn maybe_parse_show_stmt_in(&mut self) -> Result, ParserError> { + let clause = match self.parse_one_of_keywords(&[Keyword::FROM, Keyword::IN]) { + Some(Keyword::FROM) => ShowStatementInClause::FROM, + Some(Keyword::IN) => ShowStatementInClause::IN, + None => return Ok(None), + _ => return self.expected("FROM or IN", self.peek_token()), + }; + + let (parent_type, parent_name) = match self.parse_one_of_keywords(&[ + Keyword::ACCOUNT, + Keyword::DATABASE, + Keyword::SCHEMA, + Keyword::TABLE, + Keyword::VIEW, + ]) { + // If we see these next keywords it means we don't have a parent name + Some(Keyword::DATABASE) + if self.peek_keywords(&[Keyword::STARTS, Keyword::WITH]) + | self.peek_keyword(Keyword::LIMIT) => + { + (Some(ShowStatementInParentType::Database), None) + } + Some(Keyword::SCHEMA) + if self.peek_keywords(&[Keyword::STARTS, Keyword::WITH]) + | self.peek_keyword(Keyword::LIMIT) => + { + (Some(ShowStatementInParentType::Schema), None) + } + Some(parent_kw) => { + // The parent name here is still optional, for example: + // SHOW TABLES IN ACCOUNT, so parsing the object name + // may fail because the statement ends. + let parent_name = self.maybe_parse(|p| p.parse_object_name(false))?; + match parent_kw { + Keyword::ACCOUNT => (Some(ShowStatementInParentType::Account), parent_name), + Keyword::DATABASE => (Some(ShowStatementInParentType::Database), parent_name), + Keyword::SCHEMA => (Some(ShowStatementInParentType::Schema), parent_name), + Keyword::TABLE => (Some(ShowStatementInParentType::Table), parent_name), + Keyword::VIEW => (Some(ShowStatementInParentType::View), parent_name), + _ => { + return self.expected( + "one of ACCOUNT, DATABASE, SCHEMA, TABLE or VIEW", + self.peek_token(), + ) + } + } + } + None => { + // Parsing MySQL style FROM tbl_name FROM db_name + // which is equivalent to FROM tbl_name.db_name + let mut parent_name = self.parse_object_name(false)?; + if self + .parse_one_of_keywords(&[Keyword::FROM, Keyword::IN]) + .is_some() + { + parent_name.0.insert(0, self.parse_identifier(false)?); + } + (None, Some(parent_name)) + } + }; + + Ok(Some(ShowStatementIn { + clause, + parent_type, + parent_name, + })) + } + + fn maybe_parse_show_stmt_starts_with(&mut self) -> Result, ParserError> { + if self.parse_keywords(&[Keyword::STARTS, Keyword::WITH]) { + Ok(Some(self.parse_value()?)) + } else { + Ok(None) + } + } + + fn maybe_parse_show_stmt_limit(&mut self) -> Result, ParserError> { + if self.parse_keyword(Keyword::LIMIT) { + Ok(self.parse_limit()?) + } else { + Ok(None) + } + } + + fn maybe_parse_show_stmt_from(&mut self) -> Result, ParserError> { + if self.parse_keyword(Keyword::FROM) { + Ok(Some(self.parse_value()?)) + } else { + Ok(None) + } + } } impl Word { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index bef0f535c..d08e19d68 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11395,23 +11395,43 @@ fn test_try_convert() { #[test] fn test_show_dbs_schemas_tables_views() { - verified_stmt("SHOW DATABASES"); - verified_stmt("SHOW DATABASES LIKE '%abc'"); - verified_stmt("SHOW SCHEMAS"); - verified_stmt("SHOW SCHEMAS LIKE '%abc'"); - verified_stmt("SHOW TABLES"); - verified_stmt("SHOW TABLES IN db1"); - verified_stmt("SHOW TABLES IN db1 'abc'"); - verified_stmt("SHOW VIEWS"); - verified_stmt("SHOW VIEWS IN db1"); - verified_stmt("SHOW VIEWS IN db1 'abc'"); - verified_stmt("SHOW VIEWS FROM db1"); - verified_stmt("SHOW VIEWS FROM db1 'abc'"); - verified_stmt("SHOW MATERIALIZED VIEWS"); - verified_stmt("SHOW MATERIALIZED VIEWS IN db1"); - verified_stmt("SHOW MATERIALIZED VIEWS IN db1 'abc'"); - verified_stmt("SHOW MATERIALIZED VIEWS FROM db1"); - verified_stmt("SHOW MATERIALIZED VIEWS FROM db1 'abc'"); + // These statements are parsed the same by all dialects + let stmts = vec![ + "SHOW DATABASES", + "SHOW SCHEMAS", + "SHOW TABLES", + "SHOW VIEWS", + "SHOW TABLES IN db1", + "SHOW VIEWS FROM db1", + "SHOW MATERIALIZED VIEWS", + "SHOW MATERIALIZED VIEWS IN db1", + "SHOW MATERIALIZED VIEWS FROM db1", + ]; + for stmt in stmts { + verified_stmt(stmt); + } + + // These statements are parsed the same by all dialects + // except for how the parser interprets the location of + // LIKE option (infix/suffix) + let stmts = vec!["SHOW DATABASES LIKE '%abc'", "SHOW SCHEMAS LIKE '%abc'"]; + for stmt in stmts { + all_dialects_where(|d| d.supports_show_like_before_in()).verified_stmt(stmt); + all_dialects_where(|d| !d.supports_show_like_before_in()).verified_stmt(stmt); + } + + // These statements are only parsed by dialects that + // support the LIKE option in the suffix + let stmts = vec![ + "SHOW TABLES IN db1 'abc'", + "SHOW VIEWS IN db1 'abc'", + "SHOW VIEWS FROM db1 'abc'", + "SHOW MATERIALIZED VIEWS IN db1 'abc'", + "SHOW MATERIALIZED VIEWS FROM db1 'abc'", + ]; + for stmt in stmts { + all_dialects_where(|d| !d.supports_show_like_before_in()).verified_stmt(stmt); + } } #[test] diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 44b2ac6ba..8269eadc0 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -223,14 +223,22 @@ fn parse_flush() { #[test] fn parse_show_columns() { - let table_name = ObjectName(vec![Ident::new("mytable")]); assert_eq!( mysql_and_generic().verified_stmt("SHOW COLUMNS FROM mytable"), Statement::ShowColumns { extended: false, full: false, - table_name: table_name.clone(), - filter: None, + show_options: ShowStatementOptions { + show_in: Some(ShowStatementIn { + clause: ShowStatementInClause::FROM, + parent_type: None, + parent_name: Some(ObjectName(vec![Ident::new("mytable")])), + }), + filter_position: None, + limit_from: None, + limit: None, + starts_with: None, + } } ); assert_eq!( @@ -238,8 +246,17 @@ fn parse_show_columns() { Statement::ShowColumns { extended: false, full: false, - table_name: ObjectName(vec![Ident::new("mydb"), Ident::new("mytable")]), - filter: None, + show_options: ShowStatementOptions { + show_in: Some(ShowStatementIn { + clause: ShowStatementInClause::FROM, + parent_type: None, + parent_name: Some(ObjectName(vec![Ident::new("mydb"), Ident::new("mytable")])), + }), + filter_position: None, + limit_from: None, + limit: None, + starts_with: None, + } } ); assert_eq!( @@ -247,8 +264,17 @@ fn parse_show_columns() { Statement::ShowColumns { extended: true, full: false, - table_name: table_name.clone(), - filter: None, + show_options: ShowStatementOptions { + show_in: Some(ShowStatementIn { + clause: ShowStatementInClause::FROM, + parent_type: None, + parent_name: Some(ObjectName(vec![Ident::new("mytable")])), + }), + filter_position: None, + limit_from: None, + limit: None, + starts_with: None, + } } ); assert_eq!( @@ -256,8 +282,17 @@ fn parse_show_columns() { Statement::ShowColumns { extended: false, full: true, - table_name: table_name.clone(), - filter: None, + show_options: ShowStatementOptions { + show_in: Some(ShowStatementIn { + clause: ShowStatementInClause::FROM, + parent_type: None, + parent_name: Some(ObjectName(vec![Ident::new("mytable")])), + }), + filter_position: None, + limit_from: None, + limit: None, + starts_with: None, + } } ); assert_eq!( @@ -265,8 +300,19 @@ fn parse_show_columns() { Statement::ShowColumns { extended: false, full: false, - table_name: table_name.clone(), - filter: Some(ShowStatementFilter::Like("pattern".into())), + show_options: ShowStatementOptions { + show_in: Some(ShowStatementIn { + clause: ShowStatementInClause::FROM, + parent_type: None, + parent_name: Some(ObjectName(vec![Ident::new("mytable")])), + }), + filter_position: Some(ShowStatementFilterPosition::Suffix( + ShowStatementFilter::Like("pattern".into()) + )), + limit_from: None, + limit: None, + starts_with: None, + } } ); assert_eq!( @@ -274,18 +320,27 @@ fn parse_show_columns() { Statement::ShowColumns { extended: false, full: false, - table_name, - filter: Some(ShowStatementFilter::Where( - mysql_and_generic().verified_expr("1 = 2") - )), + show_options: ShowStatementOptions { + show_in: Some(ShowStatementIn { + clause: ShowStatementInClause::FROM, + parent_type: None, + parent_name: Some(ObjectName(vec![Ident::new("mytable")])), + }), + filter_position: Some(ShowStatementFilterPosition::Suffix( + ShowStatementFilter::Where(mysql_and_generic().verified_expr("1 = 2")) + )), + limit_from: None, + limit: None, + starts_with: None, + } } ); mysql_and_generic() .one_statement_parses_to("SHOW FIELDS FROM mytable", "SHOW COLUMNS FROM mytable"); mysql_and_generic() - .one_statement_parses_to("SHOW COLUMNS IN mytable", "SHOW COLUMNS FROM mytable"); + .one_statement_parses_to("SHOW COLUMNS IN mytable", "SHOW COLUMNS IN mytable"); mysql_and_generic() - .one_statement_parses_to("SHOW FIELDS IN mytable", "SHOW COLUMNS FROM mytable"); + .one_statement_parses_to("SHOW FIELDS IN mytable", "SHOW COLUMNS IN mytable"); mysql_and_generic().one_statement_parses_to( "SHOW COLUMNS FROM mytable FROM mydb", "SHOW COLUMNS FROM mydb.mytable", @@ -327,63 +382,111 @@ fn parse_show_tables() { assert_eq!( mysql_and_generic().verified_stmt("SHOW TABLES"), Statement::ShowTables { + terse: false, + history: false, extended: false, full: false, - clause: None, - db_name: None, - filter: None, + external: false, + show_options: ShowStatementOptions { + starts_with: None, + limit: None, + limit_from: None, + show_in: None, + filter_position: None + } } ); assert_eq!( mysql_and_generic().verified_stmt("SHOW TABLES FROM mydb"), Statement::ShowTables { + terse: false, + history: false, extended: false, full: false, - clause: Some(ShowClause::FROM), - db_name: Some(Ident::new("mydb")), - filter: None, + external: false, + show_options: ShowStatementOptions { + starts_with: None, + limit: None, + limit_from: None, + show_in: Some(ShowStatementIn { + clause: ShowStatementInClause::FROM, + parent_type: None, + parent_name: Some(ObjectName(vec![Ident::new("mydb")])), + }), + filter_position: None + } } ); assert_eq!( mysql_and_generic().verified_stmt("SHOW EXTENDED TABLES"), Statement::ShowTables { + terse: false, + history: false, extended: true, full: false, - clause: None, - db_name: None, - filter: None, + external: false, + show_options: ShowStatementOptions { + starts_with: None, + limit: None, + limit_from: None, + show_in: None, + filter_position: None + } } ); assert_eq!( mysql_and_generic().verified_stmt("SHOW FULL TABLES"), Statement::ShowTables { + terse: false, + history: false, extended: false, full: true, - clause: None, - db_name: None, - filter: None, + external: false, + show_options: ShowStatementOptions { + starts_with: None, + limit: None, + limit_from: None, + show_in: None, + filter_position: None + } } ); assert_eq!( mysql_and_generic().verified_stmt("SHOW TABLES LIKE 'pattern'"), Statement::ShowTables { + terse: false, + history: false, extended: false, full: false, - clause: None, - db_name: None, - filter: Some(ShowStatementFilter::Like("pattern".into())), + external: false, + show_options: ShowStatementOptions { + starts_with: None, + limit: None, + limit_from: None, + show_in: None, + filter_position: Some(ShowStatementFilterPosition::Suffix( + ShowStatementFilter::Like("pattern".into()) + )) + } } ); assert_eq!( mysql_and_generic().verified_stmt("SHOW TABLES WHERE 1 = 2"), Statement::ShowTables { + terse: false, + history: false, extended: false, full: false, - clause: None, - db_name: None, - filter: Some(ShowStatementFilter::Where( - mysql_and_generic().verified_expr("1 = 2") - )), + external: false, + show_options: ShowStatementOptions { + starts_with: None, + limit: None, + limit_from: None, + show_in: None, + filter_position: Some(ShowStatementFilterPosition::Suffix( + ShowStatementFilter::Where(mysql_and_generic().verified_expr("1 = 2")) + )) + } } ); mysql_and_generic().verified_stmt("SHOW TABLES IN mydb"); diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index c17c7b958..1f1c00e7a 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -2781,3 +2781,68 @@ fn test_parentheses_overflow() { snowflake_with_recursion_limit(max_nesting_level).parse_sql_statements(sql.as_str()); assert_eq!(parsed.err(), Some(ParserError::RecursionLimitExceeded)); } + +#[test] +fn test_show_databases() { + snowflake().verified_stmt("SHOW DATABASES"); + snowflake().verified_stmt("SHOW DATABASES HISTORY"); + snowflake().verified_stmt("SHOW DATABASES LIKE '%abc%'"); + snowflake().verified_stmt("SHOW DATABASES STARTS WITH 'demo_db'"); + snowflake().verified_stmt("SHOW DATABASES LIMIT 12"); + snowflake() + .verified_stmt("SHOW DATABASES HISTORY LIKE '%aa' STARTS WITH 'demo' LIMIT 20 FROM 'abc'"); + snowflake().verified_stmt("SHOW DATABASES IN ACCOUNT abc"); +} + +#[test] +fn test_parse_show_schemas() { + snowflake().verified_stmt("SHOW SCHEMAS"); + snowflake().verified_stmt("SHOW SCHEMAS IN ACCOUNT"); + snowflake().verified_stmt("SHOW SCHEMAS IN ACCOUNT abc"); + snowflake().verified_stmt("SHOW SCHEMAS IN DATABASE"); + snowflake().verified_stmt("SHOW SCHEMAS IN DATABASE xyz"); + snowflake().verified_stmt("SHOW SCHEMAS HISTORY LIKE '%xa%'"); + snowflake().verified_stmt("SHOW SCHEMAS STARTS WITH 'abc' LIMIT 20"); + snowflake().verified_stmt("SHOW SCHEMAS IN DATABASE STARTS WITH 'abc' LIMIT 20 FROM 'xyz'"); +} + +#[test] +fn test_parse_show_tables() { + snowflake().verified_stmt("SHOW TABLES"); + snowflake().verified_stmt("SHOW TABLES IN ACCOUNT"); + snowflake().verified_stmt("SHOW TABLES IN DATABASE"); + snowflake().verified_stmt("SHOW TABLES IN DATABASE xyz"); + snowflake().verified_stmt("SHOW TABLES IN SCHEMA"); + snowflake().verified_stmt("SHOW TABLES IN SCHEMA xyz"); + snowflake().verified_stmt("SHOW TABLES HISTORY LIKE '%xa%'"); + snowflake().verified_stmt("SHOW TABLES STARTS WITH 'abc' LIMIT 20"); + snowflake().verified_stmt("SHOW TABLES IN SCHEMA STARTS WITH 'abc' LIMIT 20 FROM 'xyz'"); + snowflake().verified_stmt("SHOW EXTERNAL TABLES"); + snowflake().verified_stmt("SHOW EXTERNAL TABLES IN ACCOUNT"); + snowflake().verified_stmt("SHOW EXTERNAL TABLES IN DATABASE"); + snowflake().verified_stmt("SHOW EXTERNAL TABLES IN DATABASE xyz"); + snowflake().verified_stmt("SHOW EXTERNAL TABLES IN SCHEMA"); + snowflake().verified_stmt("SHOW EXTERNAL TABLES IN SCHEMA xyz"); + snowflake().verified_stmt("SHOW EXTERNAL TABLES STARTS WITH 'abc' LIMIT 20"); + snowflake() + .verified_stmt("SHOW EXTERNAL TABLES IN SCHEMA STARTS WITH 'abc' LIMIT 20 FROM 'xyz'"); +} + +#[test] +fn test_show_views() { + snowflake().verified_stmt("SHOW VIEWS"); + snowflake().verified_stmt("SHOW VIEWS IN ACCOUNT"); + snowflake().verified_stmt("SHOW VIEWS IN DATABASE"); + snowflake().verified_stmt("SHOW VIEWS IN DATABASE xyz"); + snowflake().verified_stmt("SHOW VIEWS IN SCHEMA"); + snowflake().verified_stmt("SHOW VIEWS IN SCHEMA xyz"); + snowflake().verified_stmt("SHOW VIEWS STARTS WITH 'abc' LIMIT 20"); + snowflake().verified_stmt("SHOW VIEWS IN SCHEMA STARTS WITH 'abc' LIMIT 20 FROM 'xyz'"); +} + +#[test] +fn test_parse_show_columns_sql() { + snowflake().verified_stmt("SHOW COLUMNS IN TABLE"); + snowflake().verified_stmt("SHOW COLUMNS IN TABLE abc"); + snowflake().verified_stmt("SHOW COLUMNS LIKE '%xyz%' IN TABLE abc"); +} From 6d907d3adc36da6ebafd76c9abd5761f19a5ac0b Mon Sep 17 00:00:00 2001 From: hulk Date: Wed, 13 Nov 2024 21:23:33 +0800 Subject: [PATCH 50/80] Add support of COMMENT ON syntax for Snowflake (#1516) --- src/ast/mod.rs | 8 ++++ src/dialect/generic.rs | 4 ++ src/dialect/mod.rs | 7 ++- src/dialect/postgresql.rs | 45 +++---------------- src/dialect/snowflake.rs | 5 +++ src/parser/mod.rs | 47 ++++++++++++++++++++ tests/sqlparser_common.rs | 88 +++++++++++++++++++++++++++++++++++++ tests/sqlparser_postgres.rs | 62 -------------------------- 8 files changed, 164 insertions(+), 102 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 848e6bdbd..505386fbf 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -1884,6 +1884,10 @@ pub enum CommentObject { Column, Table, Extension, + Schema, + Database, + User, + Role, } impl fmt::Display for CommentObject { @@ -1892,6 +1896,10 @@ impl fmt::Display for CommentObject { CommentObject::Column => f.write_str("COLUMN"), CommentObject::Table => f.write_str("TABLE"), CommentObject::Extension => f.write_str("EXTENSION"), + CommentObject::Schema => f.write_str("SCHEMA"), + CommentObject::Database => f.write_str("DATABASE"), + CommentObject::User => f.write_str("USER"), + CommentObject::Role => f.write_str("ROLE"), } } } diff --git a/src/dialect/generic.rs b/src/dialect/generic.rs index 0a5464c9c..8cfac217b 100644 --- a/src/dialect/generic.rs +++ b/src/dialect/generic.rs @@ -111,4 +111,8 @@ impl Dialect for GenericDialect { fn supports_try_convert(&self) -> bool { true } + + fn supports_comment_on(&self) -> bool { + true + } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index f37c0d85c..d95d7c70a 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -611,7 +611,7 @@ pub trait Dialect: Debug + Any { false } - /// Returns true if this dialect expects the the `TOP` option + /// Returns true if this dialect expects the `TOP` option /// before the `ALL`/`DISTINCT` options in a `SELECT` statement. fn supports_top_before_distinct(&self) -> bool { false @@ -628,6 +628,11 @@ pub trait Dialect: Debug + Any { fn supports_show_like_before_in(&self) -> bool { false } + + /// Returns true if this dialect supports the `COMMENT` statement + fn supports_comment_on(&self) -> bool { + false + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index 72841c604..5af1ab853 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -28,7 +28,7 @@ // limitations under the License. use log::debug; -use crate::ast::{CommentObject, ObjectName, Statement, UserDefinedTypeRepresentation}; +use crate::ast::{ObjectName, Statement, UserDefinedTypeRepresentation}; use crate::dialect::{Dialect, Precedence}; use crate::keywords::Keyword; use crate::parser::{Parser, ParserError}; @@ -136,9 +136,7 @@ impl Dialect for PostgreSqlDialect { } fn parse_statement(&self, parser: &mut Parser) -> Option> { - if parser.parse_keyword(Keyword::COMMENT) { - Some(parse_comment(parser)) - } else if parser.parse_keyword(Keyword::CREATE) { + if parser.parse_keyword(Keyword::CREATE) { parser.prev_token(); // unconsume the CREATE in case we don't end up parsing anything parse_create(parser) } else { @@ -206,42 +204,11 @@ impl Dialect for PostgreSqlDialect { fn supports_factorial_operator(&self) -> bool { true } -} - -pub fn parse_comment(parser: &mut Parser) -> Result { - let if_exists = parser.parse_keywords(&[Keyword::IF, Keyword::EXISTS]); - - parser.expect_keyword(Keyword::ON)?; - let token = parser.next_token(); - - let (object_type, object_name) = match token.token { - Token::Word(w) if w.keyword == Keyword::COLUMN => { - let object_name = parser.parse_object_name(false)?; - (CommentObject::Column, object_name) - } - Token::Word(w) if w.keyword == Keyword::TABLE => { - let object_name = parser.parse_object_name(false)?; - (CommentObject::Table, object_name) - } - Token::Word(w) if w.keyword == Keyword::EXTENSION => { - let object_name = parser.parse_object_name(false)?; - (CommentObject::Extension, object_name) - } - _ => parser.expected("comment object_type", token)?, - }; - parser.expect_keyword(Keyword::IS)?; - let comment = if parser.parse_keyword(Keyword::NULL) { - None - } else { - Some(parser.parse_literal_string()?) - }; - Ok(Statement::Comment { - object_type, - object_name, - comment, - if_exists, - }) + /// see + fn supports_comment_on(&self) -> bool { + true + } } pub fn parse_create(parser: &mut Parser) -> Option> { diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index 98e8f5e2f..b584ed9b4 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -96,6 +96,11 @@ impl Dialect for SnowflakeDialect { true } + /// See [doc](https://docs.snowflake.com/en/sql-reference/sql/comment) + fn supports_comment_on(&self) -> bool { + true + } + fn parse_statement(&self, parser: &mut Parser) -> Option> { if parser.parse_keyword(Keyword::CREATE) { // possibly CREATE STAGE diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 7f6961ae8..756f4d68b 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -551,6 +551,8 @@ impl<'a> Parser<'a> { Keyword::OPTIMIZE if dialect_of!(self is ClickHouseDialect | GenericDialect) => { self.parse_optimize_table() } + // `COMMENT` is snowflake specific https://docs.snowflake.com/en/sql-reference/sql/comment + Keyword::COMMENT if self.dialect.supports_comment_on() => self.parse_comment(), _ => self.expected("an SQL statement", next_token), }, Token::LParen => { @@ -561,6 +563,51 @@ impl<'a> Parser<'a> { } } + pub fn parse_comment(&mut self) -> Result { + let if_exists = self.parse_keywords(&[Keyword::IF, Keyword::EXISTS]); + + self.expect_keyword(Keyword::ON)?; + let token = self.next_token(); + + let (object_type, object_name) = match token.token { + Token::Word(w) if w.keyword == Keyword::COLUMN => { + (CommentObject::Column, self.parse_object_name(false)?) + } + Token::Word(w) if w.keyword == Keyword::TABLE => { + (CommentObject::Table, self.parse_object_name(false)?) + } + Token::Word(w) if w.keyword == Keyword::EXTENSION => { + (CommentObject::Extension, self.parse_object_name(false)?) + } + Token::Word(w) if w.keyword == Keyword::SCHEMA => { + (CommentObject::Schema, self.parse_object_name(false)?) + } + Token::Word(w) if w.keyword == Keyword::DATABASE => { + (CommentObject::Database, self.parse_object_name(false)?) + } + Token::Word(w) if w.keyword == Keyword::USER => { + (CommentObject::User, self.parse_object_name(false)?) + } + Token::Word(w) if w.keyword == Keyword::ROLE => { + (CommentObject::Role, self.parse_object_name(false)?) + } + _ => self.expected("comment object_type", token)?, + }; + + self.expect_keyword(Keyword::IS)?; + let comment = if self.parse_keyword(Keyword::NULL) { + None + } else { + Some(self.parse_literal_string()?) + }; + Ok(Statement::Comment { + object_type, + object_name, + comment, + if_exists, + }) + } + pub fn parse_flush(&mut self) -> Result { let mut channel = None; let mut tables: Vec = vec![]; diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index d08e19d68..25bf306ad 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11629,3 +11629,91 @@ fn parse_factorial_operator() { ); } } + +#[test] +fn parse_comments() { + match all_dialects_where(|d| d.supports_comment_on()) + .verified_stmt("COMMENT ON COLUMN tab.name IS 'comment'") + { + Statement::Comment { + object_type, + object_name, + comment: Some(comment), + if_exists, + } => { + assert_eq!("comment", comment); + assert_eq!("tab.name", object_name.to_string()); + assert_eq!(CommentObject::Column, object_type); + assert!(!if_exists); + } + _ => unreachable!(), + } + + let object_types = [ + ("COLUMN", CommentObject::Column), + ("EXTENSION", CommentObject::Extension), + ("TABLE", CommentObject::Table), + ("SCHEMA", CommentObject::Schema), + ("DATABASE", CommentObject::Database), + ("USER", CommentObject::User), + ("ROLE", CommentObject::Role), + ]; + for (keyword, expected_object_type) in object_types.iter() { + match all_dialects_where(|d| d.supports_comment_on()) + .verified_stmt(format!("COMMENT IF EXISTS ON {keyword} db.t0 IS 'comment'").as_str()) + { + Statement::Comment { + object_type, + object_name, + comment: Some(comment), + if_exists, + } => { + assert_eq!("comment", comment); + assert_eq!("db.t0", object_name.to_string()); + assert_eq!(*expected_object_type, object_type); + assert!(if_exists); + } + _ => unreachable!(), + } + } + + match all_dialects_where(|d| d.supports_comment_on()) + .verified_stmt("COMMENT IF EXISTS ON TABLE public.tab IS NULL") + { + Statement::Comment { + object_type, + object_name, + comment: None, + if_exists, + } => { + assert_eq!("public.tab", object_name.to_string()); + assert_eq!(CommentObject::Table, object_type); + assert!(if_exists); + } + _ => unreachable!(), + } + + // missing IS statement + assert_eq!( + all_dialects_where(|d| d.supports_comment_on()) + .parse_sql_statements("COMMENT ON TABLE t0") + .unwrap_err(), + ParserError::ParserError("Expected: IS, found: EOF".to_string()) + ); + + // missing comment literal + assert_eq!( + all_dialects_where(|d| d.supports_comment_on()) + .parse_sql_statements("COMMENT ON TABLE t0 IS") + .unwrap_err(), + ParserError::ParserError("Expected: literal string, found: EOF".to_string()) + ); + + // unknown object type + assert_eq!( + all_dialects_where(|d| d.supports_comment_on()) + .parse_sql_statements("COMMENT ON UNKNOWN t0 IS 'comment'") + .unwrap_err(), + ParserError::ParserError("Expected: comment object_type, found: UNKNOWN".to_string()) + ); +} diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 100c8eeb2..a6c480cd7 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -2891,68 +2891,6 @@ fn test_composite_value() { ); } -#[test] -fn parse_comments() { - match pg().verified_stmt("COMMENT ON COLUMN tab.name IS 'comment'") { - Statement::Comment { - object_type, - object_name, - comment: Some(comment), - if_exists, - } => { - assert_eq!("comment", comment); - assert_eq!("tab.name", object_name.to_string()); - assert_eq!(CommentObject::Column, object_type); - assert!(!if_exists); - } - _ => unreachable!(), - } - - match pg().verified_stmt("COMMENT ON EXTENSION plpgsql IS 'comment'") { - Statement::Comment { - object_type, - object_name, - comment: Some(comment), - if_exists, - } => { - assert_eq!("comment", comment); - assert_eq!("plpgsql", object_name.to_string()); - assert_eq!(CommentObject::Extension, object_type); - assert!(!if_exists); - } - _ => unreachable!(), - } - - match pg().verified_stmt("COMMENT ON TABLE public.tab IS 'comment'") { - Statement::Comment { - object_type, - object_name, - comment: Some(comment), - if_exists, - } => { - assert_eq!("comment", comment); - assert_eq!("public.tab", object_name.to_string()); - assert_eq!(CommentObject::Table, object_type); - assert!(!if_exists); - } - _ => unreachable!(), - } - - match pg().verified_stmt("COMMENT IF EXISTS ON TABLE public.tab IS NULL") { - Statement::Comment { - object_type, - object_name, - comment: None, - if_exists, - } => { - assert_eq!("public.tab", object_name.to_string()); - assert_eq!(CommentObject::Table, object_type); - assert!(if_exists); - } - _ => unreachable!(), - } -} - #[test] fn parse_quoted_identifier() { pg_and_generic().verified_stmt(r#"SELECT "quoted "" ident""#); From 2bb81444bd94d2c02f7336c6b9941bd79769fb7f Mon Sep 17 00:00:00 2001 From: wugeer <1284057728@qq.com> Date: Thu, 14 Nov 2024 01:36:13 +0800 Subject: [PATCH 51/80] Add support for MYSQL's `CREATE TABLE SELECT` expr (#1515) --- src/dialect/mod.rs | 5 +++++ src/dialect/mysql.rs | 5 +++++ src/parser/mod.rs | 5 +++++ tests/sqlparser_common.rs | 33 ++++++++++++++++++++++++++++++++- 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index d95d7c70a..a732aa5a0 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -633,6 +633,11 @@ pub trait Dialect: Debug + Any { fn supports_comment_on(&self) -> bool { false } + + /// Returns true if the dialect supports the `CREATE TABLE SELECT` statement + fn supports_create_table_select(&self) -> bool { + false + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/mysql.rs b/src/dialect/mysql.rs index d1bf33345..197ce48d4 100644 --- a/src/dialect/mysql.rs +++ b/src/dialect/mysql.rs @@ -97,6 +97,11 @@ impl Dialect for MySqlDialect { fn supports_limit_comma(&self) -> bool { true } + + /// see + fn supports_create_table_select(&self) -> bool { + true + } } /// `LOCK TABLES` diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 756f4d68b..4115bbc9a 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -5990,6 +5990,11 @@ impl<'a> Parser<'a> { // Parse optional `AS ( query )` let query = if self.parse_keyword(Keyword::AS) { Some(self.parse_query()?) + } else if self.dialect.supports_create_table_select() && self.parse_keyword(Keyword::SELECT) + { + // rewind the SELECT keyword + self.prev_token(); + Some(self.parse_query()?) } else { None }; diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 25bf306ad..daf65edf1 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -6501,7 +6501,17 @@ fn parse_multiple_statements() { ); test_with("DELETE FROM foo", "SELECT", " bar"); test_with("INSERT INTO foo VALUES (1)", "SELECT", " bar"); - test_with("CREATE TABLE foo (baz INT)", "SELECT", " bar"); + // Since MySQL supports the `CREATE TABLE SELECT` syntax, this needs to be handled separately + let res = parse_sql_statements("CREATE TABLE foo (baz INT); SELECT bar"); + assert_eq!( + vec![ + one_statement_parses_to("CREATE TABLE foo (baz INT)", ""), + one_statement_parses_to("SELECT bar", ""), + ], + res.unwrap() + ); + // Check that extra semicolon at the end is stripped by normalization: + one_statement_parses_to("CREATE TABLE foo (baz INT);", "CREATE TABLE foo (baz INT)"); // Make sure that empty statements do not cause an error: let res = parse_sql_statements(";;"); assert_eq!(0, res.unwrap().len()); @@ -11717,3 +11727,24 @@ fn parse_comments() { ParserError::ParserError("Expected: comment object_type, found: UNKNOWN".to_string()) ); } + +#[test] +fn parse_create_table_select() { + let dialects = all_dialects_where(|d| d.supports_create_table_select()); + let sql_1 = r#"CREATE TABLE foo (baz INT) SELECT bar"#; + let expected = r#"CREATE TABLE foo (baz INT) AS SELECT bar"#; + let _ = dialects.one_statement_parses_to(sql_1, expected); + + let sql_2 = r#"CREATE TABLE foo (baz INT, name STRING) SELECT bar, oth_name FROM test.table_a"#; + let expected = + r#"CREATE TABLE foo (baz INT, name STRING) AS SELECT bar, oth_name FROM test.table_a"#; + let _ = dialects.one_statement_parses_to(sql_2, expected); + + let dialects = all_dialects_where(|d| !d.supports_create_table_select()); + for sql in [sql_1, sql_2] { + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError("Expected: end of statement, found: SELECT".to_string()) + ); + } +} From 62eaee62dc12fe001992650bf5b330f065b92c07 Mon Sep 17 00:00:00 2001 From: gaoqiangz <38213294+gaoqiangz@users.noreply.github.com> Date: Fri, 15 Nov 2024 04:32:57 +0800 Subject: [PATCH 52/80] Add support for MSSQL's `XQuery` methods (#1500) Co-authored-by: Ifeanyi Ubah --- src/ast/mod.rs | 39 ++++++++++++++++++++++ src/dialect/mod.rs | 9 +++++ src/dialect/mssql.rs | 4 +++ src/parser/mod.rs | 39 ++++++++++++++++++++++ tests/sqlparser_common.rs | 70 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 161 insertions(+) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 505386fbf..b0ac6bc41 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -808,6 +808,23 @@ pub enum Expr { }, /// Scalar function call e.g. `LEFT(foo, 5)` Function(Function), + /// Arbitrary expr method call + /// + /// Syntax: + /// + /// `.....` + /// + /// > `arbitrary-expr` can be any expression including a function call. + /// + /// Example: + /// + /// ```sql + /// SELECT (SELECT ',' + name FROM sys.objects FOR XML PATH(''), TYPE).value('.','NVARCHAR(MAX)') + /// SELECT CONVERT(XML,'abc').value('.','NVARCHAR(MAX)').value('.','NVARCHAR(MAX)') + /// ``` + /// + /// (mssql): + Method(Method), /// `CASE [] WHEN THEN ... [ELSE ] END` /// /// Note we only recognize a complete single expression as ``, @@ -1464,6 +1481,7 @@ impl fmt::Display for Expr { write!(f, " '{}'", &value::escape_single_quote_string(value)) } Expr::Function(fun) => write!(f, "{fun}"), + Expr::Method(method) => write!(f, "{method}"), Expr::Case { operand, conditions, @@ -5609,6 +5627,27 @@ impl fmt::Display for FunctionArgumentClause { } } +/// A method call +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct Method { + pub expr: Box, + // always non-empty + pub method_chain: Vec, +} + +impl fmt::Display for Method { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{}.{}", + self.expr, + display_separated(&self.method_chain, ".") + ) + } +} + #[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index a732aa5a0..ee3fd4714 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -279,6 +279,15 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if the dialect supports method calls, for example: + /// + /// ```sql + /// SELECT (SELECT ',' + name FROM sys.objects FOR XML PATH(''), TYPE).value('.','NVARCHAR(MAX)') + /// ``` + fn supports_methods(&self) -> bool { + false + } + /// Returns true if the dialect supports multiple variable assignment /// using parentheses in a `SET` variable declaration. /// diff --git a/src/dialect/mssql.rs b/src/dialect/mssql.rs index 8aab0bc8a..39ce9c125 100644 --- a/src/dialect/mssql.rs +++ b/src/dialect/mssql.rs @@ -62,4 +62,8 @@ impl Dialect for MsSqlDialect { fn supports_boolean_literals(&self) -> bool { false } + + fn supports_methods(&self) -> bool { + true + } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 4115bbc9a..a66a627bc 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1317,6 +1317,7 @@ impl<'a> Parser<'a> { } }; self.expect_token(&Token::RParen)?; + let expr = self.try_parse_method(expr)?; if !self.consume_token(&Token::Period) { Ok(expr) } else { @@ -1346,6 +1347,9 @@ impl<'a> Parser<'a> { } _ => self.expected("an expression", next_token), }?; + + let expr = self.try_parse_method(expr)?; + if self.parse_keyword(Keyword::COLLATE) { Ok(Expr::Collate { expr: Box::new(expr), @@ -1403,6 +1407,41 @@ impl<'a> Parser<'a> { }) } + /// Parses method call expression + fn try_parse_method(&mut self, expr: Expr) -> Result { + if !self.dialect.supports_methods() { + return Ok(expr); + } + let method_chain = self.maybe_parse(|p| { + let mut method_chain = Vec::new(); + while p.consume_token(&Token::Period) { + let tok = p.next_token(); + let name = match tok.token { + Token::Word(word) => word.to_ident(), + _ => return p.expected("identifier", tok), + }; + let func = match p.parse_function(ObjectName(vec![name]))? { + Expr::Function(func) => func, + _ => return p.expected("function", p.peek_token()), + }; + method_chain.push(func); + } + if !method_chain.is_empty() { + Ok(method_chain) + } else { + p.expected("function", p.peek_token()) + } + })?; + if let Some(method_chain) = method_chain { + Ok(Expr::Method(Method { + expr: Box::new(expr), + method_chain, + })) + } else { + Ok(expr) + } + } + pub fn parse_function(&mut self, name: ObjectName) -> Result { self.expect_token(&Token::LParen)?; diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index daf65edf1..4fdbf7d51 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11403,6 +11403,76 @@ fn test_try_convert() { dialects.verified_expr("TRY_CONVERT('foo', VARCHAR(MAX))"); } +#[test] +fn parse_method_select() { + let dialects = all_dialects_where(|d| d.supports_methods()); + let _ = dialects.verified_only_select( + "SELECT LEFT('abc', 1).value('.', 'NVARCHAR(MAX)').value('.', 'NVARCHAR(MAX)') AS T", + ); + let _ = dialects.verified_only_select("SELECT STUFF((SELECT ',' + name FROM sys.objects FOR XML PATH(''), TYPE).value('.', 'NVARCHAR(MAX)'), 1, 1, '') AS T"); + let _ = dialects + .verified_only_select("SELECT CAST(column AS XML).value('.', 'NVARCHAR(MAX)') AS T"); + + // `CONVERT` support + let dialects = all_dialects_where(|d| { + d.supports_methods() && d.supports_try_convert() && d.convert_type_before_value() + }); + let _ = dialects.verified_only_select("SELECT CONVERT(XML, 'abc').value('.', 'NVARCHAR(MAX)').value('.', 'NVARCHAR(MAX)') AS T"); +} + +#[test] +fn parse_method_expr() { + let dialects = all_dialects_where(|d| d.supports_methods()); + let expr = dialects + .verified_expr("LEFT('abc', 1).value('.', 'NVARCHAR(MAX)').value('.', 'NVARCHAR(MAX)')"); + match expr { + Expr::Method(Method { expr, method_chain }) => { + assert!(matches!(*expr, Expr::Function(_))); + assert!(matches!( + method_chain[..], + [Function { .. }, Function { .. }] + )); + } + _ => unreachable!(), + } + let expr = dialects.verified_expr( + "(SELECT ',' + name FROM sys.objects FOR XML PATH(''), TYPE).value('.', 'NVARCHAR(MAX)')", + ); + match expr { + Expr::Method(Method { expr, method_chain }) => { + assert!(matches!(*expr, Expr::Subquery(_))); + assert!(matches!(method_chain[..], [Function { .. }])); + } + _ => unreachable!(), + } + let expr = dialects.verified_expr("CAST(column AS XML).value('.', 'NVARCHAR(MAX)')"); + match expr { + Expr::Method(Method { expr, method_chain }) => { + assert!(matches!(*expr, Expr::Cast { .. })); + assert!(matches!(method_chain[..], [Function { .. }])); + } + _ => unreachable!(), + } + + // `CONVERT` support + let dialects = all_dialects_where(|d| { + d.supports_methods() && d.supports_try_convert() && d.convert_type_before_value() + }); + let expr = dialects.verified_expr( + "CONVERT(XML, 'abc').value('.', 'NVARCHAR(MAX)').value('.', 'NVARCHAR(MAX)')", + ); + match expr { + Expr::Method(Method { expr, method_chain }) => { + assert!(matches!(*expr, Expr::Convert { .. })); + assert!(matches!( + method_chain[..], + [Function { .. }, Function { .. }] + )); + } + _ => unreachable!(), + } +} + #[test] fn test_show_dbs_schemas_tables_views() { // These statements are parsed the same by all dialects From 724a1d1aba575fb04a2df54ca8425b39ea753938 Mon Sep 17 00:00:00 2001 From: wugeer <1284057728@qq.com> Date: Fri, 15 Nov 2024 22:53:31 +0800 Subject: [PATCH 53/80] Add support for Hive's `LOAD DATA` expr (#1520) Co-authored-by: Ifeanyi Ubah --- src/ast/mod.rs | 54 ++++++++++ src/dialect/duckdb.rs | 5 + src/dialect/generic.rs | 4 + src/dialect/hive.rs | 5 + src/dialect/mod.rs | 10 ++ src/keywords.rs | 1 + src/parser/mod.rs | 52 ++++++++-- tests/sqlparser_common.rs | 203 +++++++++++++++++++++++++++++++++++++- 8 files changed, 323 insertions(+), 11 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index b0ac6bc41..39c742153 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -3347,6 +3347,22 @@ pub enum Statement { channel: Ident, payload: Option, }, + /// ```sql + /// LOAD DATA [LOCAL] INPATH 'filepath' [OVERWRITE] INTO TABLE tablename + /// [PARTITION (partcol1=val1, partcol2=val2 ...)] + /// [INPUTFORMAT 'inputformat' SERDE 'serde'] + /// ``` + /// Loading files into tables + /// + /// See Hive + LoadData { + local: bool, + inpath: String, + overwrite: bool, + table_name: ObjectName, + partitioned: Option>, + table_format: Option, + }, } impl fmt::Display for Statement { @@ -3949,6 +3965,36 @@ impl fmt::Display for Statement { Ok(()) } Statement::CreateTable(create_table) => create_table.fmt(f), + Statement::LoadData { + local, + inpath, + overwrite, + table_name, + partitioned, + table_format, + } => { + write!( + f, + "LOAD DATA {local}INPATH '{inpath}' {overwrite}INTO TABLE {table_name}", + local = if *local { "LOCAL " } else { "" }, + inpath = inpath, + overwrite = if *overwrite { "OVERWRITE " } else { "" }, + table_name = table_name, + )?; + if let Some(ref parts) = &partitioned { + if !parts.is_empty() { + write!(f, " PARTITION ({})", display_comma_separated(parts))?; + } + } + if let Some(HiveLoadDataFormat { + serde, + input_format, + }) = &table_format + { + write!(f, " INPUTFORMAT {input_format} SERDE {serde}")?; + } + Ok(()) + } Statement::CreateVirtualTable { name, if_not_exists, @@ -5855,6 +5901,14 @@ pub enum HiveRowFormat { DELIMITED { delimiters: Vec }, } +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct HiveLoadDataFormat { + pub serde: Expr, + pub input_format: Expr, +} + #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] diff --git a/src/dialect/duckdb.rs b/src/dialect/duckdb.rs index e1b8db118..905b04e36 100644 --- a/src/dialect/duckdb.rs +++ b/src/dialect/duckdb.rs @@ -66,4 +66,9 @@ impl Dialect for DuckDbDialect { fn supports_explain_with_utility_options(&self) -> bool { true } + + /// See DuckDB + fn supports_load_extension(&self) -> bool { + true + } } diff --git a/src/dialect/generic.rs b/src/dialect/generic.rs index 8cfac217b..4998e0f4b 100644 --- a/src/dialect/generic.rs +++ b/src/dialect/generic.rs @@ -115,4 +115,8 @@ impl Dialect for GenericDialect { fn supports_comment_on(&self) -> bool { true } + + fn supports_load_extension(&self) -> bool { + true + } } diff --git a/src/dialect/hive.rs b/src/dialect/hive.rs index b97bf69be..571f9b9ba 100644 --- a/src/dialect/hive.rs +++ b/src/dialect/hive.rs @@ -56,4 +56,9 @@ impl Dialect for HiveDialect { fn supports_bang_not_operator(&self) -> bool { true } + + /// See Hive + fn supports_load_data(&self) -> bool { + true + } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index ee3fd4714..956a58986 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -620,6 +620,16 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if the dialect supports the `LOAD DATA` statement + fn supports_load_data(&self) -> bool { + false + } + + /// Returns true if the dialect supports the `LOAD extension` statement + fn supports_load_extension(&self) -> bool { + false + } + /// Returns true if this dialect expects the `TOP` option /// before the `ALL`/`DISTINCT` options in a `SELECT` statement. fn supports_top_before_distinct(&self) -> bool { diff --git a/src/keywords.rs b/src/keywords.rs index 9cdc90ce2..790268219 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -389,6 +389,7 @@ define_keywords!( INITIALLY, INNER, INOUT, + INPATH, INPUT, INPUTFORMAT, INSENSITIVE, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a66a627bc..a583112a7 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -543,10 +543,7 @@ impl<'a> Parser<'a> { Keyword::INSTALL if dialect_of!(self is DuckDbDialect | GenericDialect) => { self.parse_install() } - // `LOAD` is duckdb specific https://duckdb.org/docs/extensions/overview - Keyword::LOAD if dialect_of!(self is DuckDbDialect | GenericDialect) => { - self.parse_load() - } + Keyword::LOAD => self.parse_load(), // `OPTIMIZE` is clickhouse specific https://clickhouse.tech/docs/en/sql-reference/statements/optimize/ Keyword::OPTIMIZE if dialect_of!(self is ClickHouseDialect | GenericDialect) => { self.parse_optimize_table() @@ -11222,6 +11219,22 @@ impl<'a> Parser<'a> { } } + pub fn parse_load_data_table_format( + &mut self, + ) -> Result, ParserError> { + if self.parse_keyword(Keyword::INPUTFORMAT) { + let input_format = self.parse_expr()?; + self.expect_keyword(Keyword::SERDE)?; + let serde = self.parse_expr()?; + Ok(Some(HiveLoadDataFormat { + input_format, + serde, + })) + } else { + Ok(None) + } + } + /// Parse an UPDATE statement, returning a `Box`ed SetExpr /// /// This is used to reduce the size of the stack frames in debug builds @@ -12224,10 +12237,35 @@ impl<'a> Parser<'a> { Ok(Statement::Install { extension_name }) } - /// `LOAD [extension_name]` + /// Parse a SQL LOAD statement pub fn parse_load(&mut self) -> Result { - let extension_name = self.parse_identifier(false)?; - Ok(Statement::Load { extension_name }) + if self.dialect.supports_load_extension() { + let extension_name = self.parse_identifier(false)?; + Ok(Statement::Load { extension_name }) + } else if self.parse_keyword(Keyword::DATA) && self.dialect.supports_load_data() { + let local = self.parse_one_of_keywords(&[Keyword::LOCAL]).is_some(); + self.expect_keyword(Keyword::INPATH)?; + let inpath = self.parse_literal_string()?; + let overwrite = self.parse_one_of_keywords(&[Keyword::OVERWRITE]).is_some(); + self.expect_keyword(Keyword::INTO)?; + self.expect_keyword(Keyword::TABLE)?; + let table_name = self.parse_object_name(false)?; + let partitioned = self.parse_insert_partition()?; + let table_format = self.parse_load_data_table_format()?; + Ok(Statement::LoadData { + local, + inpath, + overwrite, + table_name, + partitioned, + table_format, + }) + } else { + self.expected( + "`DATA` or an extension name after `LOAD`", + self.peek_token(), + ) + } } /// ```sql diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 4fdbf7d51..2ffb5f44b 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11583,13 +11583,208 @@ fn parse_notify_channel() { dialects.parse_sql_statements(sql).unwrap_err(), ParserError::ParserError("Expected: an SQL statement, found: NOTIFY".to_string()) ); - assert_eq!( - dialects.parse_sql_statements(sql).unwrap_err(), - ParserError::ParserError("Expected: an SQL statement, found: NOTIFY".to_string()) - ); } } +#[test] +fn parse_load_data() { + let dialects = all_dialects_where(|d| d.supports_load_data()); + let only_supports_load_extension_dialects = + all_dialects_where(|d| !d.supports_load_data() && d.supports_load_extension()); + let not_supports_load_dialects = + all_dialects_where(|d| !d.supports_load_data() && !d.supports_load_extension()); + + let sql = "LOAD DATA INPATH '/local/path/to/data.txt' INTO TABLE test.my_table"; + match dialects.verified_stmt(sql) { + Statement::LoadData { + local, + inpath, + overwrite, + table_name, + partitioned, + table_format, + } => { + assert_eq!(false, local); + assert_eq!("/local/path/to/data.txt", inpath); + assert_eq!(false, overwrite); + assert_eq!( + ObjectName(vec![Ident::new("test"), Ident::new("my_table")]), + table_name + ); + assert_eq!(None, partitioned); + assert_eq!(None, table_format); + } + _ => unreachable!(), + }; + + // with OVERWRITE keyword + let sql = "LOAD DATA INPATH '/local/path/to/data.txt' OVERWRITE INTO TABLE my_table"; + match dialects.verified_stmt(sql) { + Statement::LoadData { + local, + inpath, + overwrite, + table_name, + partitioned, + table_format, + } => { + assert_eq!(false, local); + assert_eq!("/local/path/to/data.txt", inpath); + assert_eq!(true, overwrite); + assert_eq!(ObjectName(vec![Ident::new("my_table")]), table_name); + assert_eq!(None, partitioned); + assert_eq!(None, table_format); + } + _ => unreachable!(), + }; + + assert_eq!( + only_supports_load_extension_dialects + .parse_sql_statements(sql) + .unwrap_err(), + ParserError::ParserError("Expected: end of statement, found: INPATH".to_string()) + ); + assert_eq!( + not_supports_load_dialects + .parse_sql_statements(sql) + .unwrap_err(), + ParserError::ParserError( + "Expected: `DATA` or an extension name after `LOAD`, found: INPATH".to_string() + ) + ); + + // with LOCAL keyword + let sql = "LOAD DATA LOCAL INPATH '/local/path/to/data.txt' INTO TABLE test.my_table"; + match dialects.verified_stmt(sql) { + Statement::LoadData { + local, + inpath, + overwrite, + table_name, + partitioned, + table_format, + } => { + assert_eq!(true, local); + assert_eq!("/local/path/to/data.txt", inpath); + assert_eq!(false, overwrite); + assert_eq!( + ObjectName(vec![Ident::new("test"), Ident::new("my_table")]), + table_name + ); + assert_eq!(None, partitioned); + assert_eq!(None, table_format); + } + _ => unreachable!(), + }; + + assert_eq!( + only_supports_load_extension_dialects + .parse_sql_statements(sql) + .unwrap_err(), + ParserError::ParserError("Expected: end of statement, found: LOCAL".to_string()) + ); + assert_eq!( + not_supports_load_dialects + .parse_sql_statements(sql) + .unwrap_err(), + ParserError::ParserError( + "Expected: `DATA` or an extension name after `LOAD`, found: LOCAL".to_string() + ) + ); + + // with PARTITION clause + let sql = "LOAD DATA LOCAL INPATH '/local/path/to/data.txt' INTO TABLE my_table PARTITION (year = 2024, month = 11)"; + match dialects.verified_stmt(sql) { + Statement::LoadData { + local, + inpath, + overwrite, + table_name, + partitioned, + table_format, + } => { + assert_eq!(true, local); + assert_eq!("/local/path/to/data.txt", inpath); + assert_eq!(false, overwrite); + assert_eq!(ObjectName(vec![Ident::new("my_table")]), table_name); + assert_eq!( + Some(vec![ + Expr::BinaryOp { + left: Box::new(Expr::Identifier(Ident::new("year"))), + op: BinaryOperator::Eq, + right: Box::new(Expr::Value(Value::Number("2024".parse().unwrap(), false))), + }, + Expr::BinaryOp { + left: Box::new(Expr::Identifier(Ident::new("month"))), + op: BinaryOperator::Eq, + right: Box::new(Expr::Value(Value::Number("11".parse().unwrap(), false))), + } + ]), + partitioned + ); + assert_eq!(None, table_format); + } + _ => unreachable!(), + }; + + // with PARTITION clause + let sql = "LOAD DATA LOCAL INPATH '/local/path/to/data.txt' OVERWRITE INTO TABLE good.my_table PARTITION (year = 2024, month = 11) INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'"; + match dialects.verified_stmt(sql) { + Statement::LoadData { + local, + inpath, + overwrite, + table_name, + partitioned, + table_format, + } => { + assert_eq!(true, local); + assert_eq!("/local/path/to/data.txt", inpath); + assert_eq!(true, overwrite); + assert_eq!( + ObjectName(vec![Ident::new("good"), Ident::new("my_table")]), + table_name + ); + assert_eq!( + Some(vec![ + Expr::BinaryOp { + left: Box::new(Expr::Identifier(Ident::new("year"))), + op: BinaryOperator::Eq, + right: Box::new(Expr::Value(Value::Number("2024".parse().unwrap(), false))), + }, + Expr::BinaryOp { + left: Box::new(Expr::Identifier(Ident::new("month"))), + op: BinaryOperator::Eq, + right: Box::new(Expr::Value(Value::Number("11".parse().unwrap(), false))), + } + ]), + partitioned + ); + assert_eq!( + Some(HiveLoadDataFormat { + serde: Expr::Value(Value::SingleQuotedString( + "org.apache.hadoop.hive.serde2.OpenCSVSerde".to_string() + )), + input_format: Expr::Value(Value::SingleQuotedString( + "org.apache.hadoop.mapred.TextInputFormat".to_string() + )) + }), + table_format + ); + } + _ => unreachable!(), + }; + + // negative test case + let sql = "LOAD DATA2 LOCAL INPATH '/local/path/to/data.txt' INTO TABLE test.my_table"; + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError( + "Expected: `DATA` or an extension name after `LOAD`, found: DATA2".to_string() + ) + ); +} + #[test] fn test_select_top() { let dialects = all_dialects_where(|d| d.supports_top_before_distinct()); From 4a5f20e9111900eed7480ac92d23402d4c10f1d6 Mon Sep 17 00:00:00 2001 From: hulk Date: Mon, 18 Nov 2024 20:29:28 +0800 Subject: [PATCH 54/80] Fix ClickHouse document link from `Russian` to `English` (#1527) --- src/ast/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 39c742153..fa9e53a5a 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -3167,7 +3167,7 @@ pub enum Statement { /// KILL [CONNECTION | QUERY | MUTATION] /// ``` /// - /// See + /// See /// See Kill { modifier: Option, From a67a4f3cbe2cb6e266f3914db8520890bfa7e198 Mon Sep 17 00:00:00 2001 From: delamarch3 <68732277+delamarch3@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:30:20 +0000 Subject: [PATCH 55/80] Support ANTI and SEMI joins without LEFT/RIGHT (#1528) --- src/ast/query.rs | 18 ++++++++++++++++++ src/keywords.rs | 2 ++ src/parser/mod.rs | 10 ++++++++++ tests/sqlparser_common.rs | 16 ++++++++++++++++ 4 files changed, 46 insertions(+) diff --git a/src/ast/query.rs b/src/ast/query.rs index 60ebe3765..2160da0d0 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -1694,6 +1694,13 @@ impl fmt::Display for Join { suffix(constraint) ), JoinOperator::CrossJoin => write!(f, " CROSS JOIN {}", self.relation), + JoinOperator::Semi(constraint) => write!( + f, + " {}SEMI JOIN {}{}", + prefix(constraint), + self.relation, + suffix(constraint) + ), JoinOperator::LeftSemi(constraint) => write!( f, " {}LEFT SEMI JOIN {}{}", @@ -1708,6 +1715,13 @@ impl fmt::Display for Join { self.relation, suffix(constraint) ), + JoinOperator::Anti(constraint) => write!( + f, + " {}ANTI JOIN {}{}", + prefix(constraint), + self.relation, + suffix(constraint) + ), JoinOperator::LeftAnti(constraint) => write!( f, " {}LEFT ANTI JOIN {}{}", @@ -1746,10 +1760,14 @@ pub enum JoinOperator { RightOuter(JoinConstraint), FullOuter(JoinConstraint), CrossJoin, + /// SEMI (non-standard) + Semi(JoinConstraint), /// LEFT SEMI (non-standard) LeftSemi(JoinConstraint), /// RIGHT SEMI (non-standard) RightSemi(JoinConstraint), + /// ANTI (non-standard) + Anti(JoinConstraint), /// LEFT ANTI (non-standard) LeftAnti(JoinConstraint), /// RIGHT ANTI (non-standard) diff --git a/src/keywords.rs b/src/keywords.rs index 790268219..fdf2bf35c 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -892,6 +892,8 @@ pub const RESERVED_FOR_TABLE_ALIAS: &[Keyword] = &[ Keyword::CLUSTER, Keyword::DISTRIBUTE, Keyword::GLOBAL, + Keyword::ANTI, + Keyword::SEMI, // for MSSQL-specific OUTER APPLY (seems reserved in most dialects) Keyword::OUTER, Keyword::SET, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a583112a7..1a094c147 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -10025,6 +10025,16 @@ impl<'a> Parser<'a> { } } } + Keyword::ANTI => { + let _ = self.next_token(); // consume ANTI + self.expect_keyword(Keyword::JOIN)?; + JoinOperator::Anti + } + Keyword::SEMI => { + let _ = self.next_token(); // consume SEMI + self.expect_keyword(Keyword::JOIN)?; + JoinOperator::Semi + } Keyword::FULL => { let _ = self.next_token(); // consume FULL let _ = self.parse_keyword(Keyword::OUTER); // [ OUTER ] diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 2ffb5f44b..77496781c 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -6013,6 +6013,10 @@ fn parse_joins_on() { JoinOperator::RightOuter )] ); + assert_eq!( + only(&verified_only_select("SELECT * FROM t1 SEMI JOIN t2 ON c1 = c2").from).joins, + vec![join_with_constraint("t2", None, false, JoinOperator::Semi)] + ); assert_eq!( only(&verified_only_select("SELECT * FROM t1 LEFT SEMI JOIN t2 ON c1 = c2").from).joins, vec![join_with_constraint( @@ -6031,6 +6035,10 @@ fn parse_joins_on() { JoinOperator::RightSemi )] ); + assert_eq!( + only(&verified_only_select("SELECT * FROM t1 ANTI JOIN t2 ON c1 = c2").from).joins, + vec![join_with_constraint("t2", None, false, JoinOperator::Anti)] + ); assert_eq!( only(&verified_only_select("SELECT * FROM t1 LEFT ANTI JOIN t2 ON c1 = c2").from).joins, vec![join_with_constraint( @@ -6117,6 +6125,10 @@ fn parse_joins_using() { only(&verified_only_select("SELECT * FROM t1 RIGHT JOIN t2 USING(c1)").from).joins, vec![join_with_constraint("t2", None, JoinOperator::RightOuter)] ); + assert_eq!( + only(&verified_only_select("SELECT * FROM t1 SEMI JOIN t2 USING(c1)").from).joins, + vec![join_with_constraint("t2", None, JoinOperator::Semi)] + ); assert_eq!( only(&verified_only_select("SELECT * FROM t1 LEFT SEMI JOIN t2 USING(c1)").from).joins, vec![join_with_constraint("t2", None, JoinOperator::LeftSemi)] @@ -6125,6 +6137,10 @@ fn parse_joins_using() { only(&verified_only_select("SELECT * FROM t1 RIGHT SEMI JOIN t2 USING(c1)").from).joins, vec![join_with_constraint("t2", None, JoinOperator::RightSemi)] ); + assert_eq!( + only(&verified_only_select("SELECT * FROM t1 ANTI JOIN t2 USING(c1)").from).joins, + vec![join_with_constraint("t2", None, JoinOperator::Anti)] + ); assert_eq!( only(&verified_only_select("SELECT * FROM t1 LEFT ANTI JOIN t2 USING(c1)").from).joins, vec![join_with_constraint("t2", None, JoinOperator::LeftAnti)] From 4c629e8520b68eb289b34882aa326d80a6f8e022 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Mon, 18 Nov 2024 13:30:53 +0100 Subject: [PATCH 56/80] support sqlite's OR clauses in update statements (#1530) --- src/ast/dml.rs | 4 ++-- src/ast/mod.rs | 19 +++++++++++++------ src/parser/mod.rs | 39 +++++++++++++++++++++------------------ tests/sqlparser_common.rs | 21 +++++++++++++++++++++ tests/sqlparser_mysql.rs | 1 + tests/sqlparser_sqlite.rs | 1 + 6 files changed, 59 insertions(+), 26 deletions(-) diff --git a/src/ast/dml.rs b/src/ast/dml.rs index 2932fafb5..22309c8f8 100644 --- a/src/ast/dml.rs +++ b/src/ast/dml.rs @@ -505,8 +505,8 @@ impl Display for Insert { self.table_name.to_string() }; - if let Some(action) = self.or { - write!(f, "INSERT OR {action} INTO {table_name} ")?; + if let Some(on_conflict) = self.or { + write!(f, "INSERT {on_conflict} INTO {table_name} ")?; } else { write!( f, diff --git a/src/ast/mod.rs b/src/ast/mod.rs index fa9e53a5a..ad59f0779 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2396,6 +2396,8 @@ pub enum Statement { selection: Option, /// RETURNING returning: Option>, + /// SQLite-specific conflict resolution clause + or: Option, }, /// ```sql /// DELETE @@ -3691,8 +3693,13 @@ impl fmt::Display for Statement { from, selection, returning, + or, } => { - write!(f, "UPDATE {table}")?; + write!(f, "UPDATE ")?; + if let Some(or) = or { + write!(f, "{or} ")?; + } + write!(f, "{table}")?; if !assignments.is_empty() { write!(f, " SET {}", display_comma_separated(assignments))?; } @@ -6304,11 +6311,11 @@ impl fmt::Display for SqliteOnConflict { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { use SqliteOnConflict::*; match self { - Rollback => write!(f, "ROLLBACK"), - Abort => write!(f, "ABORT"), - Fail => write!(f, "FAIL"), - Ignore => write!(f, "IGNORE"), - Replace => write!(f, "REPLACE"), + Rollback => write!(f, "OR ROLLBACK"), + Abort => write!(f, "OR ABORT"), + Fail => write!(f, "OR FAIL"), + Ignore => write!(f, "OR IGNORE"), + Replace => write!(f, "OR REPLACE"), } } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 1a094c147..0c8237889 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -11042,24 +11042,7 @@ impl<'a> Parser<'a> { /// Parse an INSERT statement pub fn parse_insert(&mut self) -> Result { - let or = if !dialect_of!(self is SQLiteDialect) { - None - } else if self.parse_keywords(&[Keyword::OR, Keyword::REPLACE]) { - Some(SqliteOnConflict::Replace) - } else if self.parse_keywords(&[Keyword::OR, Keyword::ROLLBACK]) { - Some(SqliteOnConflict::Rollback) - } else if self.parse_keywords(&[Keyword::OR, Keyword::ABORT]) { - Some(SqliteOnConflict::Abort) - } else if self.parse_keywords(&[Keyword::OR, Keyword::FAIL]) { - Some(SqliteOnConflict::Fail) - } else if self.parse_keywords(&[Keyword::OR, Keyword::IGNORE]) { - Some(SqliteOnConflict::Ignore) - } else if self.parse_keyword(Keyword::REPLACE) { - Some(SqliteOnConflict::Replace) - } else { - None - }; - + let or = self.parse_conflict_clause(); let priority = if !dialect_of!(self is MySqlDialect | GenericDialect) { None } else if self.parse_keyword(Keyword::LOW_PRIORITY) { @@ -11218,6 +11201,24 @@ impl<'a> Parser<'a> { } } + fn parse_conflict_clause(&mut self) -> Option { + if self.parse_keywords(&[Keyword::OR, Keyword::REPLACE]) { + Some(SqliteOnConflict::Replace) + } else if self.parse_keywords(&[Keyword::OR, Keyword::ROLLBACK]) { + Some(SqliteOnConflict::Rollback) + } else if self.parse_keywords(&[Keyword::OR, Keyword::ABORT]) { + Some(SqliteOnConflict::Abort) + } else if self.parse_keywords(&[Keyword::OR, Keyword::FAIL]) { + Some(SqliteOnConflict::Fail) + } else if self.parse_keywords(&[Keyword::OR, Keyword::IGNORE]) { + Some(SqliteOnConflict::Ignore) + } else if self.parse_keyword(Keyword::REPLACE) { + Some(SqliteOnConflict::Replace) + } else { + None + } + } + pub fn parse_insert_partition(&mut self) -> Result>, ParserError> { if self.parse_keyword(Keyword::PARTITION) { self.expect_token(&Token::LParen)?; @@ -11253,6 +11254,7 @@ impl<'a> Parser<'a> { } pub fn parse_update(&mut self) -> Result { + let or = self.parse_conflict_clause(); let table = self.parse_table_and_joins()?; self.expect_keyword(Keyword::SET)?; let assignments = self.parse_comma_separated(Parser::parse_assignment)?; @@ -11279,6 +11281,7 @@ impl<'a> Parser<'a> { from, selection, returning, + or, }) } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 77496781c..283071e9b 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -443,6 +443,7 @@ fn parse_update_set_from() { ])), }), returning: None, + or: None, } ); } @@ -457,6 +458,7 @@ fn parse_update_with_table_alias() { from: _from, selection, returning, + or: None, } => { assert_eq!( TableWithJoins { @@ -505,6 +507,25 @@ fn parse_update_with_table_alias() { } } +#[test] +fn parse_update_or() { + let expect_or_clause = |sql: &str, expected_action: SqliteOnConflict| match verified_stmt(sql) { + Statement::Update { or, .. } => assert_eq!(or, Some(expected_action)), + other => unreachable!("Expected update with or, got {:?}", other), + }; + expect_or_clause( + "UPDATE OR REPLACE t SET n = n + 1", + SqliteOnConflict::Replace, + ); + expect_or_clause( + "UPDATE OR ROLLBACK t SET n = n + 1", + SqliteOnConflict::Rollback, + ); + expect_or_clause("UPDATE OR ABORT t SET n = n + 1", SqliteOnConflict::Abort); + expect_or_clause("UPDATE OR FAIL t SET n = n + 1", SqliteOnConflict::Fail); + expect_or_clause("UPDATE OR IGNORE t SET n = n + 1", SqliteOnConflict::Ignore); +} + #[test] fn parse_select_with_table_alias_as() { // AS is optional diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 8269eadc0..2a876cff2 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -1970,6 +1970,7 @@ fn parse_update_with_joins() { from: _from, selection, returning, + or: None, } => { assert_eq!( TableWithJoins { diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index 6f8bbb2d8..6f8e654dc 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -465,6 +465,7 @@ fn parse_update_tuple_row_values() { assert_eq!( sqlite().verified_stmt("UPDATE x SET (a, b) = (1, 2)"), Statement::Update { + or: None, assignments: vec![Assignment { target: AssignmentTarget::Tuple(vec![ ObjectName(vec![Ident::new("a"),]), From f961efc0c92c271a75663b1b32b1da5b49d86db3 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Mon, 18 Nov 2024 15:02:22 +0100 Subject: [PATCH 57/80] support column type definitions in table aliases (#1526) --- src/ast/mod.rs | 4 +-- src/ast/query.rs | 37 +++++++++++++++++++++++- src/parser/mod.rs | 19 +++++++++++-- tests/sqlparser_common.rs | 59 +++++++++++++++++++++++++++++++++------ 4 files changed, 105 insertions(+), 14 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index ad59f0779..fc6a1b4f1 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -60,8 +60,8 @@ pub use self::query::{ OrderBy, OrderByExpr, PivotValueSource, ProjectionSelect, Query, RenameSelectItem, RepetitionQuantifier, ReplaceSelectElement, ReplaceSelectItem, RowsPerMatch, Select, SelectInto, SelectItem, SetExpr, SetOperator, SetQuantifier, Setting, SymbolDefinition, Table, - TableAlias, TableFactor, TableFunctionArgs, TableVersion, TableWithJoins, Top, TopQuantity, - ValueTableMode, Values, WildcardAdditionalOptions, With, WithFill, + TableAlias, TableAliasColumnDef, TableFactor, TableFunctionArgs, TableVersion, TableWithJoins, + Top, TopQuantity, ValueTableMode, Values, WildcardAdditionalOptions, With, WithFill, }; pub use self::trigger::{ diff --git a/src/ast/query.rs b/src/ast/query.rs index 2160da0d0..078bbc841 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -1597,7 +1597,7 @@ impl fmt::Display for TableFactor { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct TableAlias { pub name: Ident, - pub columns: Vec, + pub columns: Vec, } impl fmt::Display for TableAlias { @@ -1610,6 +1610,41 @@ impl fmt::Display for TableAlias { } } +/// SQL column definition in a table expression alias. +/// Most of the time, the data type is not specified. +/// But some table-valued functions do require specifying the data type. +/// +/// See +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct TableAliasColumnDef { + /// Column name alias + pub name: Ident, + /// Some table-valued functions require specifying the data type in the alias. + pub data_type: Option, +} + +impl TableAliasColumnDef { + /// Create a new table alias column definition with only a name and no type + pub fn from_name>(name: S) -> Self { + TableAliasColumnDef { + name: Ident::new(name), + data_type: None, + } + } +} + +impl fmt::Display for TableAliasColumnDef { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.name)?; + if let Some(ref data_type) = self.data_type { + write!(f, " {}", data_type)?; + } + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 0c8237889..82347f58d 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8270,7 +8270,7 @@ impl<'a> Parser<'a> { ) -> Result, ParserError> { match self.parse_optional_alias(reserved_kwds)? { Some(name) => { - let columns = self.parse_parenthesized_column_list(Optional, false)?; + let columns = self.parse_table_alias_column_defs()?; Ok(Some(TableAlias { name, columns })) } None => Ok(None), @@ -8607,6 +8607,21 @@ impl<'a> Parser<'a> { } } + /// Parse a parenthesized comma-separated list of table alias column definitions. + fn parse_table_alias_column_defs(&mut self) -> Result, ParserError> { + if self.consume_token(&Token::LParen) { + let cols = self.parse_comma_separated(|p| { + let name = p.parse_identifier(false)?; + let data_type = p.maybe_parse(|p| p.parse_data_type())?; + Ok(TableAliasColumnDef { name, data_type }) + })?; + self.expect_token(&Token::RParen)?; + Ok(cols) + } else { + Ok(vec![]) + } + } + pub fn parse_precision(&mut self) -> Result { self.expect_token(&Token::LParen)?; let n = self.parse_literal_uint()?; @@ -9174,7 +9189,7 @@ impl<'a> Parser<'a> { materialized: is_materialized, } } else { - let columns = self.parse_parenthesized_column_list(Optional, false)?; + let columns = self.parse_table_alias_column_defs()?; self.expect_keyword(Keyword::AS)?; let mut is_materialized = None; if dialect_of!(self is PostgreSqlDialect) { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 283071e9b..9fb467102 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -553,7 +553,11 @@ fn parse_select_with_table_alias() { name: ObjectName(vec![Ident::new("lineitem")]), alias: Some(TableAlias { name: Ident::new("l"), - columns: vec![Ident::new("A"), Ident::new("B"), Ident::new("C"),], + columns: vec![ + TableAliasColumnDef::from_name("A"), + TableAliasColumnDef::from_name("B"), + TableAliasColumnDef::from_name("C"), + ], }), args: None, with_hints: vec![], @@ -5597,6 +5601,40 @@ fn parse_table_function() { ); } +#[test] +fn parse_select_with_alias_and_column_defs() { + let sql = r#"SELECT * FROM jsonb_to_record('{"a": "x", "b": 2}'::JSONB) AS x (a TEXT, b INT)"#; + let select = verified_only_select(sql); + + match only(&select.from) { + TableWithJoins { + relation: TableFactor::Table { + alias: Some(alias), .. + }, + .. + } => { + assert_eq!(alias.name.value, "x"); + assert_eq!( + alias.columns, + vec![ + TableAliasColumnDef { + name: Ident::new("a"), + data_type: Some(DataType::Text), + }, + TableAliasColumnDef { + name: Ident::new("b"), + data_type: Some(DataType::Int(None)), + }, + ] + ); + } + _ => unreachable!( + "Expecting only TableWithJoins with TableFactor::Table, got {:#?}", + select.from + ), + } +} + #[test] fn parse_unnest() { let sql = "SELECT UNNEST(make_array(1, 2, 3))"; @@ -6372,7 +6410,10 @@ fn parse_cte_renamed_columns() { let sql = "WITH cte (col1, col2) AS (SELECT foo, bar FROM baz) SELECT * FROM cte"; let query = all_dialects().verified_query(sql); assert_eq!( - vec![Ident::new("col1"), Ident::new("col2")], + vec![ + TableAliasColumnDef::from_name("col1"), + TableAliasColumnDef::from_name("col2") + ], query .with .unwrap() @@ -6401,10 +6442,7 @@ fn parse_recursive_cte() { value: "nums".to_string(), quote_style: None, }, - columns: vec![Ident { - value: "val".to_string(), - quote_style: None, - }], + columns: vec![TableAliasColumnDef::from_name("val")], }, query: Box::new(cte_query), from: None, @@ -9347,7 +9385,10 @@ fn parse_pivot_table() { value: "p".to_string(), quote_style: None }, - columns: vec![Ident::new("c"), Ident::new("d")], + columns: vec![ + TableAliasColumnDef::from_name("c"), + TableAliasColumnDef::from_name("d"), + ], }), } ); @@ -9408,8 +9449,8 @@ fn parse_unpivot_table() { name: Ident::new("u"), columns: ["product", "quarter", "quantity"] .into_iter() - .map(Ident::new) - .collect() + .map(TableAliasColumnDef::from_name) + .collect(), }), } ); From 92be237cfc082ecfd67cd8bbc53b74d8cd6ce46f Mon Sep 17 00:00:00 2001 From: gaoqiangz <38213294+gaoqiangz@users.noreply.github.com> Date: Mon, 18 Nov 2024 22:22:18 +0800 Subject: [PATCH 58/80] Add support for MSSQL's `JSON_ARRAY`/`JSON_OBJECT` expr (#1507) Co-authored-by: Ifeanyi Ubah --- src/ast/mod.rs | 56 ++++- src/dialect/duckdb.rs | 4 + src/dialect/generic.rs | 4 + src/dialect/mod.rs | 25 ++- src/dialect/mssql.rs | 12 ++ src/keywords.rs | 1 + src/parser/mod.rs | 115 ++++++---- tests/sqlparser_common.rs | 3 +- tests/sqlparser_mssql.rs | 441 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 617 insertions(+), 44 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index fc6a1b4f1..89e70bdd4 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -5449,6 +5449,8 @@ pub enum FunctionArgOperator { RightArrow, /// function(arg1 := value1) Assignment, + /// function(arg1 : value1) + Colon, } impl fmt::Display for FunctionArgOperator { @@ -5457,6 +5459,7 @@ impl fmt::Display for FunctionArgOperator { FunctionArgOperator::Equals => f.write_str("="), FunctionArgOperator::RightArrow => f.write_str("=>"), FunctionArgOperator::Assignment => f.write_str(":="), + FunctionArgOperator::Colon => f.write_str(":"), } } } @@ -5465,11 +5468,22 @@ impl fmt::Display for FunctionArgOperator { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum FunctionArg { + /// `name` is identifier + /// + /// Enabled when `Dialect::supports_named_fn_args_with_expr_name` returns 'false' Named { name: Ident, arg: FunctionArgExpr, operator: FunctionArgOperator, }, + /// `name` is arbitrary expression + /// + /// Enabled when `Dialect::supports_named_fn_args_with_expr_name` returns 'true' + ExprNamed { + name: Expr, + arg: FunctionArgExpr, + operator: FunctionArgOperator, + }, Unnamed(FunctionArgExpr), } @@ -5481,6 +5495,11 @@ impl fmt::Display for FunctionArg { arg, operator, } => write!(f, "{name} {operator} {arg}"), + FunctionArg::ExprNamed { + name, + arg, + operator, + } => write!(f, "{name} {operator} {arg}"), FunctionArg::Unnamed(unnamed_arg) => write!(f, "{unnamed_arg}"), } } @@ -5619,7 +5638,10 @@ impl fmt::Display for FunctionArgumentList { } write!(f, "{}", display_comma_separated(&self.args))?; if !self.clauses.is_empty() { - write!(f, " {}", display_separated(&self.clauses, " "))?; + if !self.args.is_empty() { + write!(f, " ")?; + } + write!(f, "{}", display_separated(&self.clauses, " "))?; } Ok(()) } @@ -5661,6 +5683,11 @@ pub enum FunctionArgumentClause { /// /// [`GROUP_CONCAT`]: https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_group-concat Separator(Value), + /// The json-null-clause to the [`JSON_ARRAY`]/[`JSON_OBJECT`] function in MSSQL. + /// + /// [`JSON_ARRAY`]: + /// [`JSON_OBJECT`]: + JsonNullClause(JsonNullClause), } impl fmt::Display for FunctionArgumentClause { @@ -5676,6 +5703,7 @@ impl fmt::Display for FunctionArgumentClause { FunctionArgumentClause::OnOverflow(on_overflow) => write!(f, "{on_overflow}"), FunctionArgumentClause::Having(bound) => write!(f, "{bound}"), FunctionArgumentClause::Separator(sep) => write!(f, "SEPARATOR {sep}"), + FunctionArgumentClause::JsonNullClause(null_clause) => write!(f, "{null_clause}"), } } } @@ -7564,6 +7592,32 @@ impl fmt::Display for ShowStatementIn { } } +/// MSSQL's json null clause +/// +/// ```plaintext +/// ::= +/// NULL ON NULL +/// | ABSENT ON NULL +/// ``` +/// +/// +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum JsonNullClause { + NullOnNull, + AbsentOnNull, +} + +impl Display for JsonNullClause { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JsonNullClause::NullOnNull => write!(f, "NULL ON NULL"), + JsonNullClause::AbsentOnNull => write!(f, "ABSENT ON NULL"), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/dialect/duckdb.rs b/src/dialect/duckdb.rs index 905b04e36..a2699d850 100644 --- a/src/dialect/duckdb.rs +++ b/src/dialect/duckdb.rs @@ -47,6 +47,10 @@ impl Dialect for DuckDbDialect { true } + fn supports_named_fn_args_with_assignment_operator(&self) -> bool { + true + } + // DuckDB uses this syntax for `STRUCT`s. // // https://duckdb.org/docs/sql/data_types/struct.html#creating-structs diff --git a/src/dialect/generic.rs b/src/dialect/generic.rs index 4998e0f4b..e3beeae7f 100644 --- a/src/dialect/generic.rs +++ b/src/dialect/generic.rs @@ -119,4 +119,8 @@ impl Dialect for GenericDialect { fn supports_load_extension(&self) -> bool { true } + + fn supports_named_fn_args_with_assignment_operator(&self) -> bool { + true + } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 956a58986..39ea98c69 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -231,11 +231,34 @@ pub trait Dialect: Debug + Any { false } - /// Returns true if the dialect supports named arguments of the form FUN(a = '1', b = '2'). + /// Returns true if the dialect supports named arguments of the form `FUN(a = '1', b = '2')`. fn supports_named_fn_args_with_eq_operator(&self) -> bool { false } + /// Returns true if the dialect supports named arguments of the form `FUN(a : '1', b : '2')`. + fn supports_named_fn_args_with_colon_operator(&self) -> bool { + false + } + + /// Returns true if the dialect supports named arguments of the form `FUN(a := '1', b := '2')`. + fn supports_named_fn_args_with_assignment_operator(&self) -> bool { + false + } + + /// Returns true if the dialect supports named arguments of the form `FUN(a => '1', b => '2')`. + fn supports_named_fn_args_with_rarrow_operator(&self) -> bool { + true + } + + /// Returns true if dialect supports argument name as arbitrary expression. + /// e.g. `FUN(LOWER('a'):'1', b:'2')` + /// Such function arguments are represented in the AST by the `FunctionArg::ExprNamed` variant, + /// otherwise use the `FunctionArg::Named` variant (compatible reason). + fn supports_named_fn_args_with_expr_name(&self) -> bool { + false + } + /// Returns true if the dialect supports identifiers starting with a numeric /// prefix such as tables named `59901_user_login` fn supports_numeric_prefix(&self) -> bool { diff --git a/src/dialect/mssql.rs b/src/dialect/mssql.rs index 39ce9c125..2d0ef027f 100644 --- a/src/dialect/mssql.rs +++ b/src/dialect/mssql.rs @@ -66,4 +66,16 @@ impl Dialect for MsSqlDialect { fn supports_methods(&self) -> bool { true } + + fn supports_named_fn_args_with_colon_operator(&self) -> bool { + true + } + + fn supports_named_fn_args_with_expr_name(&self) -> bool { + true + } + + fn supports_named_fn_args_with_rarrow_operator(&self) -> bool { + false + } } diff --git a/src/keywords.rs b/src/keywords.rs index fdf2bf35c..29115a0d2 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -74,6 +74,7 @@ macro_rules! define_keywords { define_keywords!( ABORT, ABS, + ABSENT, ABSOLUTE, ACCESS, ACCOUNT, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 82347f58d..35ad95803 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -11321,45 +11321,58 @@ impl<'a> Parser<'a> { } pub fn parse_function_args(&mut self) -> Result { - if self.peek_nth_token(1) == Token::RArrow { - let name = self.parse_identifier(false)?; - - self.expect_token(&Token::RArrow)?; - let arg = self.parse_wildcard_expr()?.into(); - - Ok(FunctionArg::Named { - name, - arg, - operator: FunctionArgOperator::RightArrow, - }) - } else if self.dialect.supports_named_fn_args_with_eq_operator() - && self.peek_nth_token(1) == Token::Eq - { - let name = self.parse_identifier(false)?; - - self.expect_token(&Token::Eq)?; - let arg = self.parse_wildcard_expr()?.into(); - - Ok(FunctionArg::Named { - name, - arg, - operator: FunctionArgOperator::Equals, - }) - } else if dialect_of!(self is DuckDbDialect | GenericDialect) - && self.peek_nth_token(1) == Token::Assignment - { - let name = self.parse_identifier(false)?; - - self.expect_token(&Token::Assignment)?; - let arg = self.parse_expr()?.into(); - - Ok(FunctionArg::Named { - name, - arg, - operator: FunctionArgOperator::Assignment, - }) + let arg = if self.dialect.supports_named_fn_args_with_expr_name() { + self.maybe_parse(|p| { + let name = p.parse_expr()?; + let operator = p.parse_function_named_arg_operator()?; + let arg = p.parse_wildcard_expr()?.into(); + Ok(FunctionArg::ExprNamed { + name, + arg, + operator, + }) + })? } else { - Ok(FunctionArg::Unnamed(self.parse_wildcard_expr()?.into())) + self.maybe_parse(|p| { + let name = p.parse_identifier(false)?; + let operator = p.parse_function_named_arg_operator()?; + let arg = p.parse_wildcard_expr()?.into(); + Ok(FunctionArg::Named { + name, + arg, + operator, + }) + })? + }; + if let Some(arg) = arg { + return Ok(arg); + } + Ok(FunctionArg::Unnamed(self.parse_wildcard_expr()?.into())) + } + + fn parse_function_named_arg_operator(&mut self) -> Result { + let tok = self.next_token(); + match tok.token { + Token::RArrow if self.dialect.supports_named_fn_args_with_rarrow_operator() => { + Ok(FunctionArgOperator::RightArrow) + } + Token::Eq if self.dialect.supports_named_fn_args_with_eq_operator() => { + Ok(FunctionArgOperator::Equals) + } + Token::Assignment + if self + .dialect + .supports_named_fn_args_with_assignment_operator() => + { + Ok(FunctionArgOperator::Assignment) + } + Token::Colon if self.dialect.supports_named_fn_args_with_colon_operator() => { + Ok(FunctionArgOperator::Colon) + } + _ => { + self.prev_token(); + self.expected("argument operator", tok) + } } } @@ -11403,19 +11416,24 @@ impl<'a> Parser<'a> { /// FIRST_VALUE(x IGNORE NULL); /// ``` fn parse_function_argument_list(&mut self) -> Result { + let mut clauses = vec![]; + + // For MSSQL empty argument list with json-null-clause case, e.g. `JSON_ARRAY(NULL ON NULL)` + if let Some(null_clause) = self.parse_json_null_clause() { + clauses.push(FunctionArgumentClause::JsonNullClause(null_clause)); + } + if self.consume_token(&Token::RParen) { return Ok(FunctionArgumentList { duplicate_treatment: None, args: vec![], - clauses: vec![], + clauses, }); } let duplicate_treatment = self.parse_duplicate_treatment()?; let args = self.parse_comma_separated(Parser::parse_function_args)?; - let mut clauses = vec![]; - if self.dialect.supports_window_function_null_treatment_arg() { if let Some(null_treatment) = self.parse_null_treatment()? { clauses.push(FunctionArgumentClause::IgnoreOrRespectNulls(null_treatment)); @@ -11456,6 +11474,10 @@ impl<'a> Parser<'a> { clauses.push(FunctionArgumentClause::OnOverflow(on_overflow)); } + if let Some(null_clause) = self.parse_json_null_clause() { + clauses.push(FunctionArgumentClause::JsonNullClause(null_clause)); + } + self.expect_token(&Token::RParen)?; Ok(FunctionArgumentList { duplicate_treatment, @@ -11464,6 +11486,17 @@ impl<'a> Parser<'a> { }) } + /// Parses MSSQL's json-null-clause + fn parse_json_null_clause(&mut self) -> Option { + if self.parse_keywords(&[Keyword::ABSENT, Keyword::ON, Keyword::NULL]) { + Some(JsonNullClause::AbsentOnNull) + } else if self.parse_keywords(&[Keyword::NULL, Keyword::ON, Keyword::NULL]) { + Some(JsonNullClause::NullOnNull) + } else { + None + } + } + fn parse_duplicate_treatment(&mut self) -> Result, ParserError> { let loc = self.peek_token().location; match ( diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 9fb467102..ecdca6b1b 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -4402,8 +4402,9 @@ fn parse_explain_query_plan() { #[test] fn parse_named_argument_function() { + let dialects = all_dialects_where(|d| d.supports_named_fn_args_with_rarrow_operator()); let sql = "SELECT FUN(a => '1', b => '2') FROM foo"; - let select = verified_only_select(sql); + let select = dialects.verified_only_select(sql); assert_eq!( &Expr::Function(Function { diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index c28f89e37..73fd99cf3 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -784,6 +784,447 @@ fn parse_for_json_expect_ast() { ); } +#[test] +fn parse_mssql_json_object() { + let select = ms().verified_only_select("SELECT JSON_OBJECT('name' : 'value', 'type' : 1)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, .. }), + .. + }) => assert_eq!( + &[ + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("name".into())), + arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( + "value".into() + ))), + operator: FunctionArgOperator::Colon + }, + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("type".into())), + arg: FunctionArgExpr::Expr(Expr::Value(number("1"))), + operator: FunctionArgOperator::Colon + } + ], + &args[..] + ), + _ => unreachable!(), + } + let select = ms() + .verified_only_select("SELECT JSON_OBJECT('name' : 'value', 'type' : NULL ABSENT ON NULL)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert_eq!( + &[ + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("name".into())), + arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( + "value".into() + ))), + operator: FunctionArgOperator::Colon + }, + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("type".into())), + arg: FunctionArgExpr::Expr(Expr::Value(Value::Null)), + operator: FunctionArgOperator::Colon + } + ], + &args[..] + ); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::AbsentOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select("SELECT JSON_OBJECT(NULL ON NULL)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert!(args.is_empty()); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::NullOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select("SELECT JSON_OBJECT(ABSENT ON NULL)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert!(args.is_empty()); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::AbsentOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select( + "SELECT JSON_OBJECT('name' : 'value', 'type' : JSON_ARRAY(1, 2) ABSENT ON NULL)", + ); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert_eq!( + &FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("name".into())), + arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( + "value".into() + ))), + operator: FunctionArgOperator::Colon + }, + &args[0] + ); + assert!(matches!( + args[1], + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString(_)), + arg: FunctionArgExpr::Expr(Expr::Function(_)), + operator: FunctionArgOperator::Colon + } + )); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::AbsentOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select( + "SELECT JSON_OBJECT('name' : 'value', 'type' : JSON_OBJECT('type_id' : 1, 'name' : 'a') NULL ON NULL)", + ); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert_eq!( + &FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("name".into())), + arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( + "value".into() + ))), + operator: FunctionArgOperator::Colon + }, + &args[0] + ); + assert!(matches!( + args[1], + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString(_)), + arg: FunctionArgExpr::Expr(Expr::Function(_)), + operator: FunctionArgOperator::Colon + } + )); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::NullOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select( + "SELECT JSON_OBJECT('user_name' : USER_NAME(), LOWER(@id_key) : @id_value, 'sid' : (SELECT @@SPID) ABSENT ON NULL)", + ); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert!(matches!( + args[0], + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString(_)), + arg: FunctionArgExpr::Expr(Expr::Function(_)), + operator: FunctionArgOperator::Colon + } + )); + assert!(matches!( + args[1], + FunctionArg::ExprNamed { + name: Expr::Function(_), + arg: FunctionArgExpr::Expr(Expr::Identifier(_)), + operator: FunctionArgOperator::Colon + } + )); + assert!(matches!( + args[2], + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString(_)), + arg: FunctionArgExpr::Expr(Expr::Subquery(_)), + operator: FunctionArgOperator::Colon + } + )); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::AbsentOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select( + "SELECT s.session_id, JSON_OBJECT('security_id' : s.security_id, 'login' : s.login_name, 'status' : s.status) AS info \ + FROM sys.dm_exec_sessions AS s \ + WHERE s.is_user_process = 1", + ); + match &select.projection[1] { + SelectItem::ExprWithAlias { + expr: + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, .. }), + .. + }), + .. + } => { + assert!(matches!( + args[0], + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString(_)), + arg: FunctionArgExpr::Expr(Expr::CompoundIdentifier(_)), + operator: FunctionArgOperator::Colon + } + )); + assert!(matches!( + args[1], + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString(_)), + arg: FunctionArgExpr::Expr(Expr::CompoundIdentifier(_)), + operator: FunctionArgOperator::Colon + } + )); + assert!(matches!( + args[2], + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString(_)), + arg: FunctionArgExpr::Expr(Expr::CompoundIdentifier(_)), + operator: FunctionArgOperator::Colon + } + )); + } + _ => unreachable!(), + } +} + +#[test] +fn parse_mssql_json_array() { + let select = ms().verified_only_select("SELECT JSON_ARRAY('a', 1, NULL, 2 NULL ON NULL)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert_eq!( + &[ + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + Value::SingleQuotedString("a".into()) + ))), + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(number("1")))), + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(Value::Null))), + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(number("2")))), + ], + &args[..] + ); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::NullOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select("SELECT JSON_ARRAY('a', 1, NULL, 2 ABSENT ON NULL)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert_eq!( + &[ + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + Value::SingleQuotedString("a".into()) + ))), + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(number("1")))), + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(Value::Null))), + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(number("2")))), + ], + &args[..] + ); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::AbsentOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select("SELECT JSON_ARRAY(NULL ON NULL)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert!(args.is_empty()); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::NullOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select("SELECT JSON_ARRAY(ABSENT ON NULL)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert!(args.is_empty()); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::AbsentOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select( + "SELECT JSON_ARRAY('a', JSON_OBJECT('name' : 'value', 'type' : 1) NULL ON NULL)", + ); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert_eq!( + &FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + Value::SingleQuotedString("a".into()) + ))), + &args[0] + ); + assert!(matches!( + args[1], + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Function(_))) + )); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::NullOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = ms().verified_only_select( + "SELECT JSON_ARRAY('a', JSON_OBJECT('name' : 'value', 'type' : 1), JSON_ARRAY(1, NULL, 2 NULL ON NULL))", + ); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, .. }), + .. + }) => { + assert_eq!( + &FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + Value::SingleQuotedString("a".into()) + ))), + &args[0] + ); + assert!(matches!( + args[1], + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Function(_))) + )); + assert!(matches!( + args[2], + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Function(_))) + )); + } + _ => unreachable!(), + } + let select = ms().verified_only_select("SELECT JSON_ARRAY(1, @id_value, (SELECT @@SPID))"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, .. }), + .. + }) => { + assert_eq!( + &FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(number("1")))), + &args[0] + ); + assert!(matches!( + args[1], + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(_))) + )); + assert!(matches!( + args[2], + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Subquery(_))) + )); + } + _ => unreachable!(), + } + let select = ms().verified_only_select( + "SELECT s.session_id, JSON_ARRAY(s.host_name, s.program_name, s.client_interface_name NULL ON NULL) AS info \ + FROM sys.dm_exec_sessions AS s \ + WHERE s.is_user_process = 1", + ); + match &select.projection[1] { + SelectItem::ExprWithAlias { + expr: + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }), + .. + } => { + assert!(matches!( + args[0], + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::CompoundIdentifier(_))) + )); + assert!(matches!( + args[1], + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::CompoundIdentifier(_))) + )); + assert!(matches!( + args[2], + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::CompoundIdentifier(_))) + )); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::NullOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } +} + #[test] fn parse_ampersand_arobase() { // In SQL Server, a&@b means (a) & (@b), in PostgreSQL it means (a) &@ (b) From 73947a5f021128cfccd47293ca65aa5c4e83f598 Mon Sep 17 00:00:00 2001 From: wugeer <1284057728@qq.com> Date: Wed, 20 Nov 2024 05:14:28 +0800 Subject: [PATCH 59/80] Add support for PostgreSQL `UNLISTEN` syntax and Add support for Postgres `LOAD extension` expr (#1531) Co-authored-by: Ifeanyi Ubah --- src/ast/mod.rs | 11 ++++++ src/dialect/mod.rs | 9 +---- src/dialect/postgresql.rs | 12 +++--- src/keywords.rs | 1 + src/parser/mod.rs | 22 +++++++++-- tests/sqlparser_common.rs | 77 +++++++++++++++++++++++++++++++++++++-- tests/sqlparser_duckdb.rs | 14 ------- 7 files changed, 113 insertions(+), 33 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 89e70bdd4..9185c9df4 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -3340,6 +3340,13 @@ pub enum Statement { /// See Postgres LISTEN { channel: Ident }, /// ```sql + /// UNLISTEN + /// ``` + /// stop listening for a notification + /// + /// See Postgres + UNLISTEN { channel: Ident }, + /// ```sql /// NOTIFY channel [ , payload ] /// ``` /// send a notification event together with an optional “payload” string to channel @@ -4948,6 +4955,10 @@ impl fmt::Display for Statement { write!(f, "LISTEN {channel}")?; Ok(()) } + Statement::UNLISTEN { channel } => { + write!(f, "UNLISTEN {channel}")?; + Ok(()) + } Statement::NOTIFY { channel, payload } => { write!(f, "NOTIFY {channel}")?; if let Some(payload) = payload { diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 39ea98c69..985cad749 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -633,13 +633,8 @@ pub trait Dialect: Debug + Any { false } - /// Returns true if the dialect supports the `LISTEN` statement - fn supports_listen(&self) -> bool { - false - } - - /// Returns true if the dialect supports the `NOTIFY` statement - fn supports_notify(&self) -> bool { + /// Returns true if the dialect supports the `LISTEN`, `UNLISTEN` and `NOTIFY` statements + fn supports_listen_notify(&self) -> bool { false } diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index 5af1ab853..559586e3f 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -191,12 +191,9 @@ impl Dialect for PostgreSqlDialect { } /// see - fn supports_listen(&self) -> bool { - true - } - + /// see /// see - fn supports_notify(&self) -> bool { + fn supports_listen_notify(&self) -> bool { true } @@ -209,6 +206,11 @@ impl Dialect for PostgreSqlDialect { fn supports_comment_on(&self) -> bool { true } + + /// See + fn supports_load_extension(&self) -> bool { + true + } } pub fn parse_create(parser: &mut Parser) -> Option> { diff --git a/src/keywords.rs b/src/keywords.rs index 29115a0d2..fc2a2927c 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -799,6 +799,7 @@ define_keywords!( UNION, UNIQUE, UNKNOWN, + UNLISTEN, UNLOAD, UNLOCK, UNLOGGED, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 35ad95803..35c763e93 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -532,10 +532,11 @@ impl<'a> Parser<'a> { Keyword::EXECUTE | Keyword::EXEC => self.parse_execute(), Keyword::PREPARE => self.parse_prepare(), Keyword::MERGE => self.parse_merge(), - // `LISTEN` and `NOTIFY` are Postgres-specific + // `LISTEN`, `UNLISTEN` and `NOTIFY` are Postgres-specific // syntaxes. They are used for Postgres statement. - Keyword::LISTEN if self.dialect.supports_listen() => self.parse_listen(), - Keyword::NOTIFY if self.dialect.supports_notify() => self.parse_notify(), + Keyword::LISTEN if self.dialect.supports_listen_notify() => self.parse_listen(), + Keyword::UNLISTEN if self.dialect.supports_listen_notify() => self.parse_unlisten(), + Keyword::NOTIFY if self.dialect.supports_listen_notify() => self.parse_notify(), // `PRAGMA` is sqlite specific https://www.sqlite.org/pragma.html Keyword::PRAGMA => self.parse_pragma(), Keyword::UNLOAD => self.parse_unload(), @@ -999,6 +1000,21 @@ impl<'a> Parser<'a> { Ok(Statement::LISTEN { channel }) } + pub fn parse_unlisten(&mut self) -> Result { + let channel = if self.consume_token(&Token::Mul) { + Ident::new(Expr::Wildcard.to_string()) + } else { + match self.parse_identifier(false) { + Ok(expr) => expr, + _ => { + self.prev_token(); + return self.expected("wildcard or identifier", self.peek_token()); + } + } + }; + Ok(Statement::UNLISTEN { channel }) + } + pub fn parse_notify(&mut self) -> Result { let channel = self.parse_identifier(false)?; let payload = if self.consume_token(&Token::Comma) { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index ecdca6b1b..3d9ba5da2 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11595,7 +11595,7 @@ fn test_show_dbs_schemas_tables_views() { #[test] fn parse_listen_channel() { - let dialects = all_dialects_where(|d| d.supports_listen()); + let dialects = all_dialects_where(|d| d.supports_listen_notify()); match dialects.verified_stmt("LISTEN test1") { Statement::LISTEN { channel } => { @@ -11609,7 +11609,7 @@ fn parse_listen_channel() { ParserError::ParserError("Expected: identifier, found: *".to_string()) ); - let dialects = all_dialects_where(|d| !d.supports_listen()); + let dialects = all_dialects_where(|d| !d.supports_listen_notify()); assert_eq!( dialects.parse_sql_statements("LISTEN test1").unwrap_err(), @@ -11617,9 +11617,40 @@ fn parse_listen_channel() { ); } +#[test] +fn parse_unlisten_channel() { + let dialects = all_dialects_where(|d| d.supports_listen_notify()); + + match dialects.verified_stmt("UNLISTEN test1") { + Statement::UNLISTEN { channel } => { + assert_eq!(Ident::new("test1"), channel); + } + _ => unreachable!(), + }; + + match dialects.verified_stmt("UNLISTEN *") { + Statement::UNLISTEN { channel } => { + assert_eq!(Ident::new("*"), channel); + } + _ => unreachable!(), + }; + + assert_eq!( + dialects.parse_sql_statements("UNLISTEN +").unwrap_err(), + ParserError::ParserError("Expected: wildcard or identifier, found: +".to_string()) + ); + + let dialects = all_dialects_where(|d| !d.supports_listen_notify()); + + assert_eq!( + dialects.parse_sql_statements("UNLISTEN test1").unwrap_err(), + ParserError::ParserError("Expected: an SQL statement, found: UNLISTEN".to_string()) + ); +} + #[test] fn parse_notify_channel() { - let dialects = all_dialects_where(|d| d.supports_notify()); + let dialects = all_dialects_where(|d| d.supports_listen_notify()); match dialects.verified_stmt("NOTIFY test1") { Statement::NOTIFY { channel, payload } => { @@ -11655,7 +11686,7 @@ fn parse_notify_channel() { "NOTIFY test1", "NOTIFY test1, 'this is a test notification'", ]; - let dialects = all_dialects_where(|d| !d.supports_notify()); + let dialects = all_dialects_where(|d| !d.supports_listen_notify()); for &sql in &sql_statements { assert_eq!( @@ -11864,6 +11895,44 @@ fn parse_load_data() { ); } +#[test] +fn test_load_extension() { + let dialects = all_dialects_where(|d| d.supports_load_extension()); + let not_supports_load_extension_dialects = all_dialects_where(|d| !d.supports_load_extension()); + let sql = "LOAD my_extension"; + + match dialects.verified_stmt(sql) { + Statement::Load { extension_name } => { + assert_eq!(Ident::new("my_extension"), extension_name); + } + _ => unreachable!(), + }; + + assert_eq!( + not_supports_load_extension_dialects + .parse_sql_statements(sql) + .unwrap_err(), + ParserError::ParserError( + "Expected: `DATA` or an extension name after `LOAD`, found: my_extension".to_string() + ) + ); + + let sql = "LOAD 'filename'"; + + match dialects.verified_stmt(sql) { + Statement::Load { extension_name } => { + assert_eq!( + Ident { + value: "filename".to_string(), + quote_style: Some('\'') + }, + extension_name + ); + } + _ => unreachable!(), + }; +} + #[test] fn test_select_top() { let dialects = all_dialects_where(|d| d.supports_top_before_distinct()); diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index d68f37713..a2db5c282 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -359,20 +359,6 @@ fn test_duckdb_install() { ); } -#[test] -fn test_duckdb_load_extension() { - let stmt = duckdb().verified_stmt("LOAD my_extension"); - assert_eq!( - Statement::Load { - extension_name: Ident { - value: "my_extension".to_string(), - quote_style: None - } - }, - stmt - ); -} - #[test] fn test_duckdb_struct_literal() { //struct literal syntax https://duckdb.org/docs/sql/data_types/struct#creating-structs From fad2ddd6417e2e4d149c298d1977088124a30358 Mon Sep 17 00:00:00 2001 From: Michael Victor Zink Date: Tue, 19 Nov 2024 21:55:38 -0800 Subject: [PATCH 60/80] Parse byte/bit string literals in MySQL and Postgres (#1532) --- src/tokenizer.rs | 5 +++-- tests/sqlparser_mysql.rs | 11 +++++++++++ tests/sqlparser_postgres.rs | 11 +++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 4186ec824..05aaf1e28 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -704,8 +704,9 @@ impl<'a> Tokenizer<'a> { } Ok(Some(Token::Whitespace(Whitespace::Newline))) } - // BigQuery uses b or B for byte string literal - b @ 'B' | b @ 'b' if dialect_of!(self is BigQueryDialect | GenericDialect) => { + // BigQuery and MySQL use b or B for byte string literal, Postgres for bit strings + b @ 'B' | b @ 'b' if dialect_of!(self is BigQueryDialect | PostgreSqlDialect | MySqlDialect | GenericDialect) => + { chars.next(); // consume match chars.peek() { Some('\'') => { diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 2a876cff2..ce3296737 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -2960,3 +2960,14 @@ fn parse_logical_xor() { select.projection[3] ); } + +#[test] +fn parse_bitstring_literal() { + let select = mysql_and_generic().verified_only_select("SELECT B'111'"); + assert_eq!( + select.projection, + vec![SelectItem::UnnamedExpr(Expr::Value( + Value::SingleQuotedByteStringLiteral("111".to_string()) + ))] + ); +} diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index a6c480cd7..2e2c4403c 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -5098,3 +5098,14 @@ fn parse_create_type_as_enum() { _ => unreachable!(), } } + +#[test] +fn parse_bitstring_literal() { + let select = pg_and_generic().verified_only_select("SELECT B'111'"); + assert_eq!( + select.projection, + vec![SelectItem::UnnamedExpr(Expr::Value( + Value::SingleQuotedByteStringLiteral("111".to_string()) + ))] + ); +} From a1150223af6083ca25c083d0af2ec2fd08507599 Mon Sep 17 00:00:00 2001 From: Michael Victor Zink Date: Fri, 22 Nov 2024 11:06:42 -0800 Subject: [PATCH 61/80] Allow example CLI to read from stdin (#1536) --- examples/cli.rs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/examples/cli.rs b/examples/cli.rs index 8a5d6501e..0252fca74 100644 --- a/examples/cli.rs +++ b/examples/cli.rs @@ -17,9 +17,11 @@ #![warn(clippy::all)] -/// A small command-line app to run the parser. -/// Run with `cargo run --example cli` +//! A small command-line app to run the parser. +//! Run with `cargo run --example cli` + use std::fs; +use std::io::{stdin, Read}; use simple_logger::SimpleLogger; use sqlparser::dialect::*; @@ -38,6 +40,9 @@ $ cargo run --example cli FILENAME.sql [--dialectname] To print the parse results as JSON: $ cargo run --feature json_example --example cli FILENAME.sql [--dialectname] +To read from stdin instead of a file: +$ cargo run --example cli - [--dialectname] + "#, ); @@ -57,9 +62,18 @@ $ cargo run --feature json_example --example cli FILENAME.sql [--dialectname] s => panic!("Unexpected parameter: {s}"), }; - println!("Parsing from file '{}' using {:?}", &filename, dialect); - let contents = fs::read_to_string(&filename) - .unwrap_or_else(|_| panic!("Unable to read the file {}", &filename)); + let contents = if filename == "-" { + println!("Parsing from stdin using {:?}", dialect); + let mut buf = Vec::new(); + stdin() + .read_to_end(&mut buf) + .expect("failed to read from stdin"); + String::from_utf8(buf).expect("stdin content wasn't valid utf8") + } else { + println!("Parsing from file '{}' using {:?}", &filename, dialect); + fs::read_to_string(&filename) + .unwrap_or_else(|_| panic!("Unable to read the file {}", &filename)) + }; let without_bom = if contents.chars().next().unwrap() as u64 != 0xfeff { contents.as_str() } else { From 10519003ed06defa48b1c9ecc734b3d5c92c297d Mon Sep 17 00:00:00 2001 From: tomershaniii <65544633+tomershaniii@users.noreply.github.com> Date: Sat, 23 Nov 2024 13:33:14 +0200 Subject: [PATCH 62/80] recursive select calls are parsed with bad trailing_commas parameter (#1521) --- src/parser/mod.rs | 39 +++++++++++++++++++++++++----------- tests/sqlparser_snowflake.rs | 17 ++++++++++++++++ 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 35c763e93..c8358767b 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -3532,16 +3532,11 @@ impl<'a> Parser<'a> { // e.g. `SELECT 1, 2, FROM t` // https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#trailing_commas // https://docs.snowflake.com/en/release-notes/2024/8_11#select-supports-trailing-commas - // - // This pattern could be captured better with RAII type semantics, but it's quite a bit of - // code to add for just one case, so we'll just do it manually here. - let old_value = self.options.trailing_commas; - self.options.trailing_commas |= self.dialect.supports_projection_trailing_commas(); - let ret = self.parse_comma_separated(|p| p.parse_select_item()); - self.options.trailing_commas = old_value; + let trailing_commas = + self.options.trailing_commas | self.dialect.supports_projection_trailing_commas(); - ret + self.parse_comma_separated_with_trailing_commas(|p| p.parse_select_item(), trailing_commas) } pub fn parse_actions_list(&mut self) -> Result, ParserError> { @@ -3568,11 +3563,12 @@ impl<'a> Parser<'a> { } /// Parse the comma of a comma-separated syntax element. + /// Allows for control over trailing commas /// Returns true if there is a next element - fn is_parse_comma_separated_end(&mut self) -> bool { + fn is_parse_comma_separated_end_with_trailing_commas(&mut self, trailing_commas: bool) -> bool { if !self.consume_token(&Token::Comma) { true - } else if self.options.trailing_commas { + } else if trailing_commas { let token = self.peek_token().token; match token { Token::Word(ref kw) @@ -3590,15 +3586,34 @@ impl<'a> Parser<'a> { } } + /// Parse the comma of a comma-separated syntax element. + /// Returns true if there is a next element + fn is_parse_comma_separated_end(&mut self) -> bool { + self.is_parse_comma_separated_end_with_trailing_commas(self.options.trailing_commas) + } + /// Parse a comma-separated list of 1+ items accepted by `F` - pub fn parse_comma_separated(&mut self, mut f: F) -> Result, ParserError> + pub fn parse_comma_separated(&mut self, f: F) -> Result, ParserError> + where + F: FnMut(&mut Parser<'a>) -> Result, + { + self.parse_comma_separated_with_trailing_commas(f, self.options.trailing_commas) + } + + /// Parse a comma-separated list of 1+ items accepted by `F` + /// Allows for control over trailing commas + fn parse_comma_separated_with_trailing_commas( + &mut self, + mut f: F, + trailing_commas: bool, + ) -> Result, ParserError> where F: FnMut(&mut Parser<'a>) -> Result, { let mut values = vec![]; loop { values.push(f(self)?); - if self.is_parse_comma_separated_end() { + if self.is_parse_comma_separated_end_with_trailing_commas(trailing_commas) { break; } } diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 1f1c00e7a..1d053bb0b 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -2846,3 +2846,20 @@ fn test_parse_show_columns_sql() { snowflake().verified_stmt("SHOW COLUMNS IN TABLE abc"); snowflake().verified_stmt("SHOW COLUMNS LIKE '%xyz%' IN TABLE abc"); } + +#[test] +fn test_projection_with_nested_trailing_commas() { + let sql = "SELECT a, FROM b, LATERAL FLATTEN(input => events)"; + let _ = snowflake().parse_sql_statements(sql).unwrap(); + + //Single nesting + let sql = "SELECT (SELECT a, FROM b, LATERAL FLATTEN(input => events))"; + let _ = snowflake().parse_sql_statements(sql).unwrap(); + + //Double nesting + let sql = "SELECT (SELECT (SELECT a, FROM b, LATERAL FLATTEN(input => events)))"; + let _ = snowflake().parse_sql_statements(sql).unwrap(); + + let sql = "SELECT a, b, FROM c, (SELECT d, e, FROM f, LATERAL FLATTEN(input => events))"; + let _ = snowflake().parse_sql_statements(sql).unwrap(); +} From 62fa8604af11eaae81e3a6276bbb5cc7bc2026e5 Mon Sep 17 00:00:00 2001 From: Yoav Cohen <59807311+yoavcloud@users.noreply.github.com> Date: Sat, 23 Nov 2024 13:14:38 +0100 Subject: [PATCH 63/80] PartiQL queries in Redshift (#1534) --- src/ast/query.rs | 6 ++ src/dialect/mod.rs | 6 ++ src/dialect/redshift.rs | 5 ++ src/parser/mod.rs | 21 +++-- src/test_utils.rs | 2 + tests/sqlparser_bigquery.rs | 5 ++ tests/sqlparser_clickhouse.rs | 2 + tests/sqlparser_common.rs | 37 +++++++++ tests/sqlparser_databricks.rs | 1 + tests/sqlparser_duckdb.rs | 2 + tests/sqlparser_hive.rs | 1 + tests/sqlparser_mssql.rs | 18 ++-- tests/sqlparser_mysql.rs | 5 ++ tests/sqlparser_postgres.rs | 1 + tests/sqlparser_redshift.rs | 150 ++++++++++++++++++++++++++++++++++ tests/sqlparser_snowflake.rs | 1 + tests/sqlparser_sqlite.rs | 1 + 17 files changed, 254 insertions(+), 10 deletions(-) diff --git a/src/ast/query.rs b/src/ast/query.rs index 078bbc841..bf36c626f 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -974,6 +974,8 @@ pub enum TableFactor { with_ordinality: bool, /// [Partition selection](https://dev.mysql.com/doc/refman/8.0/en/partitioning-selection.html), supported by MySQL. partitions: Vec, + /// Optional PartiQL JsonPath: + json_path: Option, }, Derived { lateral: bool, @@ -1375,8 +1377,12 @@ impl fmt::Display for TableFactor { version, partitions, with_ordinality, + json_path, } => { write!(f, "{name}")?; + if let Some(json_path) = json_path { + write!(f, "{json_path}")?; + } if !partitions.is_empty() { write!(f, "PARTITION ({})", display_comma_separated(partitions))?; } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 985cad749..159e14717 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -675,6 +675,12 @@ pub trait Dialect: Debug + Any { fn supports_create_table_select(&self) -> bool { false } + + /// Returns true if the dialect supports PartiQL for querying semi-structured data + /// + fn supports_partiql(&self) -> bool { + false + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/redshift.rs b/src/dialect/redshift.rs index 4d0773843..48eb00ab1 100644 --- a/src/dialect/redshift.rs +++ b/src/dialect/redshift.rs @@ -74,4 +74,9 @@ impl Dialect for RedshiftSqlDialect { fn supports_top_before_distinct(&self) -> bool { true } + + /// Redshift supports PartiQL: + fn supports_partiql(&self) -> bool { + true + } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c8358767b..1bf173169 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -2936,7 +2936,7 @@ impl<'a> Parser<'a> { } else if Token::LBracket == tok { if dialect_of!(self is PostgreSqlDialect | DuckDbDialect | GenericDialect) { self.parse_subscript(expr) - } else if dialect_of!(self is SnowflakeDialect) { + } else if dialect_of!(self is SnowflakeDialect) || self.dialect.supports_partiql() { self.prev_token(); self.parse_json_access(expr) } else { @@ -3072,6 +3072,14 @@ impl<'a> Parser<'a> { } fn parse_json_access(&mut self, expr: Expr) -> Result { + let path = self.parse_json_path()?; + Ok(Expr::JsonAccess { + value: Box::new(expr), + path, + }) + } + + fn parse_json_path(&mut self) -> Result { let mut path = Vec::new(); loop { match self.next_token().token { @@ -3095,10 +3103,7 @@ impl<'a> Parser<'a> { } debug_assert!(!path.is_empty()); - Ok(Expr::JsonAccess { - value: Box::new(expr), - path: JsonPath { path }, - }) + Ok(JsonPath { path }) } pub fn parse_map_access(&mut self, expr: Expr) -> Result { @@ -10338,6 +10343,11 @@ impl<'a> Parser<'a> { } else { let name = self.parse_object_name(true)?; + let json_path = match self.peek_token().token { + Token::LBracket if self.dialect.supports_partiql() => Some(self.parse_json_path()?), + _ => None, + }; + let partitions: Vec = if dialect_of!(self is MySqlDialect | GenericDialect) && self.parse_keyword(Keyword::PARTITION) { @@ -10380,6 +10390,7 @@ impl<'a> Parser<'a> { version, partitions, with_ordinality, + json_path, }; while let Some(kw) = self.parse_one_of_keywords(&[Keyword::PIVOT, Keyword::UNPIVOT]) { diff --git a/src/test_utils.rs b/src/test_utils.rs index b35fc45c2..aaee20c5f 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -345,6 +345,7 @@ pub fn table(name: impl Into) -> TableFactor { version: None, partitions: vec![], with_ordinality: false, + json_path: None, } } @@ -360,6 +361,7 @@ pub fn table_with_alias(name: impl Into, alias: impl Into) -> Ta version: None, partitions: vec![], with_ordinality: false, + json_path: None, } } diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index 2bf470f71..d4c178bbf 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -229,6 +229,7 @@ fn parse_delete_statement() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, from[0].relation ); @@ -1373,6 +1374,7 @@ fn parse_table_identifiers() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![] },] @@ -1546,6 +1548,7 @@ fn parse_table_time_travel() { ))), partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![] },] @@ -1644,6 +1647,7 @@ fn parse_merge() { version: Default::default(), partitions: Default::default(), with_ordinality: false, + json_path: None, }, table ); @@ -1659,6 +1663,7 @@ fn parse_merge() { version: Default::default(), partitions: Default::default(), with_ordinality: false, + json_path: None, }, source ); diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index a71871115..90af12ab7 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -67,6 +67,7 @@ fn parse_map_access_expr() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }], @@ -172,6 +173,7 @@ fn parse_delimited_identifiers() { version, with_ordinality: _, partitions: _, + json_path: _, } => { assert_eq!(vec![Ident::with_quote('"', "a table")], name.0); assert_eq!(Ident::with_quote('"', "alias"), alias.unwrap().name); diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 3d9ba5da2..b41063859 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -364,6 +364,7 @@ fn parse_update_set_from() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }, @@ -394,6 +395,7 @@ fn parse_update_set_from() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }], @@ -473,6 +475,7 @@ fn parse_update_with_table_alias() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }, @@ -564,6 +567,7 @@ fn parse_select_with_table_alias() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }] @@ -601,6 +605,7 @@ fn parse_delete_statement() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, from[0].relation ); @@ -648,6 +653,7 @@ fn parse_delete_statement_for_multi_tables() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, from[0].relation ); @@ -660,6 +666,7 @@ fn parse_delete_statement_for_multi_tables() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, from[0].joins[0].relation ); @@ -686,6 +693,7 @@ fn parse_delete_statement_for_multi_tables_with_using() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, from[0].relation ); @@ -698,6 +706,7 @@ fn parse_delete_statement_for_multi_tables_with_using() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, from[1].relation ); @@ -710,6 +719,7 @@ fn parse_delete_statement_for_multi_tables_with_using() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, using[0].relation ); @@ -722,6 +732,7 @@ fn parse_delete_statement_for_multi_tables_with_using() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, using[0].joins[0].relation ); @@ -753,6 +764,7 @@ fn parse_where_delete_statement() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, from[0].relation, ); @@ -798,6 +810,7 @@ fn parse_where_delete_with_alias_statement() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, from[0].relation, ); @@ -814,6 +827,7 @@ fn parse_where_delete_with_alias_statement() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }]), @@ -4718,6 +4732,7 @@ fn test_parse_named_window() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }], @@ -5301,6 +5316,7 @@ fn parse_interval_and_or_xor() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }], @@ -5912,6 +5928,7 @@ fn parse_implicit_join() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }, @@ -5924,6 +5941,7 @@ fn parse_implicit_join() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }, @@ -5944,6 +5962,7 @@ fn parse_implicit_join() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![Join { relation: TableFactor::Table { @@ -5954,6 +5973,7 @@ fn parse_implicit_join() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, global: false, join_operator: JoinOperator::Inner(JoinConstraint::Natural), @@ -5968,6 +5988,7 @@ fn parse_implicit_join() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![Join { relation: TableFactor::Table { @@ -5978,6 +5999,7 @@ fn parse_implicit_join() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, global: false, join_operator: JoinOperator::Inner(JoinConstraint::Natural), @@ -6002,6 +6024,7 @@ fn parse_cross_join() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, global: false, join_operator: JoinOperator::CrossJoin, @@ -6027,6 +6050,7 @@ fn parse_joins_on() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, global, join_operator: f(JoinConstraint::On(Expr::BinaryOp { @@ -6154,6 +6178,7 @@ fn parse_joins_using() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, global: false, join_operator: f(JoinConstraint::Using(vec!["c1".into()])), @@ -6227,6 +6252,7 @@ fn parse_natural_join() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, global: false, join_operator: f(JoinConstraint::Natural), @@ -6496,6 +6522,7 @@ fn parse_derived_tables() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, global: false, join_operator: JoinOperator::Inner(JoinConstraint::Natural), @@ -7443,6 +7470,7 @@ fn lateral_function() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![Join { relation: TableFactor::Function { @@ -8258,6 +8286,7 @@ fn parse_merge() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, } ); assert_eq!(table, table_no_into); @@ -8285,6 +8314,7 @@ fn parse_merge() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }], @@ -9359,6 +9389,7 @@ fn parse_pivot_table() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }), aggregate_functions: vec![ expected_function("a", None), @@ -9432,6 +9463,7 @@ fn parse_unpivot_table() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }), value: Ident { value: "quantity".to_string(), @@ -9499,6 +9531,7 @@ fn parse_pivot_unpivot_table() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }), value: Ident { value: "population".to_string(), @@ -9910,6 +9943,7 @@ fn parse_unload() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }], @@ -10089,6 +10123,7 @@ fn parse_connect_by() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }], @@ -10176,6 +10211,7 @@ fn parse_connect_by() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }], @@ -10337,6 +10373,7 @@ fn test_match_recognize() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }; fn check(options: &str, expect: TableFactor) { diff --git a/tests/sqlparser_databricks.rs b/tests/sqlparser_databricks.rs index 7b917bd06..1651d517a 100644 --- a/tests/sqlparser_databricks.rs +++ b/tests/sqlparser_databricks.rs @@ -193,6 +193,7 @@ fn test_values_clause() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }), query .body diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index a2db5c282..73b0f6601 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -282,6 +282,7 @@ fn test_select_union_by_name() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }], @@ -323,6 +324,7 @@ fn test_select_union_by_name() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }], diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 10bd374c0..8d4f7a680 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -457,6 +457,7 @@ fn parse_delimited_identifiers() { version, with_ordinality: _, partitions: _, + json_path: _, } => { assert_eq!(vec![Ident::with_quote('"', "a table")], name.0); assert_eq!(Ident::with_quote('"', "alias"), alias.unwrap().name); diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 73fd99cf3..74f3c077e 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -70,6 +70,7 @@ fn parse_table_time_travel() { ))), partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![] },] @@ -218,7 +219,8 @@ fn parse_mssql_openjson() { with_hints: vec![], version: None, with_ordinality: false, - partitions: vec![] + partitions: vec![], + json_path: None, }, joins: vec![Join { relation: TableFactor::OpenJsonTable { @@ -293,7 +295,8 @@ fn parse_mssql_openjson() { with_hints: vec![], version: None, with_ordinality: false, - partitions: vec![] + partitions: vec![], + json_path: None, }, joins: vec![Join { relation: TableFactor::OpenJsonTable { @@ -368,7 +371,8 @@ fn parse_mssql_openjson() { with_hints: vec![], version: None, with_ordinality: false, - partitions: vec![] + partitions: vec![], + json_path: None, }, joins: vec![Join { relation: TableFactor::OpenJsonTable { @@ -443,7 +447,8 @@ fn parse_mssql_openjson() { with_hints: vec![], version: None, with_ordinality: false, - partitions: vec![] + partitions: vec![], + json_path: None, }, joins: vec![Join { relation: TableFactor::OpenJsonTable { @@ -496,7 +501,8 @@ fn parse_mssql_openjson() { with_hints: vec![], version: None, with_ordinality: false, - partitions: vec![] + partitions: vec![], + json_path: None, }, joins: vec![Join { relation: TableFactor::OpenJsonTable { @@ -679,6 +685,7 @@ fn parse_delimited_identifiers() { version, with_ordinality: _, partitions: _, + json_path: _, } => { assert_eq!(vec![Ident::with_quote('"', "a table")], name.0); assert_eq!(Ident::with_quote('"', "alias"), alias.unwrap().name); @@ -1314,6 +1321,7 @@ fn parse_substring_in_select() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![] }], diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index ce3296737..3d8b08630 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -1862,6 +1862,7 @@ fn parse_select_with_numeric_prefix_column_name() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![] }], @@ -1918,6 +1919,7 @@ fn parse_select_with_concatenation_of_exp_number_and_numeric_prefix_column() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![] }], @@ -1985,6 +1987,7 @@ fn parse_update_with_joins() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![Join { relation: TableFactor::Table { @@ -1998,6 +2001,7 @@ fn parse_update_with_joins() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, global: false, join_operator: JoinOperator::Inner(JoinConstraint::On(Expr::BinaryOp { @@ -2428,6 +2432,7 @@ fn parse_substring_in_select() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![] }], diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 2e2c4403c..098a3464c 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -3511,6 +3511,7 @@ fn parse_delimited_identifiers() { version, with_ordinality: _, partitions: _, + json_path: _, } => { assert_eq!(vec![Ident::with_quote('"', "a table")], name.0); assert_eq!(Ident::with_quote('"', "alias"), alias.unwrap().name); diff --git a/tests/sqlparser_redshift.rs b/tests/sqlparser_redshift.rs index a25d50605..0a084b340 100644 --- a/tests/sqlparser_redshift.rs +++ b/tests/sqlparser_redshift.rs @@ -54,6 +54,7 @@ fn test_square_brackets_over_db_schema_table_name() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], } @@ -101,6 +102,7 @@ fn test_double_quotes_over_db_schema_table_name() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], } @@ -123,6 +125,7 @@ fn parse_delimited_identifiers() { version, with_ordinality: _, partitions: _, + json_path: _, } => { assert_eq!(vec![Ident::with_quote('"', "a table")], name.0); assert_eq!(Ident::with_quote('"', "alias"), alias.unwrap().name); @@ -196,3 +199,150 @@ fn test_create_view_with_no_schema_binding() { redshift_and_generic() .verified_stmt("CREATE VIEW myevent AS SELECT eventname FROM event WITH NO SCHEMA BINDING"); } + +#[test] +fn test_redshift_json_path() { + let dialects = all_dialects_where(|d| d.supports_partiql()); + let sql = "SELECT cust.c_orders[0].o_orderkey FROM customer_orders_lineitem"; + let select = dialects.verified_only_select(sql); + + assert_eq!( + &Expr::JsonAccess { + value: Box::new(Expr::CompoundIdentifier(vec![ + Ident::new("cust"), + Ident::new("c_orders") + ])), + path: JsonPath { + path: vec![ + JsonPathElem::Bracket { + key: Expr::Value(Value::Number("0".parse().unwrap(), false)) + }, + JsonPathElem::Dot { + key: "o_orderkey".to_string(), + quoted: false + } + ] + } + }, + expr_from_projection(only(&select.projection)) + ); + + let sql = "SELECT cust.c_orders[0]['id'] FROM customer_orders_lineitem"; + let select = dialects.verified_only_select(sql); + assert_eq!( + &Expr::JsonAccess { + value: Box::new(Expr::CompoundIdentifier(vec![ + Ident::new("cust"), + Ident::new("c_orders") + ])), + path: JsonPath { + path: vec![ + JsonPathElem::Bracket { + key: Expr::Value(Value::Number("0".parse().unwrap(), false)) + }, + JsonPathElem::Bracket { + key: Expr::Value(Value::SingleQuotedString("id".to_owned())) + } + ] + } + }, + expr_from_projection(only(&select.projection)) + ); + + let sql = "SELECT db1.sc1.tbl1.col1[0]['id'] FROM customer_orders_lineitem"; + let select = dialects.verified_only_select(sql); + assert_eq!( + &Expr::JsonAccess { + value: Box::new(Expr::CompoundIdentifier(vec![ + Ident::new("db1"), + Ident::new("sc1"), + Ident::new("tbl1"), + Ident::new("col1") + ])), + path: JsonPath { + path: vec![ + JsonPathElem::Bracket { + key: Expr::Value(Value::Number("0".parse().unwrap(), false)) + }, + JsonPathElem::Bracket { + key: Expr::Value(Value::SingleQuotedString("id".to_owned())) + } + ] + } + }, + expr_from_projection(only(&select.projection)) + ); +} + +#[test] +fn test_parse_json_path_from() { + let dialects = all_dialects_where(|d| d.supports_partiql()); + let select = dialects.verified_only_select("SELECT * FROM src[0].a AS a"); + match &select.from[0].relation { + TableFactor::Table { + name, json_path, .. + } => { + assert_eq!(name, &ObjectName(vec![Ident::new("src")])); + assert_eq!( + json_path, + &Some(JsonPath { + path: vec![ + JsonPathElem::Bracket { + key: Expr::Value(Value::Number("0".parse().unwrap(), false)) + }, + JsonPathElem::Dot { + key: "a".to_string(), + quoted: false + } + ] + }) + ); + } + _ => panic!(), + } + + let select = dialects.verified_only_select("SELECT * FROM src[0].a[1].b AS a"); + match &select.from[0].relation { + TableFactor::Table { + name, json_path, .. + } => { + assert_eq!(name, &ObjectName(vec![Ident::new("src")])); + assert_eq!( + json_path, + &Some(JsonPath { + path: vec![ + JsonPathElem::Bracket { + key: Expr::Value(Value::Number("0".parse().unwrap(), false)) + }, + JsonPathElem::Dot { + key: "a".to_string(), + quoted: false + }, + JsonPathElem::Bracket { + key: Expr::Value(Value::Number("1".parse().unwrap(), false)) + }, + JsonPathElem::Dot { + key: "b".to_string(), + quoted: false + }, + ] + }) + ); + } + _ => panic!(), + } + + let select = dialects.verified_only_select("SELECT * FROM src.a.b"); + match &select.from[0].relation { + TableFactor::Table { + name, json_path, .. + } => { + assert_eq!( + name, + &ObjectName(vec![Ident::new("src"), Ident::new("a"), Ident::new("b")]) + ); + assert_eq!(json_path, &None); + } + _ => panic!(), + } +} diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 1d053bb0b..f99a00f5b 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -1190,6 +1190,7 @@ fn parse_delimited_identifiers() { version, with_ordinality: _, partitions: _, + json_path: _, } => { assert_eq!(vec![Ident::with_quote('"', "a table")], name.0); assert_eq!(Ident::with_quote('"', "alias"), alias.unwrap().name); diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index 6f8e654dc..c3cfb7a63 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -486,6 +486,7 @@ fn parse_update_tuple_row_values() { version: None, partitions: vec![], with_ordinality: false, + json_path: None, }, joins: vec![], }, From 0fb2ef331ec4acb6e77d73d2aabaee07d8e1944e Mon Sep 17 00:00:00 2001 From: Andrew Kane Date: Sun, 24 Nov 2024 04:06:31 -0800 Subject: [PATCH 64/80] Include license file in sqlparser_derive crate (#1543) --- derive/Cargo.toml | 1 + derive/LICENSE.TXT | 1 + 2 files changed, 2 insertions(+) create mode 120000 derive/LICENSE.TXT diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 0c5852c4c..3b115b950 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -28,6 +28,7 @@ license = "Apache-2.0" include = [ "src/**/*.rs", "Cargo.toml", + "LICENSE.TXT", ] edition = "2021" diff --git a/derive/LICENSE.TXT b/derive/LICENSE.TXT new file mode 120000 index 000000000..14259afe2 --- /dev/null +++ b/derive/LICENSE.TXT @@ -0,0 +1 @@ +../LICENSE.TXT \ No newline at end of file From fd21fae297c7446c3acaf676be1a24556d5bac9a Mon Sep 17 00:00:00 2001 From: Yoav Cohen <59807311+yoavcloud@users.noreply.github.com> Date: Mon, 25 Nov 2024 22:01:02 +0100 Subject: [PATCH 65/80] Fallback to identifier parsing if expression parsing fails (#1513) --- src/dialect/mod.rs | 6 + src/dialect/snowflake.rs | 12 ++ src/keywords.rs | 10 + src/parser/mod.rs | 399 ++++++++++++++++++++---------------- tests/sqlparser_common.rs | 21 +- tests/sqlparser_postgres.rs | 20 +- 6 files changed, 277 insertions(+), 191 deletions(-) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 159e14717..b622c1da3 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -681,6 +681,12 @@ pub trait Dialect: Debug + Any { fn supports_partiql(&self) -> bool { false } + + /// Returns true if the specified keyword is reserved and cannot be + /// used as an identifier without special handling like quoting. + fn is_reserved_for_identifier(&self, kw: Keyword) -> bool { + keywords::RESERVED_FOR_IDENTIFIER.contains(&kw) + } } /// This represents the operators for which precedence must be defined diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index b584ed9b4..56919fb31 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -38,6 +38,8 @@ use alloc::vec::Vec; #[cfg(not(feature = "std"))] use alloc::{format, vec}; +use super::keywords::RESERVED_FOR_IDENTIFIER; + /// A [`Dialect`] for [Snowflake](https://www.snowflake.com/) #[derive(Debug, Default)] pub struct SnowflakeDialect; @@ -214,6 +216,16 @@ impl Dialect for SnowflakeDialect { fn supports_show_like_before_in(&self) -> bool { true } + + fn is_reserved_for_identifier(&self, kw: Keyword) -> bool { + // Unreserve some keywords that Snowflake accepts as identifiers + // See: https://docs.snowflake.com/en/sql-reference/reserved-keywords + if matches!(kw, Keyword::INTERVAL) { + false + } else { + RESERVED_FOR_IDENTIFIER.contains(&kw) + } + } } /// Parse snowflake create table statement. diff --git a/src/keywords.rs b/src/keywords.rs index fc2a2927c..8c0ed588f 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -948,3 +948,13 @@ pub const RESERVED_FOR_COLUMN_ALIAS: &[Keyword] = &[ Keyword::INTO, Keyword::END, ]; + +/// Global list of reserved keywords that cannot be parsed as identifiers +/// without special handling like quoting. Parser should call `Dialect::is_reserved_for_identifier` +/// to allow for each dialect to customize the list. +pub const RESERVED_FOR_IDENTIFIER: &[Keyword] = &[ + Keyword::EXISTS, + Keyword::INTERVAL, + Keyword::STRUCT, + Keyword::TRIM, +]; diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 1bf173169..6767f358a 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1025,6 +1025,183 @@ impl<'a> Parser<'a> { Ok(Statement::NOTIFY { channel, payload }) } + // Tries to parse an expression by matching the specified word to known keywords that have a special meaning in the dialect. + // Returns `None if no match is found. + fn parse_expr_prefix_by_reserved_word( + &mut self, + w: &Word, + ) -> Result, ParserError> { + match w.keyword { + Keyword::TRUE | Keyword::FALSE if self.dialect.supports_boolean_literals() => { + self.prev_token(); + Ok(Some(Expr::Value(self.parse_value()?))) + } + Keyword::NULL => { + self.prev_token(); + Ok(Some(Expr::Value(self.parse_value()?))) + } + Keyword::CURRENT_CATALOG + | Keyword::CURRENT_USER + | Keyword::SESSION_USER + | Keyword::USER + if dialect_of!(self is PostgreSqlDialect | GenericDialect) => + { + Ok(Some(Expr::Function(Function { + name: ObjectName(vec![w.to_ident()]), + parameters: FunctionArguments::None, + args: FunctionArguments::None, + null_treatment: None, + filter: None, + over: None, + within_group: vec![], + }))) + } + Keyword::CURRENT_TIMESTAMP + | Keyword::CURRENT_TIME + | Keyword::CURRENT_DATE + | Keyword::LOCALTIME + | Keyword::LOCALTIMESTAMP => { + Ok(Some(self.parse_time_functions(ObjectName(vec![w.to_ident()]))?)) + } + Keyword::CASE => Ok(Some(self.parse_case_expr()?)), + Keyword::CONVERT => Ok(Some(self.parse_convert_expr(false)?)), + Keyword::TRY_CONVERT if self.dialect.supports_try_convert() => Ok(Some(self.parse_convert_expr(true)?)), + Keyword::CAST => Ok(Some(self.parse_cast_expr(CastKind::Cast)?)), + Keyword::TRY_CAST => Ok(Some(self.parse_cast_expr(CastKind::TryCast)?)), + Keyword::SAFE_CAST => Ok(Some(self.parse_cast_expr(CastKind::SafeCast)?)), + Keyword::EXISTS + // Support parsing Databricks has a function named `exists`. + if !dialect_of!(self is DatabricksDialect) + || matches!( + self.peek_nth_token(1).token, + Token::Word(Word { + keyword: Keyword::SELECT | Keyword::WITH, + .. + }) + ) => + { + Ok(Some(self.parse_exists_expr(false)?)) + } + Keyword::EXTRACT => Ok(Some(self.parse_extract_expr()?)), + Keyword::CEIL => Ok(Some(self.parse_ceil_floor_expr(true)?)), + Keyword::FLOOR => Ok(Some(self.parse_ceil_floor_expr(false)?)), + Keyword::POSITION if self.peek_token().token == Token::LParen => { + Ok(Some(self.parse_position_expr(w.to_ident())?)) + } + Keyword::SUBSTRING => Ok(Some(self.parse_substring_expr()?)), + Keyword::OVERLAY => Ok(Some(self.parse_overlay_expr()?)), + Keyword::TRIM => Ok(Some(self.parse_trim_expr()?)), + Keyword::INTERVAL => Ok(Some(self.parse_interval()?)), + // Treat ARRAY[1,2,3] as an array [1,2,3], otherwise try as subquery or a function call + Keyword::ARRAY if self.peek_token() == Token::LBracket => { + self.expect_token(&Token::LBracket)?; + Ok(Some(self.parse_array_expr(true)?)) + } + Keyword::ARRAY + if self.peek_token() == Token::LParen + && !dialect_of!(self is ClickHouseDialect | DatabricksDialect) => + { + self.expect_token(&Token::LParen)?; + let query = self.parse_query()?; + self.expect_token(&Token::RParen)?; + Ok(Some(Expr::Function(Function { + name: ObjectName(vec![w.to_ident()]), + parameters: FunctionArguments::None, + args: FunctionArguments::Subquery(query), + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }))) + } + Keyword::NOT => Ok(Some(self.parse_not()?)), + Keyword::MATCH if dialect_of!(self is MySqlDialect | GenericDialect) => { + Ok(Some(self.parse_match_against()?)) + } + Keyword::STRUCT if dialect_of!(self is BigQueryDialect | GenericDialect) => { + self.prev_token(); + Ok(Some(self.parse_bigquery_struct_literal()?)) + } + Keyword::PRIOR if matches!(self.state, ParserState::ConnectBy) => { + let expr = self.parse_subexpr(self.dialect.prec_value(Precedence::PlusMinus))?; + Ok(Some(Expr::Prior(Box::new(expr)))) + } + Keyword::MAP if self.peek_token() == Token::LBrace && self.dialect.support_map_literal_syntax() => { + Ok(Some(self.parse_duckdb_map_literal()?)) + } + _ => Ok(None) + } + } + + // Tries to parse an expression by a word that is not known to have a special meaning in the dialect. + fn parse_expr_prefix_by_unreserved_word(&mut self, w: &Word) -> Result { + match self.peek_token().token { + Token::LParen | Token::Period => { + let mut id_parts: Vec = vec![w.to_ident()]; + let mut ends_with_wildcard = false; + while self.consume_token(&Token::Period) { + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => id_parts.push(w.to_ident()), + Token::Mul => { + // Postgres explicitly allows funcnm(tablenm.*) and the + // function array_agg traverses this control flow + if dialect_of!(self is PostgreSqlDialect) { + ends_with_wildcard = true; + break; + } else { + return self.expected("an identifier after '.'", next_token); + } + } + Token::SingleQuotedString(s) => id_parts.push(Ident::with_quote('\'', s)), + _ => { + return self.expected("an identifier or a '*' after '.'", next_token); + } + } + } + + if ends_with_wildcard { + Ok(Expr::QualifiedWildcard(ObjectName(id_parts))) + } else if self.consume_token(&Token::LParen) { + if dialect_of!(self is SnowflakeDialect | MsSqlDialect) + && self.consume_tokens(&[Token::Plus, Token::RParen]) + { + Ok(Expr::OuterJoin(Box::new( + match <[Ident; 1]>::try_from(id_parts) { + Ok([ident]) => Expr::Identifier(ident), + Err(parts) => Expr::CompoundIdentifier(parts), + }, + ))) + } else { + self.prev_token(); + self.parse_function(ObjectName(id_parts)) + } + } else { + Ok(Expr::CompoundIdentifier(id_parts)) + } + } + // string introducer https://dev.mysql.com/doc/refman/8.0/en/charset-introducer.html + Token::SingleQuotedString(_) + | Token::DoubleQuotedString(_) + | Token::HexStringLiteral(_) + if w.value.starts_with('_') => + { + Ok(Expr::IntroducedString { + introducer: w.value.clone(), + value: self.parse_introduced_string_value()?, + }) + } + Token::Arrow if self.dialect.supports_lambda_functions() => { + self.expect_token(&Token::Arrow)?; + Ok(Expr::Lambda(LambdaFunction { + params: OneOrManyWithParens::One(w.to_ident()), + body: Box::new(self.parse_expr()?), + })) + } + _ => Ok(Expr::Identifier(w.to_ident())), + } + } + /// Parse an expression prefix. pub fn parse_prefix(&mut self) -> Result { // allow the dialect to override prefix parsing @@ -1073,176 +1250,40 @@ impl<'a> Parser<'a> { let next_token = self.next_token(); let expr = match next_token.token { - Token::Word(w) => match w.keyword { - Keyword::TRUE | Keyword::FALSE if self.dialect.supports_boolean_literals() => { - self.prev_token(); - Ok(Expr::Value(self.parse_value()?)) - } - Keyword::NULL => { - self.prev_token(); - Ok(Expr::Value(self.parse_value()?)) - } - Keyword::CURRENT_CATALOG - | Keyword::CURRENT_USER - | Keyword::SESSION_USER - | Keyword::USER - if dialect_of!(self is PostgreSqlDialect | GenericDialect) => - { - Ok(Expr::Function(Function { - name: ObjectName(vec![w.to_ident()]), - parameters: FunctionArguments::None, - args: FunctionArguments::None, - null_treatment: None, - filter: None, - over: None, - within_group: vec![], - })) - } - Keyword::CURRENT_TIMESTAMP - | Keyword::CURRENT_TIME - | Keyword::CURRENT_DATE - | Keyword::LOCALTIME - | Keyword::LOCALTIMESTAMP => { - self.parse_time_functions(ObjectName(vec![w.to_ident()])) - } - Keyword::CASE => self.parse_case_expr(), - Keyword::CONVERT => self.parse_convert_expr(false), - Keyword::TRY_CONVERT if self.dialect.supports_try_convert() => self.parse_convert_expr(true), - Keyword::CAST => self.parse_cast_expr(CastKind::Cast), - Keyword::TRY_CAST => self.parse_cast_expr(CastKind::TryCast), - Keyword::SAFE_CAST => self.parse_cast_expr(CastKind::SafeCast), - Keyword::EXISTS - // Support parsing Databricks has a function named `exists`. - if !dialect_of!(self is DatabricksDialect) - || matches!( - self.peek_nth_token(1).token, - Token::Word(Word { - keyword: Keyword::SELECT | Keyword::WITH, - .. - }) - ) => - { - self.parse_exists_expr(false) - } - Keyword::EXTRACT => self.parse_extract_expr(), - Keyword::CEIL => self.parse_ceil_floor_expr(true), - Keyword::FLOOR => self.parse_ceil_floor_expr(false), - Keyword::POSITION if self.peek_token().token == Token::LParen => { - self.parse_position_expr(w.to_ident()) - } - Keyword::SUBSTRING => self.parse_substring_expr(), - Keyword::OVERLAY => self.parse_overlay_expr(), - Keyword::TRIM => self.parse_trim_expr(), - Keyword::INTERVAL => self.parse_interval(), - // Treat ARRAY[1,2,3] as an array [1,2,3], otherwise try as subquery or a function call - Keyword::ARRAY if self.peek_token() == Token::LBracket => { - self.expect_token(&Token::LBracket)?; - self.parse_array_expr(true) - } - Keyword::ARRAY - if self.peek_token() == Token::LParen - && !dialect_of!(self is ClickHouseDialect | DatabricksDialect) => - { - self.expect_token(&Token::LParen)?; - let query = self.parse_query()?; - self.expect_token(&Token::RParen)?; - Ok(Expr::Function(Function { - name: ObjectName(vec![w.to_ident()]), - parameters: FunctionArguments::None, - args: FunctionArguments::Subquery(query), - filter: None, - null_treatment: None, - over: None, - within_group: vec![], - })) - } - Keyword::NOT => self.parse_not(), - Keyword::MATCH if dialect_of!(self is MySqlDialect | GenericDialect) => { - self.parse_match_against() - } - Keyword::STRUCT if dialect_of!(self is BigQueryDialect | GenericDialect) => { - self.prev_token(); - self.parse_bigquery_struct_literal() - } - Keyword::PRIOR if matches!(self.state, ParserState::ConnectBy) => { - let expr = self.parse_subexpr(self.dialect.prec_value(Precedence::PlusMinus))?; - Ok(Expr::Prior(Box::new(expr))) - } - Keyword::MAP if self.peek_token() == Token::LBrace && self.dialect.support_map_literal_syntax() => { - self.parse_duckdb_map_literal() - } - // Here `w` is a word, check if it's a part of a multipart - // identifier, a function call, or a simple identifier: - _ => match self.peek_token().token { - Token::LParen | Token::Period => { - let mut id_parts: Vec = vec![w.to_ident()]; - let mut ends_with_wildcard = false; - while self.consume_token(&Token::Period) { - let next_token = self.next_token(); - match next_token.token { - Token::Word(w) => id_parts.push(w.to_ident()), - Token::Mul => { - // Postgres explicitly allows funcnm(tablenm.*) and the - // function array_agg traverses this control flow - if dialect_of!(self is PostgreSqlDialect) { - ends_with_wildcard = true; - break; - } else { - return self - .expected("an identifier after '.'", next_token); - } - } - Token::SingleQuotedString(s) => { - id_parts.push(Ident::with_quote('\'', s)) - } - _ => { - return self - .expected("an identifier or a '*' after '.'", next_token); - } - } - } - - if ends_with_wildcard { - Ok(Expr::QualifiedWildcard(ObjectName(id_parts))) - } else if self.consume_token(&Token::LParen) { - if dialect_of!(self is SnowflakeDialect | MsSqlDialect) - && self.consume_tokens(&[Token::Plus, Token::RParen]) - { - Ok(Expr::OuterJoin(Box::new( - match <[Ident; 1]>::try_from(id_parts) { - Ok([ident]) => Expr::Identifier(ident), - Err(parts) => Expr::CompoundIdentifier(parts), - }, - ))) - } else { - self.prev_token(); - self.parse_function(ObjectName(id_parts)) + Token::Word(w) => { + // The word we consumed may fall into one of two cases: it has a special meaning, or not. + // For example, in Snowflake, the word `interval` may have two meanings depending on the context: + // `SELECT CURRENT_DATE() + INTERVAL '1 DAY', MAX(interval) FROM tbl;` + // ^^^^^^^^^^^^^^^^ ^^^^^^^^ + // interval expression identifier + // + // We first try to parse the word and following tokens as a special expression, and if that fails, + // we rollback and try to parse it as an identifier. + match self.try_parse(|parser| parser.parse_expr_prefix_by_reserved_word(&w)) { + // This word indicated an expression prefix and parsing was successful + Ok(Some(expr)) => Ok(expr), + + // No expression prefix associated with this word + Ok(None) => Ok(self.parse_expr_prefix_by_unreserved_word(&w)?), + + // If parsing of the word as a special expression failed, we are facing two options: + // 1. The statement is malformed, e.g. `SELECT INTERVAL '1 DAI` (`DAI` instead of `DAY`) + // 2. The word is used as an identifier, e.g. `SELECT MAX(interval) FROM tbl` + // We first try to parse the word as an identifier and if that fails + // we rollback and return the parsing error we got from trying to parse a + // special expression (to maintain backwards compatibility of parsing errors). + Err(e) => { + if !self.dialect.is_reserved_for_identifier(w.keyword) { + if let Ok(Some(expr)) = self.maybe_parse(|parser| { + parser.parse_expr_prefix_by_unreserved_word(&w) + }) { + return Ok(expr); } - } else { - Ok(Expr::CompoundIdentifier(id_parts)) } + return Err(e); } - // string introducer https://dev.mysql.com/doc/refman/8.0/en/charset-introducer.html - Token::SingleQuotedString(_) - | Token::DoubleQuotedString(_) - | Token::HexStringLiteral(_) - if w.value.starts_with('_') => - { - Ok(Expr::IntroducedString { - introducer: w.value, - value: self.parse_introduced_string_value()?, - }) - } - Token::Arrow if self.dialect.supports_lambda_functions() => { - self.expect_token(&Token::Arrow)?; - return Ok(Expr::Lambda(LambdaFunction { - params: OneOrManyWithParens::One(w.to_ident()), - body: Box::new(self.parse_expr()?), - })); - } - _ => Ok(Expr::Identifier(w.to_ident())), - }, - }, // End of Token::Word + } + } // End of Token::Word // array `[1, 2, 3]` Token::LBracket => self.parse_array_expr(false), tok @ Token::Minus | tok @ Token::Plus => { @@ -3677,18 +3718,30 @@ impl<'a> Parser<'a> { } /// Run a parser method `f`, reverting back to the current position if unsuccessful. - pub fn maybe_parse(&mut self, mut f: F) -> Result, ParserError> + /// Returns `None` if `f` returns an error + pub fn maybe_parse(&mut self, f: F) -> Result, ParserError> where F: FnMut(&mut Parser) -> Result, { - let index = self.index; - match f(self) { + match self.try_parse(f) { Ok(t) => Ok(Some(t)), - // Unwind stack if limit exceeded Err(ParserError::RecursionLimitExceeded) => Err(ParserError::RecursionLimitExceeded), - Err(_) => { + _ => Ok(None), + } + } + + /// Run a parser method `f`, reverting back to the current position if unsuccessful. + pub fn try_parse(&mut self, mut f: F) -> Result + where + F: FnMut(&mut Parser) -> Result, + { + let index = self.index; + match f(self) { + Ok(t) => Ok(t), + Err(e) => { + // Unwind stack if limit exceeded self.index = index; - Ok(None) + Err(e) } } } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index b41063859..c03370892 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -34,7 +34,7 @@ use sqlparser::dialect::{ GenericDialect, HiveDialect, MsSqlDialect, MySqlDialect, PostgreSqlDialect, RedshiftSqlDialect, SQLiteDialect, SnowflakeDialect, }; -use sqlparser::keywords::ALL_KEYWORDS; +use sqlparser::keywords::{Keyword, ALL_KEYWORDS}; use sqlparser::parser::{Parser, ParserError, ParserOptions}; use sqlparser::tokenizer::Tokenizer; use test_utils::{ @@ -5113,7 +5113,6 @@ fn parse_interval_dont_require_unit() { #[test] fn parse_interval_require_unit() { let dialects = all_dialects_where(|d| d.require_interval_qualifier()); - let sql = "SELECT INTERVAL '1 DAY'"; let err = dialects.parse_sql_statements(sql).unwrap_err(); assert_eq!( @@ -12198,3 +12197,21 @@ fn parse_create_table_select() { ); } } + +#[test] +fn test_reserved_keywords_for_identifiers() { + let dialects = all_dialects_where(|d| d.is_reserved_for_identifier(Keyword::INTERVAL)); + // Dialects that reserve the word INTERVAL will not allow it as an unquoted identifier + let sql = "SELECT MAX(interval) FROM tbl"; + assert_eq!( + dialects.parse_sql_statements(sql), + Err(ParserError::ParserError( + "Expected: an expression, found: )".to_string() + )) + ); + + // Dialects that do not reserve the word INTERVAL will allow it + let dialects = all_dialects_where(|d| !d.is_reserved_for_identifier(Keyword::INTERVAL)); + let sql = "SELECT MAX(interval) FROM tbl"; + dialects.parse_sql_statements(sql).unwrap(); +} diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 098a3464c..d27569e03 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -1352,10 +1352,7 @@ fn parse_set() { local: false, hivevar: false, variables: OneOrManyWithParens::One(ObjectName(vec![Ident::new("a")])), - value: vec![Expr::Identifier(Ident { - value: "DEFAULT".into(), - quote_style: None - })], + value: vec![Expr::Identifier(Ident::new("DEFAULT"))], } ); @@ -4229,10 +4226,7 @@ fn test_simple_postgres_insert_with_alias() { body: Box::new(SetExpr::Values(Values { explicit_row: false, rows: vec![vec![ - Expr::Identifier(Ident { - value: "DEFAULT".to_string(), - quote_style: None - }), + Expr::Identifier(Ident::new("DEFAULT")), Expr::Value(Value::Number("123".to_string(), false)) ]] })), @@ -4295,10 +4289,7 @@ fn test_simple_postgres_insert_with_alias() { body: Box::new(SetExpr::Values(Values { explicit_row: false, rows: vec![vec![ - Expr::Identifier(Ident { - value: "DEFAULT".to_string(), - quote_style: None - }), + Expr::Identifier(Ident::new("DEFAULT")), Expr::Value(Value::Number( bigdecimal::BigDecimal::new(123.into(), 0), false @@ -4363,10 +4354,7 @@ fn test_simple_insert_with_quoted_alias() { body: Box::new(SetExpr::Values(Values { explicit_row: false, rows: vec![vec![ - Expr::Identifier(Ident { - value: "DEFAULT".to_string(), - quote_style: None - }), + Expr::Identifier(Ident::new("DEFAULT")), Expr::Value(Value::SingleQuotedString("0123".to_string())) ]] })), From 525d1780e8f5c7ba6b7be327eaa788b6c8c47716 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Mon, 25 Nov 2024 22:01:23 +0100 Subject: [PATCH 66/80] support `json_object('k':'v')` in postgres (#1546) --- src/dialect/postgresql.rs | 20 +++++ tests/sqlparser_common.rs | 172 +++++++++++++++++++++++++++++++++++++- tests/sqlparser_mssql.rs | 159 ----------------------------------- 3 files changed, 191 insertions(+), 160 deletions(-) diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs index 559586e3f..dcdcc88c1 100644 --- a/src/dialect/postgresql.rs +++ b/src/dialect/postgresql.rs @@ -211,6 +211,26 @@ impl Dialect for PostgreSqlDialect { fn supports_load_extension(&self) -> bool { true } + + /// See + /// + /// Required to support the colon in: + /// ```sql + /// SELECT json_object('a': 'b') + /// ``` + fn supports_named_fn_args_with_colon_operator(&self) -> bool { + true + } + + /// See + /// + /// Required to support the label in: + /// ```sql + /// SELECT json_object('label': 'value') + /// ``` + fn supports_named_fn_args_with_expr_name(&self) -> bool { + true + } } pub fn parse_create(parser: &mut Parser) -> Option> { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index c03370892..e22877dbe 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -1471,6 +1471,173 @@ fn parse_json_ops_without_colon() { } } +#[test] +fn parse_json_object() { + let dialects = TestedDialects::new(vec![ + Box::new(MsSqlDialect {}), + Box::new(PostgreSqlDialect {}), + ]); + let select = dialects.verified_only_select("SELECT JSON_OBJECT('name' : 'value', 'type' : 1)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, .. }), + .. + }) => assert_eq!( + &[ + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("name".into())), + arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( + "value".into() + ))), + operator: FunctionArgOperator::Colon + }, + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("type".into())), + arg: FunctionArgExpr::Expr(Expr::Value(number("1"))), + operator: FunctionArgOperator::Colon + } + ], + &args[..] + ), + _ => unreachable!(), + } + let select = dialects + .verified_only_select("SELECT JSON_OBJECT('name' : 'value', 'type' : NULL ABSENT ON NULL)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert_eq!( + &[ + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("name".into())), + arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( + "value".into() + ))), + operator: FunctionArgOperator::Colon + }, + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("type".into())), + arg: FunctionArgExpr::Expr(Expr::Value(Value::Null)), + operator: FunctionArgOperator::Colon + } + ], + &args[..] + ); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::AbsentOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = dialects.verified_only_select("SELECT JSON_OBJECT(NULL ON NULL)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert!(args.is_empty()); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::NullOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = dialects.verified_only_select("SELECT JSON_OBJECT(ABSENT ON NULL)"); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert!(args.is_empty()); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::AbsentOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = dialects.verified_only_select( + "SELECT JSON_OBJECT('name' : 'value', 'type' : JSON_ARRAY(1, 2) ABSENT ON NULL)", + ); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert_eq!( + &FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("name".into())), + arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( + "value".into() + ))), + operator: FunctionArgOperator::Colon + }, + &args[0] + ); + assert!(matches!( + args[1], + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString(_)), + arg: FunctionArgExpr::Expr(Expr::Function(_)), + operator: FunctionArgOperator::Colon + } + )); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::AbsentOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } + let select = dialects.verified_only_select( + "SELECT JSON_OBJECT('name' : 'value', 'type' : JSON_OBJECT('type_id' : 1, 'name' : 'a') NULL ON NULL)", + ); + match expr_from_projection(&select.projection[0]) { + Expr::Function(Function { + args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), + .. + }) => { + assert_eq!( + &FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString("name".into())), + arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( + "value".into() + ))), + operator: FunctionArgOperator::Colon + }, + &args[0] + ); + assert!(matches!( + args[1], + FunctionArg::ExprNamed { + name: Expr::Value(Value::SingleQuotedString(_)), + arg: FunctionArgExpr::Expr(Expr::Function(_)), + operator: FunctionArgOperator::Colon + } + )); + assert_eq!( + &[FunctionArgumentClause::JsonNullClause( + JsonNullClause::NullOnNull + )], + &clauses[..] + ); + } + _ => unreachable!(), + } +} + #[test] fn parse_mod_no_spaces() { use self::Expr::*; @@ -4416,7 +4583,10 @@ fn parse_explain_query_plan() { #[test] fn parse_named_argument_function() { - let dialects = all_dialects_where(|d| d.supports_named_fn_args_with_rarrow_operator()); + let dialects = all_dialects_where(|d| { + d.supports_named_fn_args_with_rarrow_operator() + && !d.supports_named_fn_args_with_expr_name() + }); let sql = "SELECT FUN(a => '1', b => '2') FROM foo"; let select = dialects.verified_only_select(sql); diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 74f3c077e..d1d8d1248 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -793,165 +793,6 @@ fn parse_for_json_expect_ast() { #[test] fn parse_mssql_json_object() { - let select = ms().verified_only_select("SELECT JSON_OBJECT('name' : 'value', 'type' : 1)"); - match expr_from_projection(&select.projection[0]) { - Expr::Function(Function { - args: FunctionArguments::List(FunctionArgumentList { args, .. }), - .. - }) => assert_eq!( - &[ - FunctionArg::ExprNamed { - name: Expr::Value(Value::SingleQuotedString("name".into())), - arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( - "value".into() - ))), - operator: FunctionArgOperator::Colon - }, - FunctionArg::ExprNamed { - name: Expr::Value(Value::SingleQuotedString("type".into())), - arg: FunctionArgExpr::Expr(Expr::Value(number("1"))), - operator: FunctionArgOperator::Colon - } - ], - &args[..] - ), - _ => unreachable!(), - } - let select = ms() - .verified_only_select("SELECT JSON_OBJECT('name' : 'value', 'type' : NULL ABSENT ON NULL)"); - match expr_from_projection(&select.projection[0]) { - Expr::Function(Function { - args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), - .. - }) => { - assert_eq!( - &[ - FunctionArg::ExprNamed { - name: Expr::Value(Value::SingleQuotedString("name".into())), - arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( - "value".into() - ))), - operator: FunctionArgOperator::Colon - }, - FunctionArg::ExprNamed { - name: Expr::Value(Value::SingleQuotedString("type".into())), - arg: FunctionArgExpr::Expr(Expr::Value(Value::Null)), - operator: FunctionArgOperator::Colon - } - ], - &args[..] - ); - assert_eq!( - &[FunctionArgumentClause::JsonNullClause( - JsonNullClause::AbsentOnNull - )], - &clauses[..] - ); - } - _ => unreachable!(), - } - let select = ms().verified_only_select("SELECT JSON_OBJECT(NULL ON NULL)"); - match expr_from_projection(&select.projection[0]) { - Expr::Function(Function { - args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), - .. - }) => { - assert!(args.is_empty()); - assert_eq!( - &[FunctionArgumentClause::JsonNullClause( - JsonNullClause::NullOnNull - )], - &clauses[..] - ); - } - _ => unreachable!(), - } - let select = ms().verified_only_select("SELECT JSON_OBJECT(ABSENT ON NULL)"); - match expr_from_projection(&select.projection[0]) { - Expr::Function(Function { - args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), - .. - }) => { - assert!(args.is_empty()); - assert_eq!( - &[FunctionArgumentClause::JsonNullClause( - JsonNullClause::AbsentOnNull - )], - &clauses[..] - ); - } - _ => unreachable!(), - } - let select = ms().verified_only_select( - "SELECT JSON_OBJECT('name' : 'value', 'type' : JSON_ARRAY(1, 2) ABSENT ON NULL)", - ); - match expr_from_projection(&select.projection[0]) { - Expr::Function(Function { - args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), - .. - }) => { - assert_eq!( - &FunctionArg::ExprNamed { - name: Expr::Value(Value::SingleQuotedString("name".into())), - arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( - "value".into() - ))), - operator: FunctionArgOperator::Colon - }, - &args[0] - ); - assert!(matches!( - args[1], - FunctionArg::ExprNamed { - name: Expr::Value(Value::SingleQuotedString(_)), - arg: FunctionArgExpr::Expr(Expr::Function(_)), - operator: FunctionArgOperator::Colon - } - )); - assert_eq!( - &[FunctionArgumentClause::JsonNullClause( - JsonNullClause::AbsentOnNull - )], - &clauses[..] - ); - } - _ => unreachable!(), - } - let select = ms().verified_only_select( - "SELECT JSON_OBJECT('name' : 'value', 'type' : JSON_OBJECT('type_id' : 1, 'name' : 'a') NULL ON NULL)", - ); - match expr_from_projection(&select.projection[0]) { - Expr::Function(Function { - args: FunctionArguments::List(FunctionArgumentList { args, clauses, .. }), - .. - }) => { - assert_eq!( - &FunctionArg::ExprNamed { - name: Expr::Value(Value::SingleQuotedString("name".into())), - arg: FunctionArgExpr::Expr(Expr::Value(Value::SingleQuotedString( - "value".into() - ))), - operator: FunctionArgOperator::Colon - }, - &args[0] - ); - assert!(matches!( - args[1], - FunctionArg::ExprNamed { - name: Expr::Value(Value::SingleQuotedString(_)), - arg: FunctionArgExpr::Expr(Expr::Function(_)), - operator: FunctionArgOperator::Colon - } - )); - assert_eq!( - &[FunctionArgumentClause::JsonNullClause( - JsonNullClause::NullOnNull - )], - &clauses[..] - ); - } - _ => unreachable!(), - } let select = ms().verified_only_select( "SELECT JSON_OBJECT('user_name' : USER_NAME(), LOWER(@id_key) : @id_value, 'sid' : (SELECT @@SPID) ABSENT ON NULL)", ); From 0adec33b94241f19273c371057bc8ad15e849ef4 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 26 Nov 2024 11:11:56 -0500 Subject: [PATCH 67/80] Document micro benchmarks (#1555) --- README.md | 12 ++++++++++++ sqlparser_bench/Cargo.toml | 1 + sqlparser_bench/README.md | 20 ++++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 sqlparser_bench/README.md diff --git a/README.md b/README.md index 934d9d06d..f44300f55 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,18 @@ Our goal as maintainers is to facilitate the integration of various features from various contributors, but not to provide the implementations ourselves, as we simply don't have the resources. +### Benchmarking + +There are several micro benchmarks in the `sqlparser_bench` directory. +You can run them with: + +``` +git checkout main +cd sqlparser_bench +cargo bench +git checkout +cargo bench +``` ## Licensing diff --git a/sqlparser_bench/Cargo.toml b/sqlparser_bench/Cargo.toml index 9c33658a2..2c1f0ae4d 100644 --- a/sqlparser_bench/Cargo.toml +++ b/sqlparser_bench/Cargo.toml @@ -17,6 +17,7 @@ [package] name = "sqlparser_bench" +description = "Benchmarks for sqlparser" version = "0.1.0" authors = ["Dandandan "] edition = "2018" diff --git a/sqlparser_bench/README.md b/sqlparser_bench/README.md new file mode 100644 index 000000000..4cdcfb29c --- /dev/null +++ b/sqlparser_bench/README.md @@ -0,0 +1,20 @@ + + +Benchmarks for sqlparser. See [the main README](../README.md) for more information. \ No newline at end of file From 3c8fd748043188957d2dfcadb4bfcfb0e1f70c82 Mon Sep 17 00:00:00 2001 From: Mark-Oliver Junge Date: Tue, 26 Nov 2024 17:22:30 +0100 Subject: [PATCH 68/80] Implement `Spanned` to retrieve source locations on AST nodes (#1435) Co-authored-by: Ifeanyi Ubah Co-authored-by: Andrew Lamb --- README.md | 17 + docs/source_spans.md | 52 + src/ast/helpers/attached_token.rs | 82 ++ src/ast/helpers/mod.rs | 1 + src/ast/mod.rs | 84 +- src/ast/query.rs | 29 +- src/ast/spans.rs | 2178 +++++++++++++++++++++++++++++ src/parser/mod.rs | 274 ++-- src/tokenizer.rs | 146 +- tests/sqlparser_bigquery.rs | 13 + tests/sqlparser_clickhouse.rs | 10 +- tests/sqlparser_common.rs | 99 +- tests/sqlparser_duckdb.rs | 40 +- tests/sqlparser_mssql.rs | 229 ++- tests/sqlparser_mysql.rs | 64 +- tests/sqlparser_postgres.rs | 150 +- tests/sqlparser_redshift.rs | 19 +- tests/sqlparser_snowflake.rs | 4 +- 18 files changed, 3092 insertions(+), 399 deletions(-) create mode 100644 docs/source_spans.md create mode 100644 src/ast/helpers/attached_token.rs create mode 100644 src/ast/spans.rs diff --git a/README.md b/README.md index f44300f55..9a67abcf8 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,23 @@ similar semantics are represented with the same AST. We welcome PRs to fix such issues and distinguish different syntaxes in the AST. +## WIP: Extracting source locations from AST nodes + +This crate allows recovering source locations from AST nodes via the [Spanned](https://docs.rs/sqlparser/latest/sqlparser/ast/trait.Spanned.html) trait, which can be used for advanced diagnostics tooling. Note that this feature is a work in progress and many nodes report missing or inaccurate spans. Please see [this document](./docs/source_spans.md#source-span-contributing-guidelines) for information on how to contribute missing improvements. + +```rust +use sqlparser::ast::Spanned; + +// Parse SQL +let ast = Parser::parse_sql(&GenericDialect, "SELECT A FROM B").unwrap(); + +// The source span can be retrieved with start and end locations +assert_eq!(ast[0].span(), Span { + start: Location::of(1, 1), + end: Location::of(1, 16), +}); +``` + ## SQL compliance SQL was first standardized in 1987, and revisions of the standard have been diff --git a/docs/source_spans.md b/docs/source_spans.md new file mode 100644 index 000000000..136a4ced2 --- /dev/null +++ b/docs/source_spans.md @@ -0,0 +1,52 @@ + +## Breaking Changes + +These are the current breaking changes introduced by the source spans feature: + +#### Added fields for spans (must be added to any existing pattern matches) +- `Ident` now stores a `Span` +- `Select`, `With`, `Cte`, `WildcardAdditionalOptions` now store a `TokenWithLocation` + +#### Misc. +- `TokenWithLocation` stores a full `Span`, rather than just a source location. Users relying on `token.location` should use `token.location.start` instead. +## Source Span Contributing Guidelines + +For contributing source spans improvement in addition to the general [contribution guidelines](../README.md#contributing), please make sure to pay attention to the following: + + +### Source Span Design Considerations + +- `Ident` always have correct source spans +- Downstream breaking change impact is to be as minimal as possible +- To this end, use recursive merging of spans in favor of storing spans on all nodes +- Any metadata added to compute spans must not change semantics (Eq, Ord, Hash, etc.) + +The primary reason for missing and inaccurate source spans at this time is missing spans of keyword tokens and values in many structures, either due to lack of time or because adding them would break downstream significantly. + +When considering adding support for source spans on a type, consider the impact to consumers of that type and whether your change would require a consumer to do non-trivial changes to their code. + +Example of a trivial change +```rust +match node { + ast::Query { + field1, + field2, + location: _, // add a new line to ignored location +} +``` + +If adding source spans to a type would require a significant change like wrapping that type or similar, please open an issue to discuss. + +### AST Node Equality and Hashes + +When adding tokens to AST nodes, make sure to store them using the [AttachedToken](https://docs.rs/sqlparser/latest/sqlparser/ast/helpers/struct.AttachedToken.html) helper to ensure that semantically equivalent AST nodes always compare as equal and hash to the same value. F.e. `select 5` and `SELECT 5` would compare as different `Select` nodes, if the select token was stored directly. f.e. + +```rust +struct Select { + select_token: AttachedToken, // only used for spans + /// remaining fields + field1, + field2, + ... +} +``` \ No newline at end of file diff --git a/src/ast/helpers/attached_token.rs b/src/ast/helpers/attached_token.rs new file mode 100644 index 000000000..48696c336 --- /dev/null +++ b/src/ast/helpers/attached_token.rs @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; +use core::fmt::{self, Debug, Formatter}; +use core::hash::{Hash, Hasher}; + +use crate::tokenizer::{Token, TokenWithLocation}; + +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "visitor")] +use sqlparser_derive::{Visit, VisitMut}; + +/// A wrapper type for attaching tokens to AST nodes that should be ignored in comparisons and hashing. +/// This should be used when a token is not relevant for semantics, but is still needed for +/// accurate source location tracking. +#[derive(Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct AttachedToken(pub TokenWithLocation); + +impl AttachedToken { + pub fn empty() -> Self { + AttachedToken(TokenWithLocation::wrap(Token::EOF)) + } +} + +// Conditional Implementations +impl Debug for AttachedToken { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +// Blanket Implementations +impl PartialEq for AttachedToken { + fn eq(&self, _: &Self) -> bool { + true + } +} + +impl Eq for AttachedToken {} + +impl PartialOrd for AttachedToken { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for AttachedToken { + fn cmp(&self, _: &Self) -> Ordering { + Ordering::Equal + } +} + +impl Hash for AttachedToken { + fn hash(&self, _state: &mut H) { + // Do nothing + } +} + +impl From for AttachedToken { + fn from(value: TokenWithLocation) -> Self { + AttachedToken(value) + } +} diff --git a/src/ast/helpers/mod.rs b/src/ast/helpers/mod.rs index d6924ab88..a96bffc51 100644 --- a/src/ast/helpers/mod.rs +++ b/src/ast/helpers/mod.rs @@ -14,5 +14,6 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +pub mod attached_token; pub mod stmt_create_table; pub mod stmt_data_loading; diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 9185c9df4..366bf4d25 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -23,9 +23,13 @@ use alloc::{ string::{String, ToString}, vec::Vec, }; +use helpers::attached_token::AttachedToken; -use core::fmt::{self, Display}; use core::ops::Deref; +use core::{ + fmt::{self, Display}, + hash, +}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -33,6 +37,8 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; +use crate::tokenizer::Span; + pub use self::data_type::{ ArrayElemTypeDef, CharLengthUnits, CharacterLength, DataType, ExactNumberInfo, StructBracketKind, TimezoneInfo, @@ -87,6 +93,9 @@ mod dml; pub mod helpers; mod operator; mod query; +mod spans; +pub use spans::Spanned; + mod trigger; mod value; @@ -131,7 +140,7 @@ where } /// An identifier, decomposed into its value or character data and the quote style. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[derive(Debug, Clone, PartialOrd, Ord)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct Ident { @@ -140,10 +149,41 @@ pub struct Ident { /// The starting quote if any. Valid quote characters are the single quote, /// double quote, backtick, and opening square bracket. pub quote_style: Option, + /// The span of the identifier in the original SQL string. + pub span: Span, +} + +impl PartialEq for Ident { + fn eq(&self, other: &Self) -> bool { + let Ident { + value, + quote_style, + // exhaustiveness check; we ignore spans in comparisons + span: _, + } = self; + + value == &other.value && quote_style == &other.quote_style + } } +impl core::hash::Hash for Ident { + fn hash(&self, state: &mut H) { + let Ident { + value, + quote_style, + // exhaustiveness check; we ignore spans in hashes + span: _, + } = self; + + value.hash(state); + quote_style.hash(state); + } +} + +impl Eq for Ident {} + impl Ident { - /// Create a new identifier with the given value and no quotes. + /// Create a new identifier with the given value and no quotes and an empty span. pub fn new(value: S) -> Self where S: Into, @@ -151,6 +191,7 @@ impl Ident { Ident { value: value.into(), quote_style: None, + span: Span::empty(), } } @@ -164,6 +205,30 @@ impl Ident { Ident { value: value.into(), quote_style: Some(quote), + span: Span::empty(), + } + } + + pub fn with_span(span: Span, value: S) -> Self + where + S: Into, + { + Ident { + value: value.into(), + quote_style: None, + span, + } + } + + pub fn with_quote_and_span(quote: char, span: Span, value: S) -> Self + where + S: Into, + { + assert!(quote == '\'' || quote == '"' || quote == '`' || quote == '['); + Ident { + value: value.into(), + quote_style: Some(quote), + span, } } } @@ -173,6 +238,7 @@ impl From<&str> for Ident { Ident { value: value.to_string(), quote_style: None, + span: Span::empty(), } } } @@ -919,10 +985,10 @@ pub enum Expr { /// `` opt_search_modifier: Option, }, - Wildcard, + Wildcard(AttachedToken), /// Qualified wildcard, e.g. `alias.*` or `schema.table.*`. /// (Same caveats apply to `QualifiedWildcard` as to `Wildcard`.) - QualifiedWildcard(ObjectName), + QualifiedWildcard(ObjectName, AttachedToken), /// Some dialects support an older syntax for outer joins where columns are /// marked with the `(+)` operator in the WHERE clause, for example: /// @@ -1211,8 +1277,8 @@ impl fmt::Display for Expr { Expr::MapAccess { column, keys } => { write!(f, "{column}{}", display_separated(keys, "")) } - Expr::Wildcard => f.write_str("*"), - Expr::QualifiedWildcard(prefix) => write!(f, "{}.*", prefix), + Expr::Wildcard(_) => f.write_str("*"), + Expr::QualifiedWildcard(prefix, _) => write!(f, "{}.*", prefix), Expr::CompoundIdentifier(s) => write!(f, "{}", display_separated(s, ".")), Expr::IsTrue(ast) => write!(f, "{ast} IS TRUE"), Expr::IsNotTrue(ast) => write!(f, "{ast} IS NOT TRUE"), @@ -5432,8 +5498,8 @@ pub enum FunctionArgExpr { impl From for FunctionArgExpr { fn from(wildcard_expr: Expr) -> Self { match wildcard_expr { - Expr::QualifiedWildcard(prefix) => Self::QualifiedWildcard(prefix), - Expr::Wildcard => Self::Wildcard, + Expr::QualifiedWildcard(prefix, _) => Self::QualifiedWildcard(prefix), + Expr::Wildcard(_) => Self::Wildcard, expr => Self::Expr(expr), } } diff --git a/src/ast/query.rs b/src/ast/query.rs index bf36c626f..0472026a0 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -18,13 +18,17 @@ #[cfg(not(feature = "std"))] use alloc::{boxed::Box, vec::Vec}; +use helpers::attached_token::AttachedToken; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; -use crate::ast::*; +use crate::{ + ast::*, + tokenizer::{Token, TokenWithLocation}, +}; /// The most complete variant of a `SELECT` query expression, optionally /// including `WITH`, `UNION` / other set operations, and `ORDER BY`. @@ -276,6 +280,8 @@ impl fmt::Display for Table { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct Select { + /// Token for the `SELECT` keyword + pub select_token: AttachedToken, pub distinct: Option, /// MSSQL syntax: `TOP () [ PERCENT ] [ WITH TIES ]` pub top: Option, @@ -505,6 +511,8 @@ impl fmt::Display for NamedWindowDefinition { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct With { + // Token for the "WITH" keyword + pub with_token: AttachedToken, pub recursive: bool, pub cte_tables: Vec, } @@ -556,6 +564,8 @@ pub struct Cte { pub query: Box, pub from: Option, pub materialized: Option, + // Token for the closing parenthesis + pub closing_paren_token: AttachedToken, } impl fmt::Display for Cte { @@ -607,10 +617,12 @@ impl fmt::Display for IdentWithAlias { } /// Additional options for wildcards, e.g. Snowflake `EXCLUDE`/`RENAME` and Bigquery `EXCEPT`. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Default)] +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct WildcardAdditionalOptions { + /// The wildcard token `*` + pub wildcard_token: AttachedToken, /// `[ILIKE...]`. /// Snowflake syntax: pub opt_ilike: Option, @@ -628,6 +640,19 @@ pub struct WildcardAdditionalOptions { pub opt_rename: Option, } +impl Default for WildcardAdditionalOptions { + fn default() -> Self { + Self { + wildcard_token: TokenWithLocation::wrap(Token::Mul).into(), + opt_ilike: None, + opt_exclude: None, + opt_except: None, + opt_replace: None, + opt_rename: None, + } + } +} + impl fmt::Display for WildcardAdditionalOptions { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if let Some(ilike) = &self.opt_ilike { diff --git a/src/ast/spans.rs b/src/ast/spans.rs new file mode 100644 index 000000000..8e8c7b14a --- /dev/null +++ b/src/ast/spans.rs @@ -0,0 +1,2178 @@ +use core::iter; + +use crate::tokenizer::Span; + +use super::{ + AlterColumnOperation, AlterIndexOperation, AlterTableOperation, Array, Assignment, + AssignmentTarget, CloseCursor, ClusteredIndex, ColumnDef, ColumnOption, ColumnOptionDef, + ConflictTarget, ConnectBy, ConstraintCharacteristics, CopySource, CreateIndex, CreateTable, + CreateTableOptions, Cte, Delete, DoUpdate, ExceptSelectItem, ExcludeSelectItem, Expr, + ExprWithAlias, Fetch, FromTable, Function, FunctionArg, FunctionArgExpr, + FunctionArgumentClause, FunctionArgumentList, FunctionArguments, GroupByExpr, HavingBound, + IlikeSelectItem, Insert, Interpolate, InterpolateExpr, Join, JoinConstraint, JoinOperator, + JsonPath, JsonPathElem, LateralView, MatchRecognizePattern, Measure, NamedWindowDefinition, + ObjectName, Offset, OnConflict, OnConflictAction, OnInsert, OrderBy, OrderByExpr, Partition, + PivotValueSource, ProjectionSelect, Query, ReferentialAction, RenameSelectItem, + ReplaceSelectElement, ReplaceSelectItem, Select, SelectInto, SelectItem, SetExpr, SqlOption, + Statement, Subscript, SymbolDefinition, TableAlias, TableAliasColumnDef, TableConstraint, + TableFactor, TableOptionsClustered, TableWithJoins, Use, Value, Values, ViewColumnDef, + WildcardAdditionalOptions, With, WithFill, +}; + +/// Given an iterator of spans, return the [Span::union] of all spans. +fn union_spans>(iter: I) -> Span { + iter.reduce(|acc, item| acc.union(&item)) + .unwrap_or(Span::empty()) +} + +/// A trait for AST nodes that have a source span for use in diagnostics. +/// +/// Source spans are not guaranteed to be entirely accurate. They may +/// be missing keywords or other tokens. Some nodes may not have a computable +/// span at all, in which case they return [`Span::empty()`]. +/// +/// Some impl blocks may contain doc comments with information +/// on which nodes are missing spans. +pub trait Spanned { + /// Compute the source span for this AST node, by recursively + /// combining the spans of its children. + fn span(&self) -> Span; +} + +impl Spanned for Query { + fn span(&self) -> Span { + let Query { + with, + body, + order_by, + limit, + limit_by, + offset, + fetch, + locks: _, // todo + for_clause: _, // todo, mssql specific + settings: _, // todo, clickhouse specific + format_clause: _, // todo, clickhouse specific + } = self; + + union_spans( + with.iter() + .map(|i| i.span()) + .chain(core::iter::once(body.span())) + .chain(order_by.as_ref().map(|i| i.span())) + .chain(limit.as_ref().map(|i| i.span())) + .chain(limit_by.iter().map(|i| i.span())) + .chain(offset.as_ref().map(|i| i.span())) + .chain(fetch.as_ref().map(|i| i.span())), + ) + } +} + +impl Spanned for Offset { + fn span(&self) -> Span { + let Offset { + value, + rows: _, // enum + } = self; + + value.span() + } +} + +impl Spanned for Fetch { + fn span(&self) -> Span { + let Fetch { + with_ties: _, // bool + percent: _, // bool + quantity, + } = self; + + quantity.as_ref().map_or(Span::empty(), |i| i.span()) + } +} + +impl Spanned for With { + fn span(&self) -> Span { + let With { + with_token, + recursive: _, // bool + cte_tables, + } = self; + + union_spans( + core::iter::once(with_token.0.span).chain(cte_tables.iter().map(|item| item.span())), + ) + } +} + +impl Spanned for Cte { + fn span(&self) -> Span { + let Cte { + alias, + query, + from, + materialized: _, // enum + closing_paren_token, + } = self; + + union_spans( + core::iter::once(alias.span()) + .chain(core::iter::once(query.span())) + .chain(from.iter().map(|item| item.span)) + .chain(core::iter::once(closing_paren_token.0.span)), + ) + } +} + +/// # partial span +/// +/// [SetExpr::Table] is not implemented. +impl Spanned for SetExpr { + fn span(&self) -> Span { + match self { + SetExpr::Select(select) => select.span(), + SetExpr::Query(query) => query.span(), + SetExpr::SetOperation { + op: _, + set_quantifier: _, + left, + right, + } => left.span().union(&right.span()), + SetExpr::Values(values) => values.span(), + SetExpr::Insert(statement) => statement.span(), + SetExpr::Table(_) => Span::empty(), + SetExpr::Update(statement) => statement.span(), + } + } +} + +impl Spanned for Values { + fn span(&self) -> Span { + let Values { + explicit_row: _, // bool, + rows, + } = self; + + union_spans( + rows.iter() + .map(|row| union_spans(row.iter().map(|expr| expr.span()))), + ) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [Statement::CopyIntoSnowflake] +/// - [Statement::CreateSecret] +/// - [Statement::CreateRole] +/// - [Statement::AlterRole] +/// - [Statement::AttachDatabase] +/// - [Statement::AttachDuckDBDatabase] +/// - [Statement::DetachDuckDBDatabase] +/// - [Statement::Drop] +/// - [Statement::DropFunction] +/// - [Statement::DropProcedure] +/// - [Statement::DropSecret] +/// - [Statement::Declare] +/// - [Statement::CreateExtension] +/// - [Statement::Fetch] +/// - [Statement::Flush] +/// - [Statement::Discard] +/// - [Statement::SetRole] +/// - [Statement::SetVariable] +/// - [Statement::SetTimeZone] +/// - [Statement::SetNames] +/// - [Statement::SetNamesDefault] +/// - [Statement::ShowFunctions] +/// - [Statement::ShowVariable] +/// - [Statement::ShowStatus] +/// - [Statement::ShowVariables] +/// - [Statement::ShowCreate] +/// - [Statement::ShowColumns] +/// - [Statement::ShowTables] +/// - [Statement::ShowCollation] +/// - [Statement::StartTransaction] +/// - [Statement::SetTransaction] +/// - [Statement::Comment] +/// - [Statement::Commit] +/// - [Statement::Rollback] +/// - [Statement::CreateSchema] +/// - [Statement::CreateDatabase] +/// - [Statement::CreateFunction] +/// - [Statement::CreateTrigger] +/// - [Statement::DropTrigger] +/// - [Statement::CreateProcedure] +/// - [Statement::CreateMacro] +/// - [Statement::CreateStage] +/// - [Statement::Assert] +/// - [Statement::Grant] +/// - [Statement::Revoke] +/// - [Statement::Deallocate] +/// - [Statement::Execute] +/// - [Statement::Prepare] +/// - [Statement::Kill] +/// - [Statement::ExplainTable] +/// - [Statement::Explain] +/// - [Statement::Savepoint] +/// - [Statement::ReleaseSavepoint] +/// - [Statement::Merge] +/// - [Statement::Cache] +/// - [Statement::UNCache] +/// - [Statement::CreateSequence] +/// - [Statement::CreateType] +/// - [Statement::Pragma] +/// - [Statement::LockTables] +/// - [Statement::UnlockTables] +/// - [Statement::Unload] +/// - [Statement::OptimizeTable] +impl Spanned for Statement { + fn span(&self) -> Span { + match self { + Statement::Analyze { + table_name, + partitions, + for_columns: _, + columns, + cache_metadata: _, + noscan: _, + compute_statistics: _, + } => union_spans( + core::iter::once(table_name.span()) + .chain(partitions.iter().flat_map(|i| i.iter().map(|k| k.span()))) + .chain(columns.iter().map(|i| i.span)), + ), + Statement::Truncate { + table_names, + partitions, + table: _, + only: _, + identity: _, + cascade: _, + on_cluster: _, + } => union_spans( + table_names + .iter() + .map(|i| i.name.span()) + .chain(partitions.iter().flat_map(|i| i.iter().map(|k| k.span()))), + ), + Statement::Msck { + table_name, + repair: _, + partition_action: _, + } => table_name.span(), + Statement::Query(query) => query.span(), + Statement::Insert(insert) => insert.span(), + Statement::Install { extension_name } => extension_name.span, + Statement::Load { extension_name } => extension_name.span, + Statement::Directory { + overwrite: _, + local: _, + path: _, + file_format: _, + source, + } => source.span(), + Statement::Call(function) => function.span(), + Statement::Copy { + source, + to: _, + target: _, + options: _, + legacy_options: _, + values: _, + } => source.span(), + Statement::CopyIntoSnowflake { + into: _, + from_stage: _, + from_stage_alias: _, + stage_params: _, + from_transformations: _, + files: _, + pattern: _, + file_format: _, + copy_options: _, + validation_mode: _, + } => Span::empty(), + Statement::Close { cursor } => match cursor { + CloseCursor::All => Span::empty(), + CloseCursor::Specific { name } => name.span, + }, + Statement::Update { + table, + assignments, + from, + selection, + returning, + or: _, + } => union_spans( + core::iter::once(table.span()) + .chain(assignments.iter().map(|i| i.span())) + .chain(from.iter().map(|i| i.span())) + .chain(selection.iter().map(|i| i.span())) + .chain(returning.iter().flat_map(|i| i.iter().map(|k| k.span()))), + ), + Statement::Delete(delete) => delete.span(), + Statement::CreateView { + or_replace: _, + materialized: _, + name, + columns, + query, + options, + cluster_by, + comment: _, + with_no_schema_binding: _, + if_not_exists: _, + temporary: _, + to, + } => union_spans( + core::iter::once(name.span()) + .chain(columns.iter().map(|i| i.span())) + .chain(core::iter::once(query.span())) + .chain(core::iter::once(options.span())) + .chain(cluster_by.iter().map(|i| i.span)) + .chain(to.iter().map(|i| i.span())), + ), + Statement::CreateTable(create_table) => create_table.span(), + Statement::CreateVirtualTable { + name, + if_not_exists: _, + module_name, + module_args, + } => union_spans( + core::iter::once(name.span()) + .chain(core::iter::once(module_name.span)) + .chain(module_args.iter().map(|i| i.span)), + ), + Statement::CreateIndex(create_index) => create_index.span(), + Statement::CreateRole { .. } => Span::empty(), + Statement::CreateSecret { .. } => Span::empty(), + Statement::AlterTable { + name, + if_exists: _, + only: _, + operations, + location: _, + on_cluster, + } => union_spans( + core::iter::once(name.span()) + .chain(operations.iter().map(|i| i.span())) + .chain(on_cluster.iter().map(|i| i.span)), + ), + Statement::AlterIndex { name, operation } => name.span().union(&operation.span()), + Statement::AlterView { + name, + columns, + query, + with_options, + } => union_spans( + core::iter::once(name.span()) + .chain(columns.iter().map(|i| i.span)) + .chain(core::iter::once(query.span())) + .chain(with_options.iter().map(|i| i.span())), + ), + // These statements need to be implemented + Statement::AlterRole { .. } => Span::empty(), + Statement::AttachDatabase { .. } => Span::empty(), + Statement::AttachDuckDBDatabase { .. } => Span::empty(), + Statement::DetachDuckDBDatabase { .. } => Span::empty(), + Statement::Drop { .. } => Span::empty(), + Statement::DropFunction { .. } => Span::empty(), + Statement::DropProcedure { .. } => Span::empty(), + Statement::DropSecret { .. } => Span::empty(), + Statement::Declare { .. } => Span::empty(), + Statement::CreateExtension { .. } => Span::empty(), + Statement::Fetch { .. } => Span::empty(), + Statement::Flush { .. } => Span::empty(), + Statement::Discard { .. } => Span::empty(), + Statement::SetRole { .. } => Span::empty(), + Statement::SetVariable { .. } => Span::empty(), + Statement::SetTimeZone { .. } => Span::empty(), + Statement::SetNames { .. } => Span::empty(), + Statement::SetNamesDefault {} => Span::empty(), + Statement::ShowFunctions { .. } => Span::empty(), + Statement::ShowVariable { .. } => Span::empty(), + Statement::ShowStatus { .. } => Span::empty(), + Statement::ShowVariables { .. } => Span::empty(), + Statement::ShowCreate { .. } => Span::empty(), + Statement::ShowColumns { .. } => Span::empty(), + Statement::ShowTables { .. } => Span::empty(), + Statement::ShowCollation { .. } => Span::empty(), + Statement::Use(u) => u.span(), + Statement::StartTransaction { .. } => Span::empty(), + Statement::SetTransaction { .. } => Span::empty(), + Statement::Comment { .. } => Span::empty(), + Statement::Commit { .. } => Span::empty(), + Statement::Rollback { .. } => Span::empty(), + Statement::CreateSchema { .. } => Span::empty(), + Statement::CreateDatabase { .. } => Span::empty(), + Statement::CreateFunction { .. } => Span::empty(), + Statement::CreateTrigger { .. } => Span::empty(), + Statement::DropTrigger { .. } => Span::empty(), + Statement::CreateProcedure { .. } => Span::empty(), + Statement::CreateMacro { .. } => Span::empty(), + Statement::CreateStage { .. } => Span::empty(), + Statement::Assert { .. } => Span::empty(), + Statement::Grant { .. } => Span::empty(), + Statement::Revoke { .. } => Span::empty(), + Statement::Deallocate { .. } => Span::empty(), + Statement::Execute { .. } => Span::empty(), + Statement::Prepare { .. } => Span::empty(), + Statement::Kill { .. } => Span::empty(), + Statement::ExplainTable { .. } => Span::empty(), + Statement::Explain { .. } => Span::empty(), + Statement::Savepoint { .. } => Span::empty(), + Statement::ReleaseSavepoint { .. } => Span::empty(), + Statement::Merge { .. } => Span::empty(), + Statement::Cache { .. } => Span::empty(), + Statement::UNCache { .. } => Span::empty(), + Statement::CreateSequence { .. } => Span::empty(), + Statement::CreateType { .. } => Span::empty(), + Statement::Pragma { .. } => Span::empty(), + Statement::LockTables { .. } => Span::empty(), + Statement::UnlockTables => Span::empty(), + Statement::Unload { .. } => Span::empty(), + Statement::OptimizeTable { .. } => Span::empty(), + Statement::CreatePolicy { .. } => Span::empty(), + Statement::AlterPolicy { .. } => Span::empty(), + Statement::DropPolicy { .. } => Span::empty(), + Statement::ShowDatabases { .. } => Span::empty(), + Statement::ShowSchemas { .. } => Span::empty(), + Statement::ShowViews { .. } => Span::empty(), + Statement::LISTEN { .. } => Span::empty(), + Statement::NOTIFY { .. } => Span::empty(), + Statement::LoadData { .. } => Span::empty(), + Statement::UNLISTEN { .. } => Span::empty(), + } + } +} + +impl Spanned for Use { + fn span(&self) -> Span { + match self { + Use::Catalog(object_name) => object_name.span(), + Use::Schema(object_name) => object_name.span(), + Use::Database(object_name) => object_name.span(), + Use::Warehouse(object_name) => object_name.span(), + Use::Object(object_name) => object_name.span(), + Use::Default => Span::empty(), + } + } +} + +impl Spanned for CreateTable { + fn span(&self) -> Span { + let CreateTable { + or_replace: _, // bool + temporary: _, // bool + external: _, // bool + global: _, // bool + if_not_exists: _, // bool + transient: _, // bool + volatile: _, // bool + name, + columns, + constraints, + hive_distribution: _, // hive specific + hive_formats: _, // hive specific + table_properties, + with_options, + file_format: _, // enum + location: _, // string, no span + query, + without_rowid: _, // bool + like, + clone, + engine: _, // todo + comment: _, // todo, no span + auto_increment_offset: _, // u32, no span + default_charset: _, // string, no span + collation: _, // string, no span + on_commit: _, // enum + on_cluster: _, // todo, clickhouse specific + primary_key: _, // todo, clickhouse specific + order_by: _, // todo, clickhouse specific + partition_by: _, // todo, BigQuery specific + cluster_by: _, // todo, BigQuery specific + clustered_by: _, // todo, Hive specific + options: _, // todo, BigQuery specific + strict: _, // bool + copy_grants: _, // bool + enable_schema_evolution: _, // bool + change_tracking: _, // bool + data_retention_time_in_days: _, // u64, no span + max_data_extension_time_in_days: _, // u64, no span + default_ddl_collation: _, // string, no span + with_aggregation_policy: _, // todo, Snowflake specific + with_row_access_policy: _, // todo, Snowflake specific + with_tags: _, // todo, Snowflake specific + } = self; + + union_spans( + core::iter::once(name.span()) + .chain(columns.iter().map(|i| i.span())) + .chain(constraints.iter().map(|i| i.span())) + .chain(table_properties.iter().map(|i| i.span())) + .chain(with_options.iter().map(|i| i.span())) + .chain(query.iter().map(|i| i.span())) + .chain(like.iter().map(|i| i.span())) + .chain(clone.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for ColumnDef { + fn span(&self) -> Span { + let ColumnDef { + name, + data_type: _, // enum + collation, + options, + } = self; + + union_spans( + core::iter::once(name.span) + .chain(collation.iter().map(|i| i.span())) + .chain(options.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for ColumnOptionDef { + fn span(&self) -> Span { + let ColumnOptionDef { name, option } = self; + + option.span().union_opt(&name.as_ref().map(|i| i.span)) + } +} + +impl Spanned for TableConstraint { + fn span(&self) -> Span { + match self { + TableConstraint::Unique { + name, + index_name, + index_type_display: _, + index_type: _, + columns, + index_options: _, + characteristics, + } => union_spans( + name.iter() + .map(|i| i.span) + .chain(index_name.iter().map(|i| i.span)) + .chain(columns.iter().map(|i| i.span)) + .chain(characteristics.iter().map(|i| i.span())), + ), + TableConstraint::PrimaryKey { + name, + index_name, + index_type: _, + columns, + index_options: _, + characteristics, + } => union_spans( + name.iter() + .map(|i| i.span) + .chain(index_name.iter().map(|i| i.span)) + .chain(columns.iter().map(|i| i.span)) + .chain(characteristics.iter().map(|i| i.span())), + ), + TableConstraint::ForeignKey { + name, + columns, + foreign_table, + referred_columns, + on_delete, + on_update, + characteristics, + } => union_spans( + name.iter() + .map(|i| i.span) + .chain(columns.iter().map(|i| i.span)) + .chain(core::iter::once(foreign_table.span())) + .chain(referred_columns.iter().map(|i| i.span)) + .chain(on_delete.iter().map(|i| i.span())) + .chain(on_update.iter().map(|i| i.span())) + .chain(characteristics.iter().map(|i| i.span())), + ), + TableConstraint::Check { name, expr } => { + expr.span().union_opt(&name.as_ref().map(|i| i.span)) + } + TableConstraint::Index { + display_as_key: _, + name, + index_type: _, + columns, + } => union_spans( + name.iter() + .map(|i| i.span) + .chain(columns.iter().map(|i| i.span)), + ), + TableConstraint::FulltextOrSpatial { + fulltext: _, + index_type_display: _, + opt_index_name, + columns, + } => union_spans( + opt_index_name + .iter() + .map(|i| i.span) + .chain(columns.iter().map(|i| i.span)), + ), + } + } +} + +impl Spanned for CreateIndex { + fn span(&self) -> Span { + let CreateIndex { + name, + table_name, + using, + columns, + unique: _, // bool + concurrently: _, // bool + if_not_exists: _, // bool + include, + nulls_distinct: _, // bool + with, + predicate, + } = self; + + union_spans( + name.iter() + .map(|i| i.span()) + .chain(core::iter::once(table_name.span())) + .chain(using.iter().map(|i| i.span)) + .chain(columns.iter().map(|i| i.span())) + .chain(include.iter().map(|i| i.span)) + .chain(with.iter().map(|i| i.span())) + .chain(predicate.iter().map(|i| i.span())), + ) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [ColumnOption::Null] +/// - [ColumnOption::NotNull] +/// - [ColumnOption::Comment] +/// - [ColumnOption::Unique]¨ +/// - [ColumnOption::DialectSpecific] +/// - [ColumnOption::Generated] +impl Spanned for ColumnOption { + fn span(&self) -> Span { + match self { + ColumnOption::Null => Span::empty(), + ColumnOption::NotNull => Span::empty(), + ColumnOption::Default(expr) => expr.span(), + ColumnOption::Materialized(expr) => expr.span(), + ColumnOption::Ephemeral(expr) => expr.as_ref().map_or(Span::empty(), |e| e.span()), + ColumnOption::Alias(expr) => expr.span(), + ColumnOption::Unique { .. } => Span::empty(), + ColumnOption::ForeignKey { + foreign_table, + referred_columns, + on_delete, + on_update, + characteristics, + } => union_spans( + core::iter::once(foreign_table.span()) + .chain(referred_columns.iter().map(|i| i.span)) + .chain(on_delete.iter().map(|i| i.span())) + .chain(on_update.iter().map(|i| i.span())) + .chain(characteristics.iter().map(|i| i.span())), + ), + ColumnOption::Check(expr) => expr.span(), + ColumnOption::DialectSpecific(_) => Span::empty(), + ColumnOption::CharacterSet(object_name) => object_name.span(), + ColumnOption::Comment(_) => Span::empty(), + ColumnOption::OnUpdate(expr) => expr.span(), + ColumnOption::Generated { .. } => Span::empty(), + ColumnOption::Options(vec) => union_spans(vec.iter().map(|i| i.span())), + ColumnOption::Identity(..) => Span::empty(), + ColumnOption::OnConflict(..) => Span::empty(), + ColumnOption::Policy(..) => Span::empty(), + ColumnOption::Tags(..) => Span::empty(), + } + } +} + +/// # missing span +impl Spanned for ReferentialAction { + fn span(&self) -> Span { + Span::empty() + } +} + +/// # missing span +impl Spanned for ConstraintCharacteristics { + fn span(&self) -> Span { + let ConstraintCharacteristics { + deferrable: _, // bool + initially: _, // enum + enforced: _, // bool + } = self; + + Span::empty() + } +} + +/// # partial span +/// +/// Missing spans: +/// - [AlterColumnOperation::SetNotNull] +/// - [AlterColumnOperation::DropNotNull] +/// - [AlterColumnOperation::DropDefault] +/// - [AlterColumnOperation::AddGenerated] +impl Spanned for AlterColumnOperation { + fn span(&self) -> Span { + match self { + AlterColumnOperation::SetNotNull => Span::empty(), + AlterColumnOperation::DropNotNull => Span::empty(), + AlterColumnOperation::SetDefault { value } => value.span(), + AlterColumnOperation::DropDefault => Span::empty(), + AlterColumnOperation::SetDataType { + data_type: _, + using, + } => using.as_ref().map_or(Span::empty(), |u| u.span()), + AlterColumnOperation::AddGenerated { .. } => Span::empty(), + } + } +} + +impl Spanned for CopySource { + fn span(&self) -> Span { + match self { + CopySource::Table { + table_name, + columns, + } => union_spans( + core::iter::once(table_name.span()).chain(columns.iter().map(|i| i.span)), + ), + CopySource::Query(query) => query.span(), + } + } +} + +impl Spanned for Delete { + fn span(&self) -> Span { + let Delete { + tables, + from, + using, + selection, + returning, + order_by, + limit, + } = self; + + union_spans( + tables + .iter() + .map(|i| i.span()) + .chain(core::iter::once(from.span())) + .chain( + using + .iter() + .map(|u| union_spans(u.iter().map(|i| i.span()))), + ) + .chain(selection.iter().map(|i| i.span())) + .chain(returning.iter().flat_map(|i| i.iter().map(|k| k.span()))) + .chain(order_by.iter().map(|i| i.span())) + .chain(limit.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for FromTable { + fn span(&self) -> Span { + match self { + FromTable::WithFromKeyword(vec) => union_spans(vec.iter().map(|i| i.span())), + FromTable::WithoutKeyword(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +impl Spanned for ViewColumnDef { + fn span(&self) -> Span { + let ViewColumnDef { + name, + data_type: _, // todo, DataType + options, + } = self; + + union_spans( + core::iter::once(name.span) + .chain(options.iter().flat_map(|i| i.iter().map(|k| k.span()))), + ) + } +} + +impl Spanned for SqlOption { + fn span(&self) -> Span { + match self { + SqlOption::Clustered(table_options_clustered) => table_options_clustered.span(), + SqlOption::Ident(ident) => ident.span, + SqlOption::KeyValue { key, value } => key.span.union(&value.span()), + SqlOption::Partition { + column_name, + range_direction: _, + for_values, + } => union_spans( + core::iter::once(column_name.span).chain(for_values.iter().map(|i| i.span())), + ), + } + } +} + +/// # partial span +/// +/// Missing spans: +/// - [TableOptionsClustered::ColumnstoreIndex] +impl Spanned for TableOptionsClustered { + fn span(&self) -> Span { + match self { + TableOptionsClustered::ColumnstoreIndex => Span::empty(), + TableOptionsClustered::ColumnstoreIndexOrder(vec) => { + union_spans(vec.iter().map(|i| i.span)) + } + TableOptionsClustered::Index(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +impl Spanned for ClusteredIndex { + fn span(&self) -> Span { + let ClusteredIndex { + name, + asc: _, // bool + } = self; + + name.span + } +} + +impl Spanned for CreateTableOptions { + fn span(&self) -> Span { + match self { + CreateTableOptions::None => Span::empty(), + CreateTableOptions::With(vec) => union_spans(vec.iter().map(|i| i.span())), + CreateTableOptions::Options(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +/// # partial span +/// +/// Missing spans: +/// - [AlterTableOperation::OwnerTo] +impl Spanned for AlterTableOperation { + fn span(&self) -> Span { + match self { + AlterTableOperation::AddConstraint(table_constraint) => table_constraint.span(), + AlterTableOperation::AddColumn { + column_keyword: _, + if_not_exists: _, + column_def, + column_position: _, + } => column_def.span(), + AlterTableOperation::AddProjection { + if_not_exists: _, + name, + select, + } => name.span.union(&select.span()), + AlterTableOperation::DropProjection { if_exists: _, name } => name.span, + AlterTableOperation::MaterializeProjection { + if_exists: _, + name, + partition, + } => name.span.union_opt(&partition.as_ref().map(|i| i.span)), + AlterTableOperation::ClearProjection { + if_exists: _, + name, + partition, + } => name.span.union_opt(&partition.as_ref().map(|i| i.span)), + AlterTableOperation::DisableRowLevelSecurity => Span::empty(), + AlterTableOperation::DisableRule { name } => name.span, + AlterTableOperation::DisableTrigger { name } => name.span, + AlterTableOperation::DropConstraint { + if_exists: _, + name, + cascade: _, + } => name.span, + AlterTableOperation::DropColumn { + column_name, + if_exists: _, + cascade: _, + } => column_name.span, + AlterTableOperation::AttachPartition { partition } => partition.span(), + AlterTableOperation::DetachPartition { partition } => partition.span(), + AlterTableOperation::FreezePartition { + partition, + with_name, + } => partition + .span() + .union_opt(&with_name.as_ref().map(|n| n.span)), + AlterTableOperation::UnfreezePartition { + partition, + with_name, + } => partition + .span() + .union_opt(&with_name.as_ref().map(|n| n.span)), + AlterTableOperation::DropPrimaryKey => Span::empty(), + AlterTableOperation::EnableAlwaysRule { name } => name.span, + AlterTableOperation::EnableAlwaysTrigger { name } => name.span, + AlterTableOperation::EnableReplicaRule { name } => name.span, + AlterTableOperation::EnableReplicaTrigger { name } => name.span, + AlterTableOperation::EnableRowLevelSecurity => Span::empty(), + AlterTableOperation::EnableRule { name } => name.span, + AlterTableOperation::EnableTrigger { name } => name.span, + AlterTableOperation::RenamePartitions { + old_partitions, + new_partitions, + } => union_spans( + old_partitions + .iter() + .map(|i| i.span()) + .chain(new_partitions.iter().map(|i| i.span())), + ), + AlterTableOperation::AddPartitions { + if_not_exists: _, + new_partitions, + } => union_spans(new_partitions.iter().map(|i| i.span())), + AlterTableOperation::DropPartitions { + partitions, + if_exists: _, + } => union_spans(partitions.iter().map(|i| i.span())), + AlterTableOperation::RenameColumn { + old_column_name, + new_column_name, + } => old_column_name.span.union(&new_column_name.span), + AlterTableOperation::RenameTable { table_name } => table_name.span(), + AlterTableOperation::ChangeColumn { + old_name, + new_name, + data_type: _, + options, + column_position: _, + } => union_spans( + core::iter::once(old_name.span) + .chain(core::iter::once(new_name.span)) + .chain(options.iter().map(|i| i.span())), + ), + AlterTableOperation::ModifyColumn { + col_name, + data_type: _, + options, + column_position: _, + } => { + union_spans(core::iter::once(col_name.span).chain(options.iter().map(|i| i.span()))) + } + AlterTableOperation::RenameConstraint { old_name, new_name } => { + old_name.span.union(&new_name.span) + } + AlterTableOperation::AlterColumn { column_name, op } => { + column_name.span.union(&op.span()) + } + AlterTableOperation::SwapWith { table_name } => table_name.span(), + AlterTableOperation::SetTblProperties { table_properties } => { + union_spans(table_properties.iter().map(|i| i.span())) + } + AlterTableOperation::OwnerTo { .. } => Span::empty(), + } + } +} + +impl Spanned for Partition { + fn span(&self) -> Span { + match self { + Partition::Identifier(ident) => ident.span, + Partition::Expr(expr) => expr.span(), + Partition::Part(expr) => expr.span(), + Partition::Partitions(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +impl Spanned for ProjectionSelect { + fn span(&self) -> Span { + let ProjectionSelect { + projection, + order_by, + group_by, + } = self; + + union_spans( + projection + .iter() + .map(|i| i.span()) + .chain(order_by.iter().map(|i| i.span())) + .chain(group_by.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for OrderBy { + fn span(&self) -> Span { + let OrderBy { exprs, interpolate } = self; + + union_spans( + exprs + .iter() + .map(|i| i.span()) + .chain(interpolate.iter().map(|i| i.span())), + ) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [GroupByExpr::All] +impl Spanned for GroupByExpr { + fn span(&self) -> Span { + match self { + GroupByExpr::All(_) => Span::empty(), + GroupByExpr::Expressions(exprs, _modifiers) => { + union_spans(exprs.iter().map(|i| i.span())) + } + } + } +} + +impl Spanned for Interpolate { + fn span(&self) -> Span { + let Interpolate { exprs } = self; + + union_spans(exprs.iter().flat_map(|i| i.iter().map(|e| e.span()))) + } +} + +impl Spanned for InterpolateExpr { + fn span(&self) -> Span { + let InterpolateExpr { column, expr } = self; + + column.span.union_opt(&expr.as_ref().map(|e| e.span())) + } +} + +impl Spanned for AlterIndexOperation { + fn span(&self) -> Span { + match self { + AlterIndexOperation::RenameIndex { index_name } => index_name.span(), + } + } +} + +/// # partial span +/// +/// Missing spans:ever +/// - [Insert::insert_alias] +impl Spanned for Insert { + fn span(&self) -> Span { + let Insert { + or: _, // enum, sqlite specific + ignore: _, // bool + into: _, // bool + table_name, + table_alias, + columns, + overwrite: _, // bool + source, + partitioned, + after_columns, + table: _, // bool + on, + returning, + replace_into: _, // bool + priority: _, // todo, mysql specific + insert_alias: _, // todo, mysql specific + } = self; + + union_spans( + core::iter::once(table_name.span()) + .chain(table_alias.as_ref().map(|i| i.span)) + .chain(columns.iter().map(|i| i.span)) + .chain(source.as_ref().map(|q| q.span())) + .chain(partitioned.iter().flat_map(|i| i.iter().map(|k| k.span()))) + .chain(after_columns.iter().map(|i| i.span)) + .chain(on.as_ref().map(|i| i.span())) + .chain(returning.iter().flat_map(|i| i.iter().map(|k| k.span()))), + ) + } +} + +impl Spanned for OnInsert { + fn span(&self) -> Span { + match self { + OnInsert::DuplicateKeyUpdate(vec) => union_spans(vec.iter().map(|i| i.span())), + OnInsert::OnConflict(on_conflict) => on_conflict.span(), + } + } +} + +impl Spanned for OnConflict { + fn span(&self) -> Span { + let OnConflict { + conflict_target, + action, + } = self; + + action + .span() + .union_opt(&conflict_target.as_ref().map(|i| i.span())) + } +} + +impl Spanned for ConflictTarget { + fn span(&self) -> Span { + match self { + ConflictTarget::Columns(vec) => union_spans(vec.iter().map(|i| i.span)), + ConflictTarget::OnConstraint(object_name) => object_name.span(), + } + } +} + +/// # partial span +/// +/// Missing spans: +/// - [OnConflictAction::DoNothing] +impl Spanned for OnConflictAction { + fn span(&self) -> Span { + match self { + OnConflictAction::DoNothing => Span::empty(), + OnConflictAction::DoUpdate(do_update) => do_update.span(), + } + } +} + +impl Spanned for DoUpdate { + fn span(&self) -> Span { + let DoUpdate { + assignments, + selection, + } = self; + + union_spans( + assignments + .iter() + .map(|i| i.span()) + .chain(selection.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for Assignment { + fn span(&self) -> Span { + let Assignment { target, value } = self; + + target.span().union(&value.span()) + } +} + +impl Spanned for AssignmentTarget { + fn span(&self) -> Span { + match self { + AssignmentTarget::ColumnName(object_name) => object_name.span(), + AssignmentTarget::Tuple(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +/// # partial span +/// +/// Most expressions are missing keywords in their spans. +/// f.e. `IS NULL ` reports as `::span`. +/// +/// Missing spans: +/// - [Expr::TypedString] +/// - [Expr::MatchAgainst] # MySQL specific +/// - [Expr::RLike] # MySQL specific +/// - [Expr::Struct] # BigQuery specific +/// - [Expr::Named] # BigQuery specific +/// - [Expr::Dictionary] # DuckDB specific +/// - [Expr::Map] # DuckDB specific +/// - [Expr::Lambda] +impl Spanned for Expr { + fn span(&self) -> Span { + match self { + Expr::Identifier(ident) => ident.span, + Expr::CompoundIdentifier(vec) => union_spans(vec.iter().map(|i| i.span)), + Expr::CompositeAccess { expr, key } => expr.span().union(&key.span), + Expr::IsFalse(expr) => expr.span(), + Expr::IsNotFalse(expr) => expr.span(), + Expr::IsTrue(expr) => expr.span(), + Expr::IsNotTrue(expr) => expr.span(), + Expr::IsNull(expr) => expr.span(), + Expr::IsNotNull(expr) => expr.span(), + Expr::IsUnknown(expr) => expr.span(), + Expr::IsNotUnknown(expr) => expr.span(), + Expr::IsDistinctFrom(lhs, rhs) => lhs.span().union(&rhs.span()), + Expr::IsNotDistinctFrom(lhs, rhs) => lhs.span().union(&rhs.span()), + Expr::InList { + expr, + list, + negated: _, + } => union_spans( + core::iter::once(expr.span()).chain(list.iter().map(|item| item.span())), + ), + Expr::InSubquery { + expr, + subquery, + negated: _, + } => expr.span().union(&subquery.span()), + Expr::InUnnest { + expr, + array_expr, + negated: _, + } => expr.span().union(&array_expr.span()), + Expr::Between { + expr, + negated: _, + low, + high, + } => expr.span().union(&low.span()).union(&high.span()), + + Expr::BinaryOp { left, op: _, right } => left.span().union(&right.span()), + Expr::Like { + negated: _, + expr, + pattern, + escape_char: _, + any: _, + } => expr.span().union(&pattern.span()), + Expr::ILike { + negated: _, + expr, + pattern, + escape_char: _, + any: _, + } => expr.span().union(&pattern.span()), + Expr::SimilarTo { + negated: _, + expr, + pattern, + escape_char: _, + } => expr.span().union(&pattern.span()), + Expr::Ceil { expr, field: _ } => expr.span(), + Expr::Floor { expr, field: _ } => expr.span(), + Expr::Position { expr, r#in } => expr.span().union(&r#in.span()), + Expr::Overlay { + expr, + overlay_what, + overlay_from, + overlay_for, + } => expr + .span() + .union(&overlay_what.span()) + .union(&overlay_from.span()) + .union_opt(&overlay_for.as_ref().map(|i| i.span())), + Expr::Collate { expr, collation } => expr + .span() + .union(&union_spans(collation.0.iter().map(|i| i.span))), + Expr::Nested(expr) => expr.span(), + Expr::Value(value) => value.span(), + Expr::TypedString { .. } => Span::empty(), + Expr::MapAccess { column, keys } => column + .span() + .union(&union_spans(keys.iter().map(|i| i.key.span()))), + Expr::Function(function) => function.span(), + Expr::GroupingSets(vec) => { + union_spans(vec.iter().flat_map(|i| i.iter().map(|k| k.span()))) + } + Expr::Cube(vec) => union_spans(vec.iter().flat_map(|i| i.iter().map(|k| k.span()))), + Expr::Rollup(vec) => union_spans(vec.iter().flat_map(|i| i.iter().map(|k| k.span()))), + Expr::Tuple(vec) => union_spans(vec.iter().map(|i| i.span())), + Expr::Array(array) => array.span(), + Expr::MatchAgainst { .. } => Span::empty(), + Expr::JsonAccess { value, path } => value.span().union(&path.span()), + Expr::RLike { .. } => Span::empty(), + Expr::AnyOp { + left, + compare_op: _, + right, + is_some: _, + } => left.span().union(&right.span()), + Expr::AllOp { + left, + compare_op: _, + right, + } => left.span().union(&right.span()), + Expr::UnaryOp { op: _, expr } => expr.span(), + Expr::Convert { + expr, + data_type: _, + charset, + target_before_value: _, + styles, + is_try: _, + } => union_spans( + core::iter::once(expr.span()) + .chain(charset.as_ref().map(|i| i.span())) + .chain(styles.iter().map(|i| i.span())), + ), + Expr::Cast { + kind: _, + expr, + data_type: _, + format: _, + } => expr.span(), + Expr::AtTimeZone { + timestamp, + time_zone, + } => timestamp.span().union(&time_zone.span()), + Expr::Extract { + field: _, + syntax: _, + expr, + } => expr.span(), + Expr::Substring { + expr, + substring_from, + substring_for, + special: _, + } => union_spans( + core::iter::once(expr.span()) + .chain(substring_from.as_ref().map(|i| i.span())) + .chain(substring_for.as_ref().map(|i| i.span())), + ), + Expr::Trim { + expr, + trim_where: _, + trim_what, + trim_characters, + } => union_spans( + core::iter::once(expr.span()) + .chain(trim_what.as_ref().map(|i| i.span())) + .chain( + trim_characters + .as_ref() + .map(|items| union_spans(items.iter().map(|i| i.span()))), + ), + ), + Expr::IntroducedString { value, .. } => value.span(), + Expr::Case { + operand, + conditions, + results, + else_result, + } => union_spans( + operand + .as_ref() + .map(|i| i.span()) + .into_iter() + .chain(conditions.iter().map(|i| i.span())) + .chain(results.iter().map(|i| i.span())) + .chain(else_result.as_ref().map(|i| i.span())), + ), + Expr::Exists { subquery, .. } => subquery.span(), + Expr::Subquery(query) => query.span(), + Expr::Struct { .. } => Span::empty(), + Expr::Named { .. } => Span::empty(), + Expr::Dictionary(_) => Span::empty(), + Expr::Map(_) => Span::empty(), + Expr::Subscript { expr, subscript } => expr.span().union(&subscript.span()), + Expr::Interval(interval) => interval.value.span(), + Expr::Wildcard(token) => token.0.span, + Expr::QualifiedWildcard(object_name, token) => union_spans( + object_name + .0 + .iter() + .map(|i| i.span) + .chain(iter::once(token.0.span)), + ), + Expr::OuterJoin(expr) => expr.span(), + Expr::Prior(expr) => expr.span(), + Expr::Lambda(_) => Span::empty(), + Expr::Method(_) => Span::empty(), + } + } +} + +impl Spanned for Subscript { + fn span(&self) -> Span { + match self { + Subscript::Index { index } => index.span(), + Subscript::Slice { + lower_bound, + upper_bound, + stride, + } => union_spans( + [ + lower_bound.as_ref().map(|i| i.span()), + upper_bound.as_ref().map(|i| i.span()), + stride.as_ref().map(|i| i.span()), + ] + .into_iter() + .flatten(), + ), + } + } +} + +impl Spanned for ObjectName { + fn span(&self) -> Span { + let ObjectName(segments) = self; + + union_spans(segments.iter().map(|i| i.span)) + } +} + +impl Spanned for Array { + fn span(&self) -> Span { + let Array { + elem, + named: _, // bool + } = self; + + union_spans(elem.iter().map(|i| i.span())) + } +} + +impl Spanned for Function { + fn span(&self) -> Span { + let Function { + name, + parameters, + args, + filter, + null_treatment: _, // enum + over: _, // todo + within_group, + } = self; + + union_spans( + name.0 + .iter() + .map(|i| i.span) + .chain(iter::once(args.span())) + .chain(iter::once(parameters.span())) + .chain(filter.iter().map(|i| i.span())) + .chain(within_group.iter().map(|i| i.span())), + ) + } +} + +/// # partial span +/// +/// The span of [FunctionArguments::None] is empty. +impl Spanned for FunctionArguments { + fn span(&self) -> Span { + match self { + FunctionArguments::None => Span::empty(), + FunctionArguments::Subquery(query) => query.span(), + FunctionArguments::List(list) => list.span(), + } + } +} + +impl Spanned for FunctionArgumentList { + fn span(&self) -> Span { + let FunctionArgumentList { + duplicate_treatment: _, // enum + args, + clauses, + } = self; + + union_spans( + // # todo: duplicate-treatment span + args.iter() + .map(|i| i.span()) + .chain(clauses.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for FunctionArgumentClause { + fn span(&self) -> Span { + match self { + FunctionArgumentClause::IgnoreOrRespectNulls(_) => Span::empty(), + FunctionArgumentClause::OrderBy(vec) => union_spans(vec.iter().map(|i| i.expr.span())), + FunctionArgumentClause::Limit(expr) => expr.span(), + FunctionArgumentClause::OnOverflow(_) => Span::empty(), + FunctionArgumentClause::Having(HavingBound(_kind, expr)) => expr.span(), + FunctionArgumentClause::Separator(value) => value.span(), + FunctionArgumentClause::JsonNullClause(_) => Span::empty(), + } + } +} + +/// # partial span +/// +/// see Spanned impl for JsonPathElem for more information +impl Spanned for JsonPath { + fn span(&self) -> Span { + let JsonPath { path } = self; + + union_spans(path.iter().map(|i| i.span())) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [JsonPathElem::Dot] +impl Spanned for JsonPathElem { + fn span(&self) -> Span { + match self { + JsonPathElem::Dot { .. } => Span::empty(), + JsonPathElem::Bracket { key } => key.span(), + } + } +} + +impl Spanned for SelectItem { + fn span(&self) -> Span { + match self { + SelectItem::UnnamedExpr(expr) => expr.span(), + SelectItem::ExprWithAlias { expr, alias } => expr.span().union(&alias.span), + SelectItem::QualifiedWildcard(object_name, wildcard_additional_options) => union_spans( + object_name + .0 + .iter() + .map(|i| i.span) + .chain(iter::once(wildcard_additional_options.span())), + ), + SelectItem::Wildcard(wildcard_additional_options) => wildcard_additional_options.span(), + } + } +} + +impl Spanned for WildcardAdditionalOptions { + fn span(&self) -> Span { + let WildcardAdditionalOptions { + wildcard_token, + opt_ilike, + opt_exclude, + opt_except, + opt_replace, + opt_rename, + } = self; + + union_spans( + core::iter::once(wildcard_token.0.span) + .chain(opt_ilike.as_ref().map(|i| i.span())) + .chain(opt_exclude.as_ref().map(|i| i.span())) + .chain(opt_rename.as_ref().map(|i| i.span())) + .chain(opt_replace.as_ref().map(|i| i.span())) + .chain(opt_except.as_ref().map(|i| i.span())), + ) + } +} + +/// # missing span +impl Spanned for IlikeSelectItem { + fn span(&self) -> Span { + Span::empty() + } +} + +impl Spanned for ExcludeSelectItem { + fn span(&self) -> Span { + match self { + ExcludeSelectItem::Single(ident) => ident.span, + ExcludeSelectItem::Multiple(vec) => union_spans(vec.iter().map(|i| i.span)), + } + } +} + +impl Spanned for RenameSelectItem { + fn span(&self) -> Span { + match self { + RenameSelectItem::Single(ident) => ident.ident.span.union(&ident.alias.span), + RenameSelectItem::Multiple(vec) => { + union_spans(vec.iter().map(|i| i.ident.span.union(&i.alias.span))) + } + } + } +} + +impl Spanned for ExceptSelectItem { + fn span(&self) -> Span { + let ExceptSelectItem { + first_element, + additional_elements, + } = self; + + union_spans( + iter::once(first_element.span).chain(additional_elements.iter().map(|i| i.span)), + ) + } +} + +impl Spanned for ReplaceSelectItem { + fn span(&self) -> Span { + let ReplaceSelectItem { items } = self; + + union_spans(items.iter().map(|i| i.span())) + } +} + +impl Spanned for ReplaceSelectElement { + fn span(&self) -> Span { + let ReplaceSelectElement { + expr, + column_name, + as_keyword: _, // bool + } = self; + + expr.span().union(&column_name.span) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [TableFactor::JsonTable] +impl Spanned for TableFactor { + fn span(&self) -> Span { + match self { + TableFactor::Table { + name, + alias, + args: _, + with_hints: _, + version: _, + with_ordinality: _, + partitions: _, + json_path: _, + } => union_spans( + name.0 + .iter() + .map(|i| i.span) + .chain(alias.as_ref().map(|alias| { + union_spans( + iter::once(alias.name.span) + .chain(alias.columns.iter().map(|i| i.span())), + ) + })), + ), + TableFactor::Derived { + lateral: _, + subquery, + alias, + } => subquery + .span() + .union_opt(&alias.as_ref().map(|alias| alias.span())), + TableFactor::TableFunction { expr, alias } => expr + .span() + .union_opt(&alias.as_ref().map(|alias| alias.span())), + TableFactor::UNNEST { + alias, + with_offset: _, + with_offset_alias, + array_exprs, + with_ordinality: _, + } => union_spans( + alias + .iter() + .map(|i| i.span()) + .chain(array_exprs.iter().map(|i| i.span())) + .chain(with_offset_alias.as_ref().map(|i| i.span)), + ), + TableFactor::NestedJoin { + table_with_joins, + alias, + } => table_with_joins + .span() + .union_opt(&alias.as_ref().map(|alias| alias.span())), + TableFactor::Function { + lateral: _, + name, + args, + alias, + } => union_spans( + name.0 + .iter() + .map(|i| i.span) + .chain(args.iter().map(|i| i.span())) + .chain(alias.as_ref().map(|alias| alias.span())), + ), + TableFactor::JsonTable { .. } => Span::empty(), + TableFactor::Pivot { + table, + aggregate_functions, + value_column, + value_source, + default_on_null, + alias, + } => union_spans( + core::iter::once(table.span()) + .chain(aggregate_functions.iter().map(|i| i.span())) + .chain(value_column.iter().map(|i| i.span)) + .chain(core::iter::once(value_source.span())) + .chain(default_on_null.as_ref().map(|i| i.span())) + .chain(alias.as_ref().map(|i| i.span())), + ), + TableFactor::Unpivot { + table, + value, + name, + columns, + alias, + } => union_spans( + core::iter::once(table.span()) + .chain(core::iter::once(value.span)) + .chain(core::iter::once(name.span)) + .chain(columns.iter().map(|i| i.span)) + .chain(alias.as_ref().map(|alias| alias.span())), + ), + TableFactor::MatchRecognize { + table, + partition_by, + order_by, + measures, + rows_per_match: _, + after_match_skip: _, + pattern, + symbols, + alias, + } => union_spans( + core::iter::once(table.span()) + .chain(partition_by.iter().map(|i| i.span())) + .chain(order_by.iter().map(|i| i.span())) + .chain(measures.iter().map(|i| i.span())) + .chain(core::iter::once(pattern.span())) + .chain(symbols.iter().map(|i| i.span())) + .chain(alias.as_ref().map(|i| i.span())), + ), + TableFactor::OpenJsonTable { .. } => Span::empty(), + } + } +} + +impl Spanned for PivotValueSource { + fn span(&self) -> Span { + match self { + PivotValueSource::List(vec) => union_spans(vec.iter().map(|i| i.span())), + PivotValueSource::Any(vec) => union_spans(vec.iter().map(|i| i.span())), + PivotValueSource::Subquery(query) => query.span(), + } + } +} + +impl Spanned for ExprWithAlias { + fn span(&self) -> Span { + let ExprWithAlias { expr, alias } = self; + + expr.span().union_opt(&alias.as_ref().map(|i| i.span)) + } +} + +/// # missing span +impl Spanned for MatchRecognizePattern { + fn span(&self) -> Span { + Span::empty() + } +} + +impl Spanned for SymbolDefinition { + fn span(&self) -> Span { + let SymbolDefinition { symbol, definition } = self; + + symbol.span.union(&definition.span()) + } +} + +impl Spanned for Measure { + fn span(&self) -> Span { + let Measure { expr, alias } = self; + + expr.span().union(&alias.span) + } +} + +impl Spanned for OrderByExpr { + fn span(&self) -> Span { + let OrderByExpr { + expr, + asc: _, // bool + nulls_first: _, // bool + with_fill, + } = self; + + expr.span().union_opt(&with_fill.as_ref().map(|f| f.span())) + } +} + +impl Spanned for WithFill { + fn span(&self) -> Span { + let WithFill { from, to, step } = self; + + union_spans( + from.iter() + .map(|f| f.span()) + .chain(to.iter().map(|t| t.span())) + .chain(step.iter().map(|s| s.span())), + ) + } +} + +impl Spanned for FunctionArg { + fn span(&self) -> Span { + match self { + FunctionArg::Named { + name, + arg, + operator: _, + } => name.span.union(&arg.span()), + FunctionArg::Unnamed(arg) => arg.span(), + FunctionArg::ExprNamed { + name, + arg, + operator: _, + } => name.span().union(&arg.span()), + } + } +} + +/// # partial span +/// +/// Missing spans: +/// - [FunctionArgExpr::Wildcard] +impl Spanned for FunctionArgExpr { + fn span(&self) -> Span { + match self { + FunctionArgExpr::Expr(expr) => expr.span(), + FunctionArgExpr::QualifiedWildcard(object_name) => { + union_spans(object_name.0.iter().map(|i| i.span)) + } + FunctionArgExpr::Wildcard => Span::empty(), + } + } +} + +impl Spanned for TableAlias { + fn span(&self) -> Span { + let TableAlias { name, columns } = self; + + union_spans(iter::once(name.span).chain(columns.iter().map(|i| i.span()))) + } +} + +impl Spanned for TableAliasColumnDef { + fn span(&self) -> Span { + let TableAliasColumnDef { name, data_type: _ } = self; + + name.span + } +} + +/// # missing span +/// +/// The span of a `Value` is currently not implemented, as doing so +/// requires a breaking changes, which may be done in a future release. +impl Spanned for Value { + fn span(&self) -> Span { + Span::empty() // # todo: Value needs to store spans before this is possible + } +} + +impl Spanned for Join { + fn span(&self) -> Span { + let Join { + relation, + global: _, // bool + join_operator, + } = self; + + relation.span().union(&join_operator.span()) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [JoinOperator::CrossJoin] +/// - [JoinOperator::CrossApply] +/// - [JoinOperator::OuterApply] +impl Spanned for JoinOperator { + fn span(&self) -> Span { + match self { + JoinOperator::Inner(join_constraint) => join_constraint.span(), + JoinOperator::LeftOuter(join_constraint) => join_constraint.span(), + JoinOperator::RightOuter(join_constraint) => join_constraint.span(), + JoinOperator::FullOuter(join_constraint) => join_constraint.span(), + JoinOperator::CrossJoin => Span::empty(), + JoinOperator::LeftSemi(join_constraint) => join_constraint.span(), + JoinOperator::RightSemi(join_constraint) => join_constraint.span(), + JoinOperator::LeftAnti(join_constraint) => join_constraint.span(), + JoinOperator::RightAnti(join_constraint) => join_constraint.span(), + JoinOperator::CrossApply => Span::empty(), + JoinOperator::OuterApply => Span::empty(), + JoinOperator::AsOf { + match_condition, + constraint, + } => match_condition.span().union(&constraint.span()), + JoinOperator::Anti(join_constraint) => join_constraint.span(), + JoinOperator::Semi(join_constraint) => join_constraint.span(), + } + } +} + +/// # partial span +/// +/// Missing spans: +/// - [JoinConstraint::Natural] +/// - [JoinConstraint::None] +impl Spanned for JoinConstraint { + fn span(&self) -> Span { + match self { + JoinConstraint::On(expr) => expr.span(), + JoinConstraint::Using(vec) => union_spans(vec.iter().map(|i| i.span)), + JoinConstraint::Natural => Span::empty(), + JoinConstraint::None => Span::empty(), + } + } +} + +impl Spanned for TableWithJoins { + fn span(&self) -> Span { + let TableWithJoins { relation, joins } = self; + + union_spans(core::iter::once(relation.span()).chain(joins.iter().map(|item| item.span()))) + } +} + +impl Spanned for Select { + fn span(&self) -> Span { + let Select { + select_token, + distinct: _, // todo + top: _, // todo, mysql specific + projection, + into, + from, + lateral_views, + prewhere, + selection, + group_by, + cluster_by, + distribute_by, + sort_by, + having, + named_window, + qualify, + window_before_qualify: _, // bool + value_table_mode: _, // todo, BigQuery specific + connect_by, + top_before_distinct: _, + } = self; + + union_spans( + core::iter::once(select_token.0.span) + .chain(projection.iter().map(|item| item.span())) + .chain(into.iter().map(|item| item.span())) + .chain(from.iter().map(|item| item.span())) + .chain(lateral_views.iter().map(|item| item.span())) + .chain(prewhere.iter().map(|item| item.span())) + .chain(selection.iter().map(|item| item.span())) + .chain(core::iter::once(group_by.span())) + .chain(cluster_by.iter().map(|item| item.span())) + .chain(distribute_by.iter().map(|item| item.span())) + .chain(sort_by.iter().map(|item| item.span())) + .chain(having.iter().map(|item| item.span())) + .chain(named_window.iter().map(|item| item.span())) + .chain(qualify.iter().map(|item| item.span())) + .chain(connect_by.iter().map(|item| item.span())), + ) + } +} + +impl Spanned for ConnectBy { + fn span(&self) -> Span { + let ConnectBy { + condition, + relationships, + } = self; + + union_spans( + core::iter::once(condition.span()).chain(relationships.iter().map(|item| item.span())), + ) + } +} + +impl Spanned for NamedWindowDefinition { + fn span(&self) -> Span { + let NamedWindowDefinition( + ident, + _, // todo: NamedWindowExpr + ) = self; + + ident.span + } +} + +impl Spanned for LateralView { + fn span(&self) -> Span { + let LateralView { + lateral_view, + lateral_view_name, + lateral_col_alias, + outer: _, // bool + } = self; + + union_spans( + core::iter::once(lateral_view.span()) + .chain(core::iter::once(lateral_view_name.span())) + .chain(lateral_col_alias.iter().map(|i| i.span)), + ) + } +} + +impl Spanned for SelectInto { + fn span(&self) -> Span { + let SelectInto { + temporary: _, // bool + unlogged: _, // bool + table: _, // bool + name, + } = self; + + name.span() + } +} + +#[cfg(test)] +pub mod tests { + use crate::dialect::{Dialect, GenericDialect, SnowflakeDialect}; + use crate::parser::Parser; + use crate::tokenizer::Span; + + use super::*; + + struct SpanTest<'a>(Parser<'a>, &'a str); + + impl<'a> SpanTest<'a> { + fn new(dialect: &'a dyn Dialect, sql: &'a str) -> Self { + Self(Parser::new(dialect).try_with_sql(sql).unwrap(), sql) + } + + // get the subsection of the source string that corresponds to the span + // only works on single-line strings + fn get_source(&self, span: Span) -> &'a str { + // lines in spans are 1-indexed + &self.1[(span.start.column as usize - 1)..(span.end.column - 1) as usize] + } + } + + #[test] + fn test_join() { + let dialect = &GenericDialect; + let mut test = SpanTest::new( + dialect, + "SELECT id, name FROM users LEFT JOIN companies ON users.company_id = companies.id", + ); + + let query = test.0.parse_select().unwrap(); + let select_span = query.span(); + + assert_eq!( + test.get_source(select_span), + "SELECT id, name FROM users LEFT JOIN companies ON users.company_id = companies.id" + ); + + let join_span = query.from[0].joins[0].span(); + + // 'LEFT JOIN' missing + assert_eq!( + test.get_source(join_span), + "companies ON users.company_id = companies.id" + ); + } + + #[test] + pub fn test_union() { + let dialect = &GenericDialect; + let mut test = SpanTest::new( + dialect, + "SELECT a FROM postgres.public.source UNION SELECT a FROM postgres.public.source", + ); + + let query = test.0.parse_query().unwrap(); + let select_span = query.span(); + + assert_eq!( + test.get_source(select_span), + "SELECT a FROM postgres.public.source UNION SELECT a FROM postgres.public.source" + ); + } + + #[test] + pub fn test_subquery() { + let dialect = &GenericDialect; + let mut test = SpanTest::new( + dialect, + "SELECT a FROM (SELECT a FROM postgres.public.source) AS b", + ); + + let query = test.0.parse_select().unwrap(); + let select_span = query.span(); + + assert_eq!( + test.get_source(select_span), + "SELECT a FROM (SELECT a FROM postgres.public.source) AS b" + ); + + let subquery_span = query.from[0].span(); + + // left paren missing + assert_eq!( + test.get_source(subquery_span), + "SELECT a FROM postgres.public.source) AS b" + ); + } + + #[test] + pub fn test_cte() { + let dialect = &GenericDialect; + let mut test = SpanTest::new(dialect, "WITH cte_outer AS (SELECT a FROM postgres.public.source), cte_ignored AS (SELECT a FROM cte_outer), cte_inner AS (SELECT a FROM cte_outer) SELECT a FROM cte_inner"); + + let query = test.0.parse_query().unwrap(); + + let select_span = query.span(); + + assert_eq!(test.get_source(select_span), "WITH cte_outer AS (SELECT a FROM postgres.public.source), cte_ignored AS (SELECT a FROM cte_outer), cte_inner AS (SELECT a FROM cte_outer) SELECT a FROM cte_inner"); + } + + #[test] + pub fn test_snowflake_lateral_flatten() { + let dialect = &SnowflakeDialect; + let mut test = SpanTest::new(dialect, "SELECT FLATTENED.VALUE:field::TEXT AS FIELD FROM SNOWFLAKE.SCHEMA.SOURCE AS S, LATERAL FLATTEN(INPUT => S.JSON_ARRAY) AS FLATTENED"); + + let query = test.0.parse_select().unwrap(); + + let select_span = query.span(); + + assert_eq!(test.get_source(select_span), "SELECT FLATTENED.VALUE:field::TEXT AS FIELD FROM SNOWFLAKE.SCHEMA.SOURCE AS S, LATERAL FLATTEN(INPUT => S.JSON_ARRAY) AS FLATTENED"); + } + + #[test] + pub fn test_wildcard_from_cte() { + let dialect = &GenericDialect; + let mut test = SpanTest::new( + dialect, + "WITH cte AS (SELECT a FROM postgres.public.source) SELECT cte.* FROM cte", + ); + + let query = test.0.parse_query().unwrap(); + let cte_span = query.clone().with.unwrap().cte_tables[0].span(); + let cte_query_span = query.clone().with.unwrap().cte_tables[0].query.span(); + let body_span = query.body.span(); + + // the WITH keyboard is part of the query + assert_eq!( + test.get_source(cte_span), + "cte AS (SELECT a FROM postgres.public.source)" + ); + assert_eq!( + test.get_source(cte_query_span), + "SELECT a FROM postgres.public.source" + ); + + assert_eq!(test.get_source(body_span), "SELECT cte.* FROM cte"); + } +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 6767f358a..b7f5cb866 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -24,6 +24,7 @@ use core::{ fmt::{self, Display}, str::FromStr, }; +use helpers::attached_token::AttachedToken; use log::debug; @@ -371,7 +372,7 @@ impl<'a> Parser<'a> { .into_iter() .map(|token| TokenWithLocation { token, - location: Location { line: 0, column: 0 }, + span: Span::empty(), }) .collect(); self.with_tokens_with_locations(tokens_with_locations) @@ -613,7 +614,7 @@ impl<'a> Parser<'a> { let mut export = false; if !dialect_of!(self is MySqlDialect | GenericDialect) { - return parser_err!("Unsupported statement FLUSH", self.peek_token().location); + return parser_err!("Unsupported statement FLUSH", self.peek_token().span.start); } let location = if self.parse_keyword(Keyword::NO_WRITE_TO_BINLOG) { @@ -914,7 +915,7 @@ impl<'a> Parser<'a> { t @ (Token::Word(_) | Token::SingleQuotedString(_)) => { if self.peek_token().token == Token::Period { let mut id_parts: Vec = vec![match t { - Token::Word(w) => w.to_ident(), + Token::Word(w) => w.to_ident(next_token.span), Token::SingleQuotedString(s) => Ident::with_quote('\'', s), _ => unreachable!(), // We matched above }]; @@ -922,13 +923,16 @@ impl<'a> Parser<'a> { while self.consume_token(&Token::Period) { let next_token = self.next_token(); match next_token.token { - Token::Word(w) => id_parts.push(w.to_ident()), + Token::Word(w) => id_parts.push(w.to_ident(next_token.span)), Token::SingleQuotedString(s) => { // SQLite has single-quoted identifiers id_parts.push(Ident::with_quote('\'', s)) } Token::Mul => { - return Ok(Expr::QualifiedWildcard(ObjectName(id_parts))); + return Ok(Expr::QualifiedWildcard( + ObjectName(id_parts), + AttachedToken(next_token), + )); } _ => { return self @@ -939,7 +943,7 @@ impl<'a> Parser<'a> { } } Token::Mul => { - return Ok(Expr::Wildcard); + return Ok(Expr::Wildcard(AttachedToken(next_token))); } _ => (), }; @@ -1002,7 +1006,7 @@ impl<'a> Parser<'a> { pub fn parse_unlisten(&mut self) -> Result { let channel = if self.consume_token(&Token::Mul) { - Ident::new(Expr::Wildcard.to_string()) + Ident::new(Expr::Wildcard(AttachedToken::empty()).to_string()) } else { match self.parse_identifier(false) { Ok(expr) => expr, @@ -1030,6 +1034,7 @@ impl<'a> Parser<'a> { fn parse_expr_prefix_by_reserved_word( &mut self, w: &Word, + w_span: Span, ) -> Result, ParserError> { match w.keyword { Keyword::TRUE | Keyword::FALSE if self.dialect.supports_boolean_literals() => { @@ -1047,7 +1052,7 @@ impl<'a> Parser<'a> { if dialect_of!(self is PostgreSqlDialect | GenericDialect) => { Ok(Some(Expr::Function(Function { - name: ObjectName(vec![w.to_ident()]), + name: ObjectName(vec![w.to_ident(w_span)]), parameters: FunctionArguments::None, args: FunctionArguments::None, null_treatment: None, @@ -1061,7 +1066,7 @@ impl<'a> Parser<'a> { | Keyword::CURRENT_DATE | Keyword::LOCALTIME | Keyword::LOCALTIMESTAMP => { - Ok(Some(self.parse_time_functions(ObjectName(vec![w.to_ident()]))?)) + Ok(Some(self.parse_time_functions(ObjectName(vec![w.to_ident(w_span)]))?)) } Keyword::CASE => Ok(Some(self.parse_case_expr()?)), Keyword::CONVERT => Ok(Some(self.parse_convert_expr(false)?)), @@ -1086,7 +1091,7 @@ impl<'a> Parser<'a> { Keyword::CEIL => Ok(Some(self.parse_ceil_floor_expr(true)?)), Keyword::FLOOR => Ok(Some(self.parse_ceil_floor_expr(false)?)), Keyword::POSITION if self.peek_token().token == Token::LParen => { - Ok(Some(self.parse_position_expr(w.to_ident())?)) + Ok(Some(self.parse_position_expr(w.to_ident(w_span))?)) } Keyword::SUBSTRING => Ok(Some(self.parse_substring_expr()?)), Keyword::OVERLAY => Ok(Some(self.parse_overlay_expr()?)), @@ -1105,7 +1110,7 @@ impl<'a> Parser<'a> { let query = self.parse_query()?; self.expect_token(&Token::RParen)?; Ok(Some(Expr::Function(Function { - name: ObjectName(vec![w.to_ident()]), + name: ObjectName(vec![w.to_ident(w_span)]), parameters: FunctionArguments::None, args: FunctionArguments::Subquery(query), filter: None, @@ -1134,20 +1139,24 @@ impl<'a> Parser<'a> { } // Tries to parse an expression by a word that is not known to have a special meaning in the dialect. - fn parse_expr_prefix_by_unreserved_word(&mut self, w: &Word) -> Result { + fn parse_expr_prefix_by_unreserved_word( + &mut self, + w: &Word, + w_span: Span, + ) -> Result { match self.peek_token().token { Token::LParen | Token::Period => { - let mut id_parts: Vec = vec![w.to_ident()]; - let mut ends_with_wildcard = false; + let mut id_parts: Vec = vec![w.to_ident(w_span)]; + let mut ending_wildcard: Option = None; while self.consume_token(&Token::Period) { let next_token = self.next_token(); match next_token.token { - Token::Word(w) => id_parts.push(w.to_ident()), + Token::Word(w) => id_parts.push(w.to_ident(next_token.span)), Token::Mul => { // Postgres explicitly allows funcnm(tablenm.*) and the // function array_agg traverses this control flow if dialect_of!(self is PostgreSqlDialect) { - ends_with_wildcard = true; + ending_wildcard = Some(next_token); break; } else { return self.expected("an identifier after '.'", next_token); @@ -1160,8 +1169,11 @@ impl<'a> Parser<'a> { } } - if ends_with_wildcard { - Ok(Expr::QualifiedWildcard(ObjectName(id_parts))) + if let Some(wildcard_token) = ending_wildcard { + Ok(Expr::QualifiedWildcard( + ObjectName(id_parts), + AttachedToken(wildcard_token), + )) } else if self.consume_token(&Token::LParen) { if dialect_of!(self is SnowflakeDialect | MsSqlDialect) && self.consume_tokens(&[Token::Plus, Token::RParen]) @@ -1194,11 +1206,11 @@ impl<'a> Parser<'a> { Token::Arrow if self.dialect.supports_lambda_functions() => { self.expect_token(&Token::Arrow)?; Ok(Expr::Lambda(LambdaFunction { - params: OneOrManyWithParens::One(w.to_ident()), + params: OneOrManyWithParens::One(w.to_ident(w_span)), body: Box::new(self.parse_expr()?), })) } - _ => Ok(Expr::Identifier(w.to_ident())), + _ => Ok(Expr::Identifier(w.to_ident(w_span))), } } @@ -1225,7 +1237,7 @@ impl<'a> Parser<'a> { // Note also that naively `SELECT date` looks like a syntax error because the `date` type // name is not followed by a string literal, but in fact in PostgreSQL it is a valid // expression that should parse as the column name "date". - let loc = self.peek_token().location; + let loc = self.peek_token().span.start; let opt_expr = self.maybe_parse(|parser| { match parser.parse_data_type()? { DataType::Interval => parser.parse_interval(), @@ -1259,12 +1271,14 @@ impl<'a> Parser<'a> { // // We first try to parse the word and following tokens as a special expression, and if that fails, // we rollback and try to parse it as an identifier. - match self.try_parse(|parser| parser.parse_expr_prefix_by_reserved_word(&w)) { + match self.try_parse(|parser| { + parser.parse_expr_prefix_by_reserved_word(&w, next_token.span) + }) { // This word indicated an expression prefix and parsing was successful Ok(Some(expr)) => Ok(expr), // No expression prefix associated with this word - Ok(None) => Ok(self.parse_expr_prefix_by_unreserved_word(&w)?), + Ok(None) => Ok(self.parse_expr_prefix_by_unreserved_word(&w, next_token.span)?), // If parsing of the word as a special expression failed, we are facing two options: // 1. The statement is malformed, e.g. `SELECT INTERVAL '1 DAI` (`DAI` instead of `DAY`) @@ -1275,7 +1289,7 @@ impl<'a> Parser<'a> { Err(e) => { if !self.dialect.is_reserved_for_identifier(w.keyword) { if let Ok(Some(expr)) = self.maybe_parse(|parser| { - parser.parse_expr_prefix_by_unreserved_word(&w) + parser.parse_expr_prefix_by_unreserved_word(&w, next_token.span) }) { return Ok(expr); } @@ -1377,11 +1391,11 @@ impl<'a> Parser<'a> { } else { let tok = self.next_token(); let key = match tok.token { - Token::Word(word) => word.to_ident(), + Token::Word(word) => word.to_ident(tok.span), _ => { return parser_err!( format!("Expected identifier, found: {tok}"), - tok.location + tok.span.start ) } }; @@ -1471,7 +1485,7 @@ impl<'a> Parser<'a> { while p.consume_token(&Token::Period) { let tok = p.next_token(); let name = match tok.token { - Token::Word(word) => word.to_ident(), + Token::Word(word) => word.to_ident(tok.span), _ => return p.expected("identifier", tok), }; let func = match p.parse_function(ObjectName(vec![name]))? { @@ -2290,7 +2304,7 @@ impl<'a> Parser<'a> { } else if self.dialect.require_interval_qualifier() { return parser_err!( "INTERVAL requires a unit after the literal value", - self.peek_token().location + self.peek_token().span.start ); } else { None @@ -2381,7 +2395,10 @@ impl<'a> Parser<'a> { let (fields, trailing_bracket) = self.parse_struct_type_def(Self::parse_struct_field_def)?; if trailing_bracket.0 { - return parser_err!("unmatched > in STRUCT literal", self.peek_token().location); + return parser_err!( + "unmatched > in STRUCT literal", + self.peek_token().span.start + ); } self.expect_token(&Token::LParen)?; @@ -2411,7 +2428,7 @@ impl<'a> Parser<'a> { if typed_syntax { return parser_err!("Typed syntax does not allow AS", { self.prev_token(); - self.peek_token().location + self.peek_token().span.start }); } let field_name = self.parse_identifier(false)?; @@ -2464,7 +2481,7 @@ impl<'a> Parser<'a> { // we've matched all field types for the current struct. // e.g. this is invalid syntax `STRUCT>>, INT>(NULL)` if trailing_bracket.0 { - return parser_err!("unmatched > in STRUCT definition", start_token.location); + return parser_err!("unmatched > in STRUCT definition", start_token.span.start); } }; @@ -2833,7 +2850,7 @@ impl<'a> Parser<'a> { format!( "Expected one of [=, >, <, =>, =<, !=] as comparison operator, found: {op}" ), - tok.location + tok.span.start ); }; @@ -2959,7 +2976,7 @@ impl<'a> Parser<'a> { // Can only happen if `get_next_precedence` got out of sync with this function _ => parser_err!( format!("No infix parser for token {:?}", tok.token), - tok.location + tok.span.start ), } } else if Token::DoubleColon == tok { @@ -2990,7 +3007,7 @@ impl<'a> Parser<'a> { // Can only happen if `get_next_precedence` got out of sync with this function parser_err!( format!("No infix parser for token {:?}", tok.token), - tok.location + tok.span.start ) } } @@ -3298,14 +3315,14 @@ impl<'a> Parser<'a> { index += 1; if let Some(TokenWithLocation { token: Token::Whitespace(_), - location: _, + span: _, }) = token { continue; } break token.cloned().unwrap_or(TokenWithLocation { token: Token::EOF, - location: Location { line: 0, column: 0 }, + span: Span::empty(), }); }) } @@ -3318,13 +3335,13 @@ impl<'a> Parser<'a> { match self.tokens.get(index - 1) { Some(TokenWithLocation { token: Token::Whitespace(_), - location: _, + span: _, }) => continue, non_whitespace => { if n == 0 { return non_whitespace.cloned().unwrap_or(TokenWithLocation { token: Token::EOF, - location: Location { line: 0, column: 0 }, + span: Span::empty(), }); } n -= 1; @@ -3346,18 +3363,10 @@ impl<'a> Parser<'a> { .cloned() .unwrap_or(TokenWithLocation { token: Token::EOF, - location: Location { line: 0, column: 0 }, + span: Span::empty(), }) } - /// Look for all of the expected keywords in sequence, without consuming them - fn peek_keyword(&mut self, expected: Keyword) -> bool { - let index = self.index; - let matched = self.parse_keyword(expected); - self.index = index; - matched - } - /// Look for all of the expected keywords in sequence, without consuming them fn peek_keywords(&mut self, expected: &[Keyword]) -> bool { let index = self.index; @@ -3375,7 +3384,7 @@ impl<'a> Parser<'a> { match self.tokens.get(self.index - 1) { Some(TokenWithLocation { token: Token::Whitespace(_), - location: _, + span: _, }) => continue, token => { return token @@ -3401,7 +3410,7 @@ impl<'a> Parser<'a> { self.index -= 1; if let Some(TokenWithLocation { token: Token::Whitespace(_), - location: _, + span: _, }) = self.tokens.get(self.index) { continue; @@ -3414,7 +3423,7 @@ impl<'a> Parser<'a> { pub fn expected(&self, expected: &str, found: TokenWithLocation) -> Result { parser_err!( format!("Expected: {expected}, found: {found}"), - found.location + found.span.start ) } @@ -3422,15 +3431,22 @@ impl<'a> Parser<'a> { /// true. Otherwise, no tokens are consumed and returns false. #[must_use] pub fn parse_keyword(&mut self, expected: Keyword) -> bool { + self.parse_keyword_token(expected).is_some() + } + + #[must_use] + pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option { match self.peek_token().token { - Token::Word(w) if expected == w.keyword => { - self.next_token(); - true - } - _ => false, + Token::Word(w) if expected == w.keyword => Some(self.next_token()), + _ => None, } } + #[must_use] + pub fn peek_keyword(&mut self, expected: Keyword) -> bool { + matches!(self.peek_token().token, Token::Word(w) if expected == w.keyword) + } + /// If the current token is the `expected` keyword followed by /// specified tokens, consume them and returns true. /// Otherwise, no tokens are consumed and returns false. @@ -3508,9 +3524,9 @@ impl<'a> Parser<'a> { /// If the current token is the `expected` keyword, consume the token. /// Otherwise, return an error. - pub fn expect_keyword(&mut self, expected: Keyword) -> Result<(), ParserError> { - if self.parse_keyword(expected) { - Ok(()) + pub fn expect_keyword(&mut self, expected: Keyword) -> Result { + if let Some(token) = self.parse_keyword_token(expected) { + Ok(token) } else { self.expected(format!("{:?}", &expected).as_str(), self.peek_token()) } @@ -3552,9 +3568,9 @@ impl<'a> Parser<'a> { } /// Bail out if the current token is not an expected keyword, or consume it if it is - pub fn expect_token(&mut self, expected: &Token) -> Result<(), ParserError> { - if self.consume_token(expected) { - Ok(()) + pub fn expect_token(&mut self, expected: &Token) -> Result { + if self.peek_token() == *expected { + Ok(self.next_token()) } else { self.expected(&expected.to_string(), self.peek_token()) } @@ -3749,7 +3765,7 @@ impl<'a> Parser<'a> { /// Parse either `ALL`, `DISTINCT` or `DISTINCT ON (...)`. Returns [`None`] if `ALL` is parsed /// and results in a [`ParserError`] if both `ALL` and `DISTINCT` are found. pub fn parse_all_or_distinct(&mut self) -> Result, ParserError> { - let loc = self.peek_token().location; + let loc = self.peek_token().span.start; let all = self.parse_keyword(Keyword::ALL); let distinct = self.parse_keyword(Keyword::DISTINCT); if !distinct { @@ -4828,7 +4844,7 @@ impl<'a> Parser<'a> { let loc = self .tokens .get(self.index - 1) - .map_or(Location { line: 0, column: 0 }, |t| t.location); + .map_or(Location { line: 0, column: 0 }, |t| t.span.start); match keyword { Keyword::AUTHORIZATION => { if authorization_owner.is_some() { @@ -5138,7 +5154,7 @@ impl<'a> Parser<'a> { let if_exists = self.parse_keywords(&[Keyword::IF, Keyword::EXISTS]); let names = self.parse_comma_separated(|p| p.parse_object_name(false))?; - let loc = self.peek_token().location; + let loc = self.peek_token().span.start; let cascade = self.parse_keyword(Keyword::CASCADE); let restrict = self.parse_keyword(Keyword::RESTRICT); let purge = self.parse_keyword(Keyword::PURGE); @@ -6029,7 +6045,7 @@ impl<'a> Parser<'a> { let _ = self.consume_token(&Token::Eq); let next_token = self.next_token(); match next_token.token { - Token::Number(s, _) => Some(Self::parse::(s, next_token.location)?), + Token::Number(s, _) => Some(Self::parse::(s, next_token.span.start)?), _ => self.expected("literal int", next_token)?, } } else { @@ -6818,7 +6834,7 @@ impl<'a> Parser<'a> { "FULLTEXT or SPATIAL option without constraint name", TokenWithLocation { token: Token::make_keyword(&name.to_string()), - location: next_token.location, + span: next_token.span, }, ); } @@ -7527,7 +7543,7 @@ impl<'a> Parser<'a> { Expr::Function(f) => Ok(Statement::Call(f)), other => parser_err!( format!("Expected a simple procedure call but found: {other}"), - self.peek_token().location + self.peek_token().span.start ), } } else { @@ -7731,7 +7747,7 @@ impl<'a> Parser<'a> { let loc = self .tokens .get(self.index - 1) - .map_or(Location { line: 0, column: 0 }, |t| t.location); + .map_or(Location { line: 0, column: 0 }, |t| t.span.start); return parser_err!(format!("Expect a char, found {s:?}"), loc); } Ok(s.chars().next().unwrap()) @@ -7777,7 +7793,7 @@ impl<'a> Parser<'a> { /// Parse a literal value (numbers, strings, date/time, booleans) pub fn parse_value(&mut self) -> Result { let next_token = self.next_token(); - let location = next_token.location; + let span = next_token.span; match next_token.token { Token::Word(w) => match w.keyword { Keyword::TRUE if self.dialect.supports_boolean_literals() => { @@ -7794,7 +7810,7 @@ impl<'a> Parser<'a> { "A value?", TokenWithLocation { token: Token::Word(w), - location, + span, }, )?, }, @@ -7802,14 +7818,14 @@ impl<'a> Parser<'a> { "a concrete value", TokenWithLocation { token: Token::Word(w), - location, + span, }, ), }, // The call to n.parse() returns a bigdecimal when the // bigdecimal feature is enabled, and is otherwise a no-op // (i.e., it returns the input string). - Token::Number(n, l) => Ok(Value::Number(Self::parse(n, location)?, l)), + Token::Number(n, l) => Ok(Value::Number(Self::parse(n, span.start)?, l)), Token::SingleQuotedString(ref s) => Ok(Value::SingleQuotedString(s.to_string())), Token::DoubleQuotedString(ref s) => Ok(Value::DoubleQuotedString(s.to_string())), Token::TripleSingleQuotedString(ref s) => { @@ -7853,7 +7869,7 @@ impl<'a> Parser<'a> { // This because snowflake allows numbers as placeholders let next_token = self.next_token(); let ident = match next_token.token { - Token::Word(w) => Ok(w.to_ident()), + Token::Word(w) => Ok(w.to_ident(next_token.span)), Token::Number(w, false) => Ok(Ident::new(w)), _ => self.expected("placeholder", next_token), }?; @@ -7864,7 +7880,7 @@ impl<'a> Parser<'a> { "a value", TokenWithLocation { token: unexpected, - location, + span, }, ), } @@ -7904,7 +7920,7 @@ impl<'a> Parser<'a> { fn parse_introduced_string_value(&mut self) -> Result { let next_token = self.next_token(); - let location = next_token.location; + let span = next_token.span; match next_token.token { Token::SingleQuotedString(ref s) => Ok(Value::SingleQuotedString(s.to_string())), Token::DoubleQuotedString(ref s) => Ok(Value::DoubleQuotedString(s.to_string())), @@ -7913,7 +7929,7 @@ impl<'a> Parser<'a> { "a string value", TokenWithLocation { token: unexpected, - location, + span, }, ), } @@ -7923,7 +7939,7 @@ impl<'a> Parser<'a> { pub fn parse_literal_uint(&mut self) -> Result { let next_token = self.next_token(); match next_token.token { - Token::Number(s, _) => Self::parse::(s, next_token.location), + Token::Number(s, _) => Self::parse::(s, next_token.span.start), _ => self.expected("literal int", next_token), } } @@ -8322,7 +8338,7 @@ impl<'a> Parser<'a> { // (For example, in `FROM t1 JOIN` the `JOIN` will always be parsed as a keyword, // not an alias.) Token::Word(w) if after_as || !reserved_kwds.contains(&w.keyword) => { - Ok(Some(w.to_ident())) + Ok(Some(w.to_ident(next_token.span))) } // MSSQL supports single-quoted strings as aliases for columns // We accept them as table aliases too, although MSSQL does not. @@ -8392,7 +8408,7 @@ impl<'a> Parser<'a> { _ => { return parser_err!( "BUG: expected to match GroupBy modifier keyword", - self.peek_token().location + self.peek_token().span.start ) } }); @@ -8455,6 +8471,7 @@ impl<'a> Parser<'a> { .map(|value| Ident { value: value.into(), quote_style: ident.quote_style, + span: ident.span, }) .collect::>() }) @@ -8470,7 +8487,7 @@ impl<'a> Parser<'a> { loop { match self.peek_token().token { Token::Word(w) => { - idents.push(w.to_ident()); + idents.push(w.to_ident(self.peek_token().span)); } Token::EOF | Token::Eq => break, _ => {} @@ -8523,8 +8540,9 @@ impl<'a> Parser<'a> { let mut idents = vec![]; // expecting at least one word for identifier - match self.next_token().token { - Token::Word(w) => idents.push(w.to_ident()), + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => idents.push(w.to_ident(next_token.span)), Token::EOF => { return Err(ParserError::ParserError( "Empty input when parsing identifier".to_string(), @@ -8541,19 +8559,22 @@ impl<'a> Parser<'a> { loop { match self.next_token().token { // ensure that optional period is succeeded by another identifier - Token::Period => match self.next_token().token { - Token::Word(w) => idents.push(w.to_ident()), - Token::EOF => { - return Err(ParserError::ParserError( - "Trailing period in identifier".to_string(), - ))? - } - token => { - return Err(ParserError::ParserError(format!( - "Unexpected token following period in identifier: {token}" - )))? + Token::Period => { + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => idents.push(w.to_ident(next_token.span)), + Token::EOF => { + return Err(ParserError::ParserError( + "Trailing period in identifier".to_string(), + ))? + } + token => { + return Err(ParserError::ParserError(format!( + "Unexpected token following period in identifier: {token}" + )))? + } } - }, + } Token::EOF => break, token => { return Err(ParserError::ParserError(format!( @@ -8575,7 +8596,7 @@ impl<'a> Parser<'a> { let next_token = self.next_token(); match next_token.token { Token::Word(w) => { - let mut ident = w.to_ident(); + let mut ident = w.to_ident(next_token.span); // On BigQuery, hyphens are permitted in unquoted identifiers inside of a FROM or // TABLE clause [0]. @@ -9006,8 +9027,9 @@ impl<'a> Parser<'a> { /// expect the initial keyword to be already consumed pub fn parse_query(&mut self) -> Result, ParserError> { let _guard = self.recursion_counter.try_decrease()?; - let with = if self.parse_keyword(Keyword::WITH) { + let with = if let Some(with_token) = self.parse_keyword_token(Keyword::WITH) { Some(With { + with_token: with_token.into(), recursive: self.parse_keyword(Keyword::RECURSIVE), cte_tables: self.parse_comma_separated(Parser::parse_cte)?, }) @@ -9265,8 +9287,10 @@ impl<'a> Parser<'a> { } } self.expect_token(&Token::LParen)?; + let query = self.parse_query()?; - self.expect_token(&Token::RParen)?; + let closing_paren_token = self.expect_token(&Token::RParen)?; + let alias = TableAlias { name, columns: vec![], @@ -9276,6 +9300,7 @@ impl<'a> Parser<'a> { query, from: None, materialized: is_materialized, + closing_paren_token: closing_paren_token.into(), } } else { let columns = self.parse_table_alias_column_defs()?; @@ -9289,14 +9314,17 @@ impl<'a> Parser<'a> { } } self.expect_token(&Token::LParen)?; + let query = self.parse_query()?; - self.expect_token(&Token::RParen)?; + let closing_paren_token = self.expect_token(&Token::RParen)?; + let alias = TableAlias { name, columns }; Cte { alias, query, from: None, materialized: is_materialized, + closing_paren_token: closing_paren_token.into(), } }; if self.parse_keyword(Keyword::FROM) { @@ -9316,7 +9344,7 @@ impl<'a> Parser<'a> { pub fn parse_query_body(&mut self, precedence: u8) -> Result, ParserError> { // We parse the expression using a Pratt parser, as in `parse_expr()`. // Start by parsing a restricted SELECT or a `(subquery)`: - let expr = if self.parse_keyword(Keyword::SELECT) { + let expr = if self.peek_keyword(Keyword::SELECT) { SetExpr::Select(self.parse_select().map(Box::new)?) } else if self.consume_token(&Token::LParen) { // CTEs are not allowed here, but the parser currently accepts them @@ -9405,9 +9433,9 @@ impl<'a> Parser<'a> { } } - /// Parse a restricted `SELECT` statement (no CTEs / `UNION` / `ORDER BY`), - /// assuming the initial `SELECT` was already consumed + /// Parse a restricted `SELECT` statement (no CTEs / `UNION` / `ORDER BY`) pub fn parse_select(&mut self) -> Result { + let select_token = self.expect_keyword(Keyword::SELECT)?; let value_table_mode = if dialect_of!(self is BigQueryDialect) && self.parse_keyword(Keyword::AS) { if self.parse_keyword(Keyword::VALUE) { @@ -9571,6 +9599,7 @@ impl<'a> Parser<'a> { }; Ok(Select { + select_token: AttachedToken(select_token), distinct, top, top_before_distinct, @@ -10656,7 +10685,7 @@ impl<'a> Parser<'a> { return self.expected("literal number", next_token); }; self.expect_token(&Token::RBrace)?; - RepetitionQuantifier::AtMost(Self::parse(n, token.location)?) + RepetitionQuantifier::AtMost(Self::parse(n, token.span.start)?) } Token::Number(n, _) if self.consume_token(&Token::Comma) => { let next_token = self.next_token(); @@ -10664,12 +10693,12 @@ impl<'a> Parser<'a> { Token::Number(m, _) => { self.expect_token(&Token::RBrace)?; RepetitionQuantifier::Range( - Self::parse(n, token.location)?, - Self::parse(m, token.location)?, + Self::parse(n, token.span.start)?, + Self::parse(m, token.span.start)?, ) } Token::RBrace => { - RepetitionQuantifier::AtLeast(Self::parse(n, token.location)?) + RepetitionQuantifier::AtLeast(Self::parse(n, token.span.start)?) } _ => { return self.expected("} or upper bound", next_token); @@ -10678,7 +10707,7 @@ impl<'a> Parser<'a> { } Token::Number(n, _) => { self.expect_token(&Token::RBrace)?; - RepetitionQuantifier::Exactly(Self::parse(n, token.location)?) + RepetitionQuantifier::Exactly(Self::parse(n, token.span.start)?) } _ => return self.expected("quantifier range", token), } @@ -11113,7 +11142,7 @@ impl<'a> Parser<'a> { .parse_keywords(&[Keyword::GRANTED, Keyword::BY]) .then(|| self.parse_identifier(false).unwrap()); - let loc = self.peek_token().location; + let loc = self.peek_token().span.start; let cascade = self.parse_keyword(Keyword::CASCADE); let restrict = self.parse_keyword(Keyword::RESTRICT); if cascade && restrict { @@ -11132,7 +11161,10 @@ impl<'a> Parser<'a> { /// Parse an REPLACE statement pub fn parse_replace(&mut self) -> Result { if !dialect_of!(self is MySqlDialect | GenericDialect) { - return parser_err!("Unsupported statement REPLACE", self.peek_token().location); + return parser_err!( + "Unsupported statement REPLACE", + self.peek_token().span.start + ); } let mut insert = self.parse_insert()?; @@ -11593,7 +11625,7 @@ impl<'a> Parser<'a> { } fn parse_duplicate_treatment(&mut self) -> Result, ParserError> { - let loc = self.peek_token().location; + let loc = self.peek_token().span.start; match ( self.parse_keyword(Keyword::ALL), self.parse_keyword(Keyword::DISTINCT), @@ -11608,17 +11640,17 @@ impl<'a> Parser<'a> { /// Parse a comma-delimited list of projections after SELECT pub fn parse_select_item(&mut self) -> Result { match self.parse_wildcard_expr()? { - Expr::QualifiedWildcard(prefix) => Ok(SelectItem::QualifiedWildcard( + Expr::QualifiedWildcard(prefix, token) => Ok(SelectItem::QualifiedWildcard( prefix, - self.parse_wildcard_additional_options()?, + self.parse_wildcard_additional_options(token.0)?, )), - Expr::Wildcard => Ok(SelectItem::Wildcard( - self.parse_wildcard_additional_options()?, + Expr::Wildcard(token) => Ok(SelectItem::Wildcard( + self.parse_wildcard_additional_options(token.0)?, )), Expr::Identifier(v) if v.value.to_lowercase() == "from" && v.quote_style.is_none() => { parser_err!( format!("Expected an expression, found: {}", v), - self.peek_token().location + self.peek_token().span.start ) } Expr::BinaryOp { @@ -11631,7 +11663,7 @@ impl<'a> Parser<'a> { let Expr::Identifier(alias) = *left else { return parser_err!( "BUG: expected identifier expression as alias", - self.peek_token().location + self.peek_token().span.start ); }; Ok(SelectItem::ExprWithAlias { @@ -11653,6 +11685,7 @@ impl<'a> Parser<'a> { /// If it is not possible to parse it, will return an option. pub fn parse_wildcard_additional_options( &mut self, + wildcard_token: TokenWithLocation, ) -> Result { let opt_ilike = if dialect_of!(self is GenericDialect | SnowflakeDialect) { self.parse_optional_select_item_ilike()? @@ -11684,6 +11717,7 @@ impl<'a> Parser<'a> { }; Ok(WildcardAdditionalOptions { + wildcard_token: wildcard_token.into(), opt_ilike, opt_exclude, opt_except, @@ -11931,7 +11965,7 @@ impl<'a> Parser<'a> { } else { let next_token = self.next_token(); let quantity = match next_token.token { - Token::Number(s, _) => Self::parse::(s, next_token.location)?, + Token::Number(s, _) => Self::parse::(s, next_token.span.start)?, _ => self.expected("literal int", next_token)?, }; Some(TopQuantity::Constant(quantity)) @@ -12812,10 +12846,11 @@ impl<'a> Parser<'a> { } impl Word { - pub fn to_ident(&self) -> Ident { + pub fn to_ident(&self, span: Span) -> Ident { Ident { value: self.value.clone(), quote_style: self.quote_style, + span, } } } @@ -13389,14 +13424,17 @@ mod tests { Ident { value: "CATALOG".to_string(), quote_style: None, + span: Span::empty(), }, Ident { value: "F(o)o. \"bar".to_string(), quote_style: Some('"'), + span: Span::empty(), }, Ident { value: "table".to_string(), quote_style: None, + span: Span::empty(), }, ]; dialect.run_parser_method(r#"CATALOG."F(o)o. ""bar".table"#, |parser| { @@ -13409,10 +13447,12 @@ mod tests { Ident { value: "CATALOG".to_string(), quote_style: None, + span: Span::empty(), }, Ident { value: "table".to_string(), quote_style: None, + span: Span::empty(), }, ]; dialect.run_parser_method("CATALOG . table", |parser| { diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 05aaf1e28..a57ba2ec8 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -29,10 +29,10 @@ use alloc::{ vec, vec::Vec, }; -use core::fmt; use core::iter::Peekable; use core::num::NonZeroU8; use core::str::Chars; +use core::{cmp, fmt}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -422,7 +422,9 @@ impl fmt::Display for Whitespace { } /// Location in input string -#[derive(Debug, Eq, PartialEq, Clone, Copy)] +#[derive(Eq, PartialEq, Hash, Clone, Copy, Ord, PartialOrd)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct Location { /// Line number, starting from 1 pub line: u64, @@ -431,36 +433,114 @@ pub struct Location { } impl fmt::Display for Location { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if self.line == 0 { return Ok(()); } - write!( - f, - // TODO: use standard compiler location syntax (::) - " at Line: {}, Column: {}", - self.line, self.column, - ) + write!(f, " at Line: {}, Column: {}", self.line, self.column) + } +} + +impl fmt::Debug for Location { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Location({},{})", self.line, self.column) + } +} + +impl Location { + pub fn of(line: u64, column: u64) -> Self { + Self { line, column } + } + + pub fn span_to(self, end: Self) -> Span { + Span { start: self, end } + } +} + +impl From<(u64, u64)> for Location { + fn from((line, column): (u64, u64)) -> Self { + Self { line, column } + } +} + +/// A span of source code locations (start, end) +#[derive(Eq, PartialEq, Hash, Clone, PartialOrd, Ord, Copy)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct Span { + pub start: Location, + pub end: Location, +} + +impl fmt::Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Span({:?}..{:?})", self.start, self.end) + } +} + +impl Span { + // An empty span (0, 0) -> (0, 0) + // We need a const instance for pattern matching + const EMPTY: Span = Self::empty(); + + pub fn new(start: Location, end: Location) -> Span { + Span { start, end } + } + + /// Returns an empty span (0, 0) -> (0, 0) + /// Empty spans represent no knowledge of source location + pub const fn empty() -> Span { + Span { + start: Location { line: 0, column: 0 }, + end: Location { line: 0, column: 0 }, + } + } + + /// Returns the smallest Span that contains both `self` and `other` + /// If either span is [Span::empty], the other span is returned + pub fn union(&self, other: &Span) -> Span { + // If either span is empty, return the other + // this prevents propagating (0, 0) through the tree + match (self, other) { + (&Span::EMPTY, _) => *other, + (_, &Span::EMPTY) => *self, + _ => Span { + start: cmp::min(self.start, other.start), + end: cmp::max(self.end, other.end), + }, + } + } + + /// Same as [Span::union] for `Option` + /// If `other` is `None`, `self` is returned + pub fn union_opt(&self, other: &Option) -> Span { + match other { + Some(other) => self.union(other), + None => *self, + } } } /// A [Token] with [Location] attached to it -#[derive(Debug, Eq, PartialEq, Clone)] +#[derive(Debug, Clone, Hash, Ord, PartialOrd, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct TokenWithLocation { pub token: Token, - pub location: Location, + pub span: Span, } impl TokenWithLocation { - pub fn new(token: Token, line: u64, column: u64) -> TokenWithLocation { - TokenWithLocation { - token, - location: Location { line, column }, - } + pub fn new(token: Token, span: Span) -> TokenWithLocation { + TokenWithLocation { token, span } } pub fn wrap(token: Token) -> TokenWithLocation { - TokenWithLocation::new(token, 0, 0) + TokenWithLocation::new(token, Span::empty()) + } + + pub fn at(token: Token, start: Location, end: Location) -> TokenWithLocation { + TokenWithLocation::new(token, Span::new(start, end)) } } @@ -656,7 +736,9 @@ impl<'a> Tokenizer<'a> { let mut location = state.location(); while let Some(token) = self.next_token(&mut state)? { - buf.push(TokenWithLocation { token, location }); + let span = location.span_to(state.location()); + + buf.push(TokenWithLocation { token, span }); location = state.location(); } @@ -2669,18 +2751,30 @@ mod tests { .tokenize_with_location() .unwrap(); let expected = vec![ - TokenWithLocation::new(Token::make_keyword("SELECT"), 1, 1), - TokenWithLocation::new(Token::Whitespace(Whitespace::Space), 1, 7), - TokenWithLocation::new(Token::make_word("a", None), 1, 8), - TokenWithLocation::new(Token::Comma, 1, 9), - TokenWithLocation::new(Token::Whitespace(Whitespace::Newline), 1, 10), - TokenWithLocation::new(Token::Whitespace(Whitespace::Space), 2, 1), - TokenWithLocation::new(Token::make_word("b", None), 2, 2), + TokenWithLocation::at(Token::make_keyword("SELECT"), (1, 1).into(), (1, 7).into()), + TokenWithLocation::at( + Token::Whitespace(Whitespace::Space), + (1, 7).into(), + (1, 8).into(), + ), + TokenWithLocation::at(Token::make_word("a", None), (1, 8).into(), (1, 9).into()), + TokenWithLocation::at(Token::Comma, (1, 9).into(), (1, 10).into()), + TokenWithLocation::at( + Token::Whitespace(Whitespace::Newline), + (1, 10).into(), + (2, 1).into(), + ), + TokenWithLocation::at( + Token::Whitespace(Whitespace::Space), + (2, 1).into(), + (2, 2).into(), + ), + TokenWithLocation::at(Token::make_word("b", None), (2, 2).into(), (2, 3).into()), ]; compare(expected, tokens); } - fn compare(expected: Vec, actual: Vec) { + fn compare(expected: Vec, actual: Vec) { //println!("------------------------------"); //println!("tokens = {:?}", actual); //println!("expected = {:?}", expected); diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index d4c178bbf..00d12ed83 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -23,6 +23,7 @@ use std::ops::Deref; use sqlparser::ast::*; use sqlparser::dialect::{BigQueryDialect, GenericDialect}; use sqlparser::parser::{ParserError, ParserOptions}; +use sqlparser::tokenizer::Span; use test_utils::*; #[test] @@ -678,10 +679,12 @@ fn parse_typed_struct_syntax_bigquery() { Ident { value: "t".into(), quote_style: None, + span: Span::empty(), }, Ident { value: "str_col".into(), quote_style: None, + span: Span::empty(), }, ]), ], @@ -690,6 +693,7 @@ fn parse_typed_struct_syntax_bigquery() { field_name: Some(Ident { value: "x".into(), quote_style: None, + span: Span::empty(), }), field_type: DataType::Int64 }, @@ -697,6 +701,7 @@ fn parse_typed_struct_syntax_bigquery() { field_name: Some(Ident { value: "y".into(), quote_style: None, + span: Span::empty(), }), field_type: DataType::String(None) }, @@ -709,6 +714,7 @@ fn parse_typed_struct_syntax_bigquery() { values: vec![Expr::Identifier(Ident { value: "nested_col".into(), quote_style: None, + span: Span::empty(), }),], fields: vec![ StructField { @@ -740,6 +746,7 @@ fn parse_typed_struct_syntax_bigquery() { values: vec![Expr::Identifier(Ident { value: "nested_col".into(), quote_style: None, + span: Span::empty(), }),], fields: vec![ StructField { @@ -987,10 +994,12 @@ fn parse_typed_struct_syntax_bigquery_and_generic() { Ident { value: "t".into(), quote_style: None, + span: Span::empty(), }, Ident { value: "str_col".into(), quote_style: None, + span: Span::empty(), }, ]), ], @@ -999,6 +1008,7 @@ fn parse_typed_struct_syntax_bigquery_and_generic() { field_name: Some(Ident { value: "x".into(), quote_style: None, + span: Span::empty(), }), field_type: DataType::Int64 }, @@ -1006,6 +1016,7 @@ fn parse_typed_struct_syntax_bigquery_and_generic() { field_name: Some(Ident { value: "y".into(), quote_style: None, + span: Span::empty(), }), field_type: DataType::String(None) }, @@ -1018,6 +1029,7 @@ fn parse_typed_struct_syntax_bigquery_and_generic() { values: vec![Expr::Identifier(Ident { value: "nested_col".into(), quote_style: None, + span: Span::empty(), }),], fields: vec![ StructField { @@ -1049,6 +1061,7 @@ fn parse_typed_struct_syntax_bigquery_and_generic() { values: vec![Expr::Identifier(Ident { value: "nested_col".into(), quote_style: None, + span: Span::empty(), }),], fields: vec![ StructField { diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index 90af12ab7..ed0c74021 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -21,6 +21,8 @@ #[macro_use] mod test_utils; +use helpers::attached_token::AttachedToken; +use sqlparser::tokenizer::Span; use test_utils::*; use sqlparser::ast::Expr::{BinaryOp, Identifier, MapAccess}; @@ -39,12 +41,14 @@ fn parse_map_access_expr() { assert_eq!( Select { distinct: None, + select_token: AttachedToken::empty(), top: None, top_before_distinct: false, projection: vec![UnnamedExpr(MapAccess { column: Box::new(Identifier(Ident { value: "string_values".to_string(), quote_style: None, + span: Span::empty(), })), keys: vec![MapAccessKey { key: call( @@ -903,7 +907,8 @@ fn parse_create_view_with_fields_data_types() { data_type: Some(DataType::Custom( ObjectName(vec![Ident { value: "int".into(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }]), vec![] )), @@ -914,7 +919,8 @@ fn parse_create_view_with_fields_data_types() { data_type: Some(DataType::Custom( ObjectName(vec![Ident { value: "String".into(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }]), vec![] )), diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index e22877dbe..4e0cac45b 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -25,6 +25,7 @@ extern crate core; +use helpers::attached_token::AttachedToken; use matches::assert_matches; use sqlparser::ast::SelectItem::UnnamedExpr; use sqlparser::ast::TableFactor::{Pivot, Unpivot}; @@ -36,6 +37,7 @@ use sqlparser::dialect::{ }; use sqlparser::keywords::{Keyword, ALL_KEYWORDS}; use sqlparser::parser::{Parser, ParserError, ParserOptions}; +use sqlparser::tokenizer::Span; use sqlparser::tokenizer::Tokenizer; use test_utils::{ all_dialects, all_dialects_where, alter_table_op, assert_eq_vec, call, expr_from_projection, @@ -378,6 +380,7 @@ fn parse_update_set_from() { subquery: Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -1271,6 +1274,7 @@ fn parse_select_with_date_column_name() { &Expr::Identifier(Ident { value: "date".into(), quote_style: None, + span: Span::empty(), }), expr_from_projection(only(&select.projection)), ); @@ -1789,6 +1793,7 @@ fn parse_null_like() { alias: Ident { value: "col_null".to_owned(), quote_style: None, + span: Span::empty(), }, }, select.projection[0] @@ -1805,6 +1810,7 @@ fn parse_null_like() { alias: Ident { value: "null_col".to_owned(), quote_style: None, + span: Span::empty(), }, }, select.projection[1] @@ -2823,6 +2829,7 @@ fn parse_listagg() { expr: Expr::Identifier(Ident { value: "id".to_string(), quote_style: None, + span: Span::empty(), }), asc: None, nulls_first: None, @@ -2832,6 +2839,7 @@ fn parse_listagg() { expr: Expr::Identifier(Ident { value: "username".to_string(), quote_style: None, + span: Span::empty(), }), asc: None, nulls_first: None, @@ -4038,7 +4046,8 @@ fn parse_alter_table() { [SqlOption::KeyValue { key: Ident { value: "classification".to_string(), - quote_style: Some('\'') + quote_style: Some('\''), + span: Span::empty(), }, value: Expr::Value(Value::SingleQuotedString("parquet".to_string())), }], @@ -4824,6 +4833,7 @@ fn test_parse_named_window() { ORDER BY C3"; let actual_select_only = verified_only_select(sql); let expected = Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -4833,6 +4843,7 @@ fn test_parse_named_window() { name: ObjectName(vec![Ident { value: "MIN".to_string(), quote_style: None, + span: Span::empty(), }]), parameters: FunctionArguments::None, args: FunctionArguments::List(FunctionArgumentList { @@ -4841,6 +4852,7 @@ fn test_parse_named_window() { Expr::Identifier(Ident { value: "c12".to_string(), quote_style: None, + span: Span::empty(), }), ))], clauses: vec![], @@ -4850,12 +4862,14 @@ fn test_parse_named_window() { over: Some(WindowType::NamedWindow(Ident { value: "window1".to_string(), quote_style: None, + span: Span::empty(), })), within_group: vec![], }), alias: Ident { value: "min1".to_string(), quote_style: None, + span: Span::empty(), }, }, SelectItem::ExprWithAlias { @@ -4863,6 +4877,7 @@ fn test_parse_named_window() { name: ObjectName(vec![Ident { value: "MAX".to_string(), quote_style: None, + span: Span::empty(), }]), parameters: FunctionArguments::None, args: FunctionArguments::List(FunctionArgumentList { @@ -4871,6 +4886,7 @@ fn test_parse_named_window() { Expr::Identifier(Ident { value: "c12".to_string(), quote_style: None, + span: Span::empty(), }), ))], clauses: vec![], @@ -4880,12 +4896,14 @@ fn test_parse_named_window() { over: Some(WindowType::NamedWindow(Ident { value: "window2".to_string(), quote_style: None, + span: Span::empty(), })), within_group: vec![], }), alias: Ident { value: "max1".to_string(), quote_style: None, + span: Span::empty(), }, }, ], @@ -4895,6 +4913,7 @@ fn test_parse_named_window() { name: ObjectName(vec![Ident { value: "aggregate_test_100".to_string(), quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -4919,6 +4938,7 @@ fn test_parse_named_window() { Ident { value: "window1".to_string(), quote_style: None, + span: Span::empty(), }, NamedWindowExpr::WindowSpec(WindowSpec { window_name: None, @@ -4927,6 +4947,7 @@ fn test_parse_named_window() { expr: Expr::Identifier(Ident { value: "C12".to_string(), quote_style: None, + span: Span::empty(), }), asc: None, nulls_first: None, @@ -4939,12 +4960,14 @@ fn test_parse_named_window() { Ident { value: "window2".to_string(), quote_style: None, + span: Span::empty(), }, NamedWindowExpr::WindowSpec(WindowSpec { window_name: None, partition_by: vec![Expr::Identifier(Ident { value: "C11".to_string(), quote_style: None, + span: Span::empty(), })], order_by: vec![], window_frame: None, @@ -5425,6 +5448,7 @@ fn interval_disallow_interval_expr_gt() { right: Box::new(Expr::Identifier(Ident { value: "x".to_string(), quote_style: None, + span: Span::empty(), })), } ) @@ -5465,12 +5489,14 @@ fn parse_interval_and_or_xor() { let expected_ast = vec![Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, projection: vec![UnnamedExpr(Expr::Identifier(Ident { value: "col".to_string(), quote_style: None, + span: Span::empty(), }))], into: None, from: vec![TableWithJoins { @@ -5478,6 +5504,7 @@ fn parse_interval_and_or_xor() { name: ObjectName(vec![Ident { value: "test".to_string(), quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -5496,12 +5523,14 @@ fn parse_interval_and_or_xor() { left: Box::new(Expr::Identifier(Ident { value: "d3_date".to_string(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Gt, right: Box::new(Expr::BinaryOp { left: Box::new(Expr::Identifier(Ident { value: "d1_date".to_string(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Plus, right: Box::new(Expr::Interval(Interval { @@ -5520,12 +5549,14 @@ fn parse_interval_and_or_xor() { left: Box::new(Expr::Identifier(Ident { value: "d2_date".to_string(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Gt, right: Box::new(Expr::BinaryOp { left: Box::new(Expr::Identifier(Ident { value: "d1_date".to_string(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Plus, right: Box::new(Expr::Interval(Interval { @@ -5617,6 +5648,7 @@ fn parse_at_timezone() { alias: Ident { value: "hour".to_string(), quote_style: Some('"'), + span: Span::empty(), }, }, only(&select.projection), @@ -6637,12 +6669,14 @@ fn parse_recursive_cte() { name: Ident { value: "nums".to_string(), quote_style: None, + span: Span::empty(), }, columns: vec![TableAliasColumnDef::from_name("val")], }, query: Box::new(cte_query), from: None, materialized: None, + closing_paren_token: AttachedToken::empty(), }; assert_eq!(with.cte_tables.first().unwrap(), &expected); } @@ -7616,22 +7650,18 @@ fn lateral_function() { let sql = "SELECT * FROM customer LEFT JOIN LATERAL generate_series(1, customer.id)"; let actual_select_only = verified_only_select(sql); let expected = Select { + select_token: AttachedToken::empty(), distinct: None, top: None, + projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions::default())], top_before_distinct: false, - projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions { - opt_ilike: None, - opt_exclude: None, - opt_except: None, - opt_rename: None, - opt_replace: None, - })], into: None, from: vec![TableWithJoins { relation: TableFactor::Table { name: ObjectName(vec![Ident { value: "customer".to_string(), quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -8270,10 +8300,12 @@ fn parse_grant() { Ident { value: "shape".into(), quote_style: None, + span: Span::empty(), }, Ident { value: "size".into(), quote_style: None, + span: Span::empty(), }, ]) }, @@ -8467,6 +8499,7 @@ fn parse_merge() { subquery: Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -8515,6 +8548,7 @@ fn parse_merge() { name: Ident { value: "stg".to_string(), quote_style: None, + span: Span::empty(), }, columns: vec![], }), @@ -8714,7 +8748,8 @@ fn test_lock_table() { lock.of.unwrap().0, vec![Ident { value: "school".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert!(lock.nonblock.is_none()); @@ -8728,7 +8763,8 @@ fn test_lock_table() { lock.of.unwrap().0, vec![Ident { value: "school".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert!(lock.nonblock.is_none()); @@ -8742,7 +8778,8 @@ fn test_lock_table() { lock.of.unwrap().0, vec![Ident { value: "school".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert!(lock.nonblock.is_none()); @@ -8752,7 +8789,8 @@ fn test_lock_table() { lock.of.unwrap().0, vec![Ident { value: "student".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert!(lock.nonblock.is_none()); @@ -8769,7 +8807,8 @@ fn test_lock_nonblock() { lock.of.unwrap().0, vec![Ident { value: "school".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert_eq!(lock.nonblock.unwrap(), NonBlock::SkipLocked); @@ -8783,7 +8822,8 @@ fn test_lock_nonblock() { lock.of.unwrap().0, vec![Ident { value: "school".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert_eq!(lock.nonblock.unwrap(), NonBlock::Nowait); @@ -9584,7 +9624,8 @@ fn parse_pivot_table() { alias: Some(TableAlias { name: Ident { value: "p".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }, columns: vec![ TableAliasColumnDef::from_name("c"), @@ -9636,12 +9677,14 @@ fn parse_unpivot_table() { }), value: Ident { value: "quantity".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() }, name: Ident { value: "quarter".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() }, columns: ["Q1", "Q2", "Q3", "Q4"] .into_iter() @@ -9704,12 +9747,14 @@ fn parse_pivot_unpivot_table() { }), value: Ident { value: "population".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() }, name: Ident { value: "year".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() }, columns: ["population_2000", "population_2010"] .into_iter() @@ -9999,10 +10044,12 @@ fn parse_execute_stored_procedure() { Ident { value: "my_schema".to_string(), quote_style: None, + span: Span::empty(), }, Ident { value: "my_stored_procedure".to_string(), quote_style: None, + span: Span::empty(), }, ]), parameters: vec![ @@ -10098,6 +10145,7 @@ fn parse_unload() { Statement::Unload { query: Box::new(Query { body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -10143,12 +10191,14 @@ fn parse_unload() { }), to: Ident { value: "s3://...".to_string(), - quote_style: Some('\'') + quote_style: Some('\''), + span: Span::empty(), }, with: vec![SqlOption::KeyValue { key: Ident { value: "format".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }, value: Expr::Value(Value::SingleQuotedString("AVRO".to_string())) }] @@ -10275,6 +10325,7 @@ fn parse_map_access_expr() { #[test] fn parse_connect_by() { let expect_query = Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -10363,6 +10414,7 @@ fn parse_connect_by() { assert_eq!( all_dialects_where(|d| d.supports_connect_by()).verified_only_select(connect_by_3), Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -11206,6 +11258,7 @@ fn test_extract_seconds_ok() { field: DateTimeField::Custom(Ident { value: "seconds".to_string(), quote_style: None, + span: Span::empty(), }), syntax: ExtractSyntax::From, expr: Box::new(Expr::Cast { @@ -11231,6 +11284,7 @@ fn test_extract_seconds_single_quote_ok() { field: DateTimeField::Custom(Ident { value: "seconds".to_string(), quote_style: Some('\''), + span: Span::empty(), }), syntax: ExtractSyntax::From, expr: Box::new(Expr::Cast { @@ -12130,7 +12184,8 @@ fn test_load_extension() { assert_eq!( Ident { value: "filename".to_string(), - quote_style: Some('\'') + quote_style: Some('\''), + span: Span::empty(), }, extension_name ); diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index 73b0f6601..01ac0649a 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -18,6 +18,8 @@ #[macro_use] mod test_utils; +use helpers::attached_token::AttachedToken; +use sqlparser::tokenizer::Span; use test_utils::*; use sqlparser::ast::*; @@ -259,22 +261,18 @@ fn test_select_union_by_name() { op: SetOperator::Union, set_quantifier: *expected_quantifier, left: Box::::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, + projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions::default())], top_before_distinct: false, - projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions { - opt_ilike: None, - opt_exclude: None, - opt_except: None, - opt_rename: None, - opt_replace: None, - })], into: None, from: vec![TableWithJoins { relation: TableFactor::Table { name: ObjectName(vec![Ident { value: "capitals".to_string(), quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -301,22 +299,18 @@ fn test_select_union_by_name() { connect_by: None, }))), right: Box::::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, + projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions::default())], top_before_distinct: false, - projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions { - opt_ilike: None, - opt_exclude: None, - opt_except: None, - opt_rename: None, - opt_replace: None, - })], into: None, from: vec![TableWithJoins { relation: TableFactor::Table { name: ObjectName(vec![Ident { value: "weather".to_string(), quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -355,12 +349,28 @@ fn test_duckdb_install() { Statement::Install { extension_name: Ident { value: "tpch".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() } } ); } +#[test] +fn test_duckdb_load_extension() { + let stmt = duckdb().verified_stmt("LOAD my_extension"); + assert_eq!( + Statement::Load { + extension_name: Ident { + value: "my_extension".to_string(), + quote_style: None, + span: Span::empty() + } + }, + stmt + ); +} + #[test] fn test_duckdb_struct_literal() { //struct literal syntax https://duckdb.org/docs/sql/data_types/struct#creating-structs diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index d1d8d1248..31668c86a 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -22,6 +22,8 @@ #[macro_use] mod test_utils; +use helpers::attached_token::AttachedToken; +use sqlparser::tokenizer::Span; use test_utils::*; use sqlparser::ast::DataType::{Int, Text}; @@ -113,6 +115,7 @@ fn parse_create_procedure() { settings: None, format_clause: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -138,14 +141,16 @@ fn parse_create_procedure() { ProcedureParam { name: Ident { value: "@foo".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, data_type: DataType::Int(None) }, ProcedureParam { name: Ident { value: "@bar".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, data_type: DataType::Varchar(Some(CharacterLength::IntegerLength { length: 256, @@ -155,7 +160,8 @@ fn parse_create_procedure() { ]), name: ObjectName(vec![Ident { value: "test".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }]) } ) @@ -204,15 +210,9 @@ fn parse_mssql_openjson() { assert_eq!( vec![TableWithJoins { relation: TableFactor::Table { - name: ObjectName(vec![Ident { - value: "t_test_table".into(), - quote_style: None, - },]), + name: ObjectName(vec![Ident::new("t_test_table")]), alias: Some(TableAlias { - name: Ident { - value: "A".into(), - quote_style: None - }, + name: Ident::new("A"), columns: vec![] }), args: None, @@ -224,23 +224,13 @@ fn parse_mssql_openjson() { }, joins: vec![Join { relation: TableFactor::OpenJsonTable { - json_expr: Expr::CompoundIdentifier(vec![ - Ident { - value: "A".into(), - quote_style: None, - }, - Ident { - value: "param".into(), - quote_style: None, - } - ]), + json_expr: Expr::CompoundIdentifier( + vec![Ident::new("A"), Ident::new("param"),] + ), json_path: Some(Value::SingleQuotedString("$.config".into())), columns: vec![ OpenJsonTableColumn { - name: Ident { - value: "kind".into(), - quote_style: None, - }, + name: Ident::new("kind"), r#type: DataType::Varchar(Some(CharacterLength::IntegerLength { length: 20, unit: None @@ -252,6 +242,7 @@ fn parse_mssql_openjson() { name: Ident { value: "id_list".into(), quote_style: Some('['), + span: Span::empty(), }, r#type: DataType::Nvarchar(Some(CharacterLength::Max)), path: Some("$.id_list".into()), @@ -259,10 +250,7 @@ fn parse_mssql_openjson() { } ], alias: Some(TableAlias { - name: Ident { - value: "B".into(), - quote_style: None - }, + name: Ident::new("B"), columns: vec![] }) }, @@ -280,15 +268,9 @@ fn parse_mssql_openjson() { assert_eq!( vec![TableWithJoins { relation: TableFactor::Table { - name: ObjectName(vec![Ident { - value: "t_test_table".into(), - quote_style: None, - },]), + name: ObjectName(vec![Ident::new("t_test_table"),]), alias: Some(TableAlias { - name: Ident { - value: "A".into(), - quote_style: None - }, + name: Ident::new("A"), columns: vec![] }), args: None, @@ -300,23 +282,13 @@ fn parse_mssql_openjson() { }, joins: vec![Join { relation: TableFactor::OpenJsonTable { - json_expr: Expr::CompoundIdentifier(vec![ - Ident { - value: "A".into(), - quote_style: None, - }, - Ident { - value: "param".into(), - quote_style: None, - } - ]), + json_expr: Expr::CompoundIdentifier( + vec![Ident::new("A"), Ident::new("param"),] + ), json_path: None, columns: vec![ OpenJsonTableColumn { - name: Ident { - value: "kind".into(), - quote_style: None, - }, + name: Ident::new("kind"), r#type: DataType::Varchar(Some(CharacterLength::IntegerLength { length: 20, unit: None @@ -328,6 +300,7 @@ fn parse_mssql_openjson() { name: Ident { value: "id_list".into(), quote_style: Some('['), + span: Span::empty(), }, r#type: DataType::Nvarchar(Some(CharacterLength::Max)), path: Some("$.id_list".into()), @@ -335,10 +308,7 @@ fn parse_mssql_openjson() { } ], alias: Some(TableAlias { - name: Ident { - value: "B".into(), - quote_style: None - }, + name: Ident::new("B"), columns: vec![] }) }, @@ -356,15 +326,10 @@ fn parse_mssql_openjson() { assert_eq!( vec![TableWithJoins { relation: TableFactor::Table { - name: ObjectName(vec![Ident { - value: "t_test_table".into(), - quote_style: None, - },]), + name: ObjectName(vec![Ident::new("t_test_table")]), + alias: Some(TableAlias { - name: Ident { - value: "A".into(), - quote_style: None - }, + name: Ident::new("A"), columns: vec![] }), args: None, @@ -376,23 +341,13 @@ fn parse_mssql_openjson() { }, joins: vec![Join { relation: TableFactor::OpenJsonTable { - json_expr: Expr::CompoundIdentifier(vec![ - Ident { - value: "A".into(), - quote_style: None, - }, - Ident { - value: "param".into(), - quote_style: None, - } - ]), + json_expr: Expr::CompoundIdentifier( + vec![Ident::new("A"), Ident::new("param"),] + ), json_path: None, columns: vec![ OpenJsonTableColumn { - name: Ident { - value: "kind".into(), - quote_style: None, - }, + name: Ident::new("kind"), r#type: DataType::Varchar(Some(CharacterLength::IntegerLength { length: 20, unit: None @@ -404,6 +359,7 @@ fn parse_mssql_openjson() { name: Ident { value: "id_list".into(), quote_style: Some('['), + span: Span::empty(), }, r#type: DataType::Nvarchar(Some(CharacterLength::Max)), path: None, @@ -411,10 +367,7 @@ fn parse_mssql_openjson() { } ], alias: Some(TableAlias { - name: Ident { - value: "B".into(), - quote_style: None - }, + name: Ident::new("B"), columns: vec![] }) }, @@ -432,15 +385,9 @@ fn parse_mssql_openjson() { assert_eq!( vec![TableWithJoins { relation: TableFactor::Table { - name: ObjectName(vec![Ident { - value: "t_test_table".into(), - quote_style: None, - },]), + name: ObjectName(vec![Ident::new("t_test_table")]), alias: Some(TableAlias { - name: Ident { - value: "A".into(), - quote_style: None - }, + name: Ident::new("A"), columns: vec![] }), args: None, @@ -452,23 +399,13 @@ fn parse_mssql_openjson() { }, joins: vec![Join { relation: TableFactor::OpenJsonTable { - json_expr: Expr::CompoundIdentifier(vec![ - Ident { - value: "A".into(), - quote_style: None, - }, - Ident { - value: "param".into(), - quote_style: None, - } - ]), + json_expr: Expr::CompoundIdentifier( + vec![Ident::new("A"), Ident::new("param"),] + ), json_path: Some(Value::SingleQuotedString("$.config".into())), columns: vec![], alias: Some(TableAlias { - name: Ident { - value: "B".into(), - quote_style: None - }, + name: Ident::new("B"), columns: vec![] }) }, @@ -486,15 +423,9 @@ fn parse_mssql_openjson() { assert_eq!( vec![TableWithJoins { relation: TableFactor::Table { - name: ObjectName(vec![Ident { - value: "t_test_table".into(), - quote_style: None, - },]), + name: ObjectName(vec![Ident::new("t_test_table")]), alias: Some(TableAlias { - name: Ident { - value: "A".into(), - quote_style: None - }, + name: Ident::new("A"), columns: vec![] }), args: None, @@ -506,23 +437,13 @@ fn parse_mssql_openjson() { }, joins: vec![Join { relation: TableFactor::OpenJsonTable { - json_expr: Expr::CompoundIdentifier(vec![ - Ident { - value: "A".into(), - quote_style: None, - }, - Ident { - value: "param".into(), - quote_style: None, - } - ]), + json_expr: Expr::CompoundIdentifier( + vec![Ident::new("A"), Ident::new("param"),] + ), json_path: None, columns: vec![], alias: Some(TableAlias { - name: Ident { - value: "B".into(), - quote_style: None - }, + name: Ident::new("B"), columns: vec![] }) }, @@ -607,7 +528,8 @@ fn parse_mssql_create_role() { authorization_owner, Some(ObjectName(vec![Ident { value: "helena".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])) ); } @@ -623,12 +545,14 @@ fn parse_alter_role() { [Statement::AlterRole { name: Ident { value: "old_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::RenameRole { role_name: Ident { value: "new_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), } }, }] @@ -640,12 +564,14 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::AddMember { member_name: Ident { value: "new_member".into(), - quote_style: None + quote_style: None, + span: Span::empty(), } }, } @@ -657,12 +583,14 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::DropMember { member_name: Ident { value: "old_member".into(), - quote_style: None + quote_style: None, + span: Span::empty(), } }, } @@ -1137,13 +1065,15 @@ fn parse_substring_in_select() { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: Some(Distinct::Distinct), top: None, top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Substring { expr: Box::new(Expr::Identifier(Ident { value: "description".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), })), substring_from: Some(Box::new(Expr::Value(number("0")))), substring_for: Some(Box::new(Expr::Value(number("1")))), @@ -1154,7 +1084,8 @@ fn parse_substring_in_select() { relation: TableFactor::Table { name: ObjectName(vec![Ident { value: "test".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -1208,7 +1139,8 @@ fn parse_mssql_declare() { Declare { names: vec![Ident { value: "@foo".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }], data_type: None, assignment: None, @@ -1222,7 +1154,8 @@ fn parse_mssql_declare() { Declare { names: vec![Ident { value: "@bar".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }], data_type: Some(Int(None)), assignment: None, @@ -1236,7 +1169,8 @@ fn parse_mssql_declare() { Declare { names: vec![Ident { value: "@baz".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }], data_type: Some(Text), assignment: Some(MsSqlAssignment(Box::new(Expr::Value(SingleQuotedString( @@ -1260,10 +1194,7 @@ fn parse_mssql_declare() { vec![ Statement::Declare { stmts: vec![Declare { - names: vec![Ident { - value: "@bar".to_string(), - quote_style: None - }], + names: vec![Ident::new("@bar"),], data_type: Some(Int(None)), assignment: None, declare_type: None, @@ -1292,6 +1223,7 @@ fn parse_mssql_declare() { settings: None, format_clause: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -1364,10 +1296,12 @@ fn parse_create_table_with_valid_options() { key: Ident { value: "DISTRIBUTION".to_string(), quote_style: None, + span: Span::empty(), }, value: Expr::Identifier(Ident { value: "ROUND_ROBIN".to_string(), quote_style: None, + span: Span::empty(), }) }, SqlOption::Partition { @@ -1411,6 +1345,7 @@ fn parse_create_table_with_valid_options() { name: Ident { value: "column_a".to_string(), quote_style: None, + span: Span::empty(), }, asc: Some(true), }, @@ -1418,6 +1353,7 @@ fn parse_create_table_with_valid_options() { name: Ident { value: "column_b".to_string(), quote_style: None, + span: Span::empty(), }, asc: Some(false), }, @@ -1425,6 +1361,7 @@ fn parse_create_table_with_valid_options() { name: Ident { value: "column_c".to_string(), quote_style: None, + span: Span::empty(), }, asc: None, }, @@ -1438,6 +1375,7 @@ fn parse_create_table_with_valid_options() { key: Ident { value: "DISTRIBUTION".to_string(), quote_style: None, + span: Span::empty(), }, value: Expr::Function( Function { @@ -1446,6 +1384,7 @@ fn parse_create_table_with_valid_options() { Ident { value: "HASH".to_string(), quote_style: None, + span: Span::empty(), }, ], ), @@ -1460,6 +1399,7 @@ fn parse_create_table_with_valid_options() { Ident { value: "column_a".to_string(), quote_style: None, + span: Span::empty(), }, ), ), @@ -1470,6 +1410,7 @@ fn parse_create_table_with_valid_options() { Ident { value: "column_b".to_string(), quote_style: None, + span: Span::empty(), }, ), ), @@ -1504,12 +1445,14 @@ fn parse_create_table_with_valid_options() { name: ObjectName(vec![Ident { value: "mytable".to_string(), quote_style: None, + span: Span::empty(), },],), columns: vec![ ColumnDef { name: Ident { value: "column_a".to_string(), quote_style: None, + span: Span::empty(), }, data_type: Int(None,), collation: None, @@ -1519,6 +1462,7 @@ fn parse_create_table_with_valid_options() { name: Ident { value: "column_b".to_string(), quote_style: None, + span: Span::empty(), }, data_type: Int(None,), collation: None, @@ -1528,6 +1472,7 @@ fn parse_create_table_with_valid_options() { name: Ident { value: "column_c".to_string(), quote_style: None, + span: Span::empty(), }, data_type: Int(None,), collation: None, @@ -1669,11 +1614,13 @@ fn parse_create_table_with_identity_column() { name: ObjectName(vec![Ident { value: "mytable".to_string(), quote_style: None, + span: Span::empty(), },],), columns: vec![ColumnDef { name: Ident { value: "columnA".to_string(), quote_style: None, + span: Span::empty(), }, data_type: Int(None,), collation: None, diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 3d8b08630..943a61718 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -19,12 +19,14 @@ //! Test SQL syntax specific to MySQL. The parser based on the generic dialect //! is also tested (on the inputs it can handle). +use helpers::attached_token::AttachedToken; use matches::assert_matches; use sqlparser::ast::MysqlInsertPriority::{Delayed, HighPriority, LowPriority}; use sqlparser::ast::*; use sqlparser::dialect::{GenericDialect, MySqlDialect}; use sqlparser::parser::{ParserError, ParserOptions}; +use sqlparser::tokenizer::Span; use sqlparser::tokenizer::Token; use test_utils::*; @@ -142,16 +144,19 @@ fn parse_flush() { ObjectName(vec![ Ident { value: "mek".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), }, Ident { value: "table1".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), } ]), ObjectName(vec![Ident { value: "table2".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]) ] } @@ -179,16 +184,19 @@ fn parse_flush() { ObjectName(vec![ Ident { value: "mek".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), }, Ident { value: "table1".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), } ]), ObjectName(vec![Ident { value: "table2".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]) ] } @@ -205,16 +213,19 @@ fn parse_flush() { ObjectName(vec![ Ident { value: "mek".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), }, Ident { value: "table1".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), } ]), ObjectName(vec![Ident { value: "table2".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]) ] } @@ -1058,12 +1069,14 @@ fn parse_escaped_quote_identifiers_with_escape() { Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "quoted ` identifier".into(), quote_style: Some('`'), + span: Span::empty(), }))], into: None, from: vec![], @@ -1109,12 +1122,14 @@ fn parse_escaped_quote_identifiers_with_no_escape() { Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "quoted `` identifier".into(), quote_style: Some('`'), + span: Span::empty(), }))], into: None, from: vec![], @@ -1153,12 +1168,15 @@ fn parse_escaped_backticks_with_escape() { Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), + distinct: None, top: None, top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "`quoted identifier`".into(), quote_style: Some('`'), + span: Span::empty(), }))], into: None, from: vec![], @@ -1201,12 +1219,15 @@ fn parse_escaped_backticks_with_no_escape() { Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), + distinct: None, top: None, top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "``quoted identifier``".into(), quote_style: Some('`'), + span: Span::empty(), }))], into: None, from: vec![], @@ -1846,6 +1867,8 @@ fn parse_select_with_numeric_prefix_column_name() { assert_eq!( q.body, Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), + distinct: None, top: None, top_before_distinct: false, @@ -1902,6 +1925,8 @@ fn parse_select_with_concatenation_of_exp_number_and_numeric_prefix_column() { assert_eq!( q.body, Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), + distinct: None, top: None, top_before_distinct: false, @@ -2055,7 +2080,8 @@ fn parse_delete_with_order_by() { vec![OrderByExpr { expr: Expr::Identifier(Ident { value: "id".to_owned(), - quote_style: None + quote_style: None, + span: Span::empty(), }), asc: Some(false), nulls_first: None, @@ -2136,7 +2162,8 @@ fn parse_alter_table_add_column() { }, column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("foo"), - quote_style: None + quote_style: None, + span: Span::empty(), })), },] ); @@ -2187,6 +2214,7 @@ fn parse_alter_table_add_columns() { column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("foo"), quote_style: None, + span: Span::empty(), })), }, ] @@ -2247,6 +2275,7 @@ fn parse_alter_table_change_column() { column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("foo"), quote_style: None, + span: Span::empty(), })), }; let sql4 = "ALTER TABLE orders CHANGE COLUMN description desc TEXT NOT NULL AFTER foo"; @@ -2286,6 +2315,7 @@ fn parse_alter_table_change_column_with_column_position() { column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("total_count"), quote_style: None, + span: Span::empty(), })), }; @@ -2342,6 +2372,7 @@ fn parse_alter_table_modify_column() { column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("foo"), quote_style: None, + span: Span::empty(), })), }; let sql4 = "ALTER TABLE orders MODIFY COLUMN description TEXT NOT NULL AFTER foo"; @@ -2379,6 +2410,7 @@ fn parse_alter_table_modify_column_with_column_position() { column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("total_count"), quote_style: None, + span: Span::empty(), })), }; @@ -2397,6 +2429,8 @@ fn parse_alter_table_modify_column_with_column_position() { #[test] fn parse_substring_in_select() { + use sqlparser::tokenizer::Span; + let sql = "SELECT DISTINCT SUBSTRING(description, 0, 1) FROM test"; match mysql().one_statement_parses_to( sql, @@ -2407,13 +2441,15 @@ fn parse_substring_in_select() { Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: Some(Distinct::Distinct), top: None, top_before_distinct: false, projection: vec![SelectItem::UnnamedExpr(Expr::Substring { expr: Box::new(Expr::Identifier(Ident { value: "description".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), })), substring_from: Some(Box::new(Expr::Value(number("0")))), substring_for: Some(Box::new(Expr::Value(number("1")))), @@ -2424,7 +2460,8 @@ fn parse_substring_in_select() { relation: TableFactor::Table { name: ObjectName(vec![Ident { value: "test".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -2730,6 +2767,7 @@ fn parse_hex_string_introducer() { Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index d27569e03..54f77b7be 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -21,6 +21,8 @@ #[macro_use] mod test_utils; +use helpers::attached_token::AttachedToken; +use sqlparser::tokenizer::Span; use test_utils::*; use sqlparser::ast::*; @@ -1163,6 +1165,7 @@ fn parse_copy_to() { source: CopySource::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -1172,6 +1175,7 @@ fn parse_copy_to() { alias: Ident { value: "a".into(), quote_style: None, + span: Span::empty(), }, }, SelectItem::ExprWithAlias { @@ -1179,6 +1183,7 @@ fn parse_copy_to() { alias: Ident { value: "b".into(), quote_style: None, + span: Span::empty(), }, } ], @@ -1318,7 +1323,8 @@ fn parse_set() { variables: OneOrManyWithParens::One(ObjectName(vec![Ident::new("a")])), value: vec![Expr::Identifier(Ident { value: "b".into(), - quote_style: None + quote_style: None, + span: Span::empty(), })], } ); @@ -1380,7 +1386,8 @@ fn parse_set() { ])), value: vec![Expr::Identifier(Ident { value: "b".into(), - quote_style: None + quote_style: None, + span: Span::empty(), })], } ); @@ -1452,6 +1459,7 @@ fn parse_set_role() { role_name: Some(Ident { value: "rolename".to_string(), quote_style: Some('\"'), + span: Span::empty(), }), } ); @@ -1466,6 +1474,7 @@ fn parse_set_role() { role_name: Some(Ident { value: "rolename".to_string(), quote_style: Some('\''), + span: Span::empty(), }), } ); @@ -1765,7 +1774,8 @@ fn parse_pg_on_conflict() { selection: Some(Expr::BinaryOp { left: Box::new(Expr::Identifier(Ident { value: "dsize".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Gt, right: Box::new(Expr::Value(Value::Placeholder("$2".to_string()))) @@ -1802,7 +1812,8 @@ fn parse_pg_on_conflict() { selection: Some(Expr::BinaryOp { left: Box::new(Expr::Identifier(Ident { value: "dsize".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Gt, right: Box::new(Expr::Value(Value::Placeholder("$2".to_string()))) @@ -2105,14 +2116,16 @@ fn parse_array_index_expr() { subscript: Box::new(Subscript::Index { index: Expr::Identifier(Ident { value: "baz".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }) }) }), subscript: Box::new(Subscript::Index { index: Expr::Identifier(Ident { value: "fooz".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }) }) }, @@ -2504,6 +2517,7 @@ fn parse_array_subquery_expr() { op: SetOperator::Union, set_quantifier: SetQuantifier::None, left: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -2525,6 +2539,7 @@ fn parse_array_subquery_expr() { connect_by: None, }))), right: Box::new(SetExpr::Select(Box::new(Select { + select_token: AttachedToken::empty(), distinct: None, top: None, top_before_distinct: false, @@ -3123,6 +3138,7 @@ fn parse_custom_operator() { left: Box::new(Expr::Identifier(Ident { value: "relname".into(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::PGCustomBinaryOperator(vec![ "database".into(), @@ -3142,6 +3158,7 @@ fn parse_custom_operator() { left: Box::new(Expr::Identifier(Ident { value: "relname".into(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::PGCustomBinaryOperator(vec!["pg_catalog".into(), "~".into()]), right: Box::new(Expr::Value(Value::SingleQuotedString("^(table)$".into()))) @@ -3157,6 +3174,7 @@ fn parse_custom_operator() { left: Box::new(Expr::Identifier(Ident { value: "relname".into(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::PGCustomBinaryOperator(vec!["~".into()]), right: Box::new(Expr::Value(Value::SingleQuotedString("^(table)$".into()))) @@ -3307,12 +3325,14 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "old_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::RenameRole { role_name: Ident { value: "new_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), } }, } @@ -3324,7 +3344,8 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::WithOptions { options: vec![ @@ -3353,7 +3374,8 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::WithOptions { options: vec![ @@ -3376,12 +3398,14 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Set { config_name: ObjectName(vec![Ident { value: "maintenance_work_mem".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), config_value: SetConfigValue::FromCurrent, in_database: None @@ -3395,17 +3419,20 @@ fn parse_alter_role() { [Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Set { config_name: ObjectName(vec![Ident { value: "maintenance_work_mem".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), config_value: SetConfigValue::Value(Expr::Value(number("100000"))), in_database: Some(ObjectName(vec![Ident { value: "database_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])) }, }] @@ -3417,17 +3444,20 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Set { config_name: ObjectName(vec![Ident { value: "maintenance_work_mem".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), config_value: SetConfigValue::Value(Expr::Value(number("100000"))), in_database: Some(ObjectName(vec![Ident { value: "database_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])) }, } @@ -3439,17 +3469,20 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Set { config_name: ObjectName(vec![Ident { value: "maintenance_work_mem".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), config_value: SetConfigValue::Default, in_database: Some(ObjectName(vec![Ident { value: "database_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])) }, } @@ -3461,7 +3494,8 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Reset { config_name: ResetConfig::ALL, @@ -3476,16 +3510,19 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Reset { config_name: ResetConfig::ConfigName(ObjectName(vec![Ident { value: "maintenance_work_mem".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])), in_database: Some(ObjectName(vec![Ident { value: "database_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])) }, } @@ -3630,7 +3667,8 @@ fn parse_drop_function() { func_desc: vec![FunctionDesc { name: ObjectName(vec![Ident { value: "test_func".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: None }], @@ -3646,7 +3684,8 @@ fn parse_drop_function() { func_desc: vec![FunctionDesc { name: ObjectName(vec![Ident { value: "test_func".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Integer(None)), @@ -3671,7 +3710,8 @@ fn parse_drop_function() { FunctionDesc { name: ObjectName(vec![Ident { value: "test_func1".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Integer(None)), @@ -3689,7 +3729,8 @@ fn parse_drop_function() { FunctionDesc { name: ObjectName(vec![Ident { value: "test_func2".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Varchar(None)), @@ -3720,7 +3761,8 @@ fn parse_drop_procedure() { proc_desc: vec![FunctionDesc { name: ObjectName(vec![Ident { value: "test_proc".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: None }], @@ -3736,7 +3778,8 @@ fn parse_drop_procedure() { proc_desc: vec![FunctionDesc { name: ObjectName(vec![Ident { value: "test_proc".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Integer(None)), @@ -3761,7 +3804,8 @@ fn parse_drop_procedure() { FunctionDesc { name: ObjectName(vec![Ident { value: "test_proc1".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Integer(None)), @@ -3779,7 +3823,8 @@ fn parse_drop_procedure() { FunctionDesc { name: ObjectName(vec![Ident { value: "test_proc2".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Varchar(None)), @@ -3860,6 +3905,7 @@ fn parse_dollar_quoted_string() { alias: Ident { value: "col_name".into(), quote_style: None, + span: Span::empty(), }, } ); @@ -4204,20 +4250,24 @@ fn test_simple_postgres_insert_with_alias() { into: true, table_name: ObjectName(vec![Ident { value: "test_tables".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), table_alias: Some(Ident { value: "test_table".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }), columns: vec![ Ident { value: "id".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }, Ident { value: "a".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), } ], overwrite: false, @@ -4267,20 +4317,24 @@ fn test_simple_postgres_insert_with_alias() { into: true, table_name: ObjectName(vec![Ident { value: "test_tables".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), table_alias: Some(Ident { value: "test_table".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }), columns: vec![ Ident { value: "id".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }, Ident { value: "a".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), } ], overwrite: false, @@ -4332,20 +4386,24 @@ fn test_simple_insert_with_quoted_alias() { into: true, table_name: ObjectName(vec![Ident { value: "test_tables".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), table_alias: Some(Ident { value: "Test_Table".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }), columns: vec![ Ident { value: "id".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }, Ident { value: "a".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), } ], overwrite: false, @@ -5017,6 +5075,7 @@ fn check_arrow_precedence(sql: &str, arrow_operator: BinaryOperator) { left: Box::new(Expr::Identifier(Ident { value: "foo".to_string(), quote_style: None, + span: Span::empty(), })), op: arrow_operator, right: Box::new(Expr::Value(Value::SingleQuotedString("bar".to_string()))), @@ -5047,6 +5106,7 @@ fn arrow_cast_precedence() { left: Box::new(Expr::Identifier(Ident { value: "foo".to_string(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Arrow, right: Box::new(Expr::Cast { diff --git a/tests/sqlparser_redshift.rs b/tests/sqlparser_redshift.rs index 0a084b340..f0c1f0c74 100644 --- a/tests/sqlparser_redshift.rs +++ b/tests/sqlparser_redshift.rs @@ -18,6 +18,7 @@ #[macro_use] mod test_utils; +use sqlparser::tokenizer::Span; use test_utils::*; use sqlparser::ast::*; @@ -31,7 +32,8 @@ fn test_square_brackets_over_db_schema_table_name() { select.projection[0], SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "col1".to_string(), - quote_style: Some('[') + quote_style: Some('['), + span: Span::empty(), })), ); assert_eq!( @@ -41,11 +43,13 @@ fn test_square_brackets_over_db_schema_table_name() { name: ObjectName(vec![ Ident { value: "test_schema".to_string(), - quote_style: Some('[') + quote_style: Some('['), + span: Span::empty(), }, Ident { value: "test_table".to_string(), - quote_style: Some('[') + quote_style: Some('['), + span: Span::empty(), } ]), alias: None, @@ -79,7 +83,8 @@ fn test_double_quotes_over_db_schema_table_name() { select.projection[0], SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "col1".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), })), ); assert_eq!( @@ -89,11 +94,13 @@ fn test_double_quotes_over_db_schema_table_name() { name: ObjectName(vec![ Ident { value: "test_schema".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }, Ident { value: "test_table".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), } ]), alias: None, diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index f99a00f5b..08792380d 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -2762,7 +2762,9 @@ fn parse_view_column_descriptions() { #[test] fn test_parentheses_overflow() { - let max_nesting_level: usize = 30; + // TODO: increase / improve after we fix the recursion limit + // for real (see https://github.com/apache/datafusion-sqlparser-rs/issues/984) + let max_nesting_level: usize = 25; // Verify the recursion check is not too wasteful... (num of parentheses - 2 is acceptable) let slack = 2; From 5a510ac4d9715528ad5c518bf1ce0719cc813b8c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 27 Nov 2024 11:33:31 -0500 Subject: [PATCH 69/80] Fix error in benchmark queries (#1560) --- sqlparser_bench/benches/sqlparser_bench.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sqlparser_bench/benches/sqlparser_bench.rs b/sqlparser_bench/benches/sqlparser_bench.rs index 27c58b450..32a6da1bc 100644 --- a/sqlparser_bench/benches/sqlparser_bench.rs +++ b/sqlparser_bench/benches/sqlparser_bench.rs @@ -23,9 +23,9 @@ fn basic_queries(c: &mut Criterion) { let mut group = c.benchmark_group("sqlparser-rs parsing benchmark"); let dialect = GenericDialect {}; - let string = "SELECT * FROM table WHERE 1 = 1"; + let string = "SELECT * FROM my_table WHERE 1 = 1"; group.bench_function("sqlparser::select", |b| { - b.iter(|| Parser::parse_sql(&dialect, string)); + b.iter(|| Parser::parse_sql(&dialect, string).unwrap()); }); let with_query = " @@ -33,14 +33,14 @@ fn basic_queries(c: &mut Criterion) { SELECT MAX(a) AS max_a, COUNT(b) AS b_num, user_id - FROM TABLE + FROM MY_TABLE GROUP BY user_id ) - SELECT * FROM table + SELECT * FROM my_table LEFT JOIN derived USING (user_id) "; group.bench_function("sqlparser::with_select", |b| { - b.iter(|| Parser::parse_sql(&dialect, with_query)); + b.iter(|| Parser::parse_sql(&dialect, with_query).unwrap()); }); } From 6291afb2c75871edf34b3d2c01ef9249a5369c81 Mon Sep 17 00:00:00 2001 From: Ifeanyi Ubah Date: Fri, 29 Nov 2024 12:37:06 +0100 Subject: [PATCH 70/80] Fix clippy warnings on rust 1.83 (#1570) --- src/ast/ddl.rs | 13 ++++++++----- src/ast/mod.rs | 2 +- src/ast/query.rs | 2 +- src/ast/value.rs | 6 +++--- src/parser/alter.rs | 2 +- src/parser/mod.rs | 6 ++---- src/tokenizer.rs | 2 +- 7 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 21a716d25..3ced478ca 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -1327,15 +1327,18 @@ pub enum ColumnOption { /// `DEFAULT ` Default(Expr), - /// ClickHouse supports `MATERIALIZE`, `EPHEMERAL` and `ALIAS` expr to generate default values. + /// `MATERIALIZE ` /// Syntax: `b INT MATERIALIZE (a + 1)` + /// /// [ClickHouse](https://clickhouse.com/docs/en/sql-reference/statements/create/table#default_values) - - /// `MATERIALIZE ` Materialized(Expr), /// `EPHEMERAL []` + /// + /// [ClickHouse](https://clickhouse.com/docs/en/sql-reference/statements/create/table#default_values) Ephemeral(Option), /// `ALIAS ` + /// + /// [ClickHouse](https://clickhouse.com/docs/en/sql-reference/statements/create/table#default_values) Alias(Expr), /// `{ PRIMARY KEY | UNIQUE } []` @@ -1552,7 +1555,7 @@ pub enum GeneratedExpressionMode { #[must_use] fn display_constraint_name(name: &'_ Option) -> impl fmt::Display + '_ { struct ConstraintName<'a>(&'a Option); - impl<'a> fmt::Display for ConstraintName<'a> { + impl fmt::Display for ConstraintName<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if let Some(name) = self.0 { write!(f, "CONSTRAINT {name} ")?; @@ -1573,7 +1576,7 @@ fn display_option<'a, T: fmt::Display>( option: &'a Option, ) -> impl fmt::Display + 'a { struct OptionDisplay<'a, T>(&'a str, &'a str, &'a Option); - impl<'a, T: fmt::Display> fmt::Display for OptionDisplay<'a, T> { + impl fmt::Display for OptionDisplay<'_, T> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if let Some(inner) = self.2 { let (prefix, postfix) = (self.0, self.1); diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 366bf4d25..386e42fb3 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -110,7 +110,7 @@ where sep: &'static str, } -impl<'a, T> fmt::Display for DisplaySeparated<'a, T> +impl fmt::Display for DisplaySeparated<'_, T> where T: fmt::Display, { diff --git a/src/ast/query.rs b/src/ast/query.rs index 0472026a0..716ffe98c 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -1713,7 +1713,7 @@ impl fmt::Display for Join { } fn suffix(constraint: &'_ JoinConstraint) -> impl fmt::Display + '_ { struct Suffix<'a>(&'a JoinConstraint); - impl<'a> fmt::Display for Suffix<'a> { + impl fmt::Display for Suffix<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self.0 { JoinConstraint::On(expr) => write!(f, " ON {expr}"), diff --git a/src/ast/value.rs b/src/ast/value.rs index 30d956a07..28bf89ba8 100644 --- a/src/ast/value.rs +++ b/src/ast/value.rs @@ -261,7 +261,7 @@ pub struct EscapeQuotedString<'a> { quote: char, } -impl<'a> fmt::Display for EscapeQuotedString<'a> { +impl fmt::Display for EscapeQuotedString<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // EscapeQuotedString doesn't know which mode of escape was // chosen by the user. So this code must to correctly display @@ -325,7 +325,7 @@ pub fn escape_double_quote_string(s: &str) -> EscapeQuotedString<'_> { pub struct EscapeEscapedStringLiteral<'a>(&'a str); -impl<'a> fmt::Display for EscapeEscapedStringLiteral<'a> { +impl fmt::Display for EscapeEscapedStringLiteral<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { for c in self.0.chars() { match c { @@ -359,7 +359,7 @@ pub fn escape_escaped_string(s: &str) -> EscapeEscapedStringLiteral<'_> { pub struct EscapeUnicodeStringLiteral<'a>(&'a str); -impl<'a> fmt::Display for EscapeUnicodeStringLiteral<'a> { +impl fmt::Display for EscapeUnicodeStringLiteral<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { for c in self.0.chars() { match c { diff --git a/src/parser/alter.rs b/src/parser/alter.rs index 534105790..3ac4ab0c7 100644 --- a/src/parser/alter.rs +++ b/src/parser/alter.rs @@ -26,7 +26,7 @@ use crate::{ tokenizer::Token, }; -impl<'a> Parser<'a> { +impl Parser<'_> { pub fn parse_alter_role(&mut self) -> Result { if dialect_of!(self is PostgreSqlDialect) { return self.parse_pg_alter_role(); diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b7f5cb866..fe6fae8bf 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -10883,13 +10883,12 @@ impl<'a> Parser<'a> { Ok(ExprWithAlias { expr, alias }) } /// Parses an expression with an optional alias - + /// /// Examples: - + /// /// ```sql /// SUM(price) AS total_price /// ``` - /// ```sql /// SUM(price) /// ``` @@ -10905,7 +10904,6 @@ impl<'a> Parser<'a> { /// assert_eq!(Some("b".to_string()), expr_with_alias.alias.map(|x|x.value)); /// # Ok(()) /// # } - pub fn parse_expr_with_alias(&mut self) -> Result { let expr = self.parse_expr()?; let alias = if self.parse_keyword(Keyword::AS) { diff --git a/src/tokenizer.rs b/src/tokenizer.rs index a57ba2ec8..bed2d9b52 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -584,7 +584,7 @@ struct State<'a> { pub col: u64, } -impl<'a> State<'a> { +impl State<'_> { /// return the next character and advance the stream pub fn next(&mut self) -> Option { match self.peekable.next() { From 92c6e7f79b2a9b54a17566e338c915565c8267bb Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 29 Nov 2024 20:08:52 +0800 Subject: [PATCH 71/80] Support relation visitor to visit the `Option` field (#1556) --- derive/README.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++ derive/src/lib.rs | 36 +++++++++++++++++++++++++++------- src/ast/mod.rs | 1 + src/ast/visitor.rs | 11 ++++++++++- 4 files changed, 89 insertions(+), 8 deletions(-) diff --git a/derive/README.md b/derive/README.md index aa70e7c71..b5ccc69e0 100644 --- a/derive/README.md +++ b/derive/README.md @@ -151,6 +151,55 @@ visitor.post_visit_expr() visitor.post_visit_expr() ``` +If the field is a `Option` and add `#[with = "visit_xxx"]` to the field, the generated code +will try to access the field only if it is `Some`: + +```rust +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct ShowStatementIn { + pub clause: ShowStatementInClause, + pub parent_type: Option, + #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))] + pub parent_name: Option, +} +``` + +This will generate + +```rust +impl sqlparser::ast::Visit for ShowStatementIn { + fn visit( + &self, + visitor: &mut V, + ) -> ::std::ops::ControlFlow { + sqlparser::ast::Visit::visit(&self.clause, visitor)?; + sqlparser::ast::Visit::visit(&self.parent_type, visitor)?; + if let Some(value) = &self.parent_name { + visitor.pre_visit_relation(value)?; + sqlparser::ast::Visit::visit(value, visitor)?; + visitor.post_visit_relation(value)?; + } + ::std::ops::ControlFlow::Continue(()) + } +} + +impl sqlparser::ast::VisitMut for ShowStatementIn { + fn visit( + &mut self, + visitor: &mut V, + ) -> ::std::ops::ControlFlow { + sqlparser::ast::VisitMut::visit(&mut self.clause, visitor)?; + sqlparser::ast::VisitMut::visit(&mut self.parent_type, visitor)?; + if let Some(value) = &mut self.parent_name { + visitor.pre_visit_relation(value)?; + sqlparser::ast::VisitMut::visit(value, visitor)?; + visitor.post_visit_relation(value)?; + } + ::std::ops::ControlFlow::Continue(()) + } +} +``` + ## Releasing This crate's release is not automated. Instead it is released manually as needed diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 5ad1607f9..dd4d37b41 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -18,11 +18,8 @@ use proc_macro2::TokenStream; use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::spanned::Spanned; -use syn::{ - parse::{Parse, ParseStream}, - parse_macro_input, parse_quote, Attribute, Data, DeriveInput, Fields, GenericParam, Generics, - Ident, Index, LitStr, Meta, Token, -}; +use syn::{parse::{Parse, ParseStream}, parse_macro_input, parse_quote, Attribute, Data, DeriveInput, Fields, GenericParam, Generics, Ident, Index, LitStr, Meta, Token, Type, TypePath}; +use syn::{Path, PathArguments}; /// Implementation of `[#derive(Visit)]` #[proc_macro_derive(VisitMut, attributes(visit))] @@ -182,9 +179,21 @@ fn visit_children( Fields::Named(fields) => { let recurse = fields.named.iter().map(|f| { let name = &f.ident; + let is_option = is_option(&f.ty); let attributes = Attributes::parse(&f.attrs); - let (pre_visit, post_visit) = attributes.visit(quote!(&#modifier self.#name)); - quote_spanned!(f.span() => #pre_visit sqlparser::ast::#visit_trait::visit(&#modifier self.#name, visitor)?; #post_visit) + if is_option && attributes.with.is_some() { + let (pre_visit, post_visit) = attributes.visit(quote!(value)); + quote_spanned!(f.span() => + if let Some(value) = &#modifier self.#name { + #pre_visit sqlparser::ast::#visit_trait::visit(value, visitor)?; #post_visit + } + ) + } else { + let (pre_visit, post_visit) = attributes.visit(quote!(&#modifier self.#name)); + quote_spanned!(f.span() => + #pre_visit sqlparser::ast::#visit_trait::visit(&#modifier self.#name, visitor)?; #post_visit + ) + } }); quote! { #(#recurse)* @@ -256,3 +265,16 @@ fn visit_children( Data::Union(_) => unimplemented!(), } } + +fn is_option(ty: &Type) -> bool { + if let Type::Path(TypePath { path: Path { segments, .. }, .. }) = ty { + if let Some(segment) = segments.last() { + if segment.ident == "Option" { + if let PathArguments::AngleBracketed(args) = &segment.arguments { + return args.args.len() == 1; + } + } + } + } + false +} diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 386e42fb3..19da04c62 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -7653,6 +7653,7 @@ impl fmt::Display for ShowStatementInParentType { pub struct ShowStatementIn { pub clause: ShowStatementInClause, pub parent_type: Option, + #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))] pub parent_name: Option, } diff --git a/src/ast/visitor.rs b/src/ast/visitor.rs index 418e0a299..eacd268a4 100644 --- a/src/ast/visitor.rs +++ b/src/ast/visitor.rs @@ -876,7 +876,16 @@ mod tests { "POST: QUERY: SELECT * FROM monthly_sales PIVOT(SUM(a.amount) FOR a.MONTH IN ('JAN', 'FEB', 'MAR', 'APR')) AS p (c, d) ORDER BY EMPID", "POST: STATEMENT: SELECT * FROM monthly_sales PIVOT(SUM(a.amount) FOR a.MONTH IN ('JAN', 'FEB', 'MAR', 'APR')) AS p (c, d) ORDER BY EMPID", ] - ) + ), + ( + "SHOW COLUMNS FROM t1", + vec![ + "PRE: STATEMENT: SHOW COLUMNS FROM t1", + "PRE: RELATION: t1", + "POST: RELATION: t1", + "POST: STATEMENT: SHOW COLUMNS FROM t1", + ], + ), ]; for (sql, expected) in tests { let actual = do_visit(sql); From a134910a362d12acb668ddf63239525073e7340f Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 30 Nov 2024 07:55:21 -0500 Subject: [PATCH 72/80] Rename `TokenWithLocation` to `TokenWithSpan`, in backwards compatible way (#1562) --- src/ast/helpers/attached_token.rs | 10 ++--- src/ast/query.rs | 4 +- src/parser/mod.rs | 66 +++++++++++++++---------------- src/tokenizer.rs | 50 ++++++++++++----------- 4 files changed, 67 insertions(+), 63 deletions(-) diff --git a/src/ast/helpers/attached_token.rs b/src/ast/helpers/attached_token.rs index 48696c336..ed340359d 100644 --- a/src/ast/helpers/attached_token.rs +++ b/src/ast/helpers/attached_token.rs @@ -19,7 +19,7 @@ use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; use core::fmt::{self, Debug, Formatter}; use core::hash::{Hash, Hasher}; -use crate::tokenizer::{Token, TokenWithLocation}; +use crate::tokenizer::{Token, TokenWithSpan}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -33,11 +33,11 @@ use sqlparser_derive::{Visit, VisitMut}; #[derive(Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] -pub struct AttachedToken(pub TokenWithLocation); +pub struct AttachedToken(pub TokenWithSpan); impl AttachedToken { pub fn empty() -> Self { - AttachedToken(TokenWithLocation::wrap(Token::EOF)) + AttachedToken(TokenWithSpan::wrap(Token::EOF)) } } @@ -75,8 +75,8 @@ impl Hash for AttachedToken { } } -impl From for AttachedToken { - fn from(value: TokenWithLocation) -> Self { +impl From for AttachedToken { + fn from(value: TokenWithSpan) -> Self { AttachedToken(value) } } diff --git a/src/ast/query.rs b/src/ast/query.rs index 716ffe98c..f3a76d893 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -27,7 +27,7 @@ use sqlparser_derive::{Visit, VisitMut}; use crate::{ ast::*, - tokenizer::{Token, TokenWithLocation}, + tokenizer::{Token, TokenWithSpan}, }; /// The most complete variant of a `SELECT` query expression, optionally @@ -643,7 +643,7 @@ pub struct WildcardAdditionalOptions { impl Default for WildcardAdditionalOptions { fn default() -> Self { Self { - wildcard_token: TokenWithLocation::wrap(Token::Mul).into(), + wildcard_token: TokenWithSpan::wrap(Token::Mul).into(), opt_ilike: None, opt_exclude: None, opt_except: None, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index fe6fae8bf..1f8dc8ba9 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -265,7 +265,7 @@ enum ParserState { } pub struct Parser<'a> { - tokens: Vec, + tokens: Vec, /// The index of the first unprocessed token in [`Parser::tokens`]. index: usize, /// The current state of the parser. @@ -359,7 +359,7 @@ impl<'a> Parser<'a> { } /// Reset this parser to parse the specified token stream - pub fn with_tokens_with_locations(mut self, tokens: Vec) -> Self { + pub fn with_tokens_with_locations(mut self, tokens: Vec) -> Self { self.tokens = tokens; self.index = 0; self @@ -368,9 +368,9 @@ impl<'a> Parser<'a> { /// Reset this parser state to parse the specified tokens pub fn with_tokens(self, tokens: Vec) -> Self { // Put in dummy locations - let tokens_with_locations: Vec = tokens + let tokens_with_locations: Vec = tokens .into_iter() - .map(|token| TokenWithLocation { + .map(|token| TokenWithSpan { token, span: Span::empty(), }) @@ -1147,7 +1147,7 @@ impl<'a> Parser<'a> { match self.peek_token().token { Token::LParen | Token::Period => { let mut id_parts: Vec = vec![w.to_ident(w_span)]; - let mut ending_wildcard: Option = None; + let mut ending_wildcard: Option = None; while self.consume_token(&Token::Period) { let next_token = self.next_token(); match next_token.token { @@ -3273,7 +3273,7 @@ impl<'a> Parser<'a> { /// Return the first non-whitespace token that has not yet been processed /// (or None if reached end-of-file) - pub fn peek_token(&self) -> TokenWithLocation { + pub fn peek_token(&self) -> TokenWithSpan { self.peek_nth_token(0) } @@ -3308,19 +3308,19 @@ impl<'a> Parser<'a> { /// yet been processed. /// /// See [`Self::peek_token`] for an example. - pub fn peek_tokens_with_location(&self) -> [TokenWithLocation; N] { + pub fn peek_tokens_with_location(&self) -> [TokenWithSpan; N] { let mut index = self.index; core::array::from_fn(|_| loop { let token = self.tokens.get(index); index += 1; - if let Some(TokenWithLocation { + if let Some(TokenWithSpan { token: Token::Whitespace(_), span: _, }) = token { continue; } - break token.cloned().unwrap_or(TokenWithLocation { + break token.cloned().unwrap_or(TokenWithSpan { token: Token::EOF, span: Span::empty(), }); @@ -3328,18 +3328,18 @@ impl<'a> Parser<'a> { } /// Return nth non-whitespace token that has not yet been processed - pub fn peek_nth_token(&self, mut n: usize) -> TokenWithLocation { + pub fn peek_nth_token(&self, mut n: usize) -> TokenWithSpan { let mut index = self.index; loop { index += 1; match self.tokens.get(index - 1) { - Some(TokenWithLocation { + Some(TokenWithSpan { token: Token::Whitespace(_), span: _, }) => continue, non_whitespace => { if n == 0 { - return non_whitespace.cloned().unwrap_or(TokenWithLocation { + return non_whitespace.cloned().unwrap_or(TokenWithSpan { token: Token::EOF, span: Span::empty(), }); @@ -3352,16 +3352,16 @@ impl<'a> Parser<'a> { /// Return the first token, possibly whitespace, that has not yet been processed /// (or None if reached end-of-file). - pub fn peek_token_no_skip(&self) -> TokenWithLocation { + pub fn peek_token_no_skip(&self) -> TokenWithSpan { self.peek_nth_token_no_skip(0) } /// Return nth token, possibly whitespace, that has not yet been processed. - pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithLocation { + pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithSpan { self.tokens .get(self.index + n) .cloned() - .unwrap_or(TokenWithLocation { + .unwrap_or(TokenWithSpan { token: Token::EOF, span: Span::empty(), }) @@ -3378,25 +3378,25 @@ impl<'a> Parser<'a> { /// Return the first non-whitespace token that has not yet been processed /// (or None if reached end-of-file) and mark it as processed. OK to call /// repeatedly after reaching EOF. - pub fn next_token(&mut self) -> TokenWithLocation { + pub fn next_token(&mut self) -> TokenWithSpan { loop { self.index += 1; match self.tokens.get(self.index - 1) { - Some(TokenWithLocation { + Some(TokenWithSpan { token: Token::Whitespace(_), span: _, }) => continue, token => { return token .cloned() - .unwrap_or_else(|| TokenWithLocation::wrap(Token::EOF)) + .unwrap_or_else(|| TokenWithSpan::wrap(Token::EOF)) } } } } /// Return the first unprocessed token, possibly whitespace. - pub fn next_token_no_skip(&mut self) -> Option<&TokenWithLocation> { + pub fn next_token_no_skip(&mut self) -> Option<&TokenWithSpan> { self.index += 1; self.tokens.get(self.index - 1) } @@ -3408,7 +3408,7 @@ impl<'a> Parser<'a> { loop { assert!(self.index > 0); self.index -= 1; - if let Some(TokenWithLocation { + if let Some(TokenWithSpan { token: Token::Whitespace(_), span: _, }) = self.tokens.get(self.index) @@ -3420,7 +3420,7 @@ impl<'a> Parser<'a> { } /// Report `found` was encountered instead of `expected` - pub fn expected(&self, expected: &str, found: TokenWithLocation) -> Result { + pub fn expected(&self, expected: &str, found: TokenWithSpan) -> Result { parser_err!( format!("Expected: {expected}, found: {found}"), found.span.start @@ -3435,7 +3435,7 @@ impl<'a> Parser<'a> { } #[must_use] - pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option { + pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option { match self.peek_token().token { Token::Word(w) if expected == w.keyword => Some(self.next_token()), _ => None, @@ -3524,7 +3524,7 @@ impl<'a> Parser<'a> { /// If the current token is the `expected` keyword, consume the token. /// Otherwise, return an error. - pub fn expect_keyword(&mut self, expected: Keyword) -> Result { + pub fn expect_keyword(&mut self, expected: Keyword) -> Result { if let Some(token) = self.parse_keyword_token(expected) { Ok(token) } else { @@ -3568,7 +3568,7 @@ impl<'a> Parser<'a> { } /// Bail out if the current token is not an expected keyword, or consume it if it is - pub fn expect_token(&mut self, expected: &Token) -> Result { + pub fn expect_token(&mut self, expected: &Token) -> Result { if self.peek_token() == *expected { Ok(self.next_token()) } else { @@ -4107,7 +4107,7 @@ impl<'a> Parser<'a> { Keyword::ARCHIVE => Ok(Some(CreateFunctionUsing::Archive(uri))), _ => self.expected( "JAR, FILE or ARCHIVE, got {:?}", - TokenWithLocation::wrap(Token::make_keyword(format!("{keyword:?}").as_str())), + TokenWithSpan::wrap(Token::make_keyword(format!("{keyword:?}").as_str())), ), } } @@ -6832,7 +6832,7 @@ impl<'a> Parser<'a> { if let Some(name) = name { return self.expected( "FULLTEXT or SPATIAL option without constraint name", - TokenWithLocation { + TokenWithSpan { token: Token::make_keyword(&name.to_string()), span: next_token.span, }, @@ -7808,7 +7808,7 @@ impl<'a> Parser<'a> { Some('\'') => Ok(Value::SingleQuotedString(w.value)), _ => self.expected( "A value?", - TokenWithLocation { + TokenWithSpan { token: Token::Word(w), span, }, @@ -7816,7 +7816,7 @@ impl<'a> Parser<'a> { }, _ => self.expected( "a concrete value", - TokenWithLocation { + TokenWithSpan { token: Token::Word(w), span, }, @@ -7878,7 +7878,7 @@ impl<'a> Parser<'a> { } unexpected => self.expected( "a value", - TokenWithLocation { + TokenWithSpan { token: unexpected, span, }, @@ -7927,7 +7927,7 @@ impl<'a> Parser<'a> { Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())), unexpected => self.expected( "a string value", - TokenWithLocation { + TokenWithSpan { token: unexpected, span, }, @@ -8618,7 +8618,7 @@ impl<'a> Parser<'a> { let token = self .next_token_no_skip() .cloned() - .unwrap_or(TokenWithLocation::wrap(Token::EOF)); + .unwrap_or(TokenWithSpan::wrap(Token::EOF)); requires_whitespace = match token.token { Token::Word(next_word) if next_word.quote_style.is_none() => { ident.value.push_str(&next_word.value); @@ -11683,7 +11683,7 @@ impl<'a> Parser<'a> { /// If it is not possible to parse it, will return an option. pub fn parse_wildcard_additional_options( &mut self, - wildcard_token: TokenWithLocation, + wildcard_token: TokenWithSpan, ) -> Result { let opt_ilike = if dialect_of!(self is GenericDialect | SnowflakeDialect) { self.parse_optional_select_item_ilike()? @@ -12708,7 +12708,7 @@ impl<'a> Parser<'a> { } /// Consume the parser and return its underlying token buffer - pub fn into_tokens(self) -> Vec { + pub fn into_tokens(self) -> Vec { self.tokens } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index bed2d9b52..7a79445e0 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -521,42 +521,46 @@ impl Span { } } +/// Backwards compatibility struct for [`TokenWithSpan`] +#[deprecated(since = "0.53.0", note = "please use `TokenWithSpan` instead")] +pub type TokenWithLocation = TokenWithSpan; + /// A [Token] with [Location] attached to it #[derive(Debug, Clone, Hash, Ord, PartialOrd, Eq, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] -pub struct TokenWithLocation { +pub struct TokenWithSpan { pub token: Token, pub span: Span, } -impl TokenWithLocation { - pub fn new(token: Token, span: Span) -> TokenWithLocation { - TokenWithLocation { token, span } +impl TokenWithSpan { + pub fn new(token: Token, span: Span) -> TokenWithSpan { + TokenWithSpan { token, span } } - pub fn wrap(token: Token) -> TokenWithLocation { - TokenWithLocation::new(token, Span::empty()) + pub fn wrap(token: Token) -> TokenWithSpan { + TokenWithSpan::new(token, Span::empty()) } - pub fn at(token: Token, start: Location, end: Location) -> TokenWithLocation { - TokenWithLocation::new(token, Span::new(start, end)) + pub fn at(token: Token, start: Location, end: Location) -> TokenWithSpan { + TokenWithSpan::new(token, Span::new(start, end)) } } -impl PartialEq for TokenWithLocation { +impl PartialEq for TokenWithSpan { fn eq(&self, other: &Token) -> bool { &self.token == other } } -impl PartialEq for Token { - fn eq(&self, other: &TokenWithLocation) -> bool { +impl PartialEq for Token { + fn eq(&self, other: &TokenWithSpan) -> bool { self == &other.token } } -impl fmt::Display for TokenWithLocation { +impl fmt::Display for TokenWithSpan { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { self.token.fmt(f) } @@ -716,8 +720,8 @@ impl<'a> Tokenizer<'a> { } /// Tokenize the statement and produce a vector of tokens with location information - pub fn tokenize_with_location(&mut self) -> Result, TokenizerError> { - let mut tokens: Vec = vec![]; + pub fn tokenize_with_location(&mut self) -> Result, TokenizerError> { + let mut tokens: Vec = vec![]; self.tokenize_with_location_into_buf(&mut tokens) .map(|_| tokens) } @@ -726,7 +730,7 @@ impl<'a> Tokenizer<'a> { /// If an error is thrown, the buffer will contain all tokens that were successfully parsed before the error. pub fn tokenize_with_location_into_buf( &mut self, - buf: &mut Vec, + buf: &mut Vec, ) -> Result<(), TokenizerError> { let mut state = State { peekable: self.query.chars().peekable(), @@ -738,7 +742,7 @@ impl<'a> Tokenizer<'a> { while let Some(token) = self.next_token(&mut state)? { let span = location.span_to(state.location()); - buf.push(TokenWithLocation { token, span }); + buf.push(TokenWithSpan { token, span }); location = state.location(); } @@ -2751,25 +2755,25 @@ mod tests { .tokenize_with_location() .unwrap(); let expected = vec![ - TokenWithLocation::at(Token::make_keyword("SELECT"), (1, 1).into(), (1, 7).into()), - TokenWithLocation::at( + TokenWithSpan::at(Token::make_keyword("SELECT"), (1, 1).into(), (1, 7).into()), + TokenWithSpan::at( Token::Whitespace(Whitespace::Space), (1, 7).into(), (1, 8).into(), ), - TokenWithLocation::at(Token::make_word("a", None), (1, 8).into(), (1, 9).into()), - TokenWithLocation::at(Token::Comma, (1, 9).into(), (1, 10).into()), - TokenWithLocation::at( + TokenWithSpan::at(Token::make_word("a", None), (1, 8).into(), (1, 9).into()), + TokenWithSpan::at(Token::Comma, (1, 9).into(), (1, 10).into()), + TokenWithSpan::at( Token::Whitespace(Whitespace::Newline), (1, 10).into(), (2, 1).into(), ), - TokenWithLocation::at( + TokenWithSpan::at( Token::Whitespace(Whitespace::Space), (2, 1).into(), (2, 2).into(), ), - TokenWithLocation::at(Token::make_word("b", None), (2, 2).into(), (2, 3).into()), + TokenWithSpan::at(Token::make_word("b", None), (2, 2).into(), (2, 3).into()), ]; compare(expected, tokens); } From 48b0e4db4e07c6f9552e2a646cfbc699add41ae1 Mon Sep 17 00:00:00 2001 From: Michael Victor Zink Date: Sat, 30 Nov 2024 04:55:54 -0800 Subject: [PATCH 73/80] Support MySQL size variants for BLOB and TEXT columns (#1564) --- src/ast/data_type.rs | 30 ++++++++++++++++++++++++++++++ src/keywords.rs | 6 ++++++ src/parser/mod.rs | 6 ++++++ tests/sqlparser_mysql.rs | 17 +++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index bc48341c4..fbfdc2dcf 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -76,6 +76,18 @@ pub enum DataType { /// [standard]: https://jakewheat.github.io/sql-overview/sql-2016-foundation-grammar.html#binary-large-object-string-type /// [Oracle]: https://docs.oracle.com/javadb/10.8.3.0/ref/rrefblob.html Blob(Option), + /// [MySQL] blob with up to 2**8 bytes + /// + /// [MySQL]: https://dev.mysql.com/doc/refman/9.1/en/blob.html + TinyBlob, + /// [MySQL] blob with up to 2**24 bytes + /// + /// [MySQL]: https://dev.mysql.com/doc/refman/9.1/en/blob.html + MediumBlob, + /// [MySQL] blob with up to 2**32 bytes + /// + /// [MySQL]: https://dev.mysql.com/doc/refman/9.1/en/blob.html + LongBlob, /// Variable-length binary data with optional length. /// /// [bigquery]: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#bytes_type @@ -275,6 +287,18 @@ pub enum DataType { Regclass, /// Text Text, + /// [MySQL] text with up to 2**8 bytes + /// + /// [MySQL]: https://dev.mysql.com/doc/refman/9.1/en/blob.html + TinyText, + /// [MySQL] text with up to 2**24 bytes + /// + /// [MySQL]: https://dev.mysql.com/doc/refman/9.1/en/blob.html + MediumText, + /// [MySQL] text with up to 2**32 bytes + /// + /// [MySQL]: https://dev.mysql.com/doc/refman/9.1/en/blob.html + LongText, /// String with optional length. String(Option), /// A fixed-length string e.g [ClickHouse][1]. @@ -355,6 +379,9 @@ impl fmt::Display for DataType { format_type_with_optional_length(f, "VARBINARY", size, false) } DataType::Blob(size) => format_type_with_optional_length(f, "BLOB", size, false), + DataType::TinyBlob => write!(f, "TINYBLOB"), + DataType::MediumBlob => write!(f, "MEDIUMBLOB"), + DataType::LongBlob => write!(f, "LONGBLOB"), DataType::Bytes(size) => format_type_with_optional_length(f, "BYTES", size, false), DataType::Numeric(info) => { write!(f, "NUMERIC{info}") @@ -486,6 +513,9 @@ impl fmt::Display for DataType { DataType::JSONB => write!(f, "JSONB"), DataType::Regclass => write!(f, "REGCLASS"), DataType::Text => write!(f, "TEXT"), + DataType::TinyText => write!(f, "TINYTEXT"), + DataType::MediumText => write!(f, "MEDIUMTEXT"), + DataType::LongText => write!(f, "LONGTEXT"), DataType::String(size) => format_type_with_optional_length(f, "STRING", size, false), DataType::Bytea => write!(f, "BYTEA"), DataType::Array(ty) => match ty { diff --git a/src/keywords.rs b/src/keywords.rs index 8c0ed588f..4ec088941 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -453,6 +453,8 @@ define_keywords!( LOCKED, LOGIN, LOGS, + LONGBLOB, + LONGTEXT, LOWCARDINALITY, LOWER, LOW_PRIORITY, @@ -471,7 +473,9 @@ define_keywords!( MAXVALUE, MAX_DATA_EXTENSION_TIME_IN_DAYS, MEASURES, + MEDIUMBLOB, MEDIUMINT, + MEDIUMTEXT, MEMBER, MERGE, METADATA, @@ -765,7 +769,9 @@ define_keywords!( TIMEZONE_HOUR, TIMEZONE_MINUTE, TIMEZONE_REGION, + TINYBLOB, TINYINT, + TINYTEXT, TO, TOP, TOTALS, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 1f8dc8ba9..afce1f713 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8129,6 +8129,9 @@ impl<'a> Parser<'a> { Keyword::BINARY => Ok(DataType::Binary(self.parse_optional_precision()?)), Keyword::VARBINARY => Ok(DataType::Varbinary(self.parse_optional_precision()?)), Keyword::BLOB => Ok(DataType::Blob(self.parse_optional_precision()?)), + Keyword::TINYBLOB => Ok(DataType::TinyBlob), + Keyword::MEDIUMBLOB => Ok(DataType::MediumBlob), + Keyword::LONGBLOB => Ok(DataType::LongBlob), Keyword::BYTES => Ok(DataType::Bytes(self.parse_optional_precision()?)), Keyword::UUID => Ok(DataType::Uuid), Keyword::DATE => Ok(DataType::Date), @@ -8188,6 +8191,9 @@ impl<'a> Parser<'a> { Ok(DataType::FixedString(character_length)) } Keyword::TEXT => Ok(DataType::Text), + Keyword::TINYTEXT => Ok(DataType::TinyText), + Keyword::MEDIUMTEXT => Ok(DataType::MediumText), + Keyword::LONGTEXT => Ok(DataType::LongText), Keyword::BYTEA => Ok(DataType::Bytea), Keyword::NUMERIC => Ok(DataType::Numeric( self.parse_exact_number_optional_precision_scale()?, diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 943a61718..2b132331e 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -3014,3 +3014,20 @@ fn parse_bitstring_literal() { ))] ); } + +#[test] +fn parse_longblob_type() { + let sql = "CREATE TABLE foo (bar LONGBLOB)"; + let stmt = mysql_and_generic().verified_stmt(sql); + if let Statement::CreateTable(CreateTable { columns, .. }) = stmt { + assert_eq!(columns.len(), 1); + assert_eq!(columns[0].data_type, DataType::LongBlob); + } else { + unreachable!() + } + mysql_and_generic().verified_stmt("CREATE TABLE foo (bar TINYBLOB)"); + mysql_and_generic().verified_stmt("CREATE TABLE foo (bar MEDIUMBLOB)"); + mysql_and_generic().verified_stmt("CREATE TABLE foo (bar TINYTEXT)"); + mysql_and_generic().verified_stmt("CREATE TABLE foo (bar MEDIUMTEXT)"); + mysql_and_generic().verified_stmt("CREATE TABLE foo (bar LONGTEXT)"); +} From b0007389dc769783fd050a2f4e9f1a45e5f07778 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 30 Nov 2024 08:00:34 -0500 Subject: [PATCH 74/80] Increase version of sqlparser_derive from 0.2.2 to 0.3.0 (#1571) --- Cargo.toml | 2 +- derive/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 18b246e04..c4d0094f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ serde = { version = "1.0", features = ["derive"], optional = true } # of dev-dependencies because of # https://github.com/rust-lang/cargo/issues/1596 serde_json = { version = "1.0", optional = true } -sqlparser_derive = { version = "0.2.0", path = "derive", optional = true } +sqlparser_derive = { version = "0.3.0", path = "derive", optional = true } [dev-dependencies] simple_logger = "5.0" diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 3b115b950..7b6477300 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -17,8 +17,8 @@ [package] name = "sqlparser_derive" -description = "proc macro for sqlparser" -version = "0.2.2" +description = "Procedural (proc) macros for sqlparser" +version = "0.3.0" authors = ["sqlparser-rs authors"] homepage = "https://github.com/sqlparser-rs/sqlparser-rs" documentation = "https://docs.rs/sqlparser_derive/" From 96f7c0277a20d0c953f2e1026347795191370caf Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Sat, 30 Nov 2024 14:01:13 +0100 Subject: [PATCH 75/80] `json_object('k' VALUE 'v')` in postgres (#1547) Co-authored-by: Ifeanyi Ubah --- src/ast/mod.rs | 3 +++ src/parser/mod.rs | 3 +++ tests/sqlparser_postgres.rs | 13 +++++++++++++ 3 files changed, 19 insertions(+) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 19da04c62..6d35badf9 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -5528,6 +5528,8 @@ pub enum FunctionArgOperator { Assignment, /// function(arg1 : value1) Colon, + /// function(arg1 VALUE value1) + Value, } impl fmt::Display for FunctionArgOperator { @@ -5537,6 +5539,7 @@ impl fmt::Display for FunctionArgOperator { FunctionArgOperator::RightArrow => f.write_str("=>"), FunctionArgOperator::Assignment => f.write_str(":="), FunctionArgOperator::Colon => f.write_str(":"), + FunctionArgOperator::Value => f.write_str("VALUE"), } } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index afce1f713..7148ae48a 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -11482,6 +11482,9 @@ impl<'a> Parser<'a> { } fn parse_function_named_arg_operator(&mut self) -> Result { + if self.parse_keyword(Keyword::VALUE) { + return Ok(FunctionArgOperator::Value); + } let tok = self.next_token(); match tok.token { Token::RArrow if self.dialect.supports_named_fn_args_with_rarrow_operator() => { diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 54f77b7be..f94e2f540 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -2824,6 +2824,19 @@ fn test_json() { ); } +#[test] +fn test_fn_arg_with_value_operator() { + match pg().verified_expr("JSON_OBJECT('name' VALUE 'value')") { + Expr::Function(Function { args: FunctionArguments::List(FunctionArgumentList { args, .. }), .. }) => { + assert!(matches!( + &args[..], + &[FunctionArg::ExprNamed { operator: FunctionArgOperator::Value, .. }] + ), "Invalid function argument: {:?}", args); + } + other => panic!("Expected: JSON_OBJECT('name' VALUE 'value') to be parsed as a function, but got {other:?}"), + } +} + #[test] fn parse_json_table_is_not_reserved() { // JSON_TABLE is not a reserved keyword in PostgreSQL, even though it is in SQL:2023 From f4f112d7d6ffc1a9de30fc66128030b78415c67c Mon Sep 17 00:00:00 2001 From: Ayman Elkfrawy <120422207+ayman-sigma@users.noreply.github.com> Date: Sat, 30 Nov 2024 05:02:08 -0800 Subject: [PATCH 76/80] Support snowflake double dot notation for object name (#1540) --- src/dialect/mod.rs | 10 ++++++++++ src/dialect/snowflake.rs | 8 ++++++++ src/parser/mod.rs | 7 +++++++ tests/sqlparser_snowflake.rs | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index b622c1da3..a8993e685 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -365,6 +365,16 @@ pub trait Dialect: Debug + Any { self.supports_trailing_commas() } + /// Returns true if the dialect supports double dot notation for object names + /// + /// Example + /// ```sql + /// SELECT * FROM db_name..table_name + /// ``` + fn supports_object_name_double_dot_notation(&self) -> bool { + false + } + /// Dialect-specific infix parser override /// /// This method is called to parse the next infix expression. diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index 56919fb31..77d2ccff1 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -54,6 +54,14 @@ impl Dialect for SnowflakeDialect { true } + // Snowflake supports double-dot notation when the schema name is not specified + // In this case the default PUBLIC schema is used + // + // see https://docs.snowflake.com/en/sql-reference/name-resolution#resolution-when-schema-omitted-double-dot-notation + fn supports_object_name_double_dot_notation(&self) -> bool { + true + } + fn is_identifier_part(&self, ch: char) -> bool { ch.is_ascii_lowercase() || ch.is_ascii_uppercase() diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 7148ae48a..16362ebba 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8457,6 +8457,13 @@ impl<'a> Parser<'a> { pub fn parse_object_name(&mut self, in_table_clause: bool) -> Result { let mut idents = vec![]; loop { + if self.dialect.supports_object_name_double_dot_notation() + && idents.len() == 1 + && self.consume_token(&Token::Period) + { + // Empty string here means default schema + idents.push(Ident::new("")); + } idents.push(self.parse_identifier(in_table_clause)?); if !self.consume_token(&Token::Period) { break; diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 08792380d..e31811c2b 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -2866,3 +2866,35 @@ fn test_projection_with_nested_trailing_commas() { let sql = "SELECT a, b, FROM c, (SELECT d, e, FROM f, LATERAL FLATTEN(input => events))"; let _ = snowflake().parse_sql_statements(sql).unwrap(); } + +#[test] +fn test_sf_double_dot_notation() { + snowflake().verified_stmt("SELECT * FROM db_name..table_name"); + snowflake().verified_stmt("SELECT * FROM x, y..z JOIN a..b AS b ON x.id = b.id"); + + assert_eq!( + snowflake() + .parse_sql_statements("SELECT * FROM X.Y..") + .unwrap_err() + .to_string(), + "sql parser error: Expected: identifier, found: ." + ); + assert_eq!( + snowflake() + .parse_sql_statements("SELECT * FROM X..Y..Z") + .unwrap_err() + .to_string(), + "sql parser error: Expected: identifier, found: ." + ); + assert_eq!( + // Ensure we don't parse leading token + snowflake() + .parse_sql_statements("SELECT * FROM .X.Y") + .unwrap_err() + .to_string(), + "sql parser error: Expected: identifier, found: ." + ); +} + +#[test] +fn test_parse_double_dot_notation_wrong_position() {} From 4ab3ab91473d152c652e6582b63abb13535703f9 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 30 Nov 2024 08:08:55 -0500 Subject: [PATCH 77/80] Update comments / docs for `Spanned` (#1549) Co-authored-by: Ifeanyi Ubah --- README.md | 19 +++-- docs/source_spans.md | 52 ------------ src/ast/helpers/attached_token.rs | 64 +++++++++++++-- src/ast/mod.rs | 16 +++- src/ast/query.rs | 5 +- src/ast/spans.rs | 50 ++++++++--- src/lib.rs | 59 ++++++++++++- src/tokenizer.rs | 132 +++++++++++++++++++++++++++--- 8 files changed, 306 insertions(+), 91 deletions(-) delete mode 100644 docs/source_spans.md diff --git a/README.md b/README.md index 9a67abcf8..fd676d115 100644 --- a/README.md +++ b/README.md @@ -100,13 +100,18 @@ similar semantics are represented with the same AST. We welcome PRs to fix such issues and distinguish different syntaxes in the AST. -## WIP: Extracting source locations from AST nodes +## Source Locations (Work in Progress) -This crate allows recovering source locations from AST nodes via the [Spanned](https://docs.rs/sqlparser/latest/sqlparser/ast/trait.Spanned.html) trait, which can be used for advanced diagnostics tooling. Note that this feature is a work in progress and many nodes report missing or inaccurate spans. Please see [this document](./docs/source_spans.md#source-span-contributing-guidelines) for information on how to contribute missing improvements. +This crate allows recovering source locations from AST nodes via the [Spanned] +trait, which can be used for advanced diagnostics tooling. Note that this +feature is a work in progress and many nodes report missing or inaccurate spans. +Please see [this ticket] for information on how to contribute missing +improvements. -```rust -use sqlparser::ast::Spanned; +[Spanned]: https://docs.rs/sqlparser/latest/sqlparser/ast/trait.Spanned.html +[this ticket]: https://github.com/apache/datafusion-sqlparser-rs/issues/1548 +```rust // Parse SQL let ast = Parser::parse_sql(&GenericDialect, "SELECT A FROM B").unwrap(); @@ -123,9 +128,9 @@ SQL was first standardized in 1987, and revisions of the standard have been published regularly since. Most revisions have added significant new features to the language, and as a result no database claims to support the full breadth of features. This parser currently supports most of the SQL-92 syntax, plus some -syntax from newer versions that have been explicitly requested, plus some MSSQL, -PostgreSQL, and other dialect-specific syntax. Whenever possible, the [online -SQL:2016 grammar][sql-2016-grammar] is used to guide what syntax to accept. +syntax from newer versions that have been explicitly requested, plus various +other dialect-specific syntax. Whenever possible, the [online SQL:2016 +grammar][sql-2016-grammar] is used to guide what syntax to accept. Unfortunately, stating anything more specific about compliance is difficult. There is no publicly available test suite that can assess compliance diff --git a/docs/source_spans.md b/docs/source_spans.md deleted file mode 100644 index 136a4ced2..000000000 --- a/docs/source_spans.md +++ /dev/null @@ -1,52 +0,0 @@ - -## Breaking Changes - -These are the current breaking changes introduced by the source spans feature: - -#### Added fields for spans (must be added to any existing pattern matches) -- `Ident` now stores a `Span` -- `Select`, `With`, `Cte`, `WildcardAdditionalOptions` now store a `TokenWithLocation` - -#### Misc. -- `TokenWithLocation` stores a full `Span`, rather than just a source location. Users relying on `token.location` should use `token.location.start` instead. -## Source Span Contributing Guidelines - -For contributing source spans improvement in addition to the general [contribution guidelines](../README.md#contributing), please make sure to pay attention to the following: - - -### Source Span Design Considerations - -- `Ident` always have correct source spans -- Downstream breaking change impact is to be as minimal as possible -- To this end, use recursive merging of spans in favor of storing spans on all nodes -- Any metadata added to compute spans must not change semantics (Eq, Ord, Hash, etc.) - -The primary reason for missing and inaccurate source spans at this time is missing spans of keyword tokens and values in many structures, either due to lack of time or because adding them would break downstream significantly. - -When considering adding support for source spans on a type, consider the impact to consumers of that type and whether your change would require a consumer to do non-trivial changes to their code. - -Example of a trivial change -```rust -match node { - ast::Query { - field1, - field2, - location: _, // add a new line to ignored location -} -``` - -If adding source spans to a type would require a significant change like wrapping that type or similar, please open an issue to discuss. - -### AST Node Equality and Hashes - -When adding tokens to AST nodes, make sure to store them using the [AttachedToken](https://docs.rs/sqlparser/latest/sqlparser/ast/helpers/struct.AttachedToken.html) helper to ensure that semantically equivalent AST nodes always compare as equal and hash to the same value. F.e. `select 5` and `SELECT 5` would compare as different `Select` nodes, if the select token was stored directly. f.e. - -```rust -struct Select { - select_token: AttachedToken, // only used for spans - /// remaining fields - field1, - field2, - ... -} -``` \ No newline at end of file diff --git a/src/ast/helpers/attached_token.rs b/src/ast/helpers/attached_token.rs index ed340359d..6b930b513 100644 --- a/src/ast/helpers/attached_token.rs +++ b/src/ast/helpers/attached_token.rs @@ -19,7 +19,7 @@ use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; use core::fmt::{self, Debug, Formatter}; use core::hash::{Hash, Hasher}; -use crate::tokenizer::{Token, TokenWithSpan}; +use crate::tokenizer::TokenWithSpan; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -27,17 +27,65 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; -/// A wrapper type for attaching tokens to AST nodes that should be ignored in comparisons and hashing. -/// This should be used when a token is not relevant for semantics, but is still needed for -/// accurate source location tracking. +/// A wrapper over [`TokenWithSpan`]s that ignores the token and source +/// location in comparisons and hashing. +/// +/// This type is used when the token and location is not relevant for semantics, +/// but is still needed for accurate source location tracking, for example, in +/// the nodes in the [ast](crate::ast) module. +/// +/// Note: **All** `AttachedTokens` are equal. +/// +/// # Examples +/// +/// Same token, different location are equal +/// ``` +/// # use sqlparser::ast::helpers::attached_token::AttachedToken; +/// # use sqlparser::tokenizer::{Location, Span, Token, TokenWithLocation}; +/// // commas @ line 1, column 10 +/// let tok1 = TokenWithLocation::new( +/// Token::Comma, +/// Span::new(Location::new(1, 10), Location::new(1, 11)), +/// ); +/// // commas @ line 2, column 20 +/// let tok2 = TokenWithLocation::new( +/// Token::Comma, +/// Span::new(Location::new(2, 20), Location::new(2, 21)), +/// ); +/// +/// assert_ne!(tok1, tok2); // token with locations are *not* equal +/// assert_eq!(AttachedToken(tok1), AttachedToken(tok2)); // attached tokens are +/// ``` +/// +/// Different token, different location are equal 🤯 +/// +/// ``` +/// # use sqlparser::ast::helpers::attached_token::AttachedToken; +/// # use sqlparser::tokenizer::{Location, Span, Token, TokenWithLocation}; +/// // commas @ line 1, column 10 +/// let tok1 = TokenWithLocation::new( +/// Token::Comma, +/// Span::new(Location::new(1, 10), Location::new(1, 11)), +/// ); +/// // period @ line 2, column 20 +/// let tok2 = TokenWithLocation::new( +/// Token::Period, +/// Span::new(Location::new(2, 10), Location::new(2, 21)), +/// ); +/// +/// assert_ne!(tok1, tok2); // token with locations are *not* equal +/// assert_eq!(AttachedToken(tok1), AttachedToken(tok2)); // attached tokens are +/// ``` +/// // period @ line 2, column 20 #[derive(Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct AttachedToken(pub TokenWithSpan); impl AttachedToken { + /// Return a new Empty AttachedToken pub fn empty() -> Self { - AttachedToken(TokenWithSpan::wrap(Token::EOF)) + AttachedToken(TokenWithSpan::new_eof()) } } @@ -80,3 +128,9 @@ impl From for AttachedToken { AttachedToken(value) } } + +impl From for TokenWithSpan { + fn from(value: AttachedToken) -> Self { + value.0 + } +} diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 6d35badf9..e52251d52 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -596,9 +596,21 @@ pub enum CeilFloorKind { /// An SQL expression of any type. /// +/// # Semantics / Type Checking +/// /// The parser does not distinguish between expressions of different types -/// (e.g. boolean vs string), so the caller must handle expressions of -/// inappropriate type, like `WHERE 1` or `SELECT 1=1`, as necessary. +/// (e.g. boolean vs string). The caller is responsible for detecting and +/// validating types as necessary (for example `WHERE 1` vs `SELECT 1=1`) +/// See the [README.md] for more details. +/// +/// [README.md]: https://github.com/apache/datafusion-sqlparser-rs/blob/main/README.md#syntax-vs-semantics +/// +/// # Equality and Hashing Does not Include Source Locations +/// +/// The `Expr` type implements `PartialEq` and `Eq` based on the semantic value +/// of the expression (not bitwise comparison). This means that `Expr` instances +/// that are semantically equivalent but have different spans (locations in the +/// source tree) will compare as equal. #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr( diff --git a/src/ast/query.rs b/src/ast/query.rs index f3a76d893..ad7fd261e 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -282,6 +282,7 @@ impl fmt::Display for Table { pub struct Select { /// Token for the `SELECT` keyword pub select_token: AttachedToken, + /// `SELECT [DISTINCT] ...` pub distinct: Option, /// MSSQL syntax: `TOP () [ PERCENT ] [ WITH TIES ]` pub top: Option, @@ -511,7 +512,7 @@ impl fmt::Display for NamedWindowDefinition { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct With { - // Token for the "WITH" keyword + /// Token for the "WITH" keyword pub with_token: AttachedToken, pub recursive: bool, pub cte_tables: Vec, @@ -564,7 +565,7 @@ pub struct Cte { pub query: Box, pub from: Option, pub materialized: Option, - // Token for the closing parenthesis + /// Token for the closing parenthesis pub closing_paren_token: AttachedToken, } diff --git a/src/ast/spans.rs b/src/ast/spans.rs index 8e8c7b14a..1e0f1bf09 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -21,21 +21,51 @@ use super::{ /// Given an iterator of spans, return the [Span::union] of all spans. fn union_spans>(iter: I) -> Span { - iter.reduce(|acc, item| acc.union(&item)) - .unwrap_or(Span::empty()) + Span::union_iter(iter) } -/// A trait for AST nodes that have a source span for use in diagnostics. +/// Trait for AST nodes that have a source location information. /// -/// Source spans are not guaranteed to be entirely accurate. They may -/// be missing keywords or other tokens. Some nodes may not have a computable -/// span at all, in which case they return [`Span::empty()`]. +/// # Notes: +/// +/// Source [`Span`] are not yet complete. They may be missing: +/// +/// 1. keywords or other tokens +/// 2. span information entirely, in which case they return [`Span::empty()`]. +/// +/// Note Some impl blocks (rendered below) are annotated with which nodes are +/// missing spans. See [this ticket] for additional information and status. +/// +/// [this ticket]: https://github.com/apache/datafusion-sqlparser-rs/issues/1548 +/// +/// # Example +/// ``` +/// # use sqlparser::parser::{Parser, ParserError}; +/// # use sqlparser::ast::Spanned; +/// # use sqlparser::dialect::GenericDialect; +/// # use sqlparser::tokenizer::Location; +/// # fn main() -> Result<(), ParserError> { +/// let dialect = GenericDialect {}; +/// let sql = r#"SELECT * +/// FROM table_1"#; +/// let statements = Parser::new(&dialect) +/// .try_with_sql(sql)? +/// .parse_statements()?; +/// // Get the span of the first statement (SELECT) +/// let span = statements[0].span(); +/// // statement starts at line 1, column 1 (1 based, not 0 based) +/// assert_eq!(span.start, Location::new(1, 1)); +/// // statement ends on line 2, column 15 +/// assert_eq!(span.end, Location::new(2, 15)); +/// # Ok(()) +/// # } +/// ``` /// -/// Some impl blocks may contain doc comments with information -/// on which nodes are missing spans. pub trait Spanned { - /// Compute the source span for this AST node, by recursively - /// combining the spans of its children. + /// Return the [`Span`] (the minimum and maximum [`Location`]) for this AST + /// node, by recursively combining the spans of its children. + /// + /// [`Location`]: crate::tokenizer::Location fn span(&self) -> Span; } diff --git a/src/lib.rs b/src/lib.rs index 6c8987b63..5d72f9f0e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,9 @@ //! 1. [`Parser::parse_sql`] and [`Parser::new`] for the Parsing API //! 2. [`ast`] for the AST structure //! 3. [`Dialect`] for supported SQL dialects +//! 4. [`Spanned`] for source text locations (see "Source Spans" below for details) +//! +//! [`Spanned`]: ast::Spanned //! //! # Example parsing SQL text //! @@ -61,13 +64,67 @@ //! // The original SQL text can be generated from the AST //! assert_eq!(ast[0].to_string(), sql); //! ``` -//! //! [sqlparser crates.io page]: https://crates.io/crates/sqlparser //! [`Parser::parse_sql`]: crate::parser::Parser::parse_sql //! [`Parser::new`]: crate::parser::Parser::new //! [`AST`]: crate::ast //! [`ast`]: crate::ast //! [`Dialect`]: crate::dialect::Dialect +//! +//! # Source Spans +//! +//! Starting with version `0.53.0` sqlparser introduced source spans to the +//! AST. This feature provides source information for syntax errors, enabling +//! better error messages. See [issue #1548] for more information and the +//! [`Spanned`] trait to access the spans. +//! +//! [issue #1548]: https://github.com/apache/datafusion-sqlparser-rs/issues/1548 +//! [`Spanned`]: ast::Spanned +//! +//! ## Migration Guide +//! +//! For the next few releases, we will be incrementally adding source spans to the +//! AST nodes, trying to minimize the impact on existing users. Some breaking +//! changes are inevitable, and the following is a summary of the changes: +//! +//! #### New fields for spans (must be added to any existing pattern matches) +//! +//! The primary change is that new fields will be added to AST nodes to store the source `Span` or `TokenWithLocation`. +//! +//! This will require +//! 1. Adding new fields to existing pattern matches. +//! 2. Filling in the proper span information when constructing AST nodes. +//! +//! For example, since `Ident` now stores a `Span`, to construct an `Ident` you +//! must provide now provide one: +//! +//! Previously: +//! ```text +//! # use sqlparser::ast::Ident; +//! Ident { +//! value: "name".into(), +//! quote_style: None, +//! } +//! ``` +//! Now +//! ```rust +//! # use sqlparser::ast::Ident; +//! # use sqlparser::tokenizer::Span; +//! Ident { +//! value: "name".into(), +//! quote_style: None, +//! span: Span::empty(), +//! }; +//! ``` +//! +//! Similarly, when pattern matching on `Ident`, you must now account for the +//! `span` field. +//! +//! #### Misc. +//! - [`TokenWithLocation`] stores a full `Span`, rather than just a source location. +//! Users relying on `token.location` should use `token.location.start` instead. +//! +//![`TokenWithLocation`]: tokenizer::TokenWithLocation #![cfg_attr(not(feature = "std"), no_std)] #![allow(clippy::upper_case_acronyms)] diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 7a79445e0..aacfc16fa 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -422,13 +422,35 @@ impl fmt::Display for Whitespace { } /// Location in input string +/// +/// # Create an "empty" (unknown) `Location` +/// ``` +/// # use sqlparser::tokenizer::Location; +/// let location = Location::empty(); +/// ``` +/// +/// # Create a `Location` from a line and column +/// ``` +/// # use sqlparser::tokenizer::Location; +/// let location = Location::new(1, 1); +/// ``` +/// +/// # Create a `Location` from a pair +/// ``` +/// # use sqlparser::tokenizer::Location; +/// let location = Location::from((1, 1)); +/// ``` #[derive(Eq, PartialEq, Hash, Clone, Copy, Ord, PartialOrd)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct Location { - /// Line number, starting from 1 + /// Line number, starting from 1. + /// + /// Note: Line 0 is used for empty spans pub line: u64, - /// Line column, starting from 1 + /// Line column, starting from 1. + /// + /// Note: Column 0 is used for empty spans pub column: u64, } @@ -448,10 +470,25 @@ impl fmt::Debug for Location { } impl Location { - pub fn of(line: u64, column: u64) -> Self { + /// Return an "empty" / unknown location + pub fn empty() -> Self { + Self { line: 0, column: 0 } + } + + /// Create a new `Location` for a given line and column + pub fn new(line: u64, column: u64) -> Self { Self { line, column } } + /// Create a new location for a given line and column + /// + /// Alias for [`Self::new`] + // TODO: remove / deprecate in favor of` `new` for consistency? + pub fn of(line: u64, column: u64) -> Self { + Self::new(line, column) + } + + /// Combine self and `end` into a new `Span` pub fn span_to(self, end: Self) -> Span { Span { start: self, end } } @@ -463,7 +500,9 @@ impl From<(u64, u64)> for Location { } } -/// A span of source code locations (start, end) +/// A span represents a linear portion of the input string (start, end) +/// +/// See [Spanned](crate::ast::Spanned) for more information. #[derive(Eq, PartialEq, Hash, Clone, PartialOrd, Ord, Copy)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] @@ -483,12 +522,15 @@ impl Span { // We need a const instance for pattern matching const EMPTY: Span = Self::empty(); + /// Create a new span from a start and end [`Location`] pub fn new(start: Location, end: Location) -> Span { Span { start, end } } - /// Returns an empty span (0, 0) -> (0, 0) + /// Returns an empty span `(0, 0) -> (0, 0)` + /// /// Empty spans represent no knowledge of source location + /// See [Spanned](crate::ast::Spanned) for more information. pub const fn empty() -> Span { Span { start: Location { line: 0, column: 0 }, @@ -498,6 +540,19 @@ impl Span { /// Returns the smallest Span that contains both `self` and `other` /// If either span is [Span::empty], the other span is returned + /// + /// # Examples + /// ``` + /// # use sqlparser::tokenizer::{Span, Location}; + /// // line 1, column1 -> line 2, column 5 + /// let span1 = Span::new(Location::new(1, 1), Location::new(2, 5)); + /// // line 2, column 3 -> line 3, column 7 + /// let span2 = Span::new(Location::new(2, 3), Location::new(3, 7)); + /// // Union of the two is the min/max of the two spans + /// // line 1, column 1 -> line 3, column 7 + /// let union = span1.union(&span2); + /// assert_eq!(union, Span::new(Location::new(1, 1), Location::new(3, 7))); + /// ``` pub fn union(&self, other: &Span) -> Span { // If either span is empty, return the other // this prevents propagating (0, 0) through the tree @@ -512,6 +567,7 @@ impl Span { } /// Same as [Span::union] for `Option` + /// /// If `other` is `None`, `self` is returned pub fn union_opt(&self, other: &Option) -> Span { match other { @@ -519,13 +575,57 @@ impl Span { None => *self, } } + + /// Return the [Span::union] of all spans in the iterator + /// + /// If the iterator is empty, an empty span is returned + /// + /// # Example + /// ``` + /// # use sqlparser::tokenizer::{Span, Location}; + /// let spans = vec![ + /// Span::new(Location::new(1, 1), Location::new(2, 5)), + /// Span::new(Location::new(2, 3), Location::new(3, 7)), + /// Span::new(Location::new(3, 1), Location::new(4, 2)), + /// ]; + /// // line 1, column 1 -> line 4, column 2 + /// assert_eq!( + /// Span::union_iter(spans), + /// Span::new(Location::new(1, 1), Location::new(4, 2)) + /// ); + pub fn union_iter>(iter: I) -> Span { + iter.into_iter() + .reduce(|acc, item| acc.union(&item)) + .unwrap_or(Span::empty()) + } } /// Backwards compatibility struct for [`TokenWithSpan`] #[deprecated(since = "0.53.0", note = "please use `TokenWithSpan` instead")] pub type TokenWithLocation = TokenWithSpan; -/// A [Token] with [Location] attached to it +/// A [Token] with [Span] attached to it +/// +/// This is used to track the location of a token in the input string +/// +/// # Examples +/// ``` +/// # use sqlparser::tokenizer::{Location, Span, Token, TokenWithSpan}; +/// // commas @ line 1, column 10 +/// let tok1 = TokenWithSpan::new( +/// Token::Comma, +/// Span::new(Location::new(1, 10), Location::new(1, 11)), +/// ); +/// assert_eq!(tok1, Token::Comma); // can compare the token +/// +/// // commas @ line 2, column 20 +/// let tok2 = TokenWithSpan::new( +/// Token::Comma, +/// Span::new(Location::new(2, 20), Location::new(2, 21)), +/// ); +/// // same token but different locations are not equal +/// assert_ne!(tok1, tok2); +/// ``` #[derive(Debug, Clone, Hash, Ord, PartialOrd, Eq, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] @@ -535,16 +635,24 @@ pub struct TokenWithSpan { } impl TokenWithSpan { - pub fn new(token: Token, span: Span) -> TokenWithSpan { - TokenWithSpan { token, span } + /// Create a new [`TokenWithSpan`] from a [`Token`] and a [`Span`] + pub fn new(token: Token, span: Span) -> Self { + Self { token, span } + } + + /// Wrap a token with an empty span + pub fn wrap(token: Token) -> Self { + Self::new(token, Span::empty()) } - pub fn wrap(token: Token) -> TokenWithSpan { - TokenWithSpan::new(token, Span::empty()) + /// Wrap a token with a location from `start` to `end` + pub fn at(token: Token, start: Location, end: Location) -> Self { + Self::new(token, Span::new(start, end)) } - pub fn at(token: Token, start: Location, end: Location) -> TokenWithSpan { - TokenWithSpan::new(token, Span::new(start, end)) + /// Return an EOF token with no location + pub fn new_eof() -> Self { + Self::wrap(Token::EOF) } } From bd750dfadadf7eda90d1c7c69ad0a4208b0fd05a Mon Sep 17 00:00:00 2001 From: Ayman Elkfrawy <120422207+ayman-sigma@users.noreply.github.com> Date: Mon, 2 Dec 2024 07:23:48 -0800 Subject: [PATCH 78/80] Support Databricks struct literal (#1542) --- src/ast/mod.rs | 6 ++++-- src/dialect/bigquery.rs | 5 +++++ src/dialect/databricks.rs | 5 +++++ src/dialect/generic.rs | 4 ++++ src/dialect/mod.rs | 10 ++++++++++ src/parser/mod.rs | 15 ++++++++------- tests/sqlparser_databricks.rs | 35 +++++++++++++++++++++++++++++++++++ 7 files changed, 71 insertions(+), 9 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index e52251d52..d928370a0 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -931,12 +931,14 @@ pub enum Expr { Rollup(Vec>), /// ROW / TUPLE a single value, such as `SELECT (1, 2)` Tuple(Vec), - /// `BigQuery` specific `Struct` literal expression [1] + /// `Struct` literal expression /// Syntax: /// ```sql /// STRUCT<[field_name] field_type, ...>( expr1 [, ... ]) + /// + /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#struct_type) + /// [Databricks](https://docs.databricks.com/en/sql/language-manual/functions/struct.html) /// ``` - /// [1]: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#struct_type Struct { /// Struct values. values: Vec, diff --git a/src/dialect/bigquery.rs b/src/dialect/bigquery.rs index 96633552b..66d7d2061 100644 --- a/src/dialect/bigquery.rs +++ b/src/dialect/bigquery.rs @@ -72,4 +72,9 @@ impl Dialect for BigQueryDialect { fn require_interval_qualifier(&self) -> bool { true } + + // See https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#constructing_a_struct + fn supports_struct_literal(&self) -> bool { + true + } } diff --git a/src/dialect/databricks.rs b/src/dialect/databricks.rs index 4924e8077..a3476b1b8 100644 --- a/src/dialect/databricks.rs +++ b/src/dialect/databricks.rs @@ -59,4 +59,9 @@ impl Dialect for DatabricksDialect { fn require_interval_qualifier(&self) -> bool { true } + + // See https://docs.databricks.com/en/sql/language-manual/functions/struct.html + fn supports_struct_literal(&self) -> bool { + true + } } diff --git a/src/dialect/generic.rs b/src/dialect/generic.rs index e3beeae7f..61e5070fb 100644 --- a/src/dialect/generic.rs +++ b/src/dialect/generic.rs @@ -123,4 +123,8 @@ impl Dialect for GenericDialect { fn supports_named_fn_args_with_assignment_operator(&self) -> bool { true } + + fn supports_struct_literal(&self) -> bool { + true + } } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index a8993e685..f40cba719 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -375,6 +375,16 @@ pub trait Dialect: Debug + Any { false } + /// Return true if the dialect supports the STRUCT literal + /// + /// Example + /// ```sql + /// SELECT STRUCT(1 as one, 'foo' as foo, false) + /// ``` + fn supports_struct_literal(&self) -> bool { + false + } + /// Dialect-specific infix parser override /// /// This method is called to parse the next infix expression. diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 16362ebba..831098ba1 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1123,9 +1123,8 @@ impl<'a> Parser<'a> { Keyword::MATCH if dialect_of!(self is MySqlDialect | GenericDialect) => { Ok(Some(self.parse_match_against()?)) } - Keyword::STRUCT if dialect_of!(self is BigQueryDialect | GenericDialect) => { - self.prev_token(); - Ok(Some(self.parse_bigquery_struct_literal()?)) + Keyword::STRUCT if self.dialect.supports_struct_literal() => { + Ok(Some(self.parse_struct_literal()?)) } Keyword::PRIOR if matches!(self.state, ParserState::ConnectBy) => { let expr = self.parse_subexpr(self.dialect.prec_value(Precedence::PlusMinus))?; @@ -2383,7 +2382,6 @@ impl<'a> Parser<'a> { } } - /// Bigquery specific: Parse a struct literal /// Syntax /// ```sql /// -- typed @@ -2391,7 +2389,9 @@ impl<'a> Parser<'a> { /// -- typeless /// STRUCT( expr1 [AS field_name] [, ... ]) /// ``` - fn parse_bigquery_struct_literal(&mut self) -> Result { + fn parse_struct_literal(&mut self) -> Result { + // Parse the fields definition if exist `<[field_name] field_type, ...>` + self.prev_token(); let (fields, trailing_bracket) = self.parse_struct_type_def(Self::parse_struct_field_def)?; if trailing_bracket.0 { @@ -2401,6 +2401,7 @@ impl<'a> Parser<'a> { ); } + // Parse the struct values `(expr1 [, ... ])` self.expect_token(&Token::LParen)?; let values = self .parse_comma_separated(|parser| parser.parse_struct_field_expr(!fields.is_empty()))?; @@ -2409,13 +2410,13 @@ impl<'a> Parser<'a> { Ok(Expr::Struct { values, fields }) } - /// Parse an expression value for a bigquery struct [1] + /// Parse an expression value for a struct literal /// Syntax /// ```sql /// expr [AS name] /// ``` /// - /// Parameter typed_syntax is set to true if the expression + /// For biquery [1], Parameter typed_syntax is set to true if the expression /// is to be parsed as a field expression declared using typed /// struct syntax [2], and false if using typeless struct syntax [3]. /// diff --git a/tests/sqlparser_databricks.rs b/tests/sqlparser_databricks.rs index 1651d517a..d73c088a7 100644 --- a/tests/sqlparser_databricks.rs +++ b/tests/sqlparser_databricks.rs @@ -278,3 +278,38 @@ fn parse_use() { ); } } + +#[test] +fn parse_databricks_struct_function() { + assert_eq!( + databricks_and_generic() + .verified_only_select("SELECT STRUCT(1, 'foo')") + .projection[0], + SelectItem::UnnamedExpr(Expr::Struct { + values: vec![ + Expr::Value(number("1")), + Expr::Value(Value::SingleQuotedString("foo".to_string())) + ], + fields: vec![] + }) + ); + assert_eq!( + databricks_and_generic() + .verified_only_select("SELECT STRUCT(1 AS one, 'foo' AS foo, false)") + .projection[0], + SelectItem::UnnamedExpr(Expr::Struct { + values: vec![ + Expr::Named { + expr: Expr::Value(number("1")).into(), + name: Ident::new("one") + }, + Expr::Named { + expr: Expr::Value(Value::SingleQuotedString("foo".to_string())).into(), + name: Ident::new("foo") + }, + Expr::Value(Value::Boolean(false)) + ], + fields: vec![] + }) + ); +} From e16b24679a1e87dd54ce9565e87e818dce4d4a0a Mon Sep 17 00:00:00 2001 From: Philip Cristiano Date: Mon, 2 Dec 2024 12:45:14 -0500 Subject: [PATCH 79/80] Encapsulate CreateFunction (#1573) --- src/ast/ddl.rs | 129 +++++++++++++++++++++++++++++++++- src/ast/mod.rs | 133 ++---------------------------------- src/parser/mod.rs | 12 ++-- tests/sqlparser_bigquery.rs | 4 +- tests/sqlparser_hive.rs | 11 +-- tests/sqlparser_postgres.rs | 8 +-- 6 files changed, 149 insertions(+), 148 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 3ced478ca..9a7d297bc 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -30,8 +30,10 @@ use sqlparser_derive::{Visit, VisitMut}; use crate::ast::value::escape_single_quote_string; use crate::ast::{ - display_comma_separated, display_separated, DataType, Expr, Ident, MySQLColumnPosition, - ObjectName, OrderByExpr, ProjectionSelect, SequenceOptions, SqlOption, Tag, Value, + display_comma_separated, display_separated, CreateFunctionBody, CreateFunctionUsing, DataType, + Expr, FunctionBehavior, FunctionCalledOnNull, FunctionDeterminismSpecifier, FunctionParallel, + Ident, MySQLColumnPosition, ObjectName, OperateFunctionArg, OrderByExpr, ProjectionSelect, + SequenceOptions, SqlOption, Tag, Value, }; use crate::keywords::Keyword; use crate::tokenizer::Token; @@ -1819,3 +1821,126 @@ impl fmt::Display for ClusteredBy { write!(f, " INTO {} BUCKETS", self.num_buckets) } } + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct CreateFunction { + pub or_replace: bool, + pub temporary: bool, + pub if_not_exists: bool, + pub name: ObjectName, + pub args: Option>, + pub return_type: Option, + /// The expression that defines the function. + /// + /// Examples: + /// ```sql + /// AS ((SELECT 1)) + /// AS "console.log();" + /// ``` + pub function_body: Option, + /// Behavior attribute for the function + /// + /// IMMUTABLE | STABLE | VOLATILE + /// + /// [Postgres](https://www.postgresql.org/docs/current/sql-createfunction.html) + pub behavior: Option, + /// CALLED ON NULL INPUT | RETURNS NULL ON NULL INPUT | STRICT + /// + /// [Postgres](https://www.postgresql.org/docs/current/sql-createfunction.html) + pub called_on_null: Option, + /// PARALLEL { UNSAFE | RESTRICTED | SAFE } + /// + /// [Postgres](https://www.postgresql.org/docs/current/sql-createfunction.html) + pub parallel: Option, + /// USING ... (Hive only) + pub using: Option, + /// Language used in a UDF definition. + /// + /// Example: + /// ```sql + /// CREATE FUNCTION foo() LANGUAGE js AS "console.log();" + /// ``` + /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_a_javascript_udf) + pub language: Option, + /// Determinism keyword used for non-sql UDF definitions. + /// + /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#syntax_11) + pub determinism_specifier: Option, + /// List of options for creating the function. + /// + /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#syntax_11) + pub options: Option>, + /// Connection resource for a remote function. + /// + /// Example: + /// ```sql + /// CREATE FUNCTION foo() + /// RETURNS FLOAT64 + /// REMOTE WITH CONNECTION us.myconnection + /// ``` + /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_a_remote_function) + pub remote_connection: Option, +} + +impl fmt::Display for CreateFunction { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "CREATE {or_replace}{temp}FUNCTION {if_not_exists}{name}", + name = self.name, + temp = if self.temporary { "TEMPORARY " } else { "" }, + or_replace = if self.or_replace { "OR REPLACE " } else { "" }, + if_not_exists = if self.if_not_exists { + "IF NOT EXISTS " + } else { + "" + }, + )?; + if let Some(args) = &self.args { + write!(f, "({})", display_comma_separated(args))?; + } + if let Some(return_type) = &self.return_type { + write!(f, " RETURNS {return_type}")?; + } + if let Some(determinism_specifier) = &self.determinism_specifier { + write!(f, " {determinism_specifier}")?; + } + if let Some(language) = &self.language { + write!(f, " LANGUAGE {language}")?; + } + if let Some(behavior) = &self.behavior { + write!(f, " {behavior}")?; + } + if let Some(called_on_null) = &self.called_on_null { + write!(f, " {called_on_null}")?; + } + if let Some(parallel) = &self.parallel { + write!(f, " {parallel}")?; + } + if let Some(remote_connection) = &self.remote_connection { + write!(f, " REMOTE WITH CONNECTION {remote_connection}")?; + } + if let Some(CreateFunctionBody::AsBeforeOptions(function_body)) = &self.function_body { + write!(f, " AS {function_body}")?; + } + if let Some(CreateFunctionBody::Return(function_body)) = &self.function_body { + write!(f, " RETURN {function_body}")?; + } + if let Some(using) = &self.using { + write!(f, " {using}")?; + } + if let Some(options) = &self.options { + write!( + f, + " OPTIONS({})", + display_comma_separated(options.as_slice()) + )?; + } + if let Some(CreateFunctionBody::AsAfterOptions(function_body)) = &self.function_body { + write!(f, " AS {function_body}")?; + } + Ok(()) + } +} diff --git a/src/ast/mod.rs b/src/ast/mod.rs index d928370a0..ef4ccff4b 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -47,7 +47,7 @@ pub use self::dcl::{AlterRoleOperation, ResetConfig, RoleOption, SetConfigValue, pub use self::ddl::{ AlterColumnOperation, AlterIndexOperation, AlterPolicyOperation, AlterTableOperation, ClusteredBy, ColumnDef, ColumnOption, ColumnOptionDef, ColumnPolicy, ColumnPolicyProperty, - ConstraintCharacteristics, Deduplicate, DeferrableInitial, GeneratedAs, + ConstraintCharacteristics, CreateFunction, Deduplicate, DeferrableInitial, GeneratedAs, GeneratedExpressionMode, IdentityParameters, IdentityProperty, IdentityPropertyFormatKind, IdentityPropertyKind, IdentityPropertyOrder, IndexOption, IndexType, KeyOrIndexDisplay, Owner, Partition, ProcedureParam, ReferentialAction, TableConstraint, TagsColumnOption, @@ -897,7 +897,7 @@ pub enum Expr { /// Example: /// /// ```sql - /// SELECT (SELECT ',' + name FROM sys.objects FOR XML PATH(''), TYPE).value('.','NVARCHAR(MAX)') + /// SELECT (SELECT ',' + name FROM sys.objects FOR XML PATH(''), TYPE).value('.','NVARCHAR(MAX)') /// SELECT CONVERT(XML,'abc').value('.','NVARCHAR(MAX)').value('.','NVARCHAR(MAX)') /// ``` /// @@ -3003,64 +3003,7 @@ pub enum Statement { /// 1. [Hive](https://cwiki.apache.org/confluence/display/hive/languagemanual+ddl#LanguageManualDDL-Create/Drop/ReloadFunction) /// 2. [Postgres](https://www.postgresql.org/docs/15/sql-createfunction.html) /// 3. [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_function_statement) - CreateFunction { - or_replace: bool, - temporary: bool, - if_not_exists: bool, - name: ObjectName, - args: Option>, - return_type: Option, - /// The expression that defines the function. - /// - /// Examples: - /// ```sql - /// AS ((SELECT 1)) - /// AS "console.log();" - /// ``` - function_body: Option, - /// Behavior attribute for the function - /// - /// IMMUTABLE | STABLE | VOLATILE - /// - /// [Postgres](https://www.postgresql.org/docs/current/sql-createfunction.html) - behavior: Option, - /// CALLED ON NULL INPUT | RETURNS NULL ON NULL INPUT | STRICT - /// - /// [Postgres](https://www.postgresql.org/docs/current/sql-createfunction.html) - called_on_null: Option, - /// PARALLEL { UNSAFE | RESTRICTED | SAFE } - /// - /// [Postgres](https://www.postgresql.org/docs/current/sql-createfunction.html) - parallel: Option, - /// USING ... (Hive only) - using: Option, - /// Language used in a UDF definition. - /// - /// Example: - /// ```sql - /// CREATE FUNCTION foo() LANGUAGE js AS "console.log();" - /// ``` - /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_a_javascript_udf) - language: Option, - /// Determinism keyword used for non-sql UDF definitions. - /// - /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#syntax_11) - determinism_specifier: Option, - /// List of options for creating the function. - /// - /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#syntax_11) - options: Option>, - /// Connection resource for a remote function. - /// - /// Example: - /// ```sql - /// CREATE FUNCTION foo() - /// RETURNS FLOAT64 - /// REMOTE WITH CONNECTION us.myconnection - /// ``` - /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_a_remote_function) - remote_connection: Option, - }, + CreateFunction(CreateFunction), /// CREATE TRIGGER /// /// Examples: @@ -3826,75 +3769,7 @@ impl fmt::Display for Statement { } Ok(()) } - Statement::CreateFunction { - or_replace, - temporary, - if_not_exists, - name, - args, - return_type, - function_body, - language, - behavior, - called_on_null, - parallel, - using, - determinism_specifier, - options, - remote_connection, - } => { - write!( - f, - "CREATE {or_replace}{temp}FUNCTION {if_not_exists}{name}", - temp = if *temporary { "TEMPORARY " } else { "" }, - or_replace = if *or_replace { "OR REPLACE " } else { "" }, - if_not_exists = if *if_not_exists { "IF NOT EXISTS " } else { "" }, - )?; - if let Some(args) = args { - write!(f, "({})", display_comma_separated(args))?; - } - if let Some(return_type) = return_type { - write!(f, " RETURNS {return_type}")?; - } - if let Some(determinism_specifier) = determinism_specifier { - write!(f, " {determinism_specifier}")?; - } - if let Some(language) = language { - write!(f, " LANGUAGE {language}")?; - } - if let Some(behavior) = behavior { - write!(f, " {behavior}")?; - } - if let Some(called_on_null) = called_on_null { - write!(f, " {called_on_null}")?; - } - if let Some(parallel) = parallel { - write!(f, " {parallel}")?; - } - if let Some(remote_connection) = remote_connection { - write!(f, " REMOTE WITH CONNECTION {remote_connection}")?; - } - if let Some(CreateFunctionBody::AsBeforeOptions(function_body)) = function_body { - write!(f, " AS {function_body}")?; - } - if let Some(CreateFunctionBody::Return(function_body)) = function_body { - write!(f, " RETURN {function_body}")?; - } - if let Some(using) = using { - write!(f, " {using}")?; - } - if let Some(options) = options { - write!( - f, - " OPTIONS({})", - display_comma_separated(options.as_slice()) - )?; - } - if let Some(CreateFunctionBody::AsAfterOptions(function_body)) = function_body { - write!(f, " AS {function_body}")?; - } - Ok(()) - } + Statement::CreateFunction(create_function) => create_function.fmt(f), Statement::CreateTrigger { or_replace, is_constraint, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 831098ba1..90665e9f9 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -4240,7 +4240,7 @@ impl<'a> Parser<'a> { } } - Ok(Statement::CreateFunction { + Ok(Statement::CreateFunction(CreateFunction { or_replace, temporary, name, @@ -4256,7 +4256,7 @@ impl<'a> Parser<'a> { determinism_specifier: None, options: None, remote_connection: None, - }) + })) } /// Parse `CREATE FUNCTION` for [Hive] @@ -4273,7 +4273,7 @@ impl<'a> Parser<'a> { let as_ = self.parse_create_function_body_string()?; let using = self.parse_optional_create_function_using()?; - Ok(Statement::CreateFunction { + Ok(Statement::CreateFunction(CreateFunction { or_replace, temporary, name, @@ -4289,7 +4289,7 @@ impl<'a> Parser<'a> { determinism_specifier: None, options: None, remote_connection: None, - }) + })) } /// Parse `CREATE FUNCTION` for [BigQuery] @@ -4362,7 +4362,7 @@ impl<'a> Parser<'a> { None }; - Ok(Statement::CreateFunction { + Ok(Statement::CreateFunction(CreateFunction { or_replace, temporary, if_not_exists, @@ -4378,7 +4378,7 @@ impl<'a> Parser<'a> { behavior: None, called_on_null: None, parallel: None, - }) + })) } fn parse_function_arg(&mut self) -> Result { diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index 00d12ed83..2be128a8c 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -2011,7 +2011,7 @@ fn test_bigquery_create_function() { let stmt = bigquery().verified_stmt(sql); assert_eq!( stmt, - Statement::CreateFunction { + Statement::CreateFunction(CreateFunction { or_replace: true, temporary: true, if_not_exists: false, @@ -2036,7 +2036,7 @@ fn test_bigquery_create_function() { remote_connection: None, called_on_null: None, parallel: None, - } + }) ); let sqls = [ diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 8d4f7a680..546b289ac 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -21,9 +21,10 @@ //! is also tested (on the inputs it can handle). use sqlparser::ast::{ - ClusteredBy, CommentDef, CreateFunctionBody, CreateFunctionUsing, CreateTable, Expr, Function, - FunctionArgumentList, FunctionArguments, Ident, ObjectName, OneOrManyWithParens, OrderByExpr, - SelectItem, Statement, TableFactor, UnaryOperator, Use, Value, + ClusteredBy, CommentDef, CreateFunction, CreateFunctionBody, CreateFunctionUsing, CreateTable, + Expr, Function, FunctionArgumentList, FunctionArguments, Ident, ObjectName, + OneOrManyWithParens, OrderByExpr, SelectItem, Statement, TableFactor, UnaryOperator, Use, + Value, }; use sqlparser::dialect::{GenericDialect, HiveDialect, MsSqlDialect}; use sqlparser::parser::ParserError; @@ -392,13 +393,13 @@ fn set_statement_with_minus() { fn parse_create_function() { let sql = "CREATE TEMPORARY FUNCTION mydb.myfunc AS 'org.random.class.Name' USING JAR 'hdfs://somewhere.com:8020/very/far'"; match hive().verified_stmt(sql) { - Statement::CreateFunction { + Statement::CreateFunction(CreateFunction { temporary, name, function_body, using, .. - } => { + }) => { assert!(temporary); assert_eq!(name.to_string(), "mydb.myfunc"); assert_eq!( diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index f94e2f540..52fe6c403 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -3631,7 +3631,7 @@ fn parse_create_function() { let sql = "CREATE FUNCTION add(INTEGER, INTEGER) RETURNS INTEGER LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS 'select $1 + $2;'"; assert_eq!( pg_and_generic().verified_stmt(sql), - Statement::CreateFunction { + Statement::CreateFunction(CreateFunction { or_replace: false, temporary: false, name: ObjectName(vec![Ident::new("add")]), @@ -3652,7 +3652,7 @@ fn parse_create_function() { determinism_specifier: None, options: None, remote_connection: None, - } + }) ); } @@ -4987,7 +4987,7 @@ fn parse_trigger_related_functions() { assert_eq!( create_function, - Statement::CreateFunction { + Statement::CreateFunction(CreateFunction { or_replace: false, temporary: false, if_not_exists: false, @@ -5017,7 +5017,7 @@ fn parse_trigger_related_functions() { options: None, remote_connection: None } - ); + )); // Check the third statement From e0b7e7e10fb580d14c63857d26d0b561b6134756 Mon Sep 17 00:00:00 2001 From: Ayman Elkfrawy Date: Mon, 2 Dec 2024 13:38:49 -0800 Subject: [PATCH 80/80] add span for InExpr --- src/ast/spans.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ast/spans.rs b/src/ast/spans.rs index 1e0f1bf09..b84ee78c9 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -1259,6 +1259,11 @@ impl Spanned for Expr { array_expr, negated: _, } => expr.span().union(&array_expr.span()), + Expr::InExpr { + expr, + in_expr, + negated: _, + } => expr.span().union(&in_expr.span()), Expr::Between { expr, negated: _,