Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rename TokenWithLocation to TokenWithSpan, in backwards compatible way #1562

Merged
merged 1 commit into from
Nov 30, 2024
Merged

Rename TokenWithLocation to TokenWithSpan, in backwards compatible way #1562

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/ast/helpers/attached_token.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
use core::fmt::{self, Debug, Formatter};
use core::hash::{Hash, Hasher};

use crate::tokenizer::{Token, TokenWithLocation};
use crate::tokenizer::{Token, TokenWithSpan};

#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
Expand All @@ -33,11 +33,11 @@ use sqlparser_derive::{Visit, VisitMut};
#[derive(Clone)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
pub struct AttachedToken(pub TokenWithLocation);
pub struct AttachedToken(pub TokenWithSpan);

impl AttachedToken {
pub fn empty() -> Self {
AttachedToken(TokenWithLocation::wrap(Token::EOF))
AttachedToken(TokenWithSpan::wrap(Token::EOF))
}
}

Expand Down Expand Up @@ -75,8 +75,8 @@ impl Hash for AttachedToken {
}
}

impl From<TokenWithLocation> for AttachedToken {
fn from(value: TokenWithLocation) -> Self {
impl From<TokenWithSpan> for AttachedToken {
fn from(value: TokenWithSpan) -> Self {
AttachedToken(value)
}
}
4 changes: 2 additions & 2 deletions src/ast/query.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ use sqlparser_derive::{Visit, VisitMut};

use crate::{
ast::*,
tokenizer::{Token, TokenWithLocation},
tokenizer::{Token, TokenWithSpan},
};

/// The most complete variant of a `SELECT` query expression, optionally
Expand Down Expand Up @@ -643,7 +643,7 @@ pub struct WildcardAdditionalOptions {
impl Default for WildcardAdditionalOptions {
fn default() -> Self {
Self {
wildcard_token: TokenWithLocation::wrap(Token::Mul).into(),
wildcard_token: TokenWithSpan::wrap(Token::Mul).into(),
opt_ilike: None,
opt_exclude: None,
opt_except: None,
Expand Down
66 changes: 33 additions & 33 deletions src/parser/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ enum ParserState {
}

pub struct Parser<'a> {
tokens: Vec<TokenWithLocation>,
tokens: Vec<TokenWithSpan>,
/// The index of the first unprocessed token in [`Parser::tokens`].
index: usize,
/// The current state of the parser.
Expand Down Expand Up @@ -359,7 +359,7 @@ impl<'a> Parser<'a> {
}

/// Reset this parser to parse the specified token stream
pub fn with_tokens_with_locations(mut self, tokens: Vec<TokenWithLocation>) -> Self {
pub fn with_tokens_with_locations(mut self, tokens: Vec<TokenWithSpan>) -> Self {
self.tokens = tokens;
self.index = 0;
self
Expand All @@ -368,9 +368,9 @@ impl<'a> Parser<'a> {
/// Reset this parser state to parse the specified tokens
pub fn with_tokens(self, tokens: Vec<Token>) -> Self {
// Put in dummy locations
let tokens_with_locations: Vec<TokenWithLocation> = tokens
let tokens_with_locations: Vec<TokenWithSpan> = tokens
.into_iter()
.map(|token| TokenWithLocation {
.map(|token| TokenWithSpan {
token,
span: Span::empty(),
})
Expand Down Expand Up @@ -1147,7 +1147,7 @@ impl<'a> Parser<'a> {
match self.peek_token().token {
Token::LParen | Token::Period => {
let mut id_parts: Vec<Ident> = vec![w.to_ident(w_span)];
let mut ending_wildcard: Option<TokenWithLocation> = None;
let mut ending_wildcard: Option<TokenWithSpan> = None;
while self.consume_token(&Token::Period) {
let next_token = self.next_token();
match next_token.token {
Expand Down Expand Up @@ -3273,7 +3273,7 @@ impl<'a> Parser<'a> {

/// Return the first non-whitespace token that has not yet been processed
/// (or None if reached end-of-file)
pub fn peek_token(&self) -> TokenWithLocation {
pub fn peek_token(&self) -> TokenWithSpan {
self.peek_nth_token(0)
}

Expand Down Expand Up @@ -3308,38 +3308,38 @@ impl<'a> Parser<'a> {
/// yet been processed.
///
/// See [`Self::peek_token`] for an example.
pub fn peek_tokens_with_location<const N: usize>(&self) -> [TokenWithLocation; N] {
pub fn peek_tokens_with_location<const N: usize>(&self) -> [TokenWithSpan; N] {
let mut index = self.index;
core::array::from_fn(|_| loop {
let token = self.tokens.get(index);
index += 1;
if let Some(TokenWithLocation {
if let Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) = token
{
continue;
}
break token.cloned().unwrap_or(TokenWithLocation {
break token.cloned().unwrap_or(TokenWithSpan {
token: Token::EOF,
span: Span::empty(),
});
})
}

/// Return nth non-whitespace token that has not yet been processed
pub fn peek_nth_token(&self, mut n: usize) -> TokenWithLocation {
pub fn peek_nth_token(&self, mut n: usize) -> TokenWithSpan {
let mut index = self.index;
loop {
index += 1;
match self.tokens.get(index - 1) {
Some(TokenWithLocation {
Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) => continue,
non_whitespace => {
if n == 0 {
return non_whitespace.cloned().unwrap_or(TokenWithLocation {
return non_whitespace.cloned().unwrap_or(TokenWithSpan {
token: Token::EOF,
span: Span::empty(),
});
Expand All @@ -3352,16 +3352,16 @@ impl<'a> Parser<'a> {

/// Return the first token, possibly whitespace, that has not yet been processed
/// (or None if reached end-of-file).
pub fn peek_token_no_skip(&self) -> TokenWithLocation {
pub fn peek_token_no_skip(&self) -> TokenWithSpan {
self.peek_nth_token_no_skip(0)
}

/// Return nth token, possibly whitespace, that has not yet been processed.
pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithLocation {
pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithSpan {
self.tokens
.get(self.index + n)
.cloned()
.unwrap_or(TokenWithLocation {
.unwrap_or(TokenWithSpan {
token: Token::EOF,
span: Span::empty(),
})
Expand All @@ -3378,25 +3378,25 @@ impl<'a> Parser<'a> {
/// Return the first non-whitespace token that has not yet been processed
/// (or None if reached end-of-file) and mark it as processed. OK to call
/// repeatedly after reaching EOF.
pub fn next_token(&mut self) -> TokenWithLocation {
pub fn next_token(&mut self) -> TokenWithSpan {
loop {
self.index += 1;
match self.tokens.get(self.index - 1) {
Some(TokenWithLocation {
Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) => continue,
token => {
return token
.cloned()
.unwrap_or_else(|| TokenWithLocation::wrap(Token::EOF))
.unwrap_or_else(|| TokenWithSpan::wrap(Token::EOF))
}
}
}
}

/// Return the first unprocessed token, possibly whitespace.
pub fn next_token_no_skip(&mut self) -> Option<&TokenWithLocation> {
pub fn next_token_no_skip(&mut self) -> Option<&TokenWithSpan> {
self.index += 1;
self.tokens.get(self.index - 1)
}
Expand All @@ -3408,7 +3408,7 @@ impl<'a> Parser<'a> {
loop {
assert!(self.index > 0);
self.index -= 1;
if let Some(TokenWithLocation {
if let Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) = self.tokens.get(self.index)
Expand All @@ -3420,7 +3420,7 @@ impl<'a> Parser<'a> {
}

/// Report `found` was encountered instead of `expected`
pub fn expected<T>(&self, expected: &str, found: TokenWithLocation) -> Result<T, ParserError> {
pub fn expected<T>(&self, expected: &str, found: TokenWithSpan) -> Result<T, ParserError> {
parser_err!(
format!("Expected: {expected}, found: {found}"),
found.span.start
Expand All @@ -3435,7 +3435,7 @@ impl<'a> Parser<'a> {
}

#[must_use]
pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option<TokenWithLocation> {
pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option<TokenWithSpan> {
match self.peek_token().token {
Token::Word(w) if expected == w.keyword => Some(self.next_token()),
_ => None,
Expand Down Expand Up @@ -3524,7 +3524,7 @@ impl<'a> Parser<'a> {

/// If the current token is the `expected` keyword, consume the token.
/// Otherwise, return an error.
pub fn expect_keyword(&mut self, expected: Keyword) -> Result<TokenWithLocation, ParserError> {
pub fn expect_keyword(&mut self, expected: Keyword) -> Result<TokenWithSpan, ParserError> {
if let Some(token) = self.parse_keyword_token(expected) {
Ok(token)
} else {
Expand Down Expand Up @@ -3568,7 +3568,7 @@ impl<'a> Parser<'a> {
}

/// Bail out if the current token is not an expected keyword, or consume it if it is
pub fn expect_token(&mut self, expected: &Token) -> Result<TokenWithLocation, ParserError> {
pub fn expect_token(&mut self, expected: &Token) -> Result<TokenWithSpan, ParserError> {
if self.peek_token() == *expected {
Ok(self.next_token())
} else {
Expand Down Expand Up @@ -4107,7 +4107,7 @@ impl<'a> Parser<'a> {
Keyword::ARCHIVE => Ok(Some(CreateFunctionUsing::Archive(uri))),
_ => self.expected(
"JAR, FILE or ARCHIVE, got {:?}",
TokenWithLocation::wrap(Token::make_keyword(format!("{keyword:?}").as_str())),
TokenWithSpan::wrap(Token::make_keyword(format!("{keyword:?}").as_str())),
),
}
}
Expand Down Expand Up @@ -6832,7 +6832,7 @@ impl<'a> Parser<'a> {
if let Some(name) = name {
return self.expected(
"FULLTEXT or SPATIAL option without constraint name",
TokenWithLocation {
TokenWithSpan {
token: Token::make_keyword(&name.to_string()),
span: next_token.span,
},
Expand Down Expand Up @@ -7808,15 +7808,15 @@ impl<'a> Parser<'a> {
Some('\'') => Ok(Value::SingleQuotedString(w.value)),
_ => self.expected(
"A value?",
TokenWithLocation {
TokenWithSpan {
token: Token::Word(w),
span,
},
)?,
},
_ => self.expected(
"a concrete value",
TokenWithLocation {
TokenWithSpan {
token: Token::Word(w),
span,
},
Expand Down Expand Up @@ -7878,7 +7878,7 @@ impl<'a> Parser<'a> {
}
unexpected => self.expected(
"a value",
TokenWithLocation {
TokenWithSpan {
token: unexpected,
span,
},
Expand Down Expand Up @@ -7927,7 +7927,7 @@ impl<'a> Parser<'a> {
Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())),
unexpected => self.expected(
"a string value",
TokenWithLocation {
TokenWithSpan {
token: unexpected,
span,
},
Expand Down Expand Up @@ -8618,7 +8618,7 @@ impl<'a> Parser<'a> {
let token = self
.next_token_no_skip()
.cloned()
.unwrap_or(TokenWithLocation::wrap(Token::EOF));
.unwrap_or(TokenWithSpan::wrap(Token::EOF));
requires_whitespace = match token.token {
Token::Word(next_word) if next_word.quote_style.is_none() => {
ident.value.push_str(&next_word.value);
Expand Down Expand Up @@ -11685,7 +11685,7 @@ impl<'a> Parser<'a> {
/// If it is not possible to parse it, will return an option.
pub fn parse_wildcard_additional_options(
&mut self,
wildcard_token: TokenWithLocation,
wildcard_token: TokenWithSpan,
) -> Result<WildcardAdditionalOptions, ParserError> {
let opt_ilike = if dialect_of!(self is GenericDialect | SnowflakeDialect) {
self.parse_optional_select_item_ilike()?
Expand Down Expand Up @@ -12710,7 +12710,7 @@ impl<'a> Parser<'a> {
}

/// Consume the parser and return its underlying token buffer
pub fn into_tokens(self) -> Vec<TokenWithLocation> {
pub fn into_tokens(self) -> Vec<TokenWithSpan> {
self.tokens
}

Expand Down
Loading