Merge pull request #55 from gwenn/sync

Merge changes from SQLite official repository
gwenn · Apr 10, 2024 · 0e7aa1c · 0e7aa1c
2 parents ba9a7d2 + 49b18c3
commit 0e7aa1c
Show file tree

Hide file tree

Showing 2 changed files with 180 additions and 104 deletions.
diff --git a/src/lexer/sql/mod.rs b/src/lexer/sql/mod.rs
@@ -526,17 +526,12 @@ fn number(data: &[u8]) -> Result<(Option<Token<'_>>, usize), Error> {
             return Ok((Some((data, TK_INTEGER)), data.len()));
         }
     }
-    return if let Some((i, b)) = data
-        .iter()
-        .enumerate()
-        .skip(1)
-        .find(|&(_, &b)| !b.is_ascii_digit())
-    {
-        if *b == b'.' {
+    return if let Some((i, b)) = find_end_of_number(data, 1, u8::is_ascii_digit)? {
+        if b == b'.' {
             return fractional_part(data, i);
-        } else if *b == b'e' || *b == b'E' {
+        } else if b == b'e' || b == b'E' {
             return exponential_part(data, i);
-        } else if is_identifier_start(*b) {
+        } else if is_identifier_start(b) {
             return Err(Error::BadNumber(None));
         }
         Ok((Some((&data[..i], TK_INTEGER)), i))
@@ -548,14 +543,9 @@ fn number(data: &[u8]) -> Result<(Option<Token<'_>>, usize), Error> {
 fn hex_integer(data: &[u8]) -> Result<(Option<Token<'_>>, usize), Error> {
     debug_assert_eq!(data[0], b'0');
     debug_assert!(data[1] == b'x' || data[1] == b'X');
-    return if let Some((i, b)) = data
-        .iter()
-        .enumerate()
-        .skip(2)
-        .find(|&(_, &b)| !b.is_ascii_hexdigit())
-    {
+    return if let Some((i, b)) = find_end_of_number(data, 2, u8::is_ascii_hexdigit)? {
         // Must not be empty (Ox is invalid)
-        if i == 2 || is_identifier_start(*b) {
+        if i == 2 || is_identifier_start(b) {
             return Err(Error::MalformedHexInteger(None));
         }
         Ok((Some((&data[..i], TK_INTEGER)), i))
@@ -570,15 +560,10 @@ fn hex_integer(data: &[u8]) -> Result<(Option<Token<'_>>, usize), Error> {
 
 fn fractional_part(data: &[u8], i: usize) -> Result<(Option<Token<'_>>, usize), Error> {
     debug_assert_eq!(data[i], b'.');
-    return if let Some((i, b)) = data
-        .iter()
-        .enumerate()
-        .skip(i + 1)
-        .find(|&(_, &b)| !b.is_ascii_digit())
-    {
-        if *b == b'e' || *b == b'E' {
+    return if let Some((i, b)) = find_end_of_number(data, i + 1, u8::is_ascii_digit)? {
+        if b == b'e' || b == b'E' {
             return exponential_part(data, i);
-        } else if is_identifier_start(*b) {
+        } else if is_identifier_start(b) {
             return Err(Error::BadNumber(None));
         }
         Ok((Some((&data[..i], TK_FLOAT)), i))
@@ -592,13 +577,8 @@ fn exponential_part(data: &[u8], i: usize) -> Result<(Option<Token<'_>>, usize),
     // data[i] == 'e'|'E'
     return if let Some(b) = data.get(i + 1) {
         let i = if *b == b'+' || *b == b'-' { i + 1 } else { i };
-        if let Some((j, b)) = data
-            .iter()
-            .enumerate()
-            .skip(i + 1)
-            .find(|&(_, &b)| !b.is_ascii_digit())
-        {
-            if j == i + 1 || is_identifier_start(*b) {
+        if let Some((j, b)) = find_end_of_number(data, i + 1, u8::is_ascii_digit)? {
+            if j == i + 1 || is_identifier_start(b) {
                 return Err(Error::BadNumber(None));
             }
             Ok((Some((&data[..j], TK_FLOAT)), j))
@@ -613,6 +593,29 @@ fn exponential_part(data: &[u8], i: usize) -> Result<(Option<Token<'_>>, usize),
     };
 }
 
+fn find_end_of_number(
+    data: &[u8],
+    i: usize,
+    test: fn(&u8) -> bool,
+) -> Result<Option<(usize, u8)>, Error> {
+    for (j, &b) in data.iter().enumerate().skip(i) {
+        if test(&b) {
+            continue;
+        } else if b == b'_' {
+            if j >= 1
+                && data.get(j - 1).map_or(false, |p| test(p))
+                && data.get(j + 1).map_or(false, |n| test(n))
+            {
+                continue;
+            }
+            return Err(Error::BadNumber(None));
+        } else {
+            return Ok(Some((j, b)));
+        }
+    }
+    Ok(None)
+}
+
 impl Tokenizer {
     fn identifierish<'input>(&mut self, data: &'input [u8]) -> (Option<Token<'input>>, usize) {
         debug_assert!(is_identifier_start(data[0]));