From 9a8b94ef9358929d91a7507295b4442eb42e7ce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 22 Dec 2024 13:10:50 +0900 Subject: [PATCH 01/12] First successful implementation of delete planning --- core/pseudo.rs | 4 + core/storage/btree.rs | 6 + core/translate/delete.rs | 25 +++++ core/translate/emitter.rs | 216 ++++++++++++++++++++++++++++++++++++ core/translate/mod.rs | 21 +++- core/translate/optimizer.rs | 9 ++ core/translate/planner.rs | 54 ++++++++- core/types.rs | 1 + core/vdbe/explain.rs | 18 +++ core/vdbe/mod.rs | 24 +++- core/vdbe/sorter.rs | 4 + 11 files changed, 378 insertions(+), 4 deletions(-) create mode 100644 core/translate/delete.rs diff --git a/core/pseudo.rs b/core/pseudo.rs index a87647d2b..45f47856e 100644 --- a/core/pseudo.rs +++ b/core/pseudo.rs @@ -79,6 +79,10 @@ impl Cursor for PseudoCursor { Ok(CursorResult::Ok(())) } + fn delete(&mut self) -> Result> { + unimplemented!() + } + fn get_null_flag(&self) -> bool { false } diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 19ba4c7fa..d500e7727 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1753,6 +1753,12 @@ impl Cursor for BTreeCursor { Ok(CursorResult::Ok(())) } + fn delete(&mut self) -> Result> { + println!("ROWID: {:?}", self.rowid.borrow()); + return Ok(CursorResult::Ok(())); + unimplemented!() + } + fn set_null_flag(&mut self, flag: bool) { self.null_flag = flag; } diff --git a/core/translate/delete.rs b/core/translate/delete.rs new file mode 100644 index 000000000..dd34d957b --- /dev/null +++ b/core/translate/delete.rs @@ -0,0 +1,25 @@ +use crate::translate::emitter::emit_program_for_delete; +use crate::translate::optimizer::optimize_delete_plan; +use crate::translate::planner::prepare_delete_plan; +use crate::{ + schema::Schema, + storage::sqlite3_ondisk::DatabaseHeader, + vdbe::Program, +}; +use crate::{Connection, Result}; +use sqlite3_parser::ast::{Expr, QualifiedName, ResultColumn}; +use std::rc::Weak; +use std::{cell::RefCell, rc::Rc}; + +pub fn translate_delete( + schema: &Schema, + tbl_name: &QualifiedName, + where_clause: Option, + _returning: &Option>, + database_header: Rc>, + connection: Weak, +) -> Result { + let delete_plan = prepare_delete_plan(schema, tbl_name, where_clause)?; + let optimized_plan = optimize_delete_plan(delete_plan)?; + emit_program_for_delete(database_header, optimized_plan, connection) +} diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index 38311b9d9..7a32336be 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -272,6 +272,51 @@ pub fn emit_program( Ok(program.build(database_header, connection)) } +pub fn emit_program_for_delete( + database_header: Rc>, + mut plan: Plan, + connection: Weak, +) -> Result { + let (mut program, mut metadata, init_label, start_offset) = prologue()?; + + // No rows will be read from source table loops if there is a constant false condition eg. WHERE 0 + let skip_loops_label = if plan.contains_constant_false_condition { + let skip_loops_label = program.allocate_label(); + program.emit_insn_with_label_dependency( + Insn::Goto { + target_pc: skip_loops_label, + }, + skip_loops_label, + ); + Some(skip_loops_label) + } else { + None + }; + + // Initialize cursors and other resources needed for query execution + init_source_for_delete(&mut program, &plan.source, &mut metadata)?; + + // Set up main query execution loop + open_loop( + &mut program, + &mut plan.source, + &plan.referenced_tables, + &mut metadata, + )?; + + // Close the loop and handle deletion + close_loop_for_delete(&mut program, &plan.source, &mut metadata)?; + + if let Some(skip_loops_label) = skip_loops_label { + program.resolve_label(skip_loops_label, program.offset()); + } + + // Finalize program + epilogue(&mut program, &mut metadata, init_label, start_offset)?; + + Ok(program.build(database_header, connection)) +} + /// Initialize resources needed for ORDER BY processing fn init_order_by( program: &mut ProgramBuilder, @@ -466,6 +511,74 @@ fn init_source( } } +fn init_source_for_delete( + program: &mut ProgramBuilder, + source: &SourceOperator, + metadata: &mut Metadata, +) -> Result<()> { + match source { + SourceOperator::Join { .. } => { + unreachable!() + } + SourceOperator::Scan { + id, + table_reference, + .. + } => { + let cursor_id = program.alloc_cursor_id( + Some(table_reference.table_identifier.clone()), + Some(Table::BTree(table_reference.table.clone())), + ); + let root_page = table_reference.table.root_page; + let next_row_label = program.allocate_label(); + metadata.next_row_labels.insert(*id, next_row_label); + program.emit_insn(Insn::OpenWriteAsync{ + cursor_id, + root_page, + }); + program.emit_insn(Insn::OpenWriteAwait {}); + + Ok(()) + } + SourceOperator::Search { + id, + table_reference, + search, + .. + } => { + let table_cursor_id = program.alloc_cursor_id( + Some(table_reference.table_identifier.clone()), + Some(Table::BTree(table_reference.table.clone())), + ); + + let next_row_label = program.allocate_label(); + + metadata.next_row_labels.insert(*id, next_row_label); + + program.emit_insn(Insn::OpenWriteAsync { + cursor_id: table_cursor_id, + root_page: table_reference.table.root_page, + }); + program.emit_insn(Insn::OpenWriteAwait {}); + + if let Search::IndexSearch { index, .. } = search { + let index_cursor_id = program + .alloc_cursor_id(Some(index.name.clone()), Some(Table::Index(index.clone()))); + program.emit_insn(Insn::OpenWriteAsync { + cursor_id: index_cursor_id, + root_page: index.root_page, + }); + program.emit_insn(Insn::OpenWriteAwait {}); + } + + Ok(()) + } + SourceOperator::Nothing => { + Ok(()) + } + } +} + /// Set up the main query execution loop /// For example in the case of a nested table scan, this means emitting the RewindAsync instruction /// for all tables involved, outermost first. @@ -1121,6 +1234,109 @@ fn close_loop( } } +fn close_loop_for_delete( + program: &mut ProgramBuilder, + source: &SourceOperator, + metadata: &mut Metadata, +) -> Result<()> { + match source { + SourceOperator::Scan { + id, + table_reference, + iter_dir, + .. + } => { + let cursor_id = program.resolve_cursor_id(&table_reference.table_identifier); + + // Emit the instructions to delete the row + let key_reg = program.alloc_register(); + program.emit_insn(Insn::RowId { + cursor_id, + dest: key_reg, + }); + program.emit_insn(Insn::DeleteAsync { cursor_id }); + program.emit_insn(Insn::DeleteAwait { cursor_id }); + + program.resolve_label(*metadata.next_row_labels.get(id).unwrap(), program.offset()); + + // Emit the NextAsync or PrevAsync instruction to continue the loop + if iter_dir + .as_ref() + .is_some_and(|dir| *dir == IterationDirection::Backwards) + { + program.emit_insn(Insn::PrevAsync { cursor_id }); + } else { + program.emit_insn(Insn::NextAsync { cursor_id }); + } + let jump_label = metadata.scan_loop_body_labels.pop().unwrap(); + + // Emit the NextAwait or PrevAwait instruction with label dependency + if iter_dir + .as_ref() + .is_some_and(|dir| *dir == IterationDirection::Backwards) + { + program.emit_insn_with_label_dependency( + Insn::PrevAwait { + cursor_id, + pc_if_next: jump_label, + }, + jump_label, + ); + } else { + program.emit_insn_with_label_dependency( + Insn::NextAwait { + cursor_id, + pc_if_next: jump_label, + }, + jump_label, + ); + } + Ok(()) + } + SourceOperator::Search { + id, + table_reference, + search, + .. + } => { + let cursor_id = match search { + Search::RowidEq { .. } | Search::RowidSearch { .. } => { + program.resolve_cursor_id(&table_reference.table_identifier) + } + Search::IndexSearch { index, .. } => program.resolve_cursor_id(&index.name), + }; + + // Emit the instructions to delete the row + let key_reg = program.alloc_register(); + program.emit_insn(Insn::RowId { + cursor_id, + dest: key_reg, + }); + program.emit_insn(Insn::DeleteAsync { cursor_id }); + program.emit_insn(Insn::DeleteAwait { cursor_id }); + + // resolve labels after calling Delete opcodes + program.resolve_label(*metadata.next_row_labels.get(id).unwrap(), program.offset()); + + // Emit the NextAsync instruction to continue the loop + if !matches!(search, Search::RowidEq { .. }) { + program.emit_insn(Insn::NextAsync { cursor_id }); + let jump_label = metadata.scan_loop_body_labels.pop().unwrap(); + program.emit_insn_with_label_dependency( + Insn::NextAwait { + cursor_id, + pc_if_next: jump_label, + }, + jump_label, + ); + } + + Ok(()) + } + _ => Ok(()), + } +} + /// Emits the bytecode for processing a GROUP BY clause. /// This is called when the main query execution loop has finished processing, /// and we now have data in the GROUP BY sorter. diff --git a/core/translate/mod.rs b/core/translate/mod.rs index 2e5d86141..5ea44f05a 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -14,6 +14,7 @@ pub(crate) mod optimizer; pub(crate) mod plan; pub(crate) mod planner; pub(crate) mod select; +pub(crate) mod delete; use std::cell::RefCell; use std::fmt::Display; @@ -29,6 +30,7 @@ use insert::translate_insert; use select::translate_select; use sqlite3_parser::ast::fmt::ToTokens; use sqlite3_parser::ast::{self, PragmaName}; +use crate::translate::delete::translate_delete; /// Translate SQL statement into bytecode program. pub fn translate( @@ -68,7 +70,24 @@ pub fn translate( ast::Stmt::CreateVirtualTable { .. } => { bail_parse_error!("CREATE VIRTUAL TABLE not supported yet") } - ast::Stmt::Delete { .. } => bail_parse_error!("DELETE not supported yet"), + ast::Stmt::Delete { + with, + tbl_name, + indexed, + where_clause, + returning, + order_by, + limit + } => { + translate_delete( + schema, + &tbl_name, + where_clause, + &returning, + database_header, + connection + ) + } ast::Stmt::Detach(_) => bail_parse_error!("DETACH not supported yet"), ast::Stmt::DropIndex { .. } => bail_parse_error!("DROP INDEX not supported yet"), ast::Stmt::DropTable { .. } => bail_parse_error!("DROP TABLE not supported yet"), diff --git a/core/translate/optimizer.rs b/core/translate/optimizer.rs index 4763f8b1e..6ca17f217 100644 --- a/core/translate/optimizer.rs +++ b/core/translate/optimizer.rs @@ -41,6 +41,15 @@ pub fn optimize_plan(mut select_plan: Plan) -> Result { Ok(select_plan) } +pub fn optimize_delete_plan(mut delete_plan: Plan) -> Result { + use_indexes( + &mut delete_plan.source, + &delete_plan.referenced_tables, + &delete_plan.available_indexes, + )?; + Ok(delete_plan) +} + fn _operator_is_already_ordered_by( operator: &mut SourceOperator, key: &mut ast::Expr, diff --git a/core/translate/planner.rs b/core/translate/planner.rs index 14757e00a..373fa498f 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -1,11 +1,10 @@ use super::{ - optimizer::Optimizable, plan::{ Aggregate, BTreeTableReference, Direction, GroupBy, Plan, ResultSetColumn, SourceOperator, }, }; use crate::{function::Func, schema::Schema, util::normalize_ident, Result}; -use sqlite3_parser::ast::{self, FromClause, JoinType, ResultColumn}; +use sqlite3_parser::ast::{self, Expr, FromClause, JoinType, QualifiedName, ResultColumn}; pub struct OperatorIdCounter { id: usize, @@ -738,6 +737,57 @@ fn parse_join( )) } +pub fn prepare_delete_plan( + schema: &Schema, + tbl_name: &QualifiedName, + where_clause: Option, +) -> Result { + let table_name = tbl_name.name.0.clone(); + + let table = if let Some(table) = schema.get_table(&table_name) { + table + } else { + crate::bail_parse_error!("Table {} not found", table_name); + }; + + let table_ref = BTreeTableReference { + table: table.clone(), + table_identifier: table_name.clone(), + table_index: 0 + }; + + // Parse and resolve the where_clause + let mut resolved_where_clause = None; + if let Some(where_expr) = where_clause { + let mut predicates = vec![]; + break_predicate_at_and_boundaries(where_expr, &mut predicates); + for expr in predicates.iter_mut() { + bind_column_references(expr, &[table_ref.clone()])?; + } + resolved_where_clause = Some(predicates); + } + + let plan = Plan { + source: SourceOperator::Scan { + id: 0, + table_reference: table_ref.clone(), + predicates: resolved_where_clause.clone(), + iter_dir: None + }, + result_columns: vec![], + where_clause: resolved_where_clause, + group_by: None, + order_by: None, + aggregates: vec![], + limit: None, + referenced_tables: vec![table_ref], + available_indexes: vec![], + contains_constant_false_condition: false + }; + + Ok(plan) +} + fn break_predicate_at_and_boundaries(predicate: ast::Expr, out_predicates: &mut Vec) { match predicate { ast::Expr::Binary(left, ast::Operator::And, right) => { diff --git a/core/types.rs b/core/types.rs index 5f1b55d7b..c545fa514 100644 --- a/core/types.rs +++ b/core/types.rs @@ -484,6 +484,7 @@ pub trait Cursor { record: &OwnedRecord, moved_before: bool, /* Tells inserter that it doesn't need to traverse in order to find leaf page */ ) -> Result>; // + fn delete(&mut self) -> Result>; fn exists(&mut self, key: &OwnedValue) -> Result>; fn set_null_flag(&mut self, flag: bool); fn get_null_flag(&self) -> bool; diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs index ce03a53fd..94d89f94c 100644 --- a/core/vdbe/explain.rs +++ b/core/vdbe/explain.rs @@ -834,6 +834,24 @@ pub fn insn_to_str( 0, "".to_string(), ), + Insn::DeleteAsync { cursor_id } => ( + "DeleteAsync", + *cursor_id as i32, + 0, + 0, + OwnedValue::Text(Rc::new("".to_string())), + 0, + "".to_string(), + ), + Insn::DeleteAwait { cursor_id } => ( + "DeleteAwait", + *cursor_id as i32, + 0, + 0, + OwnedValue::Text(Rc::new("".to_string())), + 0, + "".to_string(), + ), Insn::NewRowid { cursor, rowid_reg, diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 362f60042..3db275f63 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -484,6 +484,14 @@ pub enum Insn { cursor_id: usize, }, + DeleteAsync { + cursor_id: CursorID, + }, + + DeleteAwait { + cursor_id: CursorID + }, + NewRowid { cursor: CursorID, // P1 rowid_reg: usize, // P2 Destination register to store the new rowid @@ -2648,7 +2656,17 @@ impl Program { } } state.pc += 1; - } + }, + Insn::DeleteAsync { cursor_id } => { + let cursor = cursors.get_mut(cursor_id).unwrap(); + return_if_io!(cursor.delete()); + state.pc += 1; + }, + Insn::DeleteAwait { cursor_id } => { + let cursor = cursors.get_mut(cursor_id).unwrap(); + cursor.wait_for_completion()?; + state.pc += 1; + }, Insn::NewRowid { cursor, rowid_reg, .. } => { @@ -3879,6 +3897,10 @@ mod tests { unimplemented!() } + fn delete(&mut self, key: &OwnedValue) -> Result> { + unimplemented!() + } + fn wait_for_completion(&mut self) -> Result<()> { unimplemented!() } diff --git a/core/vdbe/sorter.rs b/core/vdbe/sorter.rs index 0365007a4..e3962b096 100644 --- a/core/vdbe/sorter.rs +++ b/core/vdbe/sorter.rs @@ -96,6 +96,10 @@ impl Cursor for Sorter { Ok(CursorResult::Ok(())) } + fn delete(&mut self) -> Result> { + unimplemented!() + } + fn set_null_flag(&mut self, _flag: bool) { todo!(); } From a42b185ecec078f4845206c48eaa2a4dbfe99e5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 22 Dec 2024 14:22:10 +0900 Subject: [PATCH 02/12] Nit --- core/storage/btree.rs | 5 ++--- core/vdbe/explain.rs | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index d500e7727..b2761a5aa 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1754,9 +1754,8 @@ impl Cursor for BTreeCursor { } fn delete(&mut self) -> Result> { - println!("ROWID: {:?}", self.rowid.borrow()); - return Ok(CursorResult::Ok(())); - unimplemented!() + debug!("rowid: {:?}", self.rowid.borrow()); + Ok(CursorResult::Ok(())) } fn set_null_flag(&mut self, flag: bool) { diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs index 94d89f94c..a5e4f91bd 100644 --- a/core/vdbe/explain.rs +++ b/core/vdbe/explain.rs @@ -839,7 +839,7 @@ pub fn insn_to_str( *cursor_id as i32, 0, 0, - OwnedValue::Text(Rc::new("".to_string())), + OwnedValue::build_text(Rc::new("".to_string())), 0, "".to_string(), ), @@ -848,7 +848,7 @@ pub fn insn_to_str( *cursor_id as i32, 0, 0, - OwnedValue::Text(Rc::new("".to_string())), + OwnedValue::build_text(Rc::new("".to_string())), 0, "".to_string(), ), From 57c7a56e35aa855370710732726bfc5012e1d4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 22 Dec 2024 14:27:21 +0900 Subject: [PATCH 03/12] Apply fmt, clippy --- core/translate/delete.rs | 6 +----- core/translate/emitter.rs | 6 ++---- core/translate/mod.rs | 24 +++++++++++------------- core/translate/planner.rs | 12 +++++------- core/vdbe/mod.rs | 10 +++++----- 5 files changed, 24 insertions(+), 34 deletions(-) diff --git a/core/translate/delete.rs b/core/translate/delete.rs index dd34d957b..d3d5fd346 100644 --- a/core/translate/delete.rs +++ b/core/translate/delete.rs @@ -1,11 +1,7 @@ use crate::translate::emitter::emit_program_for_delete; use crate::translate::optimizer::optimize_delete_plan; use crate::translate::planner::prepare_delete_plan; -use crate::{ - schema::Schema, - storage::sqlite3_ondisk::DatabaseHeader, - vdbe::Program, -}; +use crate::{schema::Schema, storage::sqlite3_ondisk::DatabaseHeader, vdbe::Program}; use crate::{Connection, Result}; use sqlite3_parser::ast::{Expr, QualifiedName, ResultColumn}; use std::rc::Weak; diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index 7a32336be..8779dbc25 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -532,7 +532,7 @@ fn init_source_for_delete( let root_page = table_reference.table.root_page; let next_row_label = program.allocate_label(); metadata.next_row_labels.insert(*id, next_row_label); - program.emit_insn(Insn::OpenWriteAsync{ + program.emit_insn(Insn::OpenWriteAsync { cursor_id, root_page, }); @@ -573,9 +573,7 @@ fn init_source_for_delete( Ok(()) } - SourceOperator::Nothing => { - Ok(()) - } + SourceOperator::Nothing => Ok(()), } } diff --git a/core/translate/mod.rs b/core/translate/mod.rs index 5ea44f05a..381c82df1 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -7,6 +7,7 @@ //! a SELECT statement will be translated into a sequence of instructions that //! will read rows from the database and filter them according to a WHERE clause. +pub(crate) mod delete; pub(crate) mod emitter; pub(crate) mod expr; pub(crate) mod insert; @@ -14,7 +15,6 @@ pub(crate) mod optimizer; pub(crate) mod plan; pub(crate) mod planner; pub(crate) mod select; -pub(crate) mod delete; use std::cell::RefCell; use std::fmt::Display; @@ -24,13 +24,13 @@ use std::str::FromStr; use crate::schema::Schema; use crate::storage::pager::Pager; use crate::storage::sqlite3_ondisk::{DatabaseHeader, MIN_PAGE_CACHE_SIZE}; +use crate::translate::delete::translate_delete; use crate::vdbe::{builder::ProgramBuilder, Insn, Program}; use crate::{bail_parse_error, Connection, Result}; use insert::translate_insert; use select::translate_select; use sqlite3_parser::ast::fmt::ToTokens; use sqlite3_parser::ast::{self, PragmaName}; -use crate::translate::delete::translate_delete; /// Translate SQL statement into bytecode program. pub fn translate( @@ -77,17 +77,15 @@ pub fn translate( where_clause, returning, order_by, - limit - } => { - translate_delete( - schema, - &tbl_name, - where_clause, - &returning, - database_header, - connection - ) - } + limit, + } => translate_delete( + schema, + &tbl_name, + where_clause, + &returning, + database_header, + connection, + ), ast::Stmt::Detach(_) => bail_parse_error!("DETACH not supported yet"), ast::Stmt::DropIndex { .. } => bail_parse_error!("DROP INDEX not supported yet"), ast::Stmt::DropTable { .. } => bail_parse_error!("DROP TABLE not supported yet"), diff --git a/core/translate/planner.rs b/core/translate/planner.rs index 373fa498f..e613dd14d 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -1,7 +1,5 @@ -use super::{ - plan::{ - Aggregate, BTreeTableReference, Direction, GroupBy, Plan, ResultSetColumn, SourceOperator, - }, +use super::plan::{ + Aggregate, BTreeTableReference, Direction, GroupBy, Plan, ResultSetColumn, SourceOperator, }; use crate::{function::Func, schema::Schema, util::normalize_ident, Result}; use sqlite3_parser::ast::{self, Expr, FromClause, JoinType, QualifiedName, ResultColumn}; @@ -753,7 +751,7 @@ pub fn prepare_delete_plan( let table_ref = BTreeTableReference { table: table.clone(), table_identifier: table_name.clone(), - table_index: 0 + table_index: 0, }; // Parse and resolve the where_clause @@ -772,7 +770,7 @@ pub fn prepare_delete_plan( id: 0, table_reference: table_ref.clone(), predicates: resolved_where_clause.clone(), - iter_dir: None + iter_dir: None, }, result_columns: vec![], where_clause: resolved_where_clause, @@ -782,7 +780,7 @@ pub fn prepare_delete_plan( limit: None, referenced_tables: vec![table_ref], available_indexes: vec![], - contains_constant_false_condition: false + contains_constant_false_condition: false, }; Ok(plan) diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 3db275f63..26f125675 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -489,7 +489,7 @@ pub enum Insn { }, DeleteAwait { - cursor_id: CursorID + cursor_id: CursorID, }, NewRowid { @@ -2656,17 +2656,17 @@ impl Program { } } state.pc += 1; - }, + } Insn::DeleteAsync { cursor_id } => { let cursor = cursors.get_mut(cursor_id).unwrap(); return_if_io!(cursor.delete()); state.pc += 1; - }, + } Insn::DeleteAwait { cursor_id } => { let cursor = cursors.get_mut(cursor_id).unwrap(); cursor.wait_for_completion()?; state.pc += 1; - }, + } Insn::NewRowid { cursor, rowid_reg, .. } => { @@ -3897,7 +3897,7 @@ mod tests { unimplemented!() } - fn delete(&mut self, key: &OwnedValue) -> Result> { + fn delete(&mut self) -> Result> { unimplemented!() } From 9bacf80f2e29bc83e1c6859b9e70f89bdfc0ae24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 22 Dec 2024 14:41:12 +0900 Subject: [PATCH 04/12] Change to println! --- core/storage/btree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index b2761a5aa..3f6b87da3 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1754,7 +1754,7 @@ impl Cursor for BTreeCursor { } fn delete(&mut self) -> Result> { - debug!("rowid: {:?}", self.rowid.borrow()); + println!("rowid: {:?}", self.rowid.borrow()); Ok(CursorResult::Ok(())) } From 1d3ce528122fb4a81bdeaca6749958df1f517de6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 22 Dec 2024 15:11:26 +0900 Subject: [PATCH 05/12] Refactor planner and optimizer to be DRY --- core/lib.rs | 4 +- core/translate/optimizer.rs | 65 ++++++++++++-------- core/translate/planner.rs | 119 ++++++++++++++++++------------------ core/translate/select.rs | 9 ++- 4 files changed, 105 insertions(+), 92 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index 1f5668d76..e39a5fa7a 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -34,12 +34,12 @@ pub use storage::wal::WalFile; pub use storage::wal::WalFileShared; use util::parse_schema_rows; -use translate::optimizer::optimize_plan; use translate::planner::prepare_select_plan; pub use error::LimboError; pub type Result = std::result::Result; +use crate::translate::optimizer::optimize_select_plan; pub use io::OpenFlags; #[cfg(feature = "fs")] pub use io::PlatformIO; @@ -267,7 +267,7 @@ impl Connection { match stmt { ast::Stmt::Select(select) => { let plan = prepare_select_plan(&*self.schema.borrow(), select)?; - let plan = optimize_plan(plan)?; + let plan = optimize_select_plan(plan)?; println!("{}", plan); } _ => todo!(), diff --git a/core/translate/optimizer.rs b/core/translate/optimizer.rs index 6ca17f217..9ccd8e7f7 100644 --- a/core/translate/optimizer.rs +++ b/core/translate/optimizer.rs @@ -9,12 +9,25 @@ use super::plan::{ Direction, IterationDirection, Plan, Search, SourceOperator, }; +pub fn optimize_select_plan(plan: Plan) -> Result { + optimize_plan(plan, true, true, true) +} + +pub fn optimize_delete_plan(plan: Plan) -> Result { + optimize_plan(plan, false, true, false) +} + /** * Make a few passes over the plan to optimize it. * TODO: these could probably be done in less passes, * but having them separate makes them easier to understand */ -pub fn optimize_plan(mut select_plan: Plan) -> Result { +fn optimize_plan( + mut select_plan: Plan, + optimize_push_predicates: bool, + optimize_use_indexes: bool, + optimize_eliminate_unnecessary_order_by: bool, +) -> Result { eliminate_between(&mut select_plan.source, &mut select_plan.where_clause)?; if let ConstantConditionEliminationResult::ImpossibleCondition = eliminate_constants(&mut select_plan.source, &mut select_plan.where_clause)? @@ -22,32 +35,32 @@ pub fn optimize_plan(mut select_plan: Plan) -> Result { select_plan.contains_constant_false_condition = true; return Ok(select_plan); } - push_predicates( - &mut select_plan.source, - &mut select_plan.where_clause, - &select_plan.referenced_tables, - )?; - use_indexes( - &mut select_plan.source, - &select_plan.referenced_tables, - &select_plan.available_indexes, - )?; - eliminate_unnecessary_orderby( - &mut select_plan.source, - &mut select_plan.order_by, - &select_plan.referenced_tables, - &select_plan.available_indexes, - )?; - Ok(select_plan) -} -pub fn optimize_delete_plan(mut delete_plan: Plan) -> Result { - use_indexes( - &mut delete_plan.source, - &delete_plan.referenced_tables, - &delete_plan.available_indexes, - )?; - Ok(delete_plan) + if optimize_push_predicates { + push_predicates( + &mut select_plan.source, + &mut select_plan.where_clause, + &select_plan.referenced_tables, + )?; + } + + if optimize_use_indexes { + use_indexes( + &mut select_plan.source, + &select_plan.referenced_tables, + &select_plan.available_indexes, + )?; + } + + if optimize_eliminate_unnecessary_order_by { + eliminate_unnecessary_orderby( + &mut select_plan.source, + &mut select_plan.order_by, + &select_plan.referenced_tables, + &select_plan.available_indexes, + )?; + } + Ok(select_plan) } fn _operator_is_already_ordered_by( diff --git a/core/translate/planner.rs b/core/translate/planner.rs index e613dd14d..d32eba6c4 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -283,14 +283,7 @@ pub fn prepare_select_plan<'a>(schema: &Schema, select: ast::Select) -> Result

(schema: &Schema, select: ast::Select) -> Result

, +) -> Result { + let table_name = tbl_name.name.0.clone(); + + let table = if let Some(table) = schema.get_table(&table_name) { + table + } else { + crate::bail_parse_error!("Table {} not found", table_name); + }; + + let table_ref = BTreeTableReference { + table: table.clone(), + table_identifier: table_name.clone(), + table_index: 0, + }; + + // Parse and resolve the where_clause + let resolved_where_clauses = parse_where(where_clause, &[table_ref.clone()])?; + + let plan = Plan { + source: SourceOperator::Scan { + id: 0, + table_reference: table_ref.clone(), + predicates: resolved_where_clauses.clone(), + iter_dir: None, + }, + result_columns: vec![], + where_clause: resolved_where_clauses, + group_by: None, + order_by: None, + aggregates: vec![], + limit: None, // TODO: add support for limit + referenced_tables: vec![table_ref], + available_indexes: vec![], + contains_constant_false_condition: false, + }; + + Ok(plan) +} + #[allow(clippy::type_complexity)] fn parse_from( schema: &Schema, @@ -552,6 +588,22 @@ fn parse_from( Ok((operator, tables)) } +fn parse_where( + where_clause: Option, + referenced_tables: &[BTreeTableReference], +) -> Result>> { + if let Some(where_expr) = where_clause { + let mut predicates = vec![]; + break_predicate_at_and_boundaries(where_expr, &mut predicates); + for expr in predicates.iter_mut() { + bind_column_references(expr, referenced_tables)?; + } + Ok(Some(predicates)) + } else { + Ok(None) + } +} + fn parse_join( schema: &Schema, join: ast::JoinedSelectTable, @@ -735,57 +787,6 @@ fn parse_join( )) } -pub fn prepare_delete_plan( - schema: &Schema, - tbl_name: &QualifiedName, - where_clause: Option, -) -> Result { - let table_name = tbl_name.name.0.clone(); - - let table = if let Some(table) = schema.get_table(&table_name) { - table - } else { - crate::bail_parse_error!("Table {} not found", table_name); - }; - - let table_ref = BTreeTableReference { - table: table.clone(), - table_identifier: table_name.clone(), - table_index: 0, - }; - - // Parse and resolve the where_clause - let mut resolved_where_clause = None; - if let Some(where_expr) = where_clause { - let mut predicates = vec![]; - break_predicate_at_and_boundaries(where_expr, &mut predicates); - for expr in predicates.iter_mut() { - bind_column_references(expr, &[table_ref.clone()])?; - } - resolved_where_clause = Some(predicates); - } - - let plan = Plan { - source: SourceOperator::Scan { - id: 0, - table_reference: table_ref.clone(), - predicates: resolved_where_clause.clone(), - iter_dir: None, - }, - result_columns: vec![], - where_clause: resolved_where_clause, - group_by: None, - order_by: None, - aggregates: vec![], - limit: None, - referenced_tables: vec![table_ref], - available_indexes: vec![], - contains_constant_false_condition: false, - }; - - Ok(plan) -} - fn break_predicate_at_and_boundaries(predicate: ast::Expr, out_predicates: &mut Vec) { match predicate { ast::Expr::Binary(left, ast::Operator::And, right) => { diff --git a/core/translate/select.rs b/core/translate/select.rs index 6d846ded8..a0a0fa36c 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -1,15 +1,14 @@ use std::rc::Weak; use std::{cell::RefCell, rc::Rc}; +use super::emitter::emit_program; +use super::planner::prepare_select_plan; use crate::storage::sqlite3_ondisk::DatabaseHeader; +use crate::translate::optimizer::optimize_select_plan; use crate::Connection; use crate::{schema::Schema, vdbe::Program, Result}; use sqlite3_parser::ast; -use super::emitter::emit_program; -use super::optimizer::optimize_plan; -use super::planner::prepare_select_plan; - pub fn translate_select( schema: &Schema, select: ast::Select, @@ -17,6 +16,6 @@ pub fn translate_select( connection: Weak, ) -> Result { let select_plan = prepare_select_plan(schema, select)?; - let optimized_plan = optimize_plan(select_plan)?; + let optimized_plan = optimize_select_plan(select_plan)?; emit_program(database_header, optimized_plan, connection) } From e83819ef3094c882658c323fb5065f2d045484cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 22 Dec 2024 16:00:35 +0900 Subject: [PATCH 06/12] Extract the appending delete related opcodes to `emit_delete_opcodes` --- core/translate/emitter.rs | 233 +++++++++++++++----------------------- 1 file changed, 91 insertions(+), 142 deletions(-) diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index 8779dbc25..49b85e17b 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -101,6 +101,15 @@ pub struct Metadata { pub result_columns_to_skip_in_orderby_sorter: Option>, } +/// Used to distinguish database operations +#[derive(Debug, Clone)] +pub enum OperationMode { + SELECT, + INSERT, + UPDATE, + DELETE, +} + /// Initialize the program with basic setup and return initial metadata and labels fn prologue() -> Result<(ProgramBuilder, Metadata, BranchOffset, BranchOffset)> { let mut program = ProgramBuilder::new(); @@ -201,7 +210,12 @@ pub fn emit_program( if let Some(ref mut group_by) = plan.group_by { init_group_by(&mut program, group_by, &plan.aggregates, &mut metadata)?; } - init_source(&mut program, &plan.source, &mut metadata)?; + init_source( + &mut program, + &plan.source, + &mut metadata, + &OperationMode::SELECT, + )?; // Set up main query execution loop open_loop( @@ -294,7 +308,12 @@ pub fn emit_program_for_delete( }; // Initialize cursors and other resources needed for query execution - init_source_for_delete(&mut program, &plan.source, &mut metadata)?; + init_source( + &mut program, + &plan.source, + &mut metadata, + &OperationMode::DELETE, + )?; // Set up main query execution loop open_loop( @@ -304,8 +323,15 @@ pub fn emit_program_for_delete( &mut metadata, )?; + emit_delete_insns(&mut program, &plan.source)?; + // Close the loop and handle deletion - close_loop_for_delete(&mut program, &plan.source, &mut metadata)?; + close_loop( + &mut program, + &plan.source, + &mut metadata, + &plan.referenced_tables, + )?; if let Some(skip_loops_label) = skip_loops_label { program.resolve_label(skip_loops_label, program.offset()); @@ -430,6 +456,7 @@ fn init_source( program: &mut ProgramBuilder, source: &SourceOperator, metadata: &mut Metadata, + mode: &OperationMode, ) -> Result<()> { match source { SourceOperator::Join { @@ -447,10 +474,10 @@ fn init_source( }; metadata.left_joins.insert(*id, lj_metadata); } - init_source(program, left, metadata)?; - init_source(program, right, metadata)?; + init_source(program, left, metadata, mode)?; + init_source(program, right, metadata, mode)?; - return Ok(()); + Ok(()) } SourceOperator::Scan { id, @@ -464,80 +491,27 @@ fn init_source( let root_page = table_reference.table.root_page; let next_row_label = program.allocate_label(); metadata.next_row_labels.insert(*id, next_row_label); - program.emit_insn(Insn::OpenReadAsync { - cursor_id, - root_page, - }); - program.emit_insn(Insn::OpenReadAwait); - - return Ok(()); - } - SourceOperator::Search { - id, - table_reference, - search, - .. - } => { - let table_cursor_id = program.alloc_cursor_id( - Some(table_reference.table_identifier.clone()), - Some(Table::BTree(table_reference.table.clone())), - ); - - let next_row_label = program.allocate_label(); - metadata.next_row_labels.insert(*id, next_row_label); - - program.emit_insn(Insn::OpenReadAsync { - cursor_id: table_cursor_id, - root_page: table_reference.table.root_page, - }); - program.emit_insn(Insn::OpenReadAwait); - - if let Search::IndexSearch { index, .. } = search { - let index_cursor_id = program - .alloc_cursor_id(Some(index.name.clone()), Some(Table::Index(index.clone()))); - program.emit_insn(Insn::OpenReadAsync { - cursor_id: index_cursor_id, - root_page: index.root_page, - }); - program.emit_insn(Insn::OpenReadAwait); + match mode { + OperationMode::SELECT => { + program.emit_insn(Insn::OpenReadAsync { + cursor_id, + root_page, + }); + program.emit_insn(Insn::OpenReadAwait {}); + } + OperationMode::DELETE => { + program.emit_insn(Insn::OpenWriteAsync { + cursor_id, + root_page, + }); + program.emit_insn(Insn::OpenWriteAwait {}); + } + _ => { + unimplemented!() + } } - return Ok(()); - } - SourceOperator::Nothing => { - return Ok(()); - } - } -} - -fn init_source_for_delete( - program: &mut ProgramBuilder, - source: &SourceOperator, - metadata: &mut Metadata, -) -> Result<()> { - match source { - SourceOperator::Join { .. } => { - unreachable!() - } - SourceOperator::Scan { - id, - table_reference, - .. - } => { - let cursor_id = program.alloc_cursor_id( - Some(table_reference.table_identifier.clone()), - Some(Table::BTree(table_reference.table.clone())), - ); - let root_page = table_reference.table.root_page; - let next_row_label = program.allocate_label(); - metadata.next_row_labels.insert(*id, next_row_label); - program.emit_insn(Insn::OpenWriteAsync { - cursor_id, - root_page, - }); - program.emit_insn(Insn::OpenWriteAwait {}); - Ok(()) } SourceOperator::Search { @@ -555,20 +529,49 @@ fn init_source_for_delete( metadata.next_row_labels.insert(*id, next_row_label); - program.emit_insn(Insn::OpenWriteAsync { - cursor_id: table_cursor_id, - root_page: table_reference.table.root_page, - }); - program.emit_insn(Insn::OpenWriteAwait {}); + match mode { + OperationMode::SELECT => { + program.emit_insn(Insn::OpenReadAsync { + cursor_id: table_cursor_id, + root_page: table_reference.table.root_page, + }); + program.emit_insn(Insn::OpenReadAwait {}); + } + OperationMode::DELETE => { + program.emit_insn(Insn::OpenWriteAsync { + cursor_id: table_cursor_id, + root_page: table_reference.table.root_page, + }); + program.emit_insn(Insn::OpenWriteAwait {}); + } + _ => { + unimplemented!() + } + } if let Search::IndexSearch { index, .. } = search { let index_cursor_id = program .alloc_cursor_id(Some(index.name.clone()), Some(Table::Index(index.clone()))); - program.emit_insn(Insn::OpenWriteAsync { - cursor_id: index_cursor_id, - root_page: index.root_page, - }); - program.emit_insn(Insn::OpenWriteAwait {}); + + match mode { + OperationMode::SELECT => { + program.emit_insn(Insn::OpenReadAsync { + cursor_id: index_cursor_id, + root_page: index.root_page, + }); + program.emit_insn(Insn::OpenReadAwait); + } + OperationMode::DELETE => { + program.emit_insn(Insn::OpenWriteAsync { + cursor_id: index_cursor_id, + root_page: index.root_page, + }); + program.emit_insn(Insn::OpenWriteAwait {}); + } + _ => { + unimplemented!() + } + } } Ok(()) @@ -1232,11 +1235,7 @@ fn close_loop( } } -fn close_loop_for_delete( - program: &mut ProgramBuilder, - source: &SourceOperator, - metadata: &mut Metadata, -) -> Result<()> { +fn emit_delete_insns(program: &mut ProgramBuilder, source: &SourceOperator) -> Result<()> { match source { SourceOperator::Scan { id, @@ -1255,40 +1254,6 @@ fn close_loop_for_delete( program.emit_insn(Insn::DeleteAsync { cursor_id }); program.emit_insn(Insn::DeleteAwait { cursor_id }); - program.resolve_label(*metadata.next_row_labels.get(id).unwrap(), program.offset()); - - // Emit the NextAsync or PrevAsync instruction to continue the loop - if iter_dir - .as_ref() - .is_some_and(|dir| *dir == IterationDirection::Backwards) - { - program.emit_insn(Insn::PrevAsync { cursor_id }); - } else { - program.emit_insn(Insn::NextAsync { cursor_id }); - } - let jump_label = metadata.scan_loop_body_labels.pop().unwrap(); - - // Emit the NextAwait or PrevAwait instruction with label dependency - if iter_dir - .as_ref() - .is_some_and(|dir| *dir == IterationDirection::Backwards) - { - program.emit_insn_with_label_dependency( - Insn::PrevAwait { - cursor_id, - pc_if_next: jump_label, - }, - jump_label, - ); - } else { - program.emit_insn_with_label_dependency( - Insn::NextAwait { - cursor_id, - pc_if_next: jump_label, - }, - jump_label, - ); - } Ok(()) } SourceOperator::Search { @@ -1313,22 +1278,6 @@ fn close_loop_for_delete( program.emit_insn(Insn::DeleteAsync { cursor_id }); program.emit_insn(Insn::DeleteAwait { cursor_id }); - // resolve labels after calling Delete opcodes - program.resolve_label(*metadata.next_row_labels.get(id).unwrap(), program.offset()); - - // Emit the NextAsync instruction to continue the loop - if !matches!(search, Search::RowidEq { .. }) { - program.emit_insn(Insn::NextAsync { cursor_id }); - let jump_label = metadata.scan_loop_body_labels.pop().unwrap(); - program.emit_insn_with_label_dependency( - Insn::NextAwait { - cursor_id, - pc_if_next: jump_label, - }, - jump_label, - ); - } - Ok(()) } _ => Ok(()), From 6f235e6f6c03fe6ee2e3e80b266d6280d4cbdce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 22 Dec 2024 21:06:54 +0900 Subject: [PATCH 07/12] Fix comment --- core/translate/emitter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index 49b85e17b..3f3bdf1b2 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -325,7 +325,7 @@ pub fn emit_program_for_delete( emit_delete_insns(&mut program, &plan.source)?; - // Close the loop and handle deletion + // Clean up and close the main execution loop close_loop( &mut program, &plan.source, From 82c127b7a3491af8841b2ab2546fbf0fcdf2bda1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Mon, 23 Dec 2024 04:47:05 +0900 Subject: [PATCH 08/12] Remove bool args in optimize_plan --- core/translate/optimizer.rs | 80 ++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/core/translate/optimizer.rs b/core/translate/optimizer.rs index 9ccd8e7f7..9e93b1d71 100644 --- a/core/translate/optimizer.rs +++ b/core/translate/optimizer.rs @@ -9,58 +9,58 @@ use super::plan::{ Direction, IterationDirection, Plan, Search, SourceOperator, }; -pub fn optimize_select_plan(plan: Plan) -> Result { - optimize_plan(plan, true, true, true) -} - -pub fn optimize_delete_plan(plan: Plan) -> Result { - optimize_plan(plan, false, true, false) -} - /** * Make a few passes over the plan to optimize it. * TODO: these could probably be done in less passes, * but having them separate makes them easier to understand */ -fn optimize_plan( - mut select_plan: Plan, - optimize_push_predicates: bool, - optimize_use_indexes: bool, - optimize_eliminate_unnecessary_order_by: bool, -) -> Result { - eliminate_between(&mut select_plan.source, &mut select_plan.where_clause)?; +pub fn optimize_select_plan(mut plan: Plan) -> Result { + eliminate_between(&mut plan.source, &mut plan.where_clause)?; if let ConstantConditionEliminationResult::ImpossibleCondition = - eliminate_constants(&mut select_plan.source, &mut select_plan.where_clause)? + eliminate_constants(&mut plan.source, &mut plan.where_clause)? { - select_plan.contains_constant_false_condition = true; - return Ok(select_plan); + plan.contains_constant_false_condition = true; + return Ok(plan); } - if optimize_push_predicates { - push_predicates( - &mut select_plan.source, - &mut select_plan.where_clause, - &select_plan.referenced_tables, - )?; - } + push_predicates( + &mut plan.source, + &mut plan.where_clause, + &plan.referenced_tables, + )?; + + use_indexes( + &mut plan.source, + &plan.referenced_tables, + &plan.available_indexes, + )?; + + eliminate_unnecessary_orderby( + &mut plan.source, + &mut plan.order_by, + &plan.referenced_tables, + &plan.available_indexes, + )?; + + Ok(plan) +} - if optimize_use_indexes { - use_indexes( - &mut select_plan.source, - &select_plan.referenced_tables, - &select_plan.available_indexes, - )?; +pub fn optimize_delete_plan(mut plan: Plan) -> Result { + eliminate_between(&mut plan.source, &mut plan.where_clause)?; + if let ConstantConditionEliminationResult::ImpossibleCondition = + eliminate_constants(&mut plan.source, &mut plan.where_clause)? + { + plan.contains_constant_false_condition = true; + return Ok(plan); } - if optimize_eliminate_unnecessary_order_by { - eliminate_unnecessary_orderby( - &mut select_plan.source, - &mut select_plan.order_by, - &select_plan.referenced_tables, - &select_plan.available_indexes, - )?; - } - Ok(select_plan) + use_indexes( + &mut plan.source, + &plan.referenced_tables, + &plan.available_indexes, + )?; + + Ok(plan) } fn _operator_is_already_ordered_by( From f8d4edc8d7495aebde190b0955c3f35429a35916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Mon, 23 Dec 2024 04:54:40 +0900 Subject: [PATCH 09/12] Use schema.get_table(...) instead of referencing directly --- core/translate/planner.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/core/translate/planner.rs b/core/translate/planner.rs index d32eba6c4..0c417fa66 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -489,17 +489,15 @@ pub fn prepare_delete_plan( tbl_name: &QualifiedName, where_clause: Option, ) -> Result { - let table_name = tbl_name.name.0.clone(); - - let table = if let Some(table) = schema.get_table(&table_name) { - table - } else { - crate::bail_parse_error!("Table {} not found", table_name); + // let table_name = tbl_name.name.0.clone(); + let table = match schema.get_table(tbl_name.name.0.as_str()) { + Some(table) => table, + None => crate::bail_corrupt_error!("Parse error: no such table: {}", tbl_name), }; let table_ref = BTreeTableReference { table: table.clone(), - table_identifier: table_name.clone(), + table_identifier: table.name.clone(), table_index: 0, }; From 5cdcb8d78ce08226168474bd32d856d1e6147585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Mon, 23 Dec 2024 05:45:23 +0900 Subject: [PATCH 10/12] Split `Plan` into `Select` and `Delete` --- core/lib.rs | 4 ++-- core/translate/delete.rs | 8 ++++---- core/translate/emitter.rs | 21 ++++++++++++++----- core/translate/optimizer.rs | 13 +++++++++--- core/translate/plan.rs | 40 ++++++++++++++++++++++++++++++++----- core/translate/planner.rs | 13 ++++++------ core/translate/select.rs | 4 ++-- 7 files changed, 75 insertions(+), 28 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index e39a5fa7a..f49d8bd3b 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -39,7 +39,7 @@ use translate::planner::prepare_select_plan; pub use error::LimboError; pub type Result = std::result::Result; -use crate::translate::optimizer::optimize_select_plan; +use crate::translate::optimizer::optimize_plan; pub use io::OpenFlags; #[cfg(feature = "fs")] pub use io::PlatformIO; @@ -267,7 +267,7 @@ impl Connection { match stmt { ast::Stmt::Select(select) => { let plan = prepare_select_plan(&*self.schema.borrow(), select)?; - let plan = optimize_select_plan(plan)?; + let plan = optimize_plan(plan)?; println!("{}", plan); } _ => todo!(), diff --git a/core/translate/delete.rs b/core/translate/delete.rs index d3d5fd346..b0ecbdc69 100644 --- a/core/translate/delete.rs +++ b/core/translate/delete.rs @@ -1,5 +1,5 @@ -use crate::translate::emitter::emit_program_for_delete; -use crate::translate::optimizer::optimize_delete_plan; +use crate::translate::emitter::emit_program; +use crate::translate::optimizer::optimize_plan; use crate::translate::planner::prepare_delete_plan; use crate::{schema::Schema, storage::sqlite3_ondisk::DatabaseHeader, vdbe::Program}; use crate::{Connection, Result}; @@ -16,6 +16,6 @@ pub fn translate_delete( connection: Weak, ) -> Result { let delete_plan = prepare_delete_plan(schema, tbl_name, where_clause)?; - let optimized_plan = optimize_delete_plan(delete_plan)?; - emit_program_for_delete(database_header, optimized_plan, connection) + let optimized_plan = optimize_plan(delete_plan)?; + emit_program(database_header, optimized_plan, connection) } diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index 3f3bdf1b2..5e5f0d601 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -9,7 +9,7 @@ use sqlite3_parser::ast::{self}; use crate::schema::{Column, PseudoTable, Table}; use crate::storage::sqlite3_ondisk::DatabaseHeader; -use crate::translate::plan::{IterationDirection, Search}; +use crate::translate::plan::{DeletePlan, IterationDirection, Plan, Search}; use crate::types::{OwnedRecord, OwnedValue}; use crate::util::exprs_are_equivalent; use crate::vdbe::builder::ProgramBuilder; @@ -20,7 +20,7 @@ use super::expr::{ translate_aggregation, translate_aggregation_groupby, translate_condition_expr, translate_expr, ConditionMetadata, }; -use super::plan::{Aggregate, BTreeTableReference, Direction, GroupBy, Plan}; +use super::plan::{Aggregate, BTreeTableReference, Direction, GroupBy, SelectPlan}; use super::plan::{ResultSetColumn, SourceOperator}; // Metadata for handling LEFT JOIN operations @@ -175,6 +175,17 @@ pub fn emit_program( database_header: Rc>, mut plan: Plan, connection: Weak, +) -> Result { + match plan { + Plan::Select(plan) => emit_program_for_select(database_header, plan, connection), + Plan::Delete(plan) => emit_program_for_delete(database_header, plan, connection), + } +} + +fn emit_program_for_select( + database_header: Rc>, + mut plan: SelectPlan, + connection: Weak, ) -> Result { let (mut program, mut metadata, init_label, start_offset) = prologue()?; @@ -286,9 +297,9 @@ pub fn emit_program( Ok(program.build(database_header, connection)) } -pub fn emit_program_for_delete( +fn emit_program_for_delete( database_header: Rc>, - mut plan: Plan, + mut plan: DeletePlan, connection: Weak, ) -> Result { let (mut program, mut metadata, init_label, start_offset) = prologue()?; @@ -925,7 +936,7 @@ pub enum InnerLoopEmitTarget<'a> { /// At this point the cursors for all tables have been opened and rewound. fn inner_loop_emit( program: &mut ProgramBuilder, - plan: &mut Plan, + plan: &mut SelectPlan, metadata: &mut Metadata, ) -> Result<()> { // if we have a group by, we emit a record into the group by sorter. diff --git a/core/translate/optimizer.rs b/core/translate/optimizer.rs index 9e93b1d71..f86c20c26 100644 --- a/core/translate/optimizer.rs +++ b/core/translate/optimizer.rs @@ -6,15 +6,22 @@ use crate::{schema::Index, Result}; use super::plan::{ get_table_ref_bitmask_for_ast_expr, get_table_ref_bitmask_for_operator, BTreeTableReference, - Direction, IterationDirection, Plan, Search, SourceOperator, + DeletePlan, Direction, IterationDirection, Plan, Search, SelectPlan, SourceOperator, }; +pub fn optimize_plan(mut plan: Plan) -> Result { + match plan { + Plan::Select(plan) => optimize_select_plan(plan).map(Plan::Select), + Plan::Delete(plan) => optimize_delete_plan(plan).map(Plan::Delete), + } +} + /** * Make a few passes over the plan to optimize it. * TODO: these could probably be done in less passes, * but having them separate makes them easier to understand */ -pub fn optimize_select_plan(mut plan: Plan) -> Result { +fn optimize_select_plan(mut plan: SelectPlan) -> Result { eliminate_between(&mut plan.source, &mut plan.where_clause)?; if let ConstantConditionEliminationResult::ImpossibleCondition = eliminate_constants(&mut plan.source, &mut plan.where_clause)? @@ -45,7 +52,7 @@ pub fn optimize_select_plan(mut plan: Plan) -> Result { Ok(plan) } -pub fn optimize_delete_plan(mut plan: Plan) -> Result { +fn optimize_delete_plan(mut plan: DeletePlan) -> Result { eliminate_between(&mut plan.source, &mut plan.where_clause)?; if let ConstantConditionEliminationResult::ImpossibleCondition = eliminate_constants(&mut plan.source, &mut plan.where_clause)? diff --git a/core/translate/plan.rs b/core/translate/plan.rs index 8e0fa326e..4cd30d08b 100644 --- a/core/translate/plan.rs +++ b/core/translate/plan.rs @@ -1,11 +1,12 @@ use core::fmt; +use sqlite3_parser::ast; +use std::ptr::write; use std::{ fmt::{Display, Formatter}, rc::Rc, }; -use sqlite3_parser::ast; - +use crate::translate::plan::Plan::{Delete, Select}; use crate::{ function::AggFunc, schema::{BTreeTable, Column, Index}, @@ -27,7 +28,13 @@ pub struct GroupBy { } #[derive(Debug)] -pub struct Plan { +pub enum Plan { + Select(SelectPlan), + Delete(DeletePlan), +} + +#[derive(Debug)] +pub struct SelectPlan { /// A tree of sources (tables). pub source: SourceOperator, /// the columns inside SELECT ... FROM @@ -50,9 +57,32 @@ pub struct Plan { pub contains_constant_false_condition: bool, } +#[derive(Debug)] +pub struct DeletePlan { + /// A tree of sources (tables). + pub source: SourceOperator, + /// the columns inside SELECT ... FROM + pub result_columns: Vec, + /// where clause split into a vec at 'AND' boundaries. + pub where_clause: Option>, + /// order by clause + pub order_by: Option>, + /// limit clause + pub limit: Option, + /// all the tables referenced in the query + pub referenced_tables: Vec, + /// all the indexes available + pub available_indexes: Vec>, + /// query contains a constant condition that is always false + pub contains_constant_false_condition: bool, +} + impl Display for Plan { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.source) + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Select(select_plan) => write!(f, "{}", select_plan.source), + Delete(delete_plan) => write!(f, "{}", delete_plan.source), + } } } diff --git a/core/translate/planner.rs b/core/translate/planner.rs index 0c417fa66..f9b941571 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -1,5 +1,6 @@ use super::plan::{ - Aggregate, BTreeTableReference, Direction, GroupBy, Plan, ResultSetColumn, SourceOperator, + Aggregate, BTreeTableReference, DeletePlan, Direction, GroupBy, Plan, ResultSetColumn, + SelectPlan, SourceOperator, }; use crate::{function::Func, schema::Schema, util::normalize_ident, Result}; use sqlite3_parser::ast::{self, Expr, FromClause, JoinType, QualifiedName, ResultColumn}; @@ -269,7 +270,7 @@ pub fn prepare_select_plan<'a>(schema: &Schema, select: ast::Select) -> Result

(schema: &Schema, select: ast::Select) -> Result

todo!(), } @@ -504,7 +505,7 @@ pub fn prepare_delete_plan( // Parse and resolve the where_clause let resolved_where_clauses = parse_where(where_clause, &[table_ref.clone()])?; - let plan = Plan { + let plan = DeletePlan { source: SourceOperator::Scan { id: 0, table_reference: table_ref.clone(), @@ -513,16 +514,14 @@ pub fn prepare_delete_plan( }, result_columns: vec![], where_clause: resolved_where_clauses, - group_by: None, order_by: None, - aggregates: vec![], limit: None, // TODO: add support for limit referenced_tables: vec![table_ref], available_indexes: vec![], contains_constant_false_condition: false, }; - Ok(plan) + Ok(Plan::Delete(plan)) } #[allow(clippy::type_complexity)] diff --git a/core/translate/select.rs b/core/translate/select.rs index a0a0fa36c..b79560fda 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -4,7 +4,7 @@ use std::{cell::RefCell, rc::Rc}; use super::emitter::emit_program; use super::planner::prepare_select_plan; use crate::storage::sqlite3_ondisk::DatabaseHeader; -use crate::translate::optimizer::optimize_select_plan; +use crate::translate::optimizer::optimize_plan; use crate::Connection; use crate::{schema::Schema, vdbe::Program, Result}; use sqlite3_parser::ast; @@ -16,6 +16,6 @@ pub fn translate_select( connection: Weak, ) -> Result { let select_plan = prepare_select_plan(schema, select)?; - let optimized_plan = optimize_select_plan(select_plan)?; + let optimized_plan = optimize_plan(select_plan)?; emit_program(database_header, optimized_plan, connection) } From 357ab551a5d0243d5c9252c971e3ac81fbace62b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Mon, 23 Dec 2024 07:42:12 +0900 Subject: [PATCH 11/12] nit --- core/translate/planner.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/translate/planner.rs b/core/translate/planner.rs index f9b941571..70df2eee9 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -490,7 +490,6 @@ pub fn prepare_delete_plan( tbl_name: &QualifiedName, where_clause: Option, ) -> Result { - // let table_name = tbl_name.name.0.clone(); let table = match schema.get_table(tbl_name.name.0.as_str()) { Some(table) => table, None => crate::bail_corrupt_error!("Parse error: no such table: {}", tbl_name), From 906975e1ca7010fdf70391eaa15b1cb224de30d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Tue, 24 Dec 2024 12:25:04 +0900 Subject: [PATCH 12/12] Add limit support --- core/translate/delete.rs | 6 +-- core/translate/emitter.rs | 87 +++++++++++++++++++++------------------ core/translate/mod.rs | 2 +- core/translate/planner.rs | 35 +++++++++------- core/vdbe/builder.rs | 11 +++++ 5 files changed, 82 insertions(+), 59 deletions(-) diff --git a/core/translate/delete.rs b/core/translate/delete.rs index b0ecbdc69..135c9d76d 100644 --- a/core/translate/delete.rs +++ b/core/translate/delete.rs @@ -3,7 +3,7 @@ use crate::translate::optimizer::optimize_plan; use crate::translate::planner::prepare_delete_plan; use crate::{schema::Schema, storage::sqlite3_ondisk::DatabaseHeader, vdbe::Program}; use crate::{Connection, Result}; -use sqlite3_parser::ast::{Expr, QualifiedName, ResultColumn}; +use sqlite3_parser::ast::{Expr, Limit, QualifiedName}; use std::rc::Weak; use std::{cell::RefCell, rc::Rc}; @@ -11,11 +11,11 @@ pub fn translate_delete( schema: &Schema, tbl_name: &QualifiedName, where_clause: Option, - _returning: &Option>, + limit: Option, database_header: Rc>, connection: Weak, ) -> Result { - let delete_plan = prepare_delete_plan(schema, tbl_name, where_clause)?; + let delete_plan = prepare_delete_plan(schema, tbl_name, where_clause, limit)?; let optimized_plan = optimize_plan(delete_plan)?; emit_program(database_header, optimized_plan, connection) } diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index 5e5f0d601..a032aa09e 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -334,7 +334,7 @@ fn emit_program_for_delete( &mut metadata, )?; - emit_delete_insns(&mut program, &plan.source)?; + emit_delete_insns(&mut program, &plan.source, &plan.limit, &metadata)?; // Clean up and close the main execution loop close_loop( @@ -1246,53 +1246,58 @@ fn close_loop( } } -fn emit_delete_insns(program: &mut ProgramBuilder, source: &SourceOperator) -> Result<()> { - match source { +fn emit_delete_insns( + program: &mut ProgramBuilder, + source: &SourceOperator, + limit: &Option, + metadata: &Metadata, +) -> Result<()> { + let cursor_id = match source { SourceOperator::Scan { - id, - table_reference, - iter_dir, - .. - } => { - let cursor_id = program.resolve_cursor_id(&table_reference.table_identifier); - - // Emit the instructions to delete the row - let key_reg = program.alloc_register(); - program.emit_insn(Insn::RowId { - cursor_id, - dest: key_reg, - }); - program.emit_insn(Insn::DeleteAsync { cursor_id }); - program.emit_insn(Insn::DeleteAwait { cursor_id }); - - Ok(()) - } + table_reference, .. + } => program.resolve_cursor_id(&table_reference.table_identifier), SourceOperator::Search { - id, table_reference, search, .. - } => { - let cursor_id = match search { - Search::RowidEq { .. } | Search::RowidSearch { .. } => { - program.resolve_cursor_id(&table_reference.table_identifier) - } - Search::IndexSearch { index, .. } => program.resolve_cursor_id(&index.name), - }; - - // Emit the instructions to delete the row - let key_reg = program.alloc_register(); - program.emit_insn(Insn::RowId { - cursor_id, - dest: key_reg, - }); - program.emit_insn(Insn::DeleteAsync { cursor_id }); - program.emit_insn(Insn::DeleteAwait { cursor_id }); + } => match search { + Search::RowidEq { .. } | Search::RowidSearch { .. } => { + program.resolve_cursor_id(&table_reference.table_identifier) + } + Search::IndexSearch { index, .. } => program.resolve_cursor_id(&index.name), + }, + _ => return Ok(()), + }; - Ok(()) - } - _ => Ok(()), + // Emit the instructions to delete the row + let key_reg = program.alloc_register(); + program.emit_insn(Insn::RowId { + cursor_id, + dest: key_reg, + }); + program.emit_insn(Insn::DeleteAsync { cursor_id }); + program.emit_insn(Insn::DeleteAwait { cursor_id }); + if let Some(limit) = limit { + let limit_reg = program.alloc_register(); + program.emit_insn(Insn::Integer { + value: *limit as i64, + dest: limit_reg, + }); + program.mark_last_insn_constant(); + let jump_label_on_limit_reached = metadata + .termination_label_stack + .last() + .expect("termination_label_stack should not be empty."); + program.emit_insn_with_label_dependency( + Insn::DecrJumpZero { + reg: limit_reg, + target_pc: *jump_label_on_limit_reached, + }, + *jump_label_on_limit_reached, + ) } + + Ok(()) } /// Emits the bytecode for processing a GROUP BY clause. diff --git a/core/translate/mod.rs b/core/translate/mod.rs index 381c82df1..8d082c1f3 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -82,7 +82,7 @@ pub fn translate( schema, &tbl_name, where_clause, - &returning, + limit, database_header, connection, ), diff --git a/core/translate/planner.rs b/core/translate/planner.rs index 70df2eee9..8a20a4029 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -2,8 +2,10 @@ use super::plan::{ Aggregate, BTreeTableReference, DeletePlan, Direction, GroupBy, Plan, ResultSetColumn, SelectPlan, SourceOperator, }; -use crate::{function::Func, schema::Schema, util::normalize_ident, Result}; -use sqlite3_parser::ast::{self, Expr, FromClause, JoinType, QualifiedName, ResultColumn}; +use crate::{bail_parse_error, function::Func, schema::Schema, util::normalize_ident, Result}; +use sqlite3_parser::ast::{ + self, Expr, FromClause, JoinType, Limit, QualifiedName, ResultColumn, SortedColumn, +}; pub struct OperatorIdCounter { id: usize, @@ -468,15 +470,7 @@ pub fn prepare_select_plan<'a>(schema: &Schema, select: ast::Select) -> Result

{ - let l = n.parse()?; - Some(l) - } - _ => todo!(), - } - } + plan.limit = select.limit.and_then(|limit| parse_limit(limit)); // Return the unoptimized query plan Ok(Plan::Select(plan)) @@ -489,6 +483,7 @@ pub fn prepare_delete_plan( schema: &Schema, tbl_name: &QualifiedName, where_clause: Option, + limit: Option, ) -> Result { let table = match schema.get_table(tbl_name.name.0.as_str()) { Some(table) => table, @@ -500,10 +495,14 @@ pub fn prepare_delete_plan( table_identifier: table.name.clone(), table_index: 0, }; + let referenced_tables = vec![table_ref.clone()]; - // Parse and resolve the where_clause + // Parse the WHERE clause let resolved_where_clauses = parse_where(where_clause, &[table_ref.clone()])?; + // Parse the LIMIT clause + let resolved_limit = limit.and_then(|limit| parse_limit(limit)); + let plan = DeletePlan { source: SourceOperator::Scan { id: 0, @@ -514,8 +513,8 @@ pub fn prepare_delete_plan( result_columns: vec![], where_clause: resolved_where_clauses, order_by: None, - limit: None, // TODO: add support for limit - referenced_tables: vec![table_ref], + limit: resolved_limit, + referenced_tables, available_indexes: vec![], contains_constant_false_condition: false, }; @@ -783,6 +782,14 @@ fn parse_join( )) } +fn parse_limit(limit: Limit) -> Option { + if let Expr::Literal(ast::Literal::Numeric(n)) = limit.expr { + n.parse().ok() + } else { + None + } +} + fn break_predicate_at_and_boundaries(predicate: ast::Expr, out_predicates: &mut Vec) { match predicate { ast::Expr::Binary(left, ast::Operator::And, right) => { diff --git a/core/vdbe/builder.rs b/core/vdbe/builder.rs index 8dd1cd4de..e3c6dc322 100644 --- a/core/vdbe/builder.rs +++ b/core/vdbe/builder.rs @@ -144,6 +144,17 @@ impl ProgramBuilder { .push((label, insn_reference)); } + /// Resolve unresolved labels to a specific offset in the instruction list. + /// + /// This function updates all instructions that reference the given label + /// to point to the specified offset. It ensures that the label and offset + /// are valid and updates the target program counter (PC) of each instruction + /// that references the label. + /// + /// # Arguments + /// + /// * `label` - The label to resolve. + /// * `to_offset` - The offset to which the labeled instructions should be resolved to. pub fn resolve_label(&mut self, label: BranchOffset, to_offset: BranchOffset) { assert!(label < 0); assert!(to_offset >= 0);