Merge 'Support uncorrelated FROM clause subqueries' from Jussi Saurio

I will warn that this PR is quite big out of necessity, since subqueries
are, as the name implies, queries within queries, so everything that
works with a regular query should also work with a subquery, roughly
speaking.
---
- Adds support for:
    * uncorrelated subqueries in FROM clause (i.e. appear as a "table",
and do not refer to outer tables). Example of this at the end of the PR
description.
    * column and subquery aliasing (`select sub.renamed from (select
name as renamed from products) sub`)
    * inner and outer filtering of subqueries (`select sub.name from
(select name from products where name = 'joe') sub`, and,  `select
sub.name from (select name from products) sub where sub.name = 'joe'`)
    * joining between regular tables and subqueries
    * joining between multiple subqueries
    * in general working with subqueries should roughly equal working
with regular tables
- Main idea: subqueries are just wrappers of a `SelectPlan` that never
emit ResultRows, instead they `Yield` control back to the parent query,
and the parent query can copy the subquery result values into a
ResultRow. New variant `SourceOperator::Subquery` that wraps a subquery
`SelectPlan`.
- Plans can now not only refer to btree tables (`select p.name from
products`) but also subqueries (`select sub.foo from (select name as foo
from products) sub`. Hence this PR also adds support for column aliases
which didn't exist before.
    * An `Expr::Column` that refers to a regular table will result in an
`Insn::Column` (i.e. a read from disk/memory) whereas an `Expr::Column`
that refers to a subquery will result in an `Insn::Copy` (from register
to register) instead
- Subquery handling is entirely unoptimized, there's no predicate
pushdown from outer query to subqueries, or elimination of redundant
subqueries (e.g. in the trivial example `SELECT * FROM (SELECT * FROM
users) sub` the subquery can just be entirely removed)
---
This PR does not add support (yet) for:
- subqueries in result columns: `SELECT t.foo, (SELECT .......) as
column_from_subquery FROM t`
- subqueries in WHERE clauses e.g. `SELECT * FROM t1 WHERE t1.foo IN
(SELECT ...)`
- subquery-related optimizations, of which there are plenty available.
No analysis is done regarding e.g. whether predicates on the outer query
level could be pushed into the subquery, or whether the subquery could
be entirely eliminated. Both of the above can probably be done fairly
easily for a bunch of trivial cases.
---
Example bytecode with comments added:
```
limbo> EXPLAIN SELECT p.name, sub.funny_name FROM products p JOIN (
  select id, concat(name, '-lol') as funny_name from products
) sub USING (id) LIMIT 3;

addr  opcode             p1    p2    p3    p4             p5  comment
----  -----------------  ----  ----  ----  -------------  --  -------
0     Init               0     31    0                    0   Start at 31

// Coroutine implementation starts at insn 2, jump immediately to 14
1     InitCoroutine      1     14    2                    0

2     OpenReadAsync      0     3     0                    0   table=products, root=3
3     OpenReadAwait      0     0     0                    0
4     RewindAsync        0     0     0                    0
5     RewindAwait        0     13    0                    0   Rewind table products
6       RowId            0     2     0                    0   r[2]=products.rowid
7       Column           0     1     4                    0   r[4]=products.name
8       String8          0     5     0     -lol           0   r[5]='-lol'
9       Function         0     4     3     concat         0   r[3]=func(r[4..5])

// jump back to main loop of query (insn 20)
10      Yield            1     0     0                    0

11    NextAsync          0     0     0                    0
12    NextAwait          0     6     0                    0
13    EndCoroutine       1     0     0                    0
14    OpenReadAsync      1     3     0                    0   table=p, root=3
15    OpenReadAwait      0     0     0                    0
16    RewindAsync        1     0     0                    0
17    RewindAwait        1     30    0                    0   Rewind table p

// Since this subquery is the inner loop of the join, reinitialize it on every iteration of the outer loop
18      InitCoroutine    1     0     2                    0

// Jump back to the subquery implementation to assign another row into registers
19      Yield            1     28    0                    0

20      RowId            1     8     0                    0   r[8]=p.rowid

// Copy sub.id
21      Copy             2     9     0                    0   r[9]=r[2]

// p.id == sub.id?
22      Ne               8     9     27                   0   if r[8]!=r[9] goto 27
23      Column           1     1     6                    0   r[6]=p.name

// copy sub.funny_name
24      Copy             3     7     0                    0   r[7]=r[3]

25      ResultRow        6     2     0                    0   output=r[6..7]
26      DecrJumpZero     10    30    0                    0   if (--r[10]==0) goto 30
27      Goto             0     19    0                    0
28    NextAsync          1     0     0                    0
29    NextAwait          1     18    0                    0
30    Halt               0     0     0                    0
31    Transaction        0     0     0                    0
32    Integer            3     10    0                    0   r[10]=3
33    Goto               0     1     0                    0
```

Closes #566

core/lib.rs

@@ -250,7 +250,7 @@ impl Connection {
                         self.header.clone(),
                         self.pager.clone(),
                         Rc::downgrade(self),
-                        &syms,
+                        syms,
                     )?);
                     Ok(Statement::new(program, self.pager.clone()))
                 }
@@ -278,7 +278,7 @@ impl Connection {
                         self.header.clone(),
                         self.pager.clone(),
                         Rc::downgrade(self),
-                        &syms,
+                        syms,
                     )?);
                     let stmt = Statement::new(program, self.pager.clone());
                     Ok(Some(Rows { stmt }))
@@ -290,16 +290,16 @@ impl Connection {
                         self.header.clone(),
                         self.pager.clone(),
                         Rc::downgrade(self),
-                        &syms,
+                        syms,
                     )?;
                     program.explain();
                     Ok(None)
                 }
                 Cmd::ExplainQueryPlan(stmt) => {
                     match stmt {
                         ast::Stmt::Select(select) => {
-                            let plan = prepare_select_plan(&self.schema.borrow(), select)?;
-                            let plan = optimize_plan(plan)?;
+                            let mut plan = prepare_select_plan(&self.schema.borrow(), select)?;
+                            optimize_plan(&mut plan)?;
                             println!("{}", plan);
                         }
                         _ => todo!(),
@@ -327,7 +327,7 @@ impl Connection {
                         self.header.clone(),
                         self.pager.clone(),
                         Rc::downgrade(self),
-                        &syms,
+                        syms,
                     )?;
                     program.explain();
                 }
@@ -339,7 +339,7 @@ impl Connection {
                         self.header.clone(),
                         self.pager.clone(),
                         Rc::downgrade(self),
-                        &syms,
+                        syms,
                     )?;
                     let mut state = vdbe::ProgramState::new(program.max_registers);
                     program.step(&mut state, self.pager.clone())?;

core/schema.rs

@@ -71,6 +71,14 @@ impl Table {
         }
     }
 
+    pub fn get_root_page(&self) -> usize {
+        match self {
+            Table::BTree(table) => table.root_page,
+            Table::Index(_) => unimplemented!(),
+            Table::Pseudo(_) => unimplemented!(),
+        }
+    }
+
     pub fn get_name(&self) -> &str {
         match self {
             Self::BTree(table) => &table.name,
@@ -211,6 +219,10 @@ impl PseudoTable {
         Self { columns: vec![] }
     }
 
+    pub fn new_with_columns(columns: Vec<Column>) -> Self {
+        Self { columns }
+    }
+
     pub fn add_column(&mut self, name: &str, ty: Type, primary_key: bool) {
         self.columns.push(Column {
             name: normalize_ident(name),

core/translate/delete.rs

@@ -1,13 +1,16 @@
+use crate::schema::Table;
 use crate::translate::emitter::emit_program;
 use crate::translate::optimizer::optimize_plan;
-use crate::translate::plan::{BTreeTableReference, DeletePlan, Plan, SourceOperator};
+use crate::translate::plan::{DeletePlan, Plan, SourceOperator};
 use crate::translate::planner::{parse_limit, parse_where};
 use crate::{schema::Schema, storage::sqlite3_ondisk::DatabaseHeader, vdbe::Program};
 use crate::{Connection, Result, SymbolTable};
 use sqlite3_parser::ast::{Expr, Limit, QualifiedName};
 use std::rc::Weak;
 use std::{cell::RefCell, rc::Rc};
 
+use super::plan::{TableReference, TableReferenceType};
+
 pub fn translate_delete(
     schema: &Schema,
     tbl_name: &QualifiedName,
@@ -17,9 +20,9 @@ pub fn translate_delete(
     connection: Weak<Connection>,
     syms: &SymbolTable,
 ) -> Result<Program> {
-    let delete_plan = prepare_delete_plan(schema, tbl_name, where_clause, limit)?;
-    let optimized_plan = optimize_plan(delete_plan)?;
-    emit_program(database_header, optimized_plan, connection, syms)
+    let mut delete_plan = prepare_delete_plan(schema, tbl_name, where_clause, limit)?;
+    optimize_plan(&mut delete_plan)?;
+    emit_program(database_header, delete_plan, connection, syms)
 }
 
 pub fn prepare_delete_plan(
@@ -33,23 +36,24 @@ pub fn prepare_delete_plan(
         None => crate::bail_corrupt_error!("Parse error: no such table: {}", tbl_name),
     };
 
-    let table_ref = BTreeTableReference {
-        table: table.clone(),
+    let btree_table_ref = TableReference {
+        table: Table::BTree(table.clone()),
         table_identifier: table.name.clone(),
         table_index: 0,
+        reference_type: TableReferenceType::BTreeTable,
     };
-    let referenced_tables = vec![table_ref.clone()];
+    let referenced_tables = vec![btree_table_ref.clone()];
 
     // Parse the WHERE clause
-    let resolved_where_clauses = parse_where(where_clause, &[table_ref.clone()])?;
+    let resolved_where_clauses = parse_where(where_clause, &referenced_tables)?;
 
     // Parse the LIMIT clause
     let resolved_limit = limit.and_then(parse_limit);
 
     let plan = DeletePlan {
         source: SourceOperator::Scan {
             id: 0,
-            table_reference: table_ref.clone(),
+            table_reference: btree_table_ref,
             predicates: resolved_where_clauses.clone(),
             iter_dir: None,
         },