From e1f681868671a8ea725341905d12c5e90be2c12f Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Fri, 13 Oct 2023 10:47:17 +0200 Subject: [PATCH 01/38] wip: joins aggregations --- psl/psl-core/src/common/preview_features.rs | 2 + quaint/src/ast/function.rs | 11 +- quaint/src/ast/function/json_array_agg.rs | 18 +++ quaint/src/ast/function/json_build_obj.rs | 15 +++ quaint/src/ast/function/json_extract_array.rs | 2 +- quaint/src/visitor.rs | 21 ++++ .../tests/queries/simple/mod.rs | 1 + .../tests/queries/simple/one2m.rs | 57 +++++++++ .../src/interface/connection.rs | 5 +- .../src/interface/transaction.rs | 3 +- .../query-connector/src/interface.rs | 13 ++ .../src/database/connection.rs | 4 +- .../src/database/operations/read.rs | 4 + .../src/database/transaction.rs | 4 +- .../src/query_builder/mod.rs | 1 + .../src/query_builder/read.rs | 20 +++- .../src/query_builder/select.rs | 113 ++++++++++++++++++ .../query_interpreters/nested_read.rs | 2 + .../interpreter/query_interpreters/read.rs | 99 ++++++++++++++- query-engine/core/src/query_ast/read.rs | 12 +- 20 files changed, 391 insertions(+), 16 deletions(-) create mode 100644 quaint/src/ast/function/json_array_agg.rs create mode 100644 quaint/src/ast/function/json_build_obj.rs create mode 100644 query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/one2m.rs create mode 100644 query-engine/connectors/sql-query-connector/src/query_builder/select.rs diff --git a/psl/psl-core/src/common/preview_features.rs b/psl/psl-core/src/common/preview_features.rs index 544bf5b99164..318010ebdf9f 100644 --- a/psl/psl-core/src/common/preview_features.rs +++ b/psl/psl-core/src/common/preview_features.rs @@ -76,6 +76,7 @@ features!( TransactionApi, UncheckedScalarInputs, Views, + RelationJoins ); /// Generator preview features @@ -90,6 +91,7 @@ pub const ALL_PREVIEW_FEATURES: FeatureMap = FeatureMap { | PostgresqlExtensions | Tracing | Views + | RelationJoins }), deprecated: enumflags2::make_bitflags!(PreviewFeature::{ AtomicNumberOperations diff --git a/quaint/src/ast/function.rs b/quaint/src/ast/function.rs index 5b6373795485..cbed3a6ed18e 100644 --- a/quaint/src/ast/function.rs +++ b/quaint/src/ast/function.rs @@ -3,6 +3,8 @@ mod average; mod coalesce; mod concat; mod count; +mod json_array_agg; +mod json_build_obj; #[cfg(any(feature = "postgresql", feature = "mysql"))] mod json_extract; #[cfg(any(feature = "postgresql", feature = "mysql"))] @@ -28,6 +30,8 @@ pub use average::*; pub use coalesce::*; pub use concat::*; pub use count::*; +pub use json_array_agg::*; +pub use json_build_obj::*; #[cfg(any(feature = "postgresql", feature = "mysql"))] pub use json_extract::*; #[cfg(any(feature = "postgresql", feature = "mysql"))] @@ -37,6 +41,7 @@ pub use json_unquote::*; pub use lower::*; pub use maximum::*; pub use minimum::*; +use postgres_types::Json; pub use row_number::*; #[cfg(feature = "postgresql")] pub use row_to_json::*; @@ -98,6 +103,8 @@ pub(crate) enum FunctionType<'a> { JsonExtractFirstArrayElem(JsonExtractFirstArrayElem<'a>), #[cfg(any(feature = "postgresql", feature = "mysql"))] JsonUnquote(JsonUnquote<'a>), + JsonArrayAgg(JsonArrayAgg<'a>), + JsonBuildObject(JsonBuildObject<'a>), #[cfg(any(feature = "postgresql", feature = "mysql"))] TextSearch(TextSearch<'a>), #[cfg(any(feature = "postgresql", feature = "mysql"))] @@ -154,5 +161,7 @@ function!( Minimum, Maximum, Coalesce, - Concat + Concat, + JsonArrayAgg, + JsonBuildObject ); diff --git a/quaint/src/ast/function/json_array_agg.rs b/quaint/src/ast/function/json_array_agg.rs new file mode 100644 index 000000000000..b7ffc9f34848 --- /dev/null +++ b/quaint/src/ast/function/json_array_agg.rs @@ -0,0 +1,18 @@ +use crate::prelude::*; + +#[derive(Debug, Clone, PartialEq)] +pub struct JsonArrayAgg<'a> { + pub(crate) expr: Box>, +} + +/// This is an internal function used to help construct the JsonArrayBeginsWith Comparable +pub fn json_array_agg<'a, E>(expr: E) -> Function<'a> +where + E: Into>, +{ + let fun = JsonArrayAgg { + expr: Box::new(expr.into()), + }; + + fun.into() +} diff --git a/quaint/src/ast/function/json_build_obj.rs b/quaint/src/ast/function/json_build_obj.rs new file mode 100644 index 000000000000..7e2fff404e4e --- /dev/null +++ b/quaint/src/ast/function/json_build_obj.rs @@ -0,0 +1,15 @@ +use std::borrow::Cow; + +use crate::prelude::*; + +#[derive(Debug, Clone, PartialEq)] +pub struct JsonBuildObject<'a> { + pub(crate) exprs: Vec<(Cow<'a, str>, Expression<'a>)>, +} + +/// This is an internal function used to help construct the JsonArrayBeginsWith Comparable +pub fn json_build_object<'a>(exprs: Vec<(Cow<'a, str>, Expression<'a>)>) -> Function<'a> { + let fun = JsonBuildObject { exprs }; + + fun.into() +} diff --git a/quaint/src/ast/function/json_extract_array.rs b/quaint/src/ast/function/json_extract_array.rs index 974c1cebea58..5eeb7c6b4387 100644 --- a/quaint/src/ast/function/json_extract_array.rs +++ b/quaint/src/ast/function/json_extract_array.rs @@ -32,4 +32,4 @@ where }; fun.into() -} +} \ No newline at end of file diff --git a/quaint/src/visitor.rs b/quaint/src/visitor.rs index c205b49dd279..11e5cebe06cc 100644 --- a/quaint/src/visitor.rs +++ b/quaint/src/visitor.rs @@ -1106,6 +1106,27 @@ pub trait Visitor<'a> { FunctionType::Concat(concat) => { self.visit_concat(concat)?; } + FunctionType::JsonArrayAgg(array_agg) => { + self.write("JSON_AGG")?; + self.surround_with("(", ")", |s| s.visit_expression(*array_agg.expr))?; + } + FunctionType::JsonBuildObject(build_obj) => { + let len = build_obj.exprs.len(); + + self.write("JSON_BUILD_OBJECT")?; + self.surround_with("(", ")", |s| { + for (i, (name, expr)) in build_obj.exprs.into_iter().enumerate() { + s.visit_raw_value(Value::text(name))?; + s.write(", ")?; + s.visit_expression(expr)?; + if i < (len - 1) { + s.write(", ")?; + } + } + + Ok(()) + })?; + } }; if let Some(alias) = fun.alias { diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/mod.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/mod.rs index 9e9cf1951163..939468590e4c 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/mod.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/mod.rs @@ -8,4 +8,5 @@ mod json_result; mod m2m; mod mongo_incorrect_fields; mod multi_field_unique; +mod one2m; mod raw_mongo; diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/one2m.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/one2m.rs new file mode 100644 index 000000000000..fb1b21e1e3dd --- /dev/null +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/one2m.rs @@ -0,0 +1,57 @@ +use indoc::indoc; +use query_engine_tests::*; + +#[test_suite(schema(schema))] +mod one2m { + fn schema() -> String { + let schema = indoc! { + r#"model Parent { + #id(id, Int, @id) + name String + + children Child[] + } + + model Child { + #id(id, Int, @id) + name String + + parentId Int? + parent Parent? @relation(fields: [parentId], references: [id]) + } + "# + }; + + schema.to_owned() + } + + #[connector_test] + async fn simple(runner: Runner) -> TestResult<()> { + test_data(&runner).await?; + + insta::assert_snapshot!( + run_query!(runner, r#"{ findManyParent { id name children { id name } } }"#), + @r###""### + ); + + Ok(()) + } + + async fn test_data(runner: &Runner) -> TestResult<()> { + create_row( + runner, + r#"{ id: 1, name: "Bob", children: { create: [{ id: 1, name: "Hello!" }, { id: 2, name: "World!" }] } }"#, + ) + .await?; + + Ok(()) + } + + async fn create_row(runner: &Runner, data: &str) -> TestResult<()> { + runner + .query(format!("mutation {{ createOneParent(data: {}) {{ id }} }}", data)) + .await? + .assert_success(); + Ok(()) + } +} diff --git a/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs b/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs index e10c0e1f5b3b..76e634908c11 100644 --- a/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs +++ b/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs @@ -6,8 +6,8 @@ use crate::{ }; use async_trait::async_trait; use connector_interface::{ - Connection, ConnectionLike, ReadOperations, RelAggregationSelection, Transaction, UpdateType, WriteArgs, - WriteOperations, + Connection, ConnectionLike, ReadOperations, RelAggregationSelection, RelatedQuery, Transaction, UpdateType, + WriteArgs, WriteOperations, }; use mongodb::{ClientSession, Database}; use query_structure::{prelude::*, SelectionResult}; @@ -211,6 +211,7 @@ impl ReadOperations for MongoDbConnection { model: &Model, query_arguments: query_structure::QueryArguments, selected_fields: &FieldSelection, + _nested: Vec, aggregation_selections: &[RelAggregationSelection], _trace_id: Option, ) -> connector_interface::Result { diff --git a/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs b/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs index 1de0bb8c750e..5804ee75c077 100644 --- a/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs +++ b/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs @@ -4,7 +4,7 @@ use crate::{ root_queries::{aggregate, read, write}, }; use connector_interface::{ - ConnectionLike, ReadOperations, RelAggregationSelection, Transaction, UpdateType, WriteOperations, + ConnectionLike, ReadOperations, RelAggregationSelection, RelatedQuery, Transaction, UpdateType, WriteOperations, }; use mongodb::options::{Acknowledgment, ReadConcern, TransactionOptions, WriteConcern}; use query_engine_metrics::{decrement_gauge, increment_gauge, metrics, PRISMA_CLIENT_QUERIES_ACTIVE}; @@ -276,6 +276,7 @@ impl<'conn> ReadOperations for MongoDbTransaction<'conn> { model: &Model, query_arguments: query_structure::QueryArguments, selected_fields: &FieldSelection, + _nested: Vec, aggregation_selections: &[RelAggregationSelection], _trace_id: Option, ) -> connector_interface::Result { diff --git a/query-engine/connectors/query-connector/src/interface.rs b/query-engine/connectors/query-connector/src/interface.rs index 942edd1868fc..18df36803a05 100644 --- a/query-engine/connectors/query-connector/src/interface.rs +++ b/query-engine/connectors/query-connector/src/interface.rs @@ -217,6 +217,18 @@ impl RelAggregationSelection { } } +#[derive(Debug, Clone)] +pub struct RelatedQuery { + pub name: String, + pub alias: Option, + pub parent_field: RelationFieldRef, + pub args: QueryArguments, + pub selected_fields: FieldSelection, + pub nested: Option>, + pub selection_order: Vec, + pub aggregation_selections: Vec, +} + #[async_trait] pub trait ReadOperations { /// Gets a single record or `None` back from the database. @@ -245,6 +257,7 @@ pub trait ReadOperations { model: &Model, query_arguments: QueryArguments, selected_fields: &FieldSelection, + nested: Vec, aggregation_selections: &[RelAggregationSelection], trace_id: Option, ) -> crate::Result; diff --git a/query-engine/connectors/sql-query-connector/src/database/connection.rs b/query-engine/connectors/sql-query-connector/src/database/connection.rs index cb7c0a9b3128..2c8738020a17 100644 --- a/query-engine/connectors/sql-query-connector/src/database/connection.rs +++ b/query-engine/connectors/sql-query-connector/src/database/connection.rs @@ -3,7 +3,7 @@ use super::{catch, transaction::SqlConnectorTransaction}; use crate::{database::operations::*, Context, SqlError}; use async_trait::async_trait; -use connector::{ConnectionLike, RelAggregationSelection}; +use connector::{ConnectionLike, RelAggregationSelection, RelatedQuery}; use connector_interface::{ self as connector, AggregationRow, AggregationSelection, Connection, ReadOperations, RecordFilter, Transaction, WriteArgs, WriteOperations, @@ -110,6 +110,7 @@ where model: &Model, query_arguments: QueryArguments, selected_fields: &FieldSelection, + nested: Vec, aggr_selections: &[RelAggregationSelection], trace_id: Option, ) -> connector::Result { @@ -120,6 +121,7 @@ where model, query_arguments, &selected_fields.into(), + nested, aggr_selections, &ctx, ) diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index 4d33fe3d2fff..f290a0caa429 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -58,6 +58,7 @@ pub(crate) async fn get_many_records( model: &Model, mut query_arguments: QueryArguments, selected_fields: &ModelProjection, + nested: Vec, aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> crate::Result { @@ -132,11 +133,14 @@ pub(crate) async fn get_many_records( } } _ => { + query_builder::select::build(query_arguments.clone(), nested.clone(), selected_fields, &[], ctx); + let query = read::get_records( model, selected_fields.as_columns(ctx).mark_all_selected(), aggr_selections, query_arguments, + nested, ctx, ); diff --git a/query-engine/connectors/sql-query-connector/src/database/transaction.rs b/query-engine/connectors/sql-query-connector/src/database/transaction.rs index 7fa9aaf3b5bc..ba88721e15ca 100644 --- a/query-engine/connectors/sql-query-connector/src/database/transaction.rs +++ b/query-engine/connectors/sql-query-connector/src/database/transaction.rs @@ -1,7 +1,7 @@ use super::catch; use crate::{database::operations::*, Context, SqlError}; use async_trait::async_trait; -use connector::{ConnectionLike, RelAggregationSelection}; +use connector::{ConnectionLike, RelAggregationSelection, RelatedQuery}; use connector_interface::{ self as connector, AggregationRow, AggregationSelection, ReadOperations, RecordFilter, Transaction, WriteArgs, WriteOperations, @@ -91,6 +91,7 @@ impl<'tx> ReadOperations for SqlConnectorTransaction<'tx> { model: &Model, query_arguments: QueryArguments, selected_fields: &FieldSelection, + nested: Vec, aggr_selections: &[RelAggregationSelection], trace_id: Option, ) -> connector::Result { @@ -101,6 +102,7 @@ impl<'tx> ReadOperations for SqlConnectorTransaction<'tx> { model, query_arguments, &selected_fields.into(), + nested, aggr_selections, &ctx, ) diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/mod.rs b/query-engine/connectors/sql-query-connector/src/query_builder/mod.rs index b605d076eed4..7f16b84f95fd 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/mod.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod read; +pub(crate) mod select; pub(crate) mod write; use crate::context::Context; diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/read.rs b/query-engine/connectors/sql-query-connector/src/query_builder/read.rs index 3aa91288ea90..6291ce0cd165 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/read.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/read.rs @@ -2,16 +2,19 @@ use crate::{ cursor_condition, filter::FilterBuilder, model_extensions::*, nested_aggregations, ordering::OrderByBuilder, sql_trace::SqlTraceComment, Context, }; -use connector_interface::{AggregationSelection, RelAggregationSelection}; +use connector_interface::{AggregationSelection, RelAggregationSelection, RelatedQuery}; use itertools::Itertools; use quaint::ast::*; use query_structure::*; use tracing::Span; +use super::select; + pub(crate) trait SelectDefinition { fn into_select( self, _: &Model, + nested: Vec, aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> (Select<'static>, Vec>); @@ -21,11 +24,12 @@ impl SelectDefinition for Filter { fn into_select( self, model: &Model, + nested: Vec, aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> (Select<'static>, Vec>) { let args = QueryArguments::from((model.clone(), self)); - args.into_select(model, aggr_selections, ctx) + args.into_select(model, nested, aggr_selections, ctx) } } @@ -33,10 +37,11 @@ impl SelectDefinition for &Filter { fn into_select( self, model: &Model, + nested: Vec, aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> (Select<'static>, Vec>) { - self.clone().into_select(model, aggr_selections, ctx) + self.clone().into_select(model, nested, aggr_selections, ctx) } } @@ -44,6 +49,7 @@ impl SelectDefinition for Select<'static> { fn into_select( self, _: &Model, + _: Vec, _: &[RelAggregationSelection], _ctx: &Context<'_>, ) -> (Select<'static>, Vec>) { @@ -55,6 +61,7 @@ impl SelectDefinition for QueryArguments { fn into_select( self, model: &Model, + nested: Vec, aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> (Select<'static>, Vec>) { @@ -118,12 +125,13 @@ pub(crate) fn get_records( columns: impl Iterator>, aggr_selections: &[RelAggregationSelection], query: T, + nested: Vec, ctx: &Context<'_>, ) -> Select<'static> where T: SelectDefinition, { - let (select, additional_selection_set) = query.into_select(model, aggr_selections, ctx); + let (select, additional_selection_set) = query.into_select(model, nested, aggr_selections, ctx); let select = columns.fold(select, |acc, col| acc.column(col)); let select = select.append_trace(&Span::current()).add_trace_id(ctx.trace_id); @@ -166,7 +174,7 @@ pub(crate) fn aggregate( ctx: &Context<'_>, ) -> Select<'static> { let columns = extract_columns(model, selections, ctx); - let sub_query = get_records(model, columns.into_iter(), &[], args, ctx); + let sub_query = get_records(model, columns.into_iter(), &[], args, Vec::new(), ctx); let sub_table = Table::from(sub_query).alias("sub"); selections.iter().fold( @@ -223,7 +231,7 @@ pub(crate) fn group_by_aggregate( having: Option, ctx: &Context<'_>, ) -> Select<'static> { - let (base_query, _) = args.into_select(model, &[], ctx); + let (base_query, _) = args.into_select(model, Vec::new(), &[], ctx); let select_query = selections.iter().fold(base_query, |select, next_op| match next_op { AggregationSelection::Field(field) => select.column(field.as_column(ctx).set_is_selected(true)), diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs new file mode 100644 index 000000000000..4ca514a41488 --- /dev/null +++ b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs @@ -0,0 +1,113 @@ +use std::borrow::Cow; + +use crate::{ + context::Context, + model_extensions::{AsColumn, AsColumns, AsTable}, +}; + +use connector_interface::{QueryArguments, RelAggregationSelection, RelatedQuery}; +use itertools::Itertools; +use prisma_models::ModelProjection; +use quaint::{prelude::*, visitor::*}; + +/* + +SELECT + "Link"."id", + "Link"."createdAt", + "Link"."updatedAt", + "Link"."url", + "Link"."shortUrl", + "Link"."userId", + "A"."json" +FROM "Link" + LEFT JOIN LATERAL ( + SELECT JSON_AGG("json") AS "json" FROM ( + SELECT JSON_BUILD_OBJECT('linkId', "LinkOpen"."linkId", 'createdAt', "LinkOpen"."createdAt") AS "json" + FROM "LinkOpen" + WHERE "LinkOpen"."linkId" = "Link"."id" + ORDER BY "LinkOpen"."createdAt" + LIMIT 1 + ) "A" + ) "A" ON true +LIMIT 10; +*/ +pub(crate) fn build( + args: QueryArguments, + nested: Vec, + selection: &ModelProjection, + _aggr_selections: &[RelAggregationSelection], + ctx: &Context<'_>, +) { + dbg!(&selection); + dbg!(&nested); + let select = Select::from_table(args.model().as_table(ctx)); + + // scalars selection + let select = selection.fields().fold(select, |acc, selection| match selection { + prisma_models::Field::Relation(rf) => acc.value(rf.name().to_owned()), + prisma_models::Field::Scalar(sf) => acc.column(sf.as_column(ctx)), + prisma_models::Field::Composite(_) => unreachable!(), + }); + + // TODO: check how to select aggregated relations + let select = nested + .iter() + .fold(select, |acc, read| acc.value(Column::from(read.name.to_owned()))); + + let select = nested.into_iter().fold(select, |acc, nested| { + let join_select = build_nested(nested, ctx); + let join_table = Table::from(nested.parent_field.model().as_table(ctx)).left_join(select); + + }) + + for n in nested { + let select = build_nested(n, ctx); + } + + let (sql, _) = Postgres::build(select).unwrap(); + + dbg!(&sql); +} + +pub(crate) fn build_nested(nested: RelatedQuery, ctx: &Context<'_>) -> Select<'static> { + /* + ```sql + SELECT + "Link"."id", + "Link"."createdAt", + "Link"."updatedAt", + "Link"."url", + "Link"."shortUrl", + "Link"."userId", + "A"."json" + FROM "Link" + LEFT JOIN LATERAL ( + SELECT JSON_AGG("json") AS "json" FROM ( -- inner + SELECT JSON_BUILD_OBJECT('linkId', "LinkOpen"."linkId", 'createdAt', "LinkOpen"."createdAt") AS "json" + FROM "LinkOpen" + WHERE "LinkOpen"."linkId" = "Link"."id" + ORDER BY "LinkOpen"."createdAt" + LIMIT 1 + ) "A" + ) "A" ON true + LIMIT 10; + ``` + */ + let build_obj_params = nested + .selected_fields + .into_iter() + .map(|f| match f { + prisma_models::SelectedField::Scalar(sf) => { + (Cow::from(sf.name().to_owned()), Expression::from(sf.as_column(ctx))) + } + _ => unreachable!(), + }) + .collect_vec(); + let inner = Select::from_table(nested.parent_field.model().as_table(ctx)) + .value(json_build_object(build_obj_params).alias("json")); + + let select = Select::from(inner).value(json_array_agg(Column::from("json"))); + + select +} diff --git a/query-engine/core/src/interpreter/query_interpreters/nested_read.rs b/query-engine/core/src/interpreter/query_interpreters/nested_read.rs index fa4dc7c6e529..2d6cabff4727 100644 --- a/query-engine/core/src/interpreter/query_interpreters/nested_read.rs +++ b/query-engine/core/src/interpreter/query_interpreters/nested_read.rs @@ -67,6 +67,7 @@ pub(crate) async fn m2m( &query.parent_field.related_model(), args, &query.selected_fields, + Vec::new(), &query.aggregation_selections, trace_id.clone(), ) @@ -208,6 +209,7 @@ pub async fn one2m( &parent_field.related_model(), args, selected_fields, + Vec::new(), &aggr_selections, trace_id, ) diff --git a/query-engine/core/src/interpreter/query_interpreters/read.rs b/query-engine/core/src/interpreter/query_interpreters/read.rs index 464ac6651677..9f69d0b0c264 100644 --- a/query-engine/core/src/interpreter/query_interpreters/read.rs +++ b/query-engine/core/src/interpreter/query_interpreters/read.rs @@ -1,6 +1,8 @@ use super::*; use crate::{interpreter::InterpretationResult, query_ast::*, result_ast::*}; -use connector::{self, error::ConnectorError, ConnectionLike, RelAggregationRow, RelAggregationSelection}; +use connector::{ + self, error::ConnectorError, ConnectionLike, RelAggregationRow, RelAggregationSelection, RelatedQuery, +}; use futures::future::{BoxFuture, FutureExt}; use inmemory_record_processor::InMemoryRecordProcessor; use query_structure::ManyRecords; @@ -85,6 +87,20 @@ fn read_one( /// are distinct by definition if a unique is in the selection set. /// -> Unstable cursors can't reliably be fetched by the underlying datasource, so we need to process part of it in-memory. fn read_many( + tx: &mut dyn ConnectionLike, + query: ManyRecordsQuery, + trace_id: Option, +) -> BoxFuture<'_, InterpretationResult> { + let use_joins = true; + + if use_joins { + read_many_by_joins(tx, query, trace_id) + } else { + read_many_by_queries(tx, query, trace_id) + } +} + +fn read_many_by_queries( tx: &mut dyn ConnectionLike, mut query: ManyRecordsQuery, trace_id: Option, @@ -101,6 +117,7 @@ fn read_many( &query.model, query.args.clone(), &query.selected_fields, + Vec::new(), &query.aggregation_selections, trace_id, ) @@ -133,6 +150,86 @@ fn read_many( fut.boxed() } +fn read_many_by_joins( + tx: &mut dyn ConnectionLike, + mut query: ManyRecordsQuery, + trace_id: Option, +) -> BoxFuture<'_, InterpretationResult> { + let processor = if query.args.requires_inmemory_processing() { + Some(InMemoryRecordProcessor::new_from_query_args(&mut query.args)) + } else { + None + }; + + let nested = build_related_reads(&query); + + let fut = async move { + let scalars = tx + .get_many_records( + &query.model, + query.args.clone(), + &query.selected_fields, + nested, + &query.aggregation_selections, + trace_id, + ) + .await?; + + let scalars = if let Some(p) = processor { + p.apply(scalars) + } else { + scalars + }; + + let (scalars, aggregation_rows) = extract_aggregation_rows_from_scalars(scalars, query.aggregation_selections); + + if scalars.records.is_empty() && query.options.contains(QueryOption::ThrowOnEmpty) { + record_not_found() + } else { + Ok(RecordSelection { + name: query.name, + fields: query.selection_order, + scalars, + nested: Vec::new(), + model: query.model, + aggregation_rows, + } + .into()) + } + }; + + fut.boxed() +} + +fn build_related_reads(query: &ManyRecordsQuery) -> Vec { + query + .nested + .clone() + .into_iter() + .filter_map(|n| n.into_related_records_query()) + .map(to_related_query) + .collect() +} + +fn to_related_query(n: RelatedRecordsQuery) -> RelatedQuery { + RelatedQuery { + name: n.name, + alias: n.alias, + parent_field: n.parent_field, + args: n.args, + selected_fields: n.selected_fields, + nested: Some( + n.nested + .into_iter() + .filter_map(|n| n.into_related_records_query()) + .map(|n| to_related_query(n)) + .collect(), + ), + selection_order: n.selection_order, + aggregation_selections: n.aggregation_selections, + } +} + /// Queries related records for a set of parent IDs. fn read_related<'conn>( tx: &'conn mut dyn ConnectionLike, diff --git a/query-engine/core/src/query_ast/read.rs b/query-engine/core/src/query_ast/read.rs index 271ff44e3883..e840eb81e50d 100644 --- a/query-engine/core/src/query_ast/read.rs +++ b/query-engine/core/src/query_ast/read.rs @@ -8,7 +8,7 @@ use std::fmt::Display; #[allow(clippy::enum_variant_names)] #[derive(Debug, Clone)] -pub(crate) enum ReadQuery { +pub enum ReadQuery { RecordQuery(RecordQuery), ManyRecordsQuery(ManyRecordsQuery), RelatedRecordsQuery(RelatedRecordsQuery), @@ -55,6 +55,14 @@ impl ReadQuery { ReadQuery::AggregateRecordsQuery(x) => x.model.clone(), } } + + pub(crate) fn into_related_records_query(self) -> Option { + if let Self::RelatedRecordsQuery(v) = self { + Some(v) + } else { + None + } + } } impl FilteredQuery for ReadQuery { @@ -194,7 +202,7 @@ pub struct RelatedRecordsQuery { pub parent_field: RelationFieldRef, pub args: QueryArguments, pub selected_fields: FieldSelection, - pub(crate) nested: Vec, + pub nested: Vec, pub selection_order: Vec, pub aggregation_selections: Vec, From 5f2ca56ef275e850f9eb3a6a7f57903ce9313cdd Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Thu, 26 Oct 2023 18:18:41 +0200 Subject: [PATCH 02/38] wip: basic impl with lateral joins + serialization support --- libs/prisma-value/src/lib.rs | 8 + quaint/src/ast/join.rs | 9 + quaint/src/ast/select.rs | 9 + quaint/src/ast/table.rs | 9 + quaint/src/visitor.rs | 20 ++ .../tests/queries/simple/one2m.rs | 73 ++++- .../src/database/connection.rs | 35 ++- .../src/database/operations/coerce.rs | 77 ++++++ .../src/database/operations/mod.rs | 1 + .../src/database/operations/read.rs | 54 +++- .../src/query_builder/select.rs | 196 ++++++++------ .../interpreter/query_interpreters/read.rs | 46 ++-- query-engine/core/src/response_ir/internal.rs | 255 ++++++++++++++---- query-engine/core/src/result_ast/mod.rs | 37 ++- 14 files changed, 666 insertions(+), 163 deletions(-) create mode 100644 query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs diff --git a/libs/prisma-value/src/lib.rs b/libs/prisma-value/src/lib.rs index d797605ecccc..43aff0156ddd 100644 --- a/libs/prisma-value/src/lib.rs +++ b/libs/prisma-value/src/lib.rs @@ -304,6 +304,14 @@ impl PrismaValue { _ => None, } } + + pub fn as_json(&self) -> Option<&String> { + if let Self::Json(v) = self { + Some(v) + } else { + None + } + } } impl fmt::Display for PrismaValue { diff --git a/quaint/src/ast/join.rs b/quaint/src/ast/join.rs index 7387250f8703..ba1684f0ea71 100644 --- a/quaint/src/ast/join.rs +++ b/quaint/src/ast/join.rs @@ -5,6 +5,7 @@ use crate::ast::{ConditionTree, Table}; pub struct JoinData<'a> { pub(crate) table: Table<'a>, pub(crate) conditions: ConditionTree<'a>, + pub(crate) lateral: bool, } impl<'a> JoinData<'a> { @@ -13,8 +14,14 @@ impl<'a> JoinData<'a> { Self { table: table.into(), conditions: ConditionTree::NoCondition, + lateral: false, } } + + pub fn as_lateral(mut self) -> Self { + self.lateral = true; + self + } } impl<'a, T> From for JoinData<'a> @@ -73,6 +80,7 @@ where JoinData { table: self.into(), conditions: conditions.into(), + lateral: false, } } } @@ -90,6 +98,7 @@ impl<'a> Joinable<'a> for JoinData<'a> { JoinData { table: self.table, conditions, + lateral: false, } } } diff --git a/quaint/src/ast/select.rs b/quaint/src/ast/select.rs index 96d50ba645c5..127753139afd 100644 --- a/quaint/src/ast/select.rs +++ b/quaint/src/ast/select.rs @@ -391,6 +391,15 @@ impl<'a> Select<'a> { self } + pub fn left_join_lateral(self, join: J) -> Self + where + J: Into>, + { + let join_data: JoinData = join.into(); + + self.left_join(join_data.as_lateral()) + } + /// Adds `RIGHT JOIN` clause to the query. /// /// ```rust diff --git a/quaint/src/ast/table.rs b/quaint/src/ast/table.rs index 4eca73f27bc7..5ebbaf68462f 100644 --- a/quaint/src/ast/table.rs +++ b/quaint/src/ast/table.rs @@ -204,6 +204,15 @@ impl<'a> Table<'a> { self } + pub fn left_join_lateral(self, join: J) -> Self + where + J: Into>, + { + let join_data: JoinData = join.into(); + + self.left_join(join_data.as_lateral()) + } + /// Adds an `INNER JOIN` clause to the query, specifically for that table. /// Useful to positionally add a JOIN clause in case you are selecting from multiple tables. /// diff --git a/quaint/src/visitor.rs b/quaint/src/visitor.rs index 11e5cebe06cc..e5b0c27b0af7 100644 --- a/quaint/src/visitor.rs +++ b/quaint/src/visitor.rs @@ -188,18 +188,38 @@ pub trait Visitor<'a> { match j { Join::Inner(data) => { self.write(" INNER JOIN ")?; + + if data.lateral { + self.write("LATERAL ")?; + } + self.visit_join_data(data)?; } Join::Left(data) => { self.write(" LEFT JOIN ")?; + + if data.lateral { + self.write("LATERAL ")?; + } + self.visit_join_data(data)?; } Join::Right(data) => { self.write(" RIGHT JOIN ")?; + + if data.lateral { + self.write("LATERAL ")?; + } + self.visit_join_data(data)?; } Join::Full(data) => { self.write(" FULL JOIN ")?; + + if data.lateral { + self.write("LATERAL ")?; + } + self.visit_join_data(data)?; } } diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/one2m.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/one2m.rs index fb1b21e1e3dd..97f41f5f4387 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/one2m.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/one2m.rs @@ -2,7 +2,7 @@ use indoc::indoc; use query_engine_tests::*; #[test_suite(schema(schema))] -mod one2m { +mod simple { fn schema() -> String { let schema = indoc! { r#"model Parent { @@ -31,7 +31,7 @@ mod one2m { insta::assert_snapshot!( run_query!(runner, r#"{ findManyParent { id name children { id name } } }"#), - @r###""### + @r###"{"data":{"findManyParent":[{"id":1,"name":"Bob","children":[{"id":1,"name":"Hello!"},{"id":2,"name":"World!"}]}]}}"### ); Ok(()) @@ -55,3 +55,72 @@ mod one2m { Ok(()) } } + +#[test_suite(schema(schema))] +mod nested { + fn schema() -> String { + let schema = indoc! { + r#"model Parent { + #id(parentId, Int, @id) + + children Child[] + } + + model Child { + #id(childId, Int, @id) + + parentId Int? + parent Parent? @relation(fields: [parentId], references: [parentId]) + + children GrandChild[] + } + + model GrandChild { + #id(grandChildId, Int, @id) + + parentId Int? + parent Child? @relation(fields: [parentId], references: [childId]) + }"# + }; + + schema.to_owned() + } + + #[connector_test] + async fn vanilla(runner: Runner) -> TestResult<()> { + create_test_data(&runner).await?; + + insta::assert_snapshot!( + run_query!(runner, r#"{ findManyParent { parentId children { childId children { grandChildId } } } }"#), + @r###"{"data":{"findManyParent":[{"parentId":1,"children":[{"childId":1,"children":[{"grandChildId":1},{"grandChildId":2}]},{"childId":2,"children":[{"grandChildId":3}]}]}]}}"### + ); + + Ok(()) + } + + async fn create_test_data(runner: &Runner) -> TestResult<()> { + create_row( + runner, + r#"{ + parentId: 1, + children: { + create: [ + { childId: 1, children: { create: [{ grandChildId: 1 }, { grandChildId: 2 }] }}, + { childId: 2, children: { create: [{ grandChildId: 3 }] } } + ] + } + }"#, + ) + .await?; + + Ok(()) + } + + async fn create_row(runner: &Runner, data: &str) -> TestResult<()> { + runner + .query(format!("mutation {{ createOneParent(data: {}) {{ parentId }} }}", data)) + .await? + .assert_success(); + Ok(()) + } +} diff --git a/query-engine/connectors/sql-query-connector/src/database/connection.rs b/query-engine/connectors/sql-query-connector/src/database/connection.rs index 2c8738020a17..4ffa35262015 100644 --- a/query-engine/connectors/sql-query-connector/src/database/connection.rs +++ b/query-engine/connectors/sql-query-connector/src/database/connection.rs @@ -9,6 +9,7 @@ use connector_interface::{ WriteArgs, WriteOperations, }; use prisma_value::PrismaValue; +use psl::PreviewFeature; use quaint::{ connector::{IsolationLevel, TransactionCapable}, prelude::{ConnectionInfo, Queryable}, @@ -116,16 +117,30 @@ where ) -> connector::Result { catch(self.connection_info.clone(), async move { let ctx = Context::new(&self.connection_info, trace_id.as_deref()); - read::get_many_records( - &self.inner, - model, - query_arguments, - &selected_fields.into(), - nested, - aggr_selections, - &ctx, - ) - .await + + if self.features.contains(PreviewFeature::RelationJoins) { + read::get_many_records_joins( + &self.inner, + model, + query_arguments, + &selected_fields.into(), + nested, + aggr_selections, + &ctx, + ) + .await + } else { + read::get_many_records( + &self.inner, + model, + query_arguments, + &selected_fields.into(), + nested, + aggr_selections, + &ctx, + ) + .await + } }) .await } diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs new file mode 100644 index 000000000000..3af1fefc2c52 --- /dev/null +++ b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs @@ -0,0 +1,77 @@ +use connector_interface::RelatedQuery; +use prisma_models::*; +use prisma_value::PrismaValue; + +// TODO: find better name +pub(crate) fn coerce_record_with_join(record: &mut Record, rq_indexes: Vec<(usize, &RelatedQuery)>) { + for (val_idx, rq) in rq_indexes { + let val = record.values.get_mut(val_idx).unwrap(); + let json_val: serde_json::Value = serde_json::from_str(&val.as_json().unwrap()).unwrap(); + + *val = coerce_json_relation_to_pv(json_val, rq); + } +} + +// TODO: find better name +pub(crate) fn coerce_json_relation_to_pv(value: serde_json::Value, q: &RelatedQuery) -> PrismaValue { + match value { + serde_json::Value::Array(values) => PrismaValue::List( + values + .into_iter() + .map(|value| coerce_json_relation_to_pv(value, q)) + .collect(), + ), + serde_json::Value::Object(obj) => { + let mut map: Vec<(String, PrismaValue)> = Vec::with_capacity(obj.len()); + let related_model = q.parent_field.related_model(); + + for (key, value) in obj { + match related_model.fields().all().find(|f| f.db_name() == key).unwrap() { + Field::Scalar(sf) => { + map.push((key, coerce_json_scalar_to_pv(value, &sf))); + } + Field::Relation(rf) => { + // TODO: optimize this + if let Some(rq) = q + .nested + .as_ref() + .unwrap() + .iter() + .find(|rq| rq.parent_field.name() == rf.name()) + { + map.push((key, coerce_json_relation_to_pv(value, rq))); + } + } + _ => unreachable!(), + } + } + + PrismaValue::Object(map) + } + _ => unreachable!(), + } +} + +pub(crate) fn coerce_json_scalar_to_pv(value: serde_json::Value, sf: &ScalarField) -> PrismaValue { + match value { + serde_json::Value::Null => PrismaValue::Null, + serde_json::Value::Bool(b) => PrismaValue::Boolean(b), + serde_json::Value::Number(n) => match sf.type_identifier() { + TypeIdentifier::Int => PrismaValue::Int(n.as_i64().unwrap()), + TypeIdentifier::BigInt => PrismaValue::BigInt(n.as_i64().unwrap()), + TypeIdentifier::Float => todo!(), + TypeIdentifier::Decimal => todo!(), + _ => unreachable!(), + }, + serde_json::Value::String(s) => match sf.type_identifier() { + TypeIdentifier::String => PrismaValue::String(s), + TypeIdentifier::Enum(_) => PrismaValue::Enum(s), + TypeIdentifier::DateTime => PrismaValue::DateTime(parse_datetime(&s).unwrap()), + TypeIdentifier::UUID => PrismaValue::Uuid(uuid::Uuid::parse_str(&s).unwrap()), + TypeIdentifier::Bytes => PrismaValue::Bytes(decode_bytes(&s).unwrap()), + _ => unreachable!(), + }, + serde_json::Value::Array(_) => todo!(), + serde_json::Value::Object(_) => todo!(), + } +} diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/mod.rs b/query-engine/connectors/sql-query-connector/src/database/operations/mod.rs index 5a10395a788e..65a7e7712617 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/mod.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/mod.rs @@ -2,3 +2,4 @@ pub mod read; pub(crate) mod update; pub mod upsert; pub mod write; +pub mod coerce; diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index f290a0caa429..71e85ce9baf4 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -1,3 +1,4 @@ +use super::coerce::coerce_record_with_join; use crate::{ column_metadata, model_extensions::*, @@ -5,6 +6,7 @@ use crate::{ query_builder::{self, read}, Context, QueryExt, Queryable, SqlError, }; + use connector_interface::*; use futures::stream::{FuturesUnordered, StreamExt}; use quaint::ast::*; @@ -53,6 +55,56 @@ pub(crate) async fn get_single_record( Ok(record) } +pub(crate) async fn get_many_records_joins( + conn: &dyn Queryable, + _model: &Model, + query_arguments: QueryArguments, + selected_fields: &ModelProjection, + nested: Vec, + _aggr_selections: &[RelAggregationSelection], + ctx: &Context<'_>, +) -> crate::Result { + let mut field_names: Vec<_> = selected_fields.db_names().collect(); + field_names.extend(nested.iter().map(|n| n.parent_field.name().to_owned())); + + let mut idents = selected_fields.type_identifiers_with_arities(); + idents.extend(nested.iter().map(|_| (TypeIdentifier::Json, FieldArity::Required))); + + let meta = column_metadata::create(field_names.as_slice(), idents.as_slice()); + let rq_indexes = related_queries_indexes(&nested, field_names.as_slice()); + + let mut records = ManyRecords::new(field_names.clone()); + + let query = query_builder::select::build(query_arguments.clone(), nested.clone(), selected_fields, &[], ctx); + + for item in conn.filter(query.into(), meta.as_slice(), ctx).await?.into_iter() { + let mut record = Record::from(item); + + // Coerces json values to prisma values + coerce_record_with_join(&mut record, rq_indexes.clone()); + + records.push(record) + } + + Ok(records) +} + +// TODO: find better name +fn related_queries_indexes<'a>( + related_queries: &'a [RelatedQuery], + field_names: &[String], +) -> Vec<(usize, &'a RelatedQuery)> { + let mut output: Vec<(usize, &RelatedQuery)> = Vec::new(); + + for (idx, field_name) in field_names.iter().enumerate() { + if let Some(rq) = related_queries.iter().find(|rq| rq.name == *field_name) { + output.push((idx, rq)); + } + } + + output +} + pub(crate) async fn get_many_records( conn: &dyn Queryable, model: &Model, @@ -133,8 +185,6 @@ pub(crate) async fn get_many_records( } } _ => { - query_builder::select::build(query_arguments.clone(), nested.clone(), selected_fields, &[], ctx); - let query = read::get_records( model, selected_fields.as_columns(ctx).mark_all_selected(), diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs index 4ca514a41488..13bd464cde31 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs @@ -2,112 +2,156 @@ use std::borrow::Cow; use crate::{ context::Context, - model_extensions::{AsColumn, AsColumns, AsTable}, + filter::FilterBuilder, + model_extensions::{AsColumn, AsColumns, AsTable, RelationFieldExt}, }; use connector_interface::{QueryArguments, RelAggregationSelection, RelatedQuery}; use itertools::Itertools; -use prisma_models::ModelProjection; +use prisma_models::{ModelProjection, RelationField}; use quaint::{prelude::*, visitor::*}; -/* - -SELECT - "Link"."id", - "Link"."createdAt", - "Link"."updatedAt", - "Link"."url", - "Link"."shortUrl", - "Link"."userId", - "A"."json" -FROM "Link" - LEFT JOIN LATERAL ( - SELECT JSON_AGG("json") AS "json" FROM ( - SELECT JSON_BUILD_OBJECT('linkId', "LinkOpen"."linkId", 'createdAt', "LinkOpen"."createdAt") AS "json" - FROM "LinkOpen" - WHERE "LinkOpen"."linkId" = "Link"."id" - ORDER BY "LinkOpen"."createdAt" - LIMIT 1 - ) "A" - ) "A" ON true -LIMIT 10; -*/ +pub const JSON_AGG_IDENT: &str = "data"; + pub(crate) fn build( args: QueryArguments, nested: Vec, selection: &ModelProjection, _aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, -) { - dbg!(&selection); - dbg!(&nested); +) -> Select<'static> { + // SELECT ... FROM Table let select = Select::from_table(args.model().as_table(ctx)); // scalars selection - let select = selection.fields().fold(select, |acc, selection| match selection { - prisma_models::Field::Relation(rf) => acc.value(rf.name().to_owned()), - prisma_models::Field::Scalar(sf) => acc.column(sf.as_column(ctx)), - prisma_models::Field::Composite(_) => unreachable!(), - }); + let select = selection + .scalar_fields() + .fold(select, |acc, sf| acc.column(sf.as_column(ctx))); // TODO: check how to select aggregated relations - let select = nested - .iter() - .fold(select, |acc, read| acc.value(Column::from(read.name.to_owned()))); + let select = nested.iter().fold(select, |acc, read| { + acc.value(Column::from((join_alias_name(&read.parent_field), JSON_AGG_IDENT)).alias(read.name.to_owned())) + }); - let select = nested.into_iter().fold(select, |acc, nested| { - let join_select = build_nested(nested, ctx); - let join_table = Table::from(nested.parent_field.model().as_table(ctx)).left_join(select); + let select = with_nested_joins(select, nested, ctx); + let select = with_pagination_and_filters(select, args, ctx); - }) + let (sql, _) = Postgres::build(select.clone()).unwrap(); - for n in nested { - let select = build_nested(n, ctx); - } + println!("{}", sql); + + select +} - let (sql, _) = Postgres::build(select).unwrap(); +fn with_pagination_and_filters<'a>(select: Select<'a>, args: QueryArguments, ctx: &Context<'_>) -> Select<'a> { + let (filter, joins) = match args.filter { + Some(filter) => { + let (filter, joins) = FilterBuilder::with_top_level_joins().visit_filter(filter, ctx); - dbg!(&sql); + (Some(filter), joins) + } + None => (None, None), + }; + + let select = match filter { + Some(filter) => select.and_where(filter), + None => select, + }; + + let select = match joins { + Some(joins) => joins.into_iter().fold(select, |acc, join| acc.join(join.data)), + None => select, + }; + + let select = match args.take { + Some(take) => select.limit(take as usize), + None => select, + }; + + let select = match args.skip { + Some(skip) => select.offset(skip as usize), + None => select, + }; + + select } -pub(crate) fn build_nested(nested: RelatedQuery, ctx: &Context<'_>) -> Select<'static> { - /* - ```sql - SELECT - "Link"."id", - "Link"."createdAt", - "Link"."updatedAt", - "Link"."url", - "Link"."shortUrl", - "Link"."userId", - "A"."json" - FROM "Link" - LEFT JOIN LATERAL ( - SELECT JSON_AGG("json") AS "json" FROM ( -- inner - SELECT JSON_BUILD_OBJECT('linkId', "LinkOpen"."linkId", 'createdAt', "LinkOpen"."createdAt") AS "json" - FROM "LinkOpen" - WHERE "LinkOpen"."linkId" = "Link"."id" - ORDER BY "LinkOpen"."createdAt" - LIMIT 1 - ) "A" - ) "A" ON true - LIMIT 10; - ``` - */ - let build_obj_params = nested - .selected_fields - .into_iter() +pub(crate) fn build_nested(related_query: RelatedQuery, ctx: &Context<'_>) -> Select<'static> { + let mut build_obj_params = ModelProjection::from(related_query.selected_fields) + .fields() .map(|f| match f { - prisma_models::SelectedField::Scalar(sf) => { - (Cow::from(sf.name().to_owned()), Expression::from(sf.as_column(ctx))) + prisma_models::Field::Scalar(sf) => { + (Cow::from(sf.db_name().to_owned()), Expression::from(sf.as_column(ctx))) } _ => unreachable!(), }) .collect_vec(); - let inner = Select::from_table(nested.parent_field.model().as_table(ctx)) - .value(json_build_object(build_obj_params).alias("json")); - let select = Select::from(inner).value(json_array_agg(Column::from("json"))); + if let Some(nested_queries) = &related_query.nested { + for nested_query in nested_queries { + build_obj_params.push(( + Cow::from(nested_query.name.to_owned()), + Expression::from(Column::from(( + join_alias_name(&nested_query.parent_field), + JSON_AGG_IDENT, + ))), + )); + } + } + + let inner_alias = join_alias_name(&related_query.parent_field.related_field()); + + // SELECT JSON_BUILD_OBJECT() + let inner = Select::from_table(related_query.parent_field.related_model().as_table(ctx)) + .value(json_build_object(build_obj_params).alias(JSON_AGG_IDENT)); + + // SELECT + let inner = ModelProjection::from(related_query.parent_field.related_field().linking_fields()) + .as_columns(ctx) + .fold(inner, |acc, c| acc.column(c)); + + let inner = with_pagination_and_filters(inner, related_query.args, ctx); + + let inner = if let Some(nested) = related_query.nested { + with_nested_joins(inner, nested, ctx) + } else { + inner + }; + + let inner = Table::from(inner).alias(inner_alias); + + let select = Select::from_table(inner).value(json_array_agg(Column::from(JSON_AGG_IDENT)).alias(JSON_AGG_IDENT)); select } + +fn with_nested_joins<'a>(input: Select<'a>, nested: Vec, ctx: &Context<'_>) -> Select<'a> { + nested.into_iter().fold(input, |acc, nested| { + let alias = join_alias_name(&nested.parent_field); + + let join_columns = nested.parent_field.join_columns(ctx); + let related_alias = join_alias_name(&nested.parent_field.related_field()); + let related_join_columns = ModelProjection::from(nested.parent_field.related_field().linking_fields()) + .as_columns(ctx) + .map(|c| c.table(related_alias.clone())); + // WHERE Parent.id = Child.id + let join_cond = join_columns + .zip(related_join_columns) + .fold(None::, |acc, (a, b)| match acc { + Some(acc) => Some(acc.and(a.equals(b))), + None => Some(a.equals(b).into()), + }) + .unwrap(); + + // LEFT JOIN LATERAL () AS ON TRUE + let join_select = Table::from(build_nested(nested, ctx).and_where(join_cond)) + .alias(alias) + .on(ConditionTree::single(true.raw())); + + acc.left_join_lateral(join_select) + }) +} + +fn join_alias_name(rf: &RelationField) -> String { + format!("{}_{}", rf.model().name(), rf.name()) +} diff --git a/query-engine/core/src/interpreter/query_interpreters/read.rs b/query-engine/core/src/interpreter/query_interpreters/read.rs index 9f69d0b0c264..2cc6333f509d 100644 --- a/query-engine/core/src/interpreter/query_interpreters/read.rs +++ b/query-engine/core/src/interpreter/query_interpreters/read.rs @@ -152,47 +152,33 @@ fn read_many_by_queries( fn read_many_by_joins( tx: &mut dyn ConnectionLike, - mut query: ManyRecordsQuery, + query: ManyRecordsQuery, trace_id: Option, ) -> BoxFuture<'_, InterpretationResult> { - let processor = if query.args.requires_inmemory_processing() { - Some(InMemoryRecordProcessor::new_from_query_args(&mut query.args)) - } else { - None - }; - + // TODO: Hack, ideally, relations should be part of the selection set let nested = build_related_reads(&query); let fut = async move { - let scalars = tx + let records = tx .get_many_records( &query.model, query.args.clone(), &query.selected_fields, - nested, + nested.clone(), &query.aggregation_selections, trace_id, ) .await?; - let scalars = if let Some(p) = processor { - p.apply(scalars) - } else { - scalars - }; - - let (scalars, aggregation_rows) = extract_aggregation_rows_from_scalars(scalars, query.aggregation_selections); - - if scalars.records.is_empty() && query.options.contains(QueryOption::ThrowOnEmpty) { + if records.records.is_empty() && query.options.contains(QueryOption::ThrowOnEmpty) { record_not_found() } else { - Ok(RecordSelection { + Ok(RecordSelectionWithRelations { name: query.name, fields: query.selection_order, - scalars, - nested: Vec::new(), + records, + nested: build_relation_record_selection(nested), model: query.model, - aggregation_rows, } .into()) } @@ -201,6 +187,22 @@ fn read_many_by_joins( fut.boxed() } +fn build_relation_record_selection(related_queries: Vec) -> Vec { + related_queries + .into_iter() + .map(|rq| RelationRecordSelection { + name: rq.name, + fields: rq.selection_order, + model: rq.parent_field.related_model(), + nested: if let Some(nested) = rq.nested { + build_relation_record_selection(nested) + } else { + Vec::new() + }, + }) + .collect() +} + fn build_related_reads(query: &ManyRecordsQuery) -> Vec { query .nested diff --git a/query-engine/core/src/response_ir/internal.rs b/query-engine/core/src/response_ir/internal.rs index 7becb19e768b..aac14075de37 100644 --- a/query-engine/core/src/response_ir/internal.rs +++ b/query-engine/core/src/response_ir/internal.rs @@ -1,6 +1,9 @@ use super::*; use crate::{ - constants::custom_types, protocol::EngineProtocol, CoreError, QueryResult, RecordAggregations, RecordSelection, + constants::custom_types, + protocol::EngineProtocol, + result_ast::{RecordSelectionWithRelations, RelationRecordSelection}, + CoreError, QueryResult, RecordAggregations, RecordSelection, }; use connector::{AggregationResult, RelAggregationResult, RelAggregationRow}; use indexmap::IndexMap; @@ -46,6 +49,9 @@ pub(crate) fn serialize_internal( QueryResult::RecordSelection(Some(rs)) => { serialize_record_selection(*rs, field, field.field_type(), is_list, query_schema) } + QueryResult::RecordSelectionWithRelations(rs) => { + serialize_record_selection_with_relations(*rs, field, field.field_type(), is_list, query_schema) + } QueryResult::RecordAggregations(ras) => serialize_aggregations(field, ras), QueryResult::Count(c) => { // Todo needs a real implementation or needs to move to RecordAggregation @@ -216,6 +222,33 @@ fn coerce_non_numeric(value: PrismaValue, output: &OutputType<'_>) -> PrismaValu } } +fn serialize_record_selection_with_relations( + record_selection: RecordSelectionWithRelations, + field: &OutputField<'_>, + typ: &OutputType<'_>, // We additionally pass the type to allow recursing into nested type definitions of a field. + is_list: bool, + query_schema: &QuerySchema, +) -> crate::Result { + let name = record_selection.name.clone(); + + match &typ.inner { + inner if typ.is_list() => serialize_record_selection_with_relations( + record_selection, + field, + &OutputType::non_list(inner.clone()), + true, + query_schema, + ), + InnerOutputType::Object(obj) => { + let result = serialize_objects_with_relation(record_selection, obj, query_schema)?; + + process_object(field, is_list, result, name) + } + // We always serialize record selections into objects or lists on the top levels. Scalars and enums are handled separately. + _ => unreachable!(), + } +} + fn serialize_record_selection( record_selection: RecordSelection, field: &OutputField<'_>, @@ -235,54 +268,185 @@ fn serialize_record_selection( ), InnerOutputType::Object(obj) => { let result = serialize_objects(record_selection, obj, query_schema)?; - let is_optional = field.is_nullable; - // Items will be ref'ed on the top level to allow cheap clones in nested scenarios. - match (is_list, is_optional) { - // List(Opt(_)) | List(_) - (true, opt) => { - result - .into_iter() - .map(|(parent, items)| { - if !opt { - // Check that all items are non-null - if items.iter().any(|item| matches!(item, Item::Value(PrismaValue::Null))) { - return Err(CoreError::null_serialization_error(&name)); - } - } - - Ok((parent, Item::Ref(ItemRef::new(Item::list(items))))) - }) - .collect() + process_object(field, is_list, result, name) + } + + _ => unreachable!(), // We always serialize record selections into objects or lists on the top levels. Scalars and enums are handled separately. + } +} + +// TODO: rename function +fn process_object( + field: &OutputField<'_>, + is_list: bool, + result: IndexMap, Vec>, + name: String, +) -> Result, Item>, CoreError> { + let is_optional = field.is_nullable; + + // Items will be ref'ed on the top level to allow cheap clones in nested scenarios. + match (is_list, is_optional) { + // List(Opt(_)) | List(_) + (true, opt) => { + result + .into_iter() + .map(|(parent, items)| { + if !opt { + // Check that all items are non-null + if items.iter().any(|item| matches!(item, Item::Value(PrismaValue::Null))) { + return Err(CoreError::null_serialization_error(&name)); + } + } + + Ok((parent, Item::Ref(ItemRef::new(Item::list(items))))) + }) + .collect() + } + + // Opt(_) + (false, opt) => { + result + .into_iter() + .map(|(parent, mut items)| { + // As it's not a list, we require a single result + if items.len() > 1 { + items.reverse(); + let first = items.pop().unwrap(); + + // Simple return the first record in the list. + Ok((parent, Item::Ref(ItemRef::new(first)))) + } else if items.is_empty() && opt { + Ok((parent, Item::Ref(ItemRef::new(Item::Value(PrismaValue::Null))))) + } else if items.is_empty() && opt { + Err(CoreError::null_serialization_error(&name)) + } else { + Ok((parent, Item::Ref(ItemRef::new(items.pop().unwrap())))) + } + }) + .collect() + } + } +} + +// TODO: Handle errors properly +fn serialize_objects_with_relation( + result: RecordSelectionWithRelations, + typ: &ObjectType<'_>, + query_schema: &QuerySchema, +) -> crate::Result { + let mut object_mapping = UncheckedItemsWithParents::with_capacity(result.records.records.len()); + + let model = result.model; + let db_field_names = result.fields; + let nested = result.nested; + + let fields: Vec<_> = db_field_names + .iter() + .filter_map(|f| model.fields().all().find(|field| field.db_name() == f)) + .collect(); + + for record in result.records.records.into_iter() { + if !object_mapping.contains_key(&record.parent_id) { + object_mapping.insert(record.parent_id.clone(), Vec::new()); + } + + let values = record.values; + let mut object = IndexMap::with_capacity(values.len()); + + for (val, field) in values.into_iter().zip(fields.iter()) { + let out_field = typ.find_field(field.name()).unwrap(); + + match field { + Field::Scalar(_) if !out_field.field_type().is_object() => { + object.insert(field.name().to_owned(), serialize_scalar(out_field, val)?); } + Field::Relation(_) if out_field.field_type().is_list() => { + let inner_typ = out_field.field_type.as_object_type().unwrap(); + let rrs = nested.iter().find(|rrs| rrs.name == field.name()).unwrap(); - // Opt(_) - (false, opt) => { - result + let items = val + .into_list() + .unwrap() .into_iter() - .map(|(parent, mut items)| { - // As it's not a list, we require a single result - if items.len() > 1 { - items.reverse(); - let first = items.pop().unwrap(); - - // Simple return the first record in the list. - Ok((parent, Item::Ref(ItemRef::new(first)))) - } else if items.is_empty() && opt { - Ok((parent, Item::Ref(ItemRef::new(Item::Value(PrismaValue::Null))))) - } else if items.is_empty() && opt { - Err(CoreError::null_serialization_error(&name)) - } else { - Ok((parent, Item::Ref(ItemRef::new(items.pop().unwrap())))) - } - }) - .collect() + .map(|value| serialize_relation_selection(rrs, value, inner_typ, query_schema)) + .collect::>>()?; + + object.insert(field.name().to_owned(), Item::list(items)); + } + Field::Relation(_) => { + let inner_typ = out_field.field_type.as_object_type().unwrap(); + let rrs = nested.iter().find(|rrs| rrs.name == field.name()).unwrap(); + + object.insert( + field.name().to_owned(), + serialize_relation_selection(rrs, val, inner_typ, query_schema)?, + ); } + _ => (), } } - _ => unreachable!(), // We always serialize record selections into objects or lists on the top levels. Scalars and enums are handled separately. + let result = Item::Map(object); + + object_mapping.get_mut(&record.parent_id).unwrap().push(result); } + + Ok(object_mapping) +} + +fn serialize_relation_selection( + rrs: &RelationRecordSelection, + value: PrismaValue, + // parent_id: Option, + typ: &ObjectType<'_>, + query_schema: &QuerySchema, +) -> crate::Result { + let mut map = Map::new(); + + // TODO: handle errors + let mut value_obj: HashMap = HashMap::from_iter(value.into_object().unwrap().into_iter()); + let db_field_names = &rrs.fields; + let fields: Vec<_> = db_field_names + .iter() + .filter_map(|f| rrs.model.fields().all().find(|field| field.db_name() == f)) + .collect(); + + for field in fields { + let out_field = typ.find_field(field.name()).unwrap(); + let value = value_obj.remove(field.name()).unwrap(); + + match field { + Field::Scalar(_) if !out_field.field_type().is_object() => { + map.insert(field.name().to_owned(), serialize_scalar(out_field, value)?); + } + Field::Relation(_) if out_field.field_type().is_list() => { + let inner_typ = out_field.field_type.as_object_type().unwrap(); + let inner_rrs = rrs.nested.iter().find(|rrs| rrs.name == field.name()).unwrap(); + + let items = value + .into_list() + .unwrap() + .into_iter() + .map(|value| serialize_relation_selection(inner_rrs, value, inner_typ, query_schema)) + .collect::>>()?; + + map.insert(field.name().to_owned(), Item::list(items)); + } + Field::Relation(_) => { + let inner_typ = out_field.field_type.as_object_type().unwrap(); + let inner_rrs = rrs.nested.iter().find(|rrs| rrs.name == field.name()).unwrap(); + + map.insert( + field.name().to_owned(), + serialize_relation_selection(inner_rrs, value, inner_typ, query_schema)?, + ); + } + _ => (), + } + } + + Ok(Item::Map(map)) } /// Serializes the given result into objects of given type. @@ -372,16 +536,7 @@ fn serialize_objects( acc }); - // TODO: Find out how to easily determine when a result is null. - // If the object is null or completely empty, coerce into null instead. - let result = Item::Map(map); - // let result = if result.is_null_or_empty() { - // Item::Value(PrismaValue::Null) - // } else { - // result - // }; - - object_mapping.get_mut(&record.parent_id).unwrap().push(result); + object_mapping.get_mut(&record.parent_id).unwrap().push(Item::Map(map)); } Ok(object_mapping) diff --git a/query-engine/core/src/result_ast/mod.rs b/query-engine/core/src/result_ast/mod.rs index 91c58f8551a9..a54f333c90a2 100644 --- a/query-engine/core/src/result_ast/mod.rs +++ b/query-engine/core/src/result_ast/mod.rs @@ -6,12 +6,47 @@ pub(crate) enum QueryResult { Id(Option), Count(usize), RecordSelection(Option>), + RecordSelectionWithRelations(Box), Json(serde_json::Value), RecordAggregations(RecordAggregations), Unit, } -// Todo: In theory, much of this info can go into the serializer as soon as the read results are resolved in a flat tree. +#[derive(Debug, Clone)] +pub struct RecordSelectionWithRelations { + /// Name of the query. + pub(crate) name: String, + + /// Holds an ordered list of selected field names for each contained record. + pub(crate) fields: Vec, + + /// Selection results + pub(crate) records: ManyRecords, + + pub(crate) nested: Vec, + + /// The model of the contained records. + pub(crate) model: Model, +} + +impl From for QueryResult { + fn from(value: RecordSelectionWithRelations) -> Self { + QueryResult::RecordSelectionWithRelations(Box::new(value)) + } +} + +#[derive(Debug, Clone)] +pub struct RelationRecordSelection { + /// Name of the relation. + pub name: String, + /// Holds an ordered list of selected field names for each contained record. + pub fields: Vec, + /// The model of the contained records. + pub model: Model, + /// Nested relation selections + pub nested: Vec, +} + #[derive(Debug, Clone)] pub struct RecordSelection { /// Name of the query. From 0474565ed38c81133fe9470eb817a90f18ce8bc7 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Thu, 26 Oct 2023 18:40:42 +0200 Subject: [PATCH 03/38] fix: iterate over record fields in serializer --- query-engine/core/src/response_ir/internal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-engine/core/src/response_ir/internal.rs b/query-engine/core/src/response_ir/internal.rs index aac14075de37..a8f67a4e3728 100644 --- a/query-engine/core/src/response_ir/internal.rs +++ b/query-engine/core/src/response_ir/internal.rs @@ -338,7 +338,7 @@ fn serialize_objects_with_relation( let mut object_mapping = UncheckedItemsWithParents::with_capacity(result.records.records.len()); let model = result.model; - let db_field_names = result.fields; + let db_field_names = result.records.field_names; let nested = result.nested; let fields: Vec<_> = db_field_names From d1f083173abcf6e8e43e3dbff6aad54bc7e83166 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Thu, 26 Oct 2023 19:15:14 +0200 Subject: [PATCH 04/38] wip: support to-one relation and fix serializer --- .../src/database/operations/coerce.rs | 19 ++++++++++++++++++- query-engine/core/src/response_ir/internal.rs | 17 ++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs index 3af1fefc2c52..f91a91bc8c51 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs @@ -15,12 +15,29 @@ pub(crate) fn coerce_record_with_join(record: &mut Record, rq_indexes: Vec<(usiz // TODO: find better name pub(crate) fn coerce_json_relation_to_pv(value: serde_json::Value, q: &RelatedQuery) -> PrismaValue { match value { - serde_json::Value::Array(values) => PrismaValue::List( + // to-many + serde_json::Value::Array(values) if q.parent_field.is_list() => PrismaValue::List( values .into_iter() .map(|value| coerce_json_relation_to_pv(value, q)) .collect(), ), + // to-one + serde_json::Value::Array(values) => { + let coerced = values + .into_iter() + .next() + .map(|value| coerce_json_relation_to_pv(value, q)); + + // TODO(HACK): We probably want to update the sql builder instead to not aggregate to-one relations as array + // If the arary is empty, it means there's no relations + if let Some(val) = coerced { + val + // else the relation's null + } else { + PrismaValue::Null + } + } serde_json::Value::Object(obj) => { let mut map: Vec<(String, PrismaValue)> = Vec::with_capacity(obj.len()); let related_model = q.parent_field.related_model(); diff --git a/query-engine/core/src/response_ir/internal.rs b/query-engine/core/src/response_ir/internal.rs index a8f67a4e3728..f81d9de7b56c 100644 --- a/query-engine/core/src/response_ir/internal.rs +++ b/query-engine/core/src/response_ir/internal.rs @@ -13,7 +13,7 @@ use schema::{ constants::{aggregations::*, output_fields::*}, *, }; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; /// A grouping of items to their parent record. /// The item implicitly holds the information of the type of item contained. @@ -240,7 +240,9 @@ fn serialize_record_selection_with_relations( query_schema, ), InnerOutputType::Object(obj) => { + dbg!(&record_selection); let result = serialize_objects_with_relation(record_selection, obj, query_schema)?; + dbg!(&result); process_object(field, is_list, result, name) } @@ -346,6 +348,10 @@ fn serialize_objects_with_relation( .filter_map(|f| model.fields().all().find(|field| field.db_name() == f)) .collect(); + // Hack: we convert it to a hashset to support contains with &str as input + // because Vec::contains(&str) doesn't work and we don't want to allocate a string record value + let selected_db_field_names: HashSet = result.fields.into_iter().collect(); + for record in result.records.records.into_iter() { if !object_mapping.contains_key(&record.parent_id) { object_mapping.insert(record.parent_id.clone(), Vec::new()); @@ -355,6 +361,11 @@ fn serialize_objects_with_relation( let mut object = IndexMap::with_capacity(values.len()); for (val, field) in values.into_iter().zip(fields.iter()) { + // Skip fields that aren't part of the selection set + if !selected_db_field_names.contains(field.name()) { + continue; + } + let out_field = typ.find_field(field.name()).unwrap(); match field { @@ -409,12 +420,12 @@ fn serialize_relation_selection( let db_field_names = &rrs.fields; let fields: Vec<_> = db_field_names .iter() - .filter_map(|f| rrs.model.fields().all().find(|field| field.db_name() == f)) + .filter_map(|f| rrs.model.fields().all().find(|field| field.name() == f)) .collect(); for field in fields { let out_field = typ.find_field(field.name()).unwrap(); - let value = value_obj.remove(field.name()).unwrap(); + let value = value_obj.remove(field.db_name()).unwrap(); match field { Field::Scalar(_) if !out_field.field_type().is_object() => { From 4fb012fd1b121a8d74655a7af0b138185e1e2f6f Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Thu, 26 Oct 2023 19:56:25 +0200 Subject: [PATCH 05/38] add basic m2m support --- .envrc | 2 +- quaint/src/ast/function/json_extract_array.rs | 2 +- .../src/database/operations/mod.rs | 2 +- .../src/query_builder/select.rs | 105 ++++++++++++------ 4 files changed, 77 insertions(+), 34 deletions(-) diff --git a/.envrc b/.envrc index 5488da9e10e7..431c45be687c 100644 --- a/.envrc +++ b/.envrc @@ -20,7 +20,7 @@ export SIMPLE_TEST_MODE="yes" # Reduces the amount of generated `relation_link_t ### QE specific logging vars ### export QE_LOG_LEVEL=debug # Set it to "trace" to enable query-graph debugging logs # export PRISMA_RENDER_DOT_FILE=1 # Uncomment to enable rendering a dot file of the Query Graph from an executed query. -# export FMT_SQL=1 # Uncomment it to enable logging formatted SQL queries +export FMT_SQL=1 # Uncomment it to enable logging formatted SQL queries ### Uncomment to run driver adapters tests. See query-engine-driver-adapters.yml workflow for how tests run in CI. # export EXTERNAL_TEST_EXECUTOR="napi" diff --git a/quaint/src/ast/function/json_extract_array.rs b/quaint/src/ast/function/json_extract_array.rs index 5eeb7c6b4387..974c1cebea58 100644 --- a/quaint/src/ast/function/json_extract_array.rs +++ b/quaint/src/ast/function/json_extract_array.rs @@ -32,4 +32,4 @@ where }; fun.into() -} \ No newline at end of file +} diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/mod.rs b/query-engine/connectors/sql-query-connector/src/database/operations/mod.rs index 65a7e7712617..b4eadcceb221 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/mod.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/mod.rs @@ -1,5 +1,5 @@ +pub mod coerce; pub mod read; pub(crate) mod update; pub mod upsert; pub mod write; -pub mod coerce; diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs index 13bd464cde31..a87f5227f1c7 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs @@ -33,7 +33,7 @@ pub(crate) fn build( acc.value(Column::from((join_alias_name(&read.parent_field), JSON_AGG_IDENT)).alias(read.name.to_owned())) }); - let select = with_nested_joins(select, nested, ctx); + let select = with_related_queries(select, nested, ctx); let select = with_pagination_and_filters(select, args, ctx); let (sql, _) = Postgres::build(select.clone()).unwrap(); @@ -76,7 +76,67 @@ fn with_pagination_and_filters<'a>(select: Select<'a>, args: QueryArguments, ctx select } -pub(crate) fn build_nested(related_query: RelatedQuery, ctx: &Context<'_>) -> Select<'static> { +fn with_related_queries<'a>(input: Select<'a>, related_queries: Vec, ctx: &Context<'_>) -> Select<'a> { + related_queries.into_iter().fold(input, |acc, rq| { + let alias = join_alias_name(&rq.parent_field); + let is_m2m = rq.parent_field.relation().is_many_to_many(); + + let join_columns = rq + .parent_field + .join_columns(ctx) + .map(|c| c.opt_table(is_m2m.then(|| m2m_join_alias_name(&rq.parent_field)))); + let related_alias = join_alias_name(&rq.parent_field.related_field()); + let related_join_columns = ModelProjection::from(rq.parent_field.related_field().linking_fields()) + .as_columns(ctx) + .map(|c| c.table(related_alias.clone())); + // WHERE Parent.id = Child.id + let join_cond = join_columns + .zip(related_join_columns) + .fold(None::, |acc, (a, b)| match acc { + Some(acc) => Some(acc.and(a.equals(b))), + None => Some(a.equals(b).into()), + }) + .unwrap(); + + let m2m_join = build_m2m_join(&rq, ctx); + + // LEFT JOIN LATERAL () AS ON TRUE + let join_select = Table::from(build_related_query_select(rq, ctx).and_where(join_cond)) + .alias(alias) + .on(ConditionTree::single(true.raw())); + + if is_m2m { + // m2m relations need to left join on the relation table first + acc.join(m2m_join).left_join_lateral(join_select) + } else { + acc.left_join_lateral(join_select) + } + }) +} + +fn build_m2m_join(rq: &RelatedQuery, ctx: &Context<'_>) -> Join<'static> { + let m2m_table = rq + .parent_field + .as_table(ctx) + .alias(m2m_join_alias_name(&rq.parent_field)); + + let left_columns = rq.parent_field.identifier_columns(ctx); + let right_columns = ModelProjection::from(rq.parent_field.model().primary_identifier()).as_columns(ctx); + + let conditions = left_columns + .zip(right_columns) + .fold(None::, |acc, (a, b)| match acc { + Some(acc) => Some(acc.and(a.equals(b))), + None => Some(a.equals(b).into()), + }) + .unwrap(); + + let m2m_join = m2m_table.on(conditions); + + Join::Left(m2m_join) +} + +pub(crate) fn build_related_query_select(related_query: RelatedQuery, ctx: &Context<'_>) -> Select<'static> { let mut build_obj_params = ModelProjection::from(related_query.selected_fields) .fields() .map(|f| match f { @@ -113,45 +173,28 @@ pub(crate) fn build_nested(related_query: RelatedQuery, ctx: &Context<'_>) -> Se let inner = with_pagination_and_filters(inner, related_query.args, ctx); let inner = if let Some(nested) = related_query.nested { - with_nested_joins(inner, nested, ctx) + with_related_queries(inner, nested, ctx) } else { inner }; let inner = Table::from(inner).alias(inner_alias); - let select = Select::from_table(inner).value(json_array_agg(Column::from(JSON_AGG_IDENT)).alias(JSON_AGG_IDENT)); + let select = Select::from_table(inner).value( + coalesce(vec![ + json_array_agg(Column::from(JSON_AGG_IDENT)).into(), + Expression::from("[]".raw()), + ]) + .alias(JSON_AGG_IDENT), + ); select } -fn with_nested_joins<'a>(input: Select<'a>, nested: Vec, ctx: &Context<'_>) -> Select<'a> { - nested.into_iter().fold(input, |acc, nested| { - let alias = join_alias_name(&nested.parent_field); - - let join_columns = nested.parent_field.join_columns(ctx); - let related_alias = join_alias_name(&nested.parent_field.related_field()); - let related_join_columns = ModelProjection::from(nested.parent_field.related_field().linking_fields()) - .as_columns(ctx) - .map(|c| c.table(related_alias.clone())); - // WHERE Parent.id = Child.id - let join_cond = join_columns - .zip(related_join_columns) - .fold(None::, |acc, (a, b)| match acc { - Some(acc) => Some(acc.and(a.equals(b))), - None => Some(a.equals(b).into()), - }) - .unwrap(); - - // LEFT JOIN LATERAL () AS ON TRUE - let join_select = Table::from(build_nested(nested, ctx).and_where(join_cond)) - .alias(alias) - .on(ConditionTree::single(true.raw())); - - acc.left_join_lateral(join_select) - }) -} - fn join_alias_name(rf: &RelationField) -> String { format!("{}_{}", rf.model().name(), rf.name()) } + +fn m2m_join_alias_name(rf: &RelationField) -> String { + format!("{}_{}_m2m", rf.model().name(), rf.name()) +} From 90e6617957fa7e39874feadc231deb6376118feb Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Tue, 31 Oct 2023 17:59:25 +0100 Subject: [PATCH 06/38] fix m2m support and make basic ordering + pagination work --- quaint/src/ast/join.rs | 2 +- quaint/src/ast/select.rs | 2 +- quaint/src/ast/table.rs | 2 +- .../query-connector/src/interface.rs | 1 + .../src/database/operations/coerce.rs | 2 +- .../sql-query-connector/src/ordering.rs | 11 +- .../src/query_builder/select.rs | 296 +++++++++++------- .../core/src/interpreter/interpreter_impl.rs | 7 + .../interpreter/query_interpreters/read.rs | 5 +- query-engine/core/src/query_ast/read.rs | 9 + query-engine/core/src/response_ir/internal.rs | 4 +- 11 files changed, 220 insertions(+), 121 deletions(-) diff --git a/quaint/src/ast/join.rs b/quaint/src/ast/join.rs index ba1684f0ea71..158b39a0e4cf 100644 --- a/quaint/src/ast/join.rs +++ b/quaint/src/ast/join.rs @@ -18,7 +18,7 @@ impl<'a> JoinData<'a> { } } - pub fn as_lateral(mut self) -> Self { + pub fn lateral(mut self) -> Self { self.lateral = true; self } diff --git a/quaint/src/ast/select.rs b/quaint/src/ast/select.rs index 127753139afd..10722048aee7 100644 --- a/quaint/src/ast/select.rs +++ b/quaint/src/ast/select.rs @@ -397,7 +397,7 @@ impl<'a> Select<'a> { { let join_data: JoinData = join.into(); - self.left_join(join_data.as_lateral()) + self.left_join(join_data.lateral()) } /// Adds `RIGHT JOIN` clause to the query. diff --git a/quaint/src/ast/table.rs b/quaint/src/ast/table.rs index 5ebbaf68462f..d09c7ecdfefc 100644 --- a/quaint/src/ast/table.rs +++ b/quaint/src/ast/table.rs @@ -210,7 +210,7 @@ impl<'a> Table<'a> { { let join_data: JoinData = join.into(); - self.left_join(join_data.as_lateral()) + self.left_join(join_data.lateral()) } /// Adds an `INNER JOIN` clause to the query, specifically for that table. diff --git a/query-engine/connectors/query-connector/src/interface.rs b/query-engine/connectors/query-connector/src/interface.rs index 18df36803a05..aebbdbe563a1 100644 --- a/query-engine/connectors/query-connector/src/interface.rs +++ b/query-engine/connectors/query-connector/src/interface.rs @@ -217,6 +217,7 @@ impl RelAggregationSelection { } } +// TODO: rename this #[derive(Debug, Clone)] pub struct RelatedQuery { pub name: String, diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs index f91a91bc8c51..55673a40cdaf 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs @@ -15,7 +15,7 @@ pub(crate) fn coerce_record_with_join(record: &mut Record, rq_indexes: Vec<(usiz // TODO: find better name pub(crate) fn coerce_json_relation_to_pv(value: serde_json::Value, q: &RelatedQuery) -> PrismaValue { match value { - // to-many + // one-to-many serde_json::Value::Array(values) if q.parent_field.is_list() => PrismaValue::List( values .into_iter() diff --git a/query-engine/connectors/sql-query-connector/src/ordering.rs b/query-engine/connectors/sql-query-connector/src/ordering.rs index 5f61d0c3a907..310e10ec43d7 100644 --- a/query-engine/connectors/sql-query-connector/src/ordering.rs +++ b/query-engine/connectors/sql-query-connector/src/ordering.rs @@ -18,16 +18,23 @@ pub(crate) struct OrderByDefinition { #[derive(Debug, Default)] pub(crate) struct OrderByBuilder { + parent_alias: Option, // Used to generate unique join alias join_counter: usize, } +impl OrderByBuilder { + pub(crate) fn with_parent_alias(mut self, alias: Option) -> Self { + self.parent_alias = alias; + self + } +} + impl OrderByBuilder { /// Builds all expressions for an `ORDER BY` clause based on the query arguments. pub(crate) fn build(&mut self, query_arguments: &QueryArguments, ctx: &Context<'_>) -> Vec { let needs_reversed_order = query_arguments.needs_reversed_order(); - // The index is used to differentiate potentially separate relations to the same model. query_arguments .order_by .iter() @@ -201,7 +208,7 @@ impl OrderByBuilder { let order_by_column = if let Some(last_join) = joins.last() { Column::from((last_join.alias.to_owned(), order_by.field.db_name().to_owned())) } else { - order_by.field.as_column(ctx) + order_by.field.as_column(ctx).opt_table(self.parent_alias.clone()) }; (joins, order_by_column) diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs index a87f5227f1c7..b5ccae5f730d 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs @@ -4,12 +4,13 @@ use crate::{ context::Context, filter::FilterBuilder, model_extensions::{AsColumn, AsColumns, AsTable, RelationFieldExt}, + ordering::OrderByBuilder, }; -use connector_interface::{QueryArguments, RelAggregationSelection, RelatedQuery}; +use connector_interface::{Filter, QueryArguments, RelAggregationSelection, RelatedQuery}; use itertools::Itertools; -use prisma_models::{ModelProjection, RelationField}; -use quaint::{prelude::*, visitor::*}; +use prisma_models::{ModelProjection, RelationField, ScalarField}; +use quaint::prelude::*; pub const JSON_AGG_IDENT: &str = "data"; @@ -29,115 +30,54 @@ pub(crate) fn build( .fold(select, |acc, sf| acc.column(sf.as_column(ctx))); // TODO: check how to select aggregated relations + // Adds relation selections to the top-level query let select = nested.iter().fold(select, |acc, read| { - acc.value(Column::from((join_alias_name(&read.parent_field), JSON_AGG_IDENT)).alias(read.name.to_owned())) + let table_name = match read.parent_field.relation().is_many_to_many() { + true => m2m_join_alias_name(&read.parent_field), + false => join_alias_name(&read.parent_field), + }; + + acc.value(Column::from((table_name, JSON_AGG_IDENT)).alias(read.name.to_owned())) }); + // Adds joins for relations let select = with_related_queries(select, nested, ctx); - let select = with_pagination_and_filters(select, args, ctx); - - let (sql, _) = Postgres::build(select.clone()).unwrap(); - - println!("{}", sql); + let select = with_ordering(select, &args, None, ctx); + let select = with_pagination(select, args.take, args.skip); + let select = with_filters(select, args.filter, ctx); select } -fn with_pagination_and_filters<'a>(select: Select<'a>, args: QueryArguments, ctx: &Context<'_>) -> Select<'a> { - let (filter, joins) = match args.filter { - Some(filter) => { - let (filter, joins) = FilterBuilder::with_top_level_joins().visit_filter(filter, ctx); - - (Some(filter), joins) - } - None => (None, None), - }; - - let select = match filter { - Some(filter) => select.and_where(filter), - None => select, - }; - - let select = match joins { - Some(joins) => joins.into_iter().fold(select, |acc, join| acc.join(join.data)), - None => select, - }; - - let select = match args.take { - Some(take) => select.limit(take as usize), - None => select, - }; - - let select = match args.skip { - Some(skip) => select.offset(skip as usize), - None => select, - }; - - select +fn with_related_queries<'a>(input: Select<'a>, related_queries: Vec, ctx: &Context<'_>) -> Select<'a> { + related_queries + .into_iter() + .fold(input, |acc, rq| with_related_query(acc, rq, ctx)) } -fn with_related_queries<'a>(input: Select<'a>, related_queries: Vec, ctx: &Context<'_>) -> Select<'a> { - related_queries.into_iter().fold(input, |acc, rq| { - let alias = join_alias_name(&rq.parent_field); - let is_m2m = rq.parent_field.relation().is_many_to_many(); - - let join_columns = rq - .parent_field - .join_columns(ctx) - .map(|c| c.opt_table(is_m2m.then(|| m2m_join_alias_name(&rq.parent_field)))); - let related_alias = join_alias_name(&rq.parent_field.related_field()); - let related_join_columns = ModelProjection::from(rq.parent_field.related_field().linking_fields()) - .as_columns(ctx) - .map(|c| c.table(related_alias.clone())); - // WHERE Parent.id = Child.id - let join_cond = join_columns - .zip(related_join_columns) - .fold(None::, |acc, (a, b)| match acc { - Some(acc) => Some(acc.and(a.equals(b))), - None => Some(a.equals(b).into()), - }) - .unwrap(); +fn with_related_query<'a>(select: Select<'a>, rq: RelatedQuery, ctx: &Context<'_>) -> Select<'a> { + if rq.parent_field.relation().is_many_to_many() { + let m2m_join = build_m2m_join(rq, ctx); - let m2m_join = build_m2m_join(&rq, ctx); + // m2m relations need to left join on the relation table first + select.left_join(m2m_join) + } else { + let alias = join_alias_name(&rq.parent_field); // LEFT JOIN LATERAL () AS ON TRUE - let join_select = Table::from(build_related_query_select(rq, ctx).and_where(join_cond)) + let join_select = Table::from(build_related_query_select(rq, ctx)) .alias(alias) - .on(ConditionTree::single(true.raw())); + .on(ConditionTree::single(true.raw())) + .lateral(); - if is_m2m { - // m2m relations need to left join on the relation table first - acc.join(m2m_join).left_join_lateral(join_select) - } else { - acc.left_join_lateral(join_select) - } - }) + select.left_join(join_select) + } } -fn build_m2m_join(rq: &RelatedQuery, ctx: &Context<'_>) -> Join<'static> { - let m2m_table = rq - .parent_field - .as_table(ctx) - .alias(m2m_join_alias_name(&rq.parent_field)); - - let left_columns = rq.parent_field.identifier_columns(ctx); - let right_columns = ModelProjection::from(rq.parent_field.model().primary_identifier()).as_columns(ctx); - - let conditions = left_columns - .zip(right_columns) - .fold(None::, |acc, (a, b)| match acc { - Some(acc) => Some(acc.and(a.equals(b))), - None => Some(a.equals(b).into()), - }) - .unwrap(); - - let m2m_join = m2m_table.on(conditions); - - Join::Left(m2m_join) -} +fn build_related_query_select(rq: RelatedQuery, ctx: &Context<'_>) -> Select<'static> { + let mut fields_to_select: Vec = vec![]; -pub(crate) fn build_related_query_select(related_query: RelatedQuery, ctx: &Context<'_>) -> Select<'static> { - let mut build_obj_params = ModelProjection::from(related_query.selected_fields) + let mut build_obj_params = ModelProjection::from(rq.selected_fields) .fields() .map(|f| match f { prisma_models::Field::Scalar(sf) => { @@ -147,50 +87,182 @@ pub(crate) fn build_related_query_select(related_query: RelatedQuery, ctx: &Cont }) .collect_vec(); - if let Some(nested_queries) = &related_query.nested { + if let Some(nested_queries) = &rq.nested { for nested_query in nested_queries { + let table_name = match nested_query.parent_field.relation().is_many_to_many() { + true => m2m_join_alias_name(&nested_query.parent_field), + false => join_alias_name(&nested_query.parent_field), + }; + build_obj_params.push(( Cow::from(nested_query.name.to_owned()), - Expression::from(Column::from(( - join_alias_name(&nested_query.parent_field), - JSON_AGG_IDENT, - ))), + Expression::from(Column::from((table_name, JSON_AGG_IDENT))), )); } } - let inner_alias = join_alias_name(&related_query.parent_field.related_field()); + let inner_alias = join_alias_name(&rq.parent_field.related_field()); // SELECT JSON_BUILD_OBJECT() - let inner = Select::from_table(related_query.parent_field.related_model().as_table(ctx)) + let inner = Select::from_table(rq.parent_field.related_model().as_table(ctx)) .value(json_build_object(build_obj_params).alias(JSON_AGG_IDENT)); // SELECT - let inner = ModelProjection::from(related_query.parent_field.related_field().linking_fields()) + let inner = ModelProjection::from(rq.parent_field.related_field().linking_fields()) .as_columns(ctx) .fold(inner, |acc, c| acc.column(c)); - let inner = with_pagination_and_filters(inner, related_query.args, ctx); + let inner = with_join_conditions(inner, &rq.parent_field, ctx); - let inner = if let Some(nested) = related_query.nested { + let inner = if let Some(nested) = rq.nested { with_related_queries(inner, nested, ctx) } else { inner }; - let inner = Table::from(inner).alias(inner_alias); + if rq.parent_field.relation().is_many_to_many() { + // SELECT ONLY if it's a m2m table as we need to order by outside of the inner select + let inner = rq + .args + .order_by + .iter() + .flat_map(|order_by| match order_by { + prisma_models::OrderBy::Scalar(x) if x.path.is_empty() => vec![x.field.clone()], + prisma_models::OrderBy::Relevance(x) => x.fields.clone(), + _ => Vec::new(), + }) + .fold(inner, |acc, sf| acc.column(sf.as_column(ctx))); + + inner + } else { + let inner = with_ordering(inner, &rq.args, None, ctx); + let inner = with_pagination(inner, rq.args.take, rq.args.skip); + let inner = with_filters(inner, rq.args.filter, ctx); - let select = Select::from_table(inner).value( - coalesce(vec![ - json_array_agg(Column::from(JSON_AGG_IDENT)).into(), - Expression::from("[]".raw()), - ]) - .alias(JSON_AGG_IDENT), - ); + let inner = Table::from(inner).alias(inner_alias.clone()); + let middle = Select::from_table(inner).column(Column::from((inner_alias.clone(), JSON_AGG_IDENT))); + let outer = Select::from_table(Table::from(middle).alias(format!("{}_1", inner_alias))).value(json_agg()); + + outer + } +} + +fn build_m2m_join<'a>(rq: RelatedQuery, ctx: &Context<'_>) -> JoinData<'a> { + let rf = rq.parent_field.clone(); + let m2m_alias = m2m_join_alias_name(&rf); + + let left_columns = rf.related_field().m2m_columns(ctx); + let right_columns = ModelProjection::from(rf.model().primary_identifier()).as_columns(ctx); + + let conditions = left_columns + .into_iter() + .zip(right_columns) + .fold(None::, |acc, (a, b)| match acc { + Some(acc) => Some(acc.and(a.equals(b))), + None => Some(a.equals(b).into()), + }) + .unwrap(); + + let inner = Select::from_table(rf.as_table(ctx)) + .value(Column::from((join_alias_name(&rf), JSON_AGG_IDENT))) + .and_where(conditions); + + let inner = with_ordering(inner, &rq.args, Some(join_alias_name(&rq.parent_field)), ctx); + let inner = with_pagination(inner, rq.args.take, rq.args.skip); + // TODO: avoid clone? + let inner = with_filters(inner, rq.args.filter.clone(), ctx); + + let join_select = Table::from(build_related_query_select(rq, ctx)) + .alias(join_alias_name(&rf)) + .on(ConditionTree::single(true.raw())) + .lateral(); + + let inner = inner.left_join(join_select); + + let outer = Select::from_table(Table::from(inner).alias(format!("{}_1", m2m_alias))).value(json_agg()); + + Table::from(outer) + .alias(m2m_alias) + .on(ConditionTree::single(true.raw())) + .lateral() +} + +fn json_agg() -> Function<'static> { + coalesce(vec![ + json_array_agg(Column::from(JSON_AGG_IDENT)).into(), + Expression::from("[]".raw()), + ]) + .alias(JSON_AGG_IDENT) +} + +/// Builds the lateral join conditions +fn with_join_conditions<'a>(select: Select<'a>, rf: &RelationField, ctx: &Context<'_>) -> Select<'a> { + let join_columns = rf.join_columns(ctx); + // .map(|c| c.opt_table(is_m2m.then(|| m2m_join_alias_name(rf)))); + let related_join_columns = ModelProjection::from(rf.related_field().linking_fields()).as_columns(ctx); + + // WHERE Parent.id = Child.id + let conditions = join_columns + .zip(related_join_columns) + .fold(None::, |acc, (a, b)| match acc { + Some(acc) => Some(acc.and(a.equals(b))), + None => Some(a.equals(b).into()), + }) + .unwrap(); + + select.and_where(conditions) +} + +fn with_ordering<'a>( + select: Select<'a>, + args: &QueryArguments, + parent_alias: Option, + ctx: &Context<'_>, +) -> Select<'a> { + let order_by_definitions = OrderByBuilder::default() + .with_parent_alias(parent_alias) + .build(args, ctx); + + let select = order_by_definitions + .iter() + .flat_map(|j| &j.joins) + .fold(select, |acc, join| acc.join(join.clone().data)); + + order_by_definitions + .iter() + .fold(select, |acc, o| acc.order_by(o.order_definition.clone())) +} + +fn with_pagination<'a>(select: Select<'a>, take: Option, skip: Option) -> Select<'a> { + let select = match take { + Some(take) => select.limit(take as usize), + None => select, + }; + + let select = match skip { + Some(skip) => select.offset(skip as usize), + None => select, + }; select } +fn with_filters<'a>(select: Select<'a>, filter: Option, ctx: &Context<'_>) -> Select<'a> { + if let Some(filter) = filter { + let (filter, joins) = FilterBuilder::with_top_level_joins().visit_filter(filter, ctx); + let select = select.and_where(filter); + + let select = match joins { + Some(joins) => joins.into_iter().fold(select, |acc, join| acc.join(join.data)), + None => select, + }; + + select + } else { + select + } +} + fn join_alias_name(rf: &RelationField) -> String { format!("{}_{}", rf.model().name(), rf.name()) } diff --git a/query-engine/core/src/interpreter/interpreter_impl.rs b/query-engine/core/src/interpreter/interpreter_impl.rs index 8aa3d77ae761..f1011b13f8f5 100644 --- a/query-engine/core/src/interpreter/interpreter_impl.rs +++ b/query-engine/core/src/interpreter/interpreter_impl.rs @@ -72,6 +72,13 @@ impl ExpressionResult { .into_iter() .collect(), ), + QueryResult::RecordSelectionWithRelations(rsr) => Some( + rsr.records + .extract_selection_results(field_selection) + .expect("Expected record selection to contain required model ID fields.") + .into_iter() + .collect(), + ), QueryResult::RecordSelection(None) => Some(vec![]), _ => None, diff --git a/query-engine/core/src/interpreter/query_interpreters/read.rs b/query-engine/core/src/interpreter/query_interpreters/read.rs index 2cc6333f509d..e14fec7b4884 100644 --- a/query-engine/core/src/interpreter/query_interpreters/read.rs +++ b/query-engine/core/src/interpreter/query_interpreters/read.rs @@ -91,7 +91,8 @@ fn read_many( query: ManyRecordsQuery, trace_id: Option, ) -> BoxFuture<'_, InterpretationResult> { - let use_joins = true; + // use joins if we're not using cursors + let use_joins = query.args.cursor.is_none() && !query.nested.iter().any(|q| q.has_cursor()); if use_joins { read_many_by_joins(tx, query, trace_id) @@ -170,6 +171,8 @@ fn read_many_by_joins( ) .await?; + // dbg!(&records); + if records.records.is_empty() && query.options.contains(QueryOption::ThrowOnEmpty) { record_not_found() } else { diff --git a/query-engine/core/src/query_ast/read.rs b/query-engine/core/src/query_ast/read.rs index e840eb81e50d..e753ce5e9f86 100644 --- a/query-engine/core/src/query_ast/read.rs +++ b/query-engine/core/src/query_ast/read.rs @@ -63,6 +63,15 @@ impl ReadQuery { None } } + + pub(crate) fn has_cursor(&self) -> bool { + match self { + ReadQuery::RecordQuery(_) => false, + ReadQuery::ManyRecordsQuery(q) => q.args.cursor.is_some() || q.nested.iter().any(|q| q.has_cursor()), + ReadQuery::RelatedRecordsQuery(q) => q.args.cursor.is_some() || q.nested.iter().any(|q| q.has_cursor()), + ReadQuery::AggregateRecordsQuery(_) => false, + } + } } impl FilteredQuery for ReadQuery { diff --git a/query-engine/core/src/response_ir/internal.rs b/query-engine/core/src/response_ir/internal.rs index f81d9de7b56c..fd77628e353c 100644 --- a/query-engine/core/src/response_ir/internal.rs +++ b/query-engine/core/src/response_ir/internal.rs @@ -240,9 +240,9 @@ fn serialize_record_selection_with_relations( query_schema, ), InnerOutputType::Object(obj) => { - dbg!(&record_selection); + // dbg!(&record_selection); let result = serialize_objects_with_relation(record_selection, obj, query_schema)?; - dbg!(&result); + // dbg!(&result); process_object(field, is_list, result, name) } From 0ed76a78e5d57fd09ad9025a2412bb0b194a6ae1 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 16:32:33 +0100 Subject: [PATCH 07/38] add SelectedField::Relation and remove RelatedQuery --- .../mongodb-query-connector/src/error.rs | 1 + .../src/output_meta.rs | 1 + .../mongodb-query-connector/src/projection.rs | 1 + .../mongodb-query-connector/src/value.rs | 1 + .../query-connector/src/interface.rs | 15 +---- .../query-connector/src/write_args.rs | 1 + .../src/model_extensions/selection_result.rs | 3 +- .../query-structure/src/field_selection.rs | 64 ++++++++++++++++++- .../query-structure/src/filter/into_filter.rs | 3 +- .../src/projections/model_projection.rs | 2 + .../query-structure/src/selection_result.rs | 1 + 11 files changed, 76 insertions(+), 17 deletions(-) diff --git a/query-engine/connectors/mongodb-query-connector/src/error.rs b/query-engine/connectors/mongodb-query-connector/src/error.rs index f32ff78e29c9..3355f0020f36 100644 --- a/query-engine/connectors/mongodb-query-connector/src/error.rs +++ b/query-engine/connectors/mongodb-query-connector/src/error.rs @@ -278,6 +278,7 @@ impl DecorateErrorWithFieldInformationExtension for crate::Result { match selected_field { SelectedField::Scalar(sf) => self.decorate_with_scalar_field_info(sf), SelectedField::Composite(composite_sel) => self.decorate_with_composite_field_info(&composite_sel.field), + SelectedField::Relation(_) => unreachable!(), } } diff --git a/query-engine/connectors/mongodb-query-connector/src/output_meta.rs b/query-engine/connectors/mongodb-query-connector/src/output_meta.rs index 081672f9d6e6..1100c9d436ad 100644 --- a/query-engine/connectors/mongodb-query-connector/src/output_meta.rs +++ b/query-engine/connectors/mongodb-query-connector/src/output_meta.rs @@ -80,6 +80,7 @@ pub fn from_selections( }), ); } + SelectedField::Relation(_) => unreachable!(), } } diff --git a/query-engine/connectors/mongodb-query-connector/src/projection.rs b/query-engine/connectors/mongodb-query-connector/src/projection.rs index 80a6a3e792e7..cbb16e0a097c 100644 --- a/query-engine/connectors/mongodb-query-connector/src/projection.rs +++ b/query-engine/connectors/mongodb-query-connector/src/projection.rs @@ -26,6 +26,7 @@ fn path_prefixed_selection(doc: &mut Document, parent_paths: Vec, select parent_paths.push(cs.field.db_name().to_owned()); path_prefixed_selection(doc, parent_paths, cs.selections); } + query_structure::SelectedField::Relation(_) => unreachable!(), } } } diff --git a/query-engine/connectors/mongodb-query-connector/src/value.rs b/query-engine/connectors/mongodb-query-connector/src/value.rs index cf6812d59b6d..9faecaa13f4c 100644 --- a/query-engine/connectors/mongodb-query-connector/src/value.rs +++ b/query-engine/connectors/mongodb-query-connector/src/value.rs @@ -23,6 +23,7 @@ impl IntoBson for (&SelectedField, PrismaValue) { match selection { SelectedField::Scalar(sf) => (sf, value).into_bson(), SelectedField::Composite(_) => todo!(), // [Composites] todo + SelectedField::Relation(_) => unreachable!(), } } } diff --git a/query-engine/connectors/query-connector/src/interface.rs b/query-engine/connectors/query-connector/src/interface.rs index aebbdbe563a1..167e28340a7d 100644 --- a/query-engine/connectors/query-connector/src/interface.rs +++ b/query-engine/connectors/query-connector/src/interface.rs @@ -217,19 +217,6 @@ impl RelAggregationSelection { } } -// TODO: rename this -#[derive(Debug, Clone)] -pub struct RelatedQuery { - pub name: String, - pub alias: Option, - pub parent_field: RelationFieldRef, - pub args: QueryArguments, - pub selected_fields: FieldSelection, - pub nested: Option>, - pub selection_order: Vec, - pub aggregation_selections: Vec, -} - #[async_trait] pub trait ReadOperations { /// Gets a single record or `None` back from the database. @@ -258,8 +245,8 @@ pub trait ReadOperations { model: &Model, query_arguments: QueryArguments, selected_fields: &FieldSelection, - nested: Vec, aggregation_selections: &[RelAggregationSelection], + relation_load_strategy: RelationLoadStrategy, trace_id: Option, ) -> crate::Result; diff --git a/query-engine/connectors/query-connector/src/write_args.rs b/query-engine/connectors/query-connector/src/write_args.rs index e0b030975042..c89f4e51514f 100644 --- a/query-engine/connectors/query-connector/src/write_args.rs +++ b/query-engine/connectors/query-connector/src/write_args.rs @@ -327,6 +327,7 @@ impl From<(&SelectedField, PrismaValue)> for WriteOperation { match selection { SelectedField::Scalar(sf) => (sf, pv).into(), SelectedField::Composite(cs) => (&cs.field, pv).into(), + SelectedField::Relation(_) => todo!(), } } } diff --git a/query-engine/connectors/sql-query-connector/src/model_extensions/selection_result.rs b/query-engine/connectors/sql-query-connector/src/model_extensions/selection_result.rs index 51eb7768d068..152f15864c0d 100644 --- a/query-engine/connectors/sql-query-connector/src/model_extensions/selection_result.rs +++ b/query-engine/connectors/sql-query-connector/src/model_extensions/selection_result.rs @@ -36,7 +36,8 @@ impl SelectionResultExt for SelectionResult { .iter() .map(|(selection, v)| match selection { SelectedField::Scalar(sf) => sf.value(v.clone(), ctx), - SelectedField::Composite(_cf) => todo!(), // [Composites] todo + SelectedField::Composite(_cf) => todo!(), + SelectedField::Relation(_) => todo!(), // [Composites] todo }) .collect() } diff --git a/query-engine/query-structure/src/field_selection.rs b/query-engine/query-structure/src/field_selection.rs index b44529793a5a..1f2903c094bb 100644 --- a/query-engine/query-structure/src/field_selection.rs +++ b/query-engine/query-structure/src/field_selection.rs @@ -1,9 +1,10 @@ use crate::{ parent_container::ParentContainer, prisma_value_ext::PrismaValueExtensions, CompositeFieldRef, DomainError, Field, - ScalarFieldRef, SelectionResult, + QueryArguments, RelationField, ScalarField, ScalarFieldRef, SelectionResult, TypeIdentifier, }; use itertools::Itertools; use prisma_value::PrismaValue; +use psl::schema_ast::ast::FieldArity; use std::fmt::Display; /// A selection of fields from a model. @@ -31,6 +32,7 @@ impl FieldSelection { .and_then(|selection| selection.as_composite()) .map(|cs| cs.is_superset_of(other_cs)) .unwrap_or(false), + SelectedField::Relation(_) => todo!(), }) } @@ -64,6 +66,7 @@ impl FieldSelection { .map(|selection| match selection { SelectedField::Scalar(sf) => sf.clone().into(), SelectedField::Composite(cf) => cf.field.clone().into(), + SelectedField::Relation(rs) => rs.field.clone().into(), }) .collect() } @@ -76,6 +79,7 @@ impl FieldSelection { .filter_map(|selection| match selection { SelectedField::Scalar(sf) => Some(sf.clone()), SelectedField::Composite(_) => None, + SelectedField::Relation(_) => None, }) .collect::>(); @@ -139,6 +143,24 @@ impl FieldSelection { FieldSelection { selections } } + + pub fn type_identifiers_with_arities(&self) -> Vec<(TypeIdentifier, FieldArity)> { + self.selections() + .filter_map(|selection| match selection { + SelectedField::Scalar(sf) => Some(sf.type_identifier_with_arity()), + SelectedField::Relation(rf) if rf.field.is_list() => Some((TypeIdentifier::Json, FieldArity::Required)), + SelectedField::Relation(rf) => Some((TypeIdentifier::Json, rf.field.arity())), + SelectedField::Composite(_) => None, + }) + .collect() + } + + pub fn relations(&self) -> impl Iterator { + self.selections().filter_map(|selection| match selection { + SelectedField::Relation(rs) => Some(rs), + _ => None, + }) + } } /// A selected field. Can be contained on a model or composite type. @@ -147,6 +169,30 @@ impl FieldSelection { pub enum SelectedField { Scalar(ScalarFieldRef), Composite(CompositeSelection), + Relation(RelationSelection), +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RelationSelection { + pub field: RelationField, + pub args: QueryArguments, + pub selections: Vec, +} + +impl RelationSelection { + pub fn scalars(&self) -> impl Iterator { + self.selections.iter().filter_map(|selection| match selection { + SelectedField::Scalar(sf) => Some(sf), + _ => None, + }) + } + + pub fn relations(&self) -> impl Iterator { + self.selections.iter().filter_map(|selection| match selection { + SelectedField::Relation(rs) => Some(rs), + _ => None, + }) + } } impl SelectedField { @@ -154,6 +200,7 @@ impl SelectedField { match self { SelectedField::Scalar(sf) => sf.name(), SelectedField::Composite(cf) => cf.field.name(), + SelectedField::Relation(rs) => rs.field.name(), } } @@ -161,6 +208,7 @@ impl SelectedField { match self { SelectedField::Scalar(sf) => sf.db_name(), SelectedField::Composite(cs) => cs.field.db_name(), + SelectedField::Relation(rs) => rs.field.name(), } } @@ -175,6 +223,7 @@ impl SelectedField { match self { SelectedField::Scalar(sf) => sf.container(), SelectedField::Composite(cs) => cs.field.container(), + SelectedField::Relation(rs) => ParentContainer::from(rs.field.model()), } } @@ -183,8 +232,14 @@ impl SelectedField { match self { SelectedField::Scalar(sf) => value.coerce(&sf.type_identifier()), SelectedField::Composite(cs) => cs.coerce_value(value), + SelectedField::Relation(_) => todo!(), } } + + /// Returns `true` if the selected field is [`Scalar`]. + pub fn is_scalar(&self) -> bool { + matches!(self, Self::Scalar(..)) + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -203,6 +258,7 @@ impl CompositeSelection { .and_then(|selection| selection.as_composite()) .map(|cs| cs.is_superset_of(other_cs)) .unwrap_or(false), + SelectedField::Relation(_) => unreachable!(), }) } @@ -278,6 +334,12 @@ impl Display for SelectedField { cs.field, cs.selections.iter().map(|selection| format!("{selection}")).join(", ") ), + SelectedField::Relation(rs) => write!( + f, + "{} {{ {} }}", + rs.field, + rs.selections.iter().map(|selection| format!("{selection}")).join(", ") + ), } } } diff --git a/query-engine/query-structure/src/filter/into_filter.rs b/query-engine/query-structure/src/filter/into_filter.rs index b180b3b80c49..eaf4711628fe 100644 --- a/query-engine/query-structure/src/filter/into_filter.rs +++ b/query-engine/query-structure/src/filter/into_filter.rs @@ -14,7 +14,8 @@ impl IntoFilter for SelectionResult { .into_iter() .map(|(selection, value)| match selection { SelectedField::Scalar(sf) => sf.equals(value), - SelectedField::Composite(_) => todo!(), // [Composites] todo + SelectedField::Composite(_) => unreachable!(), // [Composites] todo + SelectedField::Relation(_) => unreachable!(), }) .collect(); diff --git a/query-engine/query-structure/src/projections/model_projection.rs b/query-engine/query-structure/src/projections/model_projection.rs index e17cafc896d6..9c966bd8a56f 100644 --- a/query-engine/query-structure/src/projections/model_projection.rs +++ b/query-engine/query-structure/src/projections/model_projection.rs @@ -30,6 +30,7 @@ impl From<&FieldSelection> for ModelProjection { .filter_map(|selected| match selected { SelectedField::Scalar(sf) => Some(sf.clone().into()), SelectedField::Composite(_cf) => None, + SelectedField::Relation(_) => None, }) .collect(), } @@ -112,6 +113,7 @@ impl From<&SelectionResult> for ModelProjection { .map(|(field_selection, _)| match field_selection { SelectedField::Scalar(sf) => sf.clone().into(), SelectedField::Composite(cf) => cf.field.clone().into(), + SelectedField::Relation(_) => todo!(), }) .collect::>(); diff --git a/query-engine/query-structure/src/selection_result.rs b/query-engine/query-structure/src/selection_result.rs index d6e1ef46349d..156797bc94ee 100644 --- a/query-engine/query-structure/src/selection_result.rs +++ b/query-engine/query-structure/src/selection_result.rs @@ -94,6 +94,7 @@ impl SelectionResult { .filter_map(|(selection, _)| match selection { SelectedField::Scalar(sf) => Some(sf.clone()), SelectedField::Composite(_) => None, + SelectedField::Relation(_) => todo!(), }) .collect(); From c7a753fdd070804b7a1f2372d5fa7512d1d036e1 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 16:38:03 +0100 Subject: [PATCH 08/38] add RelationLoadStrategy and feature gate joins --- .../src/cockroach_datamodel_connector.rs | 3 +- .../src/mssql_datamodel_connector.rs | 3 +- .../src/postgres_datamodel_connector.rs | 3 +- .../src/datamodel_connector/capabilities.rs | 1 + .../src/interface/connection.rs | 8 +- .../src/interface/transaction.rs | 6 +- .../src/database/connection.rs | 40 +++---- .../src/database/transaction.rs | 10 +- .../query_interpreters/nested_read.rs | 4 +- .../interpreter/query_interpreters/read.rs | 81 ++++--------- query-engine/core/src/query_ast/read.rs | 30 +++-- query-engine/core/src/query_graph/mod.rs | 1 + .../core/src/query_graph_builder/builder.rs | 10 +- .../src/query_graph_builder/read/first.rs | 17 ++- .../core/src/query_graph_builder/read/many.rs | 27 +++-- .../core/src/query_graph_builder/read/one.rs | 23 ++-- .../src/query_graph_builder/read/related.rs | 6 +- .../src/query_graph_builder/read/utils.rs | 107 +++++++++++++++--- .../src/query_graph_builder/write/create.rs | 2 +- .../src/query_graph_builder/write/delete.rs | 2 +- .../src/query_graph_builder/write/update.rs | 2 +- .../src/query_graph_builder/write/upsert.rs | 2 +- .../src/query_graph_builder/write/utils.rs | 1 + .../query-structure/src/query_arguments.rs | 8 +- query-engine/schema/src/query_schema.rs | 2 +- 25 files changed, 236 insertions(+), 163 deletions(-) diff --git a/psl/builtin-connectors/src/cockroach_datamodel_connector.rs b/psl/builtin-connectors/src/cockroach_datamodel_connector.rs index 5456deb59df6..4ab77cf45639 100644 --- a/psl/builtin-connectors/src/cockroach_datamodel_connector.rs +++ b/psl/builtin-connectors/src/cockroach_datamodel_connector.rs @@ -58,7 +58,8 @@ const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Connector FilteredInlineChildNestedToOneDisconnect | InsertReturning | UpdateReturning | - RowIn + RowIn | + LateralJoin }); const SCALAR_TYPE_DEFAULTS: &[(ScalarType, CockroachType)] = &[ diff --git a/psl/builtin-connectors/src/mssql_datamodel_connector.rs b/psl/builtin-connectors/src/mssql_datamodel_connector.rs index 46647fabe8af..d150deb4fdfc 100644 --- a/psl/builtin-connectors/src/mssql_datamodel_connector.rs +++ b/psl/builtin-connectors/src/mssql_datamodel_connector.rs @@ -52,7 +52,8 @@ const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Connector SupportsTxIsolationReadCommitted | SupportsTxIsolationRepeatableRead | SupportsTxIsolationSerializable | - SupportsTxIsolationSnapshot + SupportsTxIsolationSnapshot | + LateralJoin }); pub(crate) struct MsSqlDatamodelConnector; diff --git a/psl/builtin-connectors/src/postgres_datamodel_connector.rs b/psl/builtin-connectors/src/postgres_datamodel_connector.rs index 8fac79165c58..54053767e303 100644 --- a/psl/builtin-connectors/src/postgres_datamodel_connector.rs +++ b/psl/builtin-connectors/src/postgres_datamodel_connector.rs @@ -65,7 +65,8 @@ const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Connector NativeUpsert | InsertReturning | UpdateReturning | - RowIn + RowIn | + LateralJoin }); pub struct PostgresDatamodelConnector; diff --git a/psl/psl-core/src/datamodel_connector/capabilities.rs b/psl/psl-core/src/datamodel_connector/capabilities.rs index 1b3f557e6285..eee38e8b4888 100644 --- a/psl/psl-core/src/datamodel_connector/capabilities.rs +++ b/psl/psl-core/src/datamodel_connector/capabilities.rs @@ -104,6 +104,7 @@ capabilities!( InsertReturning, UpdateReturning, RowIn, // Connector supports (a, b) IN (c, d) expression. + LateralJoin, ); /// Contains all capabilities that the connector is able to serve. diff --git a/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs b/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs index 76e634908c11..206d0b296d40 100644 --- a/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs +++ b/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs @@ -6,11 +6,11 @@ use crate::{ }; use async_trait::async_trait; use connector_interface::{ - Connection, ConnectionLike, ReadOperations, RelAggregationSelection, RelatedQuery, Transaction, UpdateType, - WriteArgs, WriteOperations, + Connection, ConnectionLike, ReadOperations, RelAggregationSelection, Transaction, UpdateType, WriteArgs, + WriteOperations, }; use mongodb::{ClientSession, Database}; -use query_structure::{prelude::*, SelectionResult}; +use query_structure::{prelude::*, RelationLoadStrategy, SelectionResult}; use std::collections::HashMap; pub struct MongoDbConnection { @@ -211,8 +211,8 @@ impl ReadOperations for MongoDbConnection { model: &Model, query_arguments: query_structure::QueryArguments, selected_fields: &FieldSelection, - _nested: Vec, aggregation_selections: &[RelAggregationSelection], + _relation_load_strategy: RelationLoadStrategy, _trace_id: Option, ) -> connector_interface::Result { catch(async move { diff --git a/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs b/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs index 5804ee75c077..107459c4b3b9 100644 --- a/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs +++ b/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs @@ -4,11 +4,11 @@ use crate::{ root_queries::{aggregate, read, write}, }; use connector_interface::{ - ConnectionLike, ReadOperations, RelAggregationSelection, RelatedQuery, Transaction, UpdateType, WriteOperations, + ConnectionLike, ReadOperations, RelAggregationSelection, Transaction, UpdateType, WriteOperations, }; use mongodb::options::{Acknowledgment, ReadConcern, TransactionOptions, WriteConcern}; use query_engine_metrics::{decrement_gauge, increment_gauge, metrics, PRISMA_CLIENT_QUERIES_ACTIVE}; -use query_structure::SelectionResult; +use query_structure::{RelationLoadStrategy, SelectionResult}; use std::collections::HashMap; pub struct MongoDbTransaction<'conn> { @@ -276,8 +276,8 @@ impl<'conn> ReadOperations for MongoDbTransaction<'conn> { model: &Model, query_arguments: query_structure::QueryArguments, selected_fields: &FieldSelection, - _nested: Vec, aggregation_selections: &[RelAggregationSelection], + _relation_load_strategy: RelationLoadStrategy, _trace_id: Option, ) -> connector_interface::Result { catch(async move { diff --git a/query-engine/connectors/sql-query-connector/src/database/connection.rs b/query-engine/connectors/sql-query-connector/src/database/connection.rs index 4ffa35262015..a987849e8b8d 100644 --- a/query-engine/connectors/sql-query-connector/src/database/connection.rs +++ b/query-engine/connectors/sql-query-connector/src/database/connection.rs @@ -3,18 +3,17 @@ use super::{catch, transaction::SqlConnectorTransaction}; use crate::{database::operations::*, Context, SqlError}; use async_trait::async_trait; -use connector::{ConnectionLike, RelAggregationSelection, RelatedQuery}; +use connector::{ConnectionLike, RelAggregationSelection}; use connector_interface::{ self as connector, AggregationRow, AggregationSelection, Connection, ReadOperations, RecordFilter, Transaction, WriteArgs, WriteOperations, }; use prisma_value::PrismaValue; -use psl::PreviewFeature; use quaint::{ connector::{IsolationLevel, TransactionCapable}, prelude::{ConnectionInfo, Queryable}, }; -use query_structure::{prelude::*, Filter, QueryArguments, SelectionResult}; +use query_structure::{prelude::*, Filter, QueryArguments, RelationLoadStrategy, SelectionResult}; use std::{collections::HashMap, str::FromStr}; pub(crate) struct SqlConnection { @@ -111,36 +110,23 @@ where model: &Model, query_arguments: QueryArguments, selected_fields: &FieldSelection, - nested: Vec, aggr_selections: &[RelAggregationSelection], + relation_load_strategy: RelationLoadStrategy, trace_id: Option, ) -> connector::Result { catch(self.connection_info.clone(), async move { let ctx = Context::new(&self.connection_info, trace_id.as_deref()); - if self.features.contains(PreviewFeature::RelationJoins) { - read::get_many_records_joins( - &self.inner, - model, - query_arguments, - &selected_fields.into(), - nested, - aggr_selections, - &ctx, - ) - .await - } else { - read::get_many_records( - &self.inner, - model, - query_arguments, - &selected_fields.into(), - nested, - aggr_selections, - &ctx, - ) - .await - } + read::get_many_records( + &self.inner, + model, + query_arguments, + selected_fields, + aggr_selections, + relation_load_strategy, + &ctx, + ) + .await }) .await } diff --git a/query-engine/connectors/sql-query-connector/src/database/transaction.rs b/query-engine/connectors/sql-query-connector/src/database/transaction.rs index ba88721e15ca..6b5bc668b54d 100644 --- a/query-engine/connectors/sql-query-connector/src/database/transaction.rs +++ b/query-engine/connectors/sql-query-connector/src/database/transaction.rs @@ -1,14 +1,14 @@ use super::catch; use crate::{database::operations::*, Context, SqlError}; use async_trait::async_trait; -use connector::{ConnectionLike, RelAggregationSelection, RelatedQuery}; +use connector::{ConnectionLike, RelAggregationSelection}; use connector_interface::{ self as connector, AggregationRow, AggregationSelection, ReadOperations, RecordFilter, Transaction, WriteArgs, WriteOperations, }; use prisma_value::PrismaValue; use quaint::prelude::ConnectionInfo; -use query_structure::{prelude::*, Filter, QueryArguments, SelectionResult}; +use query_structure::{prelude::*, Filter, QueryArguments, RelationLoadStrategy, SelectionResult}; use std::collections::HashMap; pub struct SqlConnectorTransaction<'tx> { @@ -91,8 +91,8 @@ impl<'tx> ReadOperations for SqlConnectorTransaction<'tx> { model: &Model, query_arguments: QueryArguments, selected_fields: &FieldSelection, - nested: Vec, aggr_selections: &[RelAggregationSelection], + relation_load_strategy: RelationLoadStrategy, trace_id: Option, ) -> connector::Result { catch(self.connection_info.clone(), async move { @@ -101,9 +101,9 @@ impl<'tx> ReadOperations for SqlConnectorTransaction<'tx> { self.inner.as_queryable(), model, query_arguments, - &selected_fields.into(), - nested, + &selected_fields, aggr_selections, + relation_load_strategy, &ctx, ) .await diff --git a/query-engine/core/src/interpreter/query_interpreters/nested_read.rs b/query-engine/core/src/interpreter/query_interpreters/nested_read.rs index 2d6cabff4727..c3ec29f065aa 100644 --- a/query-engine/core/src/interpreter/query_interpreters/nested_read.rs +++ b/query-engine/core/src/interpreter/query_interpreters/nested_read.rs @@ -67,8 +67,8 @@ pub(crate) async fn m2m( &query.parent_field.related_model(), args, &query.selected_fields, - Vec::new(), &query.aggregation_selections, + RelationLoadStrategy::Query, trace_id.clone(), ) .await? @@ -209,8 +209,8 @@ pub async fn one2m( &parent_field.related_model(), args, selected_fields, - Vec::new(), &aggr_selections, + RelationLoadStrategy::Query, trace_id, ) .await? diff --git a/query-engine/core/src/interpreter/query_interpreters/read.rs b/query-engine/core/src/interpreter/query_interpreters/read.rs index e14fec7b4884..2f5e71336946 100644 --- a/query-engine/core/src/interpreter/query_interpreters/read.rs +++ b/query-engine/core/src/interpreter/query_interpreters/read.rs @@ -1,11 +1,9 @@ use super::*; use crate::{interpreter::InterpretationResult, query_ast::*, result_ast::*}; -use connector::{ - self, error::ConnectorError, ConnectionLike, RelAggregationRow, RelAggregationSelection, RelatedQuery, -}; +use connector::{self, error::ConnectorError, ConnectionLike, RelAggregationRow, RelAggregationSelection}; use futures::future::{BoxFuture, FutureExt}; use inmemory_record_processor::InMemoryRecordProcessor; -use query_structure::ManyRecords; +use query_structure::{ManyRecords, RelationLoadStrategy, RelationSelection}; use std::collections::HashMap; use user_facing_errors::KnownError; @@ -91,13 +89,9 @@ fn read_many( query: ManyRecordsQuery, trace_id: Option, ) -> BoxFuture<'_, InterpretationResult> { - // use joins if we're not using cursors - let use_joins = query.args.cursor.is_none() && !query.nested.iter().any(|q| q.has_cursor()); - - if use_joins { - read_many_by_joins(tx, query, trace_id) - } else { - read_many_by_queries(tx, query, trace_id) + match query.relation_load_strategy { + RelationLoadStrategy::Join => read_many_by_joins(tx, query, trace_id), + RelationLoadStrategy::Query => read_many_by_queries(tx, query, trace_id), } } @@ -118,8 +112,8 @@ fn read_many_by_queries( &query.model, query.args.clone(), &query.selected_fields, - Vec::new(), &query.aggregation_selections, + query.relation_load_strategy, trace_id, ) .await?; @@ -156,31 +150,26 @@ fn read_many_by_joins( query: ManyRecordsQuery, trace_id: Option, ) -> BoxFuture<'_, InterpretationResult> { - // TODO: Hack, ideally, relations should be part of the selection set - let nested = build_related_reads(&query); - let fut = async move { - let records = tx + let result = tx .get_many_records( &query.model, query.args.clone(), &query.selected_fields, - nested.clone(), &query.aggregation_selections, + query.relation_load_strategy, trace_id, ) .await?; - // dbg!(&records); - - if records.records.is_empty() && query.options.contains(QueryOption::ThrowOnEmpty) { + if result.records.is_empty() && query.options.contains(QueryOption::ThrowOnEmpty) { record_not_found() } else { Ok(RecordSelectionWithRelations { name: query.name, fields: query.selection_order, - records, - nested: build_relation_record_selection(nested), + records: result, + nested: build_relation_record_selection(query.selected_fields.relations()), model: query.model, } .into()) @@ -190,51 +179,19 @@ fn read_many_by_joins( fut.boxed() } -fn build_relation_record_selection(related_queries: Vec) -> Vec { - related_queries - .into_iter() +fn build_relation_record_selection<'a>( + selections: impl Iterator, +) -> Vec { + selections .map(|rq| RelationRecordSelection { - name: rq.name, - fields: rq.selection_order, - model: rq.parent_field.related_model(), - nested: if let Some(nested) = rq.nested { - build_relation_record_selection(nested) - } else { - Vec::new() - }, + name: rq.field.name().to_owned(), + fields: rq.selections.iter().map(|sf| sf.prisma_name().to_owned()).collect(), + model: rq.field.related_model(), + nested: build_relation_record_selection(rq.relations()), }) .collect() } -fn build_related_reads(query: &ManyRecordsQuery) -> Vec { - query - .nested - .clone() - .into_iter() - .filter_map(|n| n.into_related_records_query()) - .map(to_related_query) - .collect() -} - -fn to_related_query(n: RelatedRecordsQuery) -> RelatedQuery { - RelatedQuery { - name: n.name, - alias: n.alias, - parent_field: n.parent_field, - args: n.args, - selected_fields: n.selected_fields, - nested: Some( - n.nested - .into_iter() - .filter_map(|n| n.into_related_records_query()) - .map(|n| to_related_query(n)) - .collect(), - ), - selection_order: n.selection_order, - aggregation_selections: n.aggregation_selections, - } -} - /// Queries related records for a set of parent IDs. fn read_related<'conn>( tx: &'conn mut dyn ConnectionLike, diff --git a/query-engine/core/src/query_ast/read.rs b/query-engine/core/src/query_ast/read.rs index e753ce5e9f86..c25fc4ab6784 100644 --- a/query-engine/core/src/query_ast/read.rs +++ b/query-engine/core/src/query_ast/read.rs @@ -3,7 +3,7 @@ use super::FilteredQuery; use crate::ToGraphviz; use connector::{AggregationSelection, RelAggregationSelection}; use enumflags2::BitFlags; -use query_structure::{prelude::*, Filter, QueryArguments}; +use query_structure::{prelude::*, Filter, QueryArguments, RelationLoadStrategy}; use std::fmt::Display; #[allow(clippy::enum_variant_names)] @@ -56,14 +56,6 @@ impl ReadQuery { } } - pub(crate) fn into_related_records_query(self) -> Option { - if let Self::RelatedRecordsQuery(v) = self { - Some(v) - } else { - None - } - } - pub(crate) fn has_cursor(&self) -> bool { match self { ReadQuery::RecordQuery(_) => false, @@ -72,6 +64,15 @@ impl ReadQuery { ReadQuery::AggregateRecordsQuery(_) => false, } } + + pub(crate) fn has_distinct(&self) -> bool { + match self { + ReadQuery::RecordQuery(_) => false, + ReadQuery::ManyRecordsQuery(q) => q.args.distinct.is_some() || q.nested.iter().any(|q| q.has_cursor()), + ReadQuery::RelatedRecordsQuery(q) => q.args.distinct.is_some() || q.nested.iter().any(|q| q.has_cursor()), + ReadQuery::AggregateRecordsQuery(_) => false, + } + } } impl FilteredQuery for ReadQuery { @@ -202,6 +203,7 @@ pub struct ManyRecordsQuery { pub selection_order: Vec, pub aggregation_selections: Vec, pub options: QueryOptions, + pub relation_load_strategy: RelationLoadStrategy, } #[derive(Debug, Clone)] @@ -220,6 +222,16 @@ pub struct RelatedRecordsQuery { pub parent_results: Option>, } +impl RelatedRecordsQuery { + pub fn has_cursor(&self) -> bool { + self.args.cursor.is_some() || self.nested.iter().any(|q| q.has_cursor()) + } + + pub fn has_distinct(&self) -> bool { + self.args.distinct.is_some() || self.nested.iter().any(|q| q.has_distinct()) + } +} + #[derive(Debug, Clone)] pub struct AggregateRecordsQuery { pub name: String, diff --git a/query-engine/core/src/query_graph/mod.rs b/query-engine/core/src/query_graph/mod.rs index 6086fa243331..f1d5896d695d 100644 --- a/query-engine/core/src/query_graph/mod.rs +++ b/query-engine/core/src/query_graph/mod.rs @@ -797,6 +797,7 @@ impl QueryGraph { selection_order: vec![], aggregation_selections: vec![], options: QueryOptions::none(), + relation_load_strategy: query_structure::RelationLoadStrategy::Query, }); let reload_query = Query::Read(read_query); diff --git a/query-engine/core/src/query_graph_builder/builder.rs b/query-engine/core/src/query_graph_builder/builder.rs index b7851baf451e..1152c4974a45 100644 --- a/query-engine/core/src/query_graph_builder/builder.rs +++ b/query-engine/core/src/query_graph_builder/builder.rs @@ -74,11 +74,11 @@ impl<'a> QueryGraphBuilder<'a> { let query_schema = self.query_schema; let mut graph = match (&query_info.tag, query_info.model.map(|id| self.query_schema.internal_data_model.clone().zip(id))) { - (QueryTag::FindUnique, Some(m)) => read::find_unique(parsed_field, m).map(Into::into), - (QueryTag::FindUniqueOrThrow, Some(m)) => read::find_unique_or_throw(parsed_field, m).map(Into::into), - (QueryTag::FindFirst, Some(m)) => read::find_first(parsed_field, m).map(Into::into), - (QueryTag::FindFirstOrThrow, Some(m)) => read::find_first_or_throw(parsed_field, m).map(Into::into), - (QueryTag::FindMany, Some(m)) => read::find_many(parsed_field, m).map(Into::into), + (QueryTag::FindUnique, Some(m)) => read::find_unique(parsed_field, m, query_schema).map(Into::into), + (QueryTag::FindUniqueOrThrow, Some(m)) => read::find_unique_or_throw(parsed_field, m, query_schema).map(Into::into), + (QueryTag::FindFirst, Some(m)) => read::find_first(parsed_field, m, query_schema).map(Into::into), + (QueryTag::FindFirstOrThrow, Some(m)) => read::find_first_or_throw(parsed_field, m, query_schema).map(Into::into), + (QueryTag::FindMany, Some(m)) => read::find_many(parsed_field, m, query_schema).map(Into::into), (QueryTag::Aggregate, Some(m)) => read::aggregate(parsed_field, m).map(Into::into), (QueryTag::GroupBy, Some(m)) => read::group_by(parsed_field, m).map(Into::into), (QueryTag::CreateOne, Some(m)) => QueryGraph::root(|g| write::create_record(g, query_schema, m, parsed_field)), diff --git a/query-engine/core/src/query_graph_builder/read/first.rs b/query-engine/core/src/query_graph_builder/read/first.rs index 84c90016858a..1d1b22dc43ce 100644 --- a/query-engine/core/src/query_graph_builder/read/first.rs +++ b/query-engine/core/src/query_graph_builder/read/first.rs @@ -1,15 +1,24 @@ use query_structure::Model; +use schema::QuerySchema; use super::*; use crate::ParsedField; -pub(crate) fn find_first(field: ParsedField<'_>, model: Model) -> QueryGraphBuilderResult { - let many_query = many::find_many(field, model)?; +pub(crate) fn find_first( + field: ParsedField<'_>, + model: Model, + query_schema: &QuerySchema, +) -> QueryGraphBuilderResult { + let many_query = many::find_many(field, model, query_schema)?; try_limit_to_one(many_query) } -pub(crate) fn find_first_or_throw(field: ParsedField<'_>, model: Model) -> QueryGraphBuilderResult { - let many_query = many::find_many_or_throw(field, model)?; +pub(crate) fn find_first_or_throw( + field: ParsedField<'_>, + model: Model, + query_schema: &QuerySchema, +) -> QueryGraphBuilderResult { + let many_query = many::find_many_or_throw(field, model, query_schema)?; try_limit_to_one(many_query) } diff --git a/query-engine/core/src/query_graph_builder/read/many.rs b/query-engine/core/src/query_graph_builder/read/many.rs index 6c9242330a83..c0f6ae9f5966 100644 --- a/query-engine/core/src/query_graph_builder/read/many.rs +++ b/query-engine/core/src/query_graph_builder/read/many.rs @@ -1,13 +1,22 @@ -use super::*; +use super::{utils::get_relation_load_strategy, *}; use crate::{query_document::ParsedField, ManyRecordsQuery, QueryOption, QueryOptions, ReadQuery}; use query_structure::Model; +use schema::QuerySchema; -pub(crate) fn find_many(field: ParsedField<'_>, model: Model) -> QueryGraphBuilderResult { - find_many_with_options(field, model, QueryOptions::none()) +pub(crate) fn find_many( + field: ParsedField<'_>, + model: Model, + query_schema: &QuerySchema, +) -> QueryGraphBuilderResult { + find_many_with_options(field, model, QueryOptions::none(), query_schema) } -pub(crate) fn find_many_or_throw(field: ParsedField<'_>, model: Model) -> QueryGraphBuilderResult { - find_many_with_options(field, model, QueryOption::ThrowOnEmpty.into()) +pub(crate) fn find_many_or_throw( + field: ParsedField<'_>, + model: Model, + query_schema: &QuerySchema, +) -> QueryGraphBuilderResult { + find_many_with_options(field, model, QueryOption::ThrowOnEmpty.into(), query_schema) } #[inline] @@ -15,6 +24,7 @@ fn find_many_with_options( field: ParsedField<'_>, model: Model, options: QueryOptions, + query_schema: &QuerySchema, ) -> QueryGraphBuilderResult { let args = extractors::extract_query_args(field.arguments, &model)?; let name = field.name; @@ -23,13 +33,15 @@ fn find_many_with_options( let (aggr_fields_pairs, nested_fields) = extractors::extract_nested_rel_aggr_selections(nested_fields); let aggregation_selections = utils::collect_relation_aggr_selections(aggr_fields_pairs, &model)?; let selection_order: Vec = utils::collect_selection_order(&nested_fields); - let selected_fields = utils::collect_selected_fields(&nested_fields, args.distinct.clone(), &model); - let nested = utils::collect_nested_queries(nested_fields, &model)?; + let selected_fields = utils::collect_selected_fields(&nested_fields, args.distinct.clone(), &model, query_schema)?; + let nested = utils::collect_nested_queries(nested_fields, &model, query_schema)?; let model = model; let selected_fields = utils::merge_relation_selections(selected_fields, None, &nested); let selected_fields = utils::merge_cursor_fields(selected_fields, &args.cursor); + let relation_load_strategy = get_relation_load_strategy(&args, &nested, &aggregation_selections, query_schema); + Ok(ReadQuery::ManyRecordsQuery(ManyRecordsQuery { name, alias, @@ -40,5 +52,6 @@ fn find_many_with_options( selection_order, aggregation_selections, options, + relation_load_strategy, })) } diff --git a/query-engine/core/src/query_graph_builder/read/one.rs b/query-engine/core/src/query_graph_builder/read/one.rs index d71c2535bb2f..fa5465324837 100644 --- a/query-engine/core/src/query_graph_builder/read/one.rs +++ b/query-engine/core/src/query_graph_builder/read/one.rs @@ -1,15 +1,23 @@ use super::*; use crate::{query_document::*, QueryOption, QueryOptions, ReadQuery, RecordQuery}; use query_structure::Model; -use schema::constants::args; +use schema::{constants::args, QuerySchema}; use std::convert::TryInto; -pub(crate) fn find_unique(field: ParsedField<'_>, model: Model) -> QueryGraphBuilderResult { - find_unique_with_options(field, model, QueryOptions::none()) +pub(crate) fn find_unique( + field: ParsedField<'_>, + model: Model, + query_schema: &QuerySchema, +) -> QueryGraphBuilderResult { + find_unique_with_options(field, model, QueryOptions::none(), query_schema) } -pub(crate) fn find_unique_or_throw(field: ParsedField<'_>, model: Model) -> QueryGraphBuilderResult { - find_unique_with_options(field, model, QueryOption::ThrowOnEmpty.into()) +pub(crate) fn find_unique_or_throw( + field: ParsedField<'_>, + model: Model, + query_schema: &QuerySchema, +) -> QueryGraphBuilderResult { + find_unique_with_options(field, model, QueryOption::ThrowOnEmpty.into(), query_schema) } /// Builds a read query from a parsed incoming read query field. @@ -18,6 +26,7 @@ fn find_unique_with_options( mut field: ParsedField<'_>, model: Model, options: QueryOptions, + query_schema: &QuerySchema, ) -> QueryGraphBuilderResult { let filter = match field.arguments.lookup(args::WHERE) { Some(where_arg) => { @@ -34,8 +43,8 @@ fn find_unique_with_options( let (aggr_fields_pairs, nested_fields) = extractors::extract_nested_rel_aggr_selections(nested_fields); let aggregation_selections = utils::collect_relation_aggr_selections(aggr_fields_pairs, &model)?; let selection_order: Vec = utils::collect_selection_order(&nested_fields); - let selected_fields = utils::collect_selected_fields(&nested_fields, None, &model); - let nested = utils::collect_nested_queries(nested_fields, &model)?; + let selected_fields = utils::collect_selected_fields(&nested_fields, None, &model, query_schema)?; + let nested = utils::collect_nested_queries(nested_fields, &model, query_schema)?; let selected_fields = utils::merge_relation_selections(selected_fields, None, &nested); Ok(ReadQuery::RecordQuery(RecordQuery { diff --git a/query-engine/core/src/query_graph_builder/read/related.rs b/query-engine/core/src/query_graph_builder/read/related.rs index 9c73699b0477..7ebed8a7a06c 100644 --- a/query-engine/core/src/query_graph_builder/read/related.rs +++ b/query-engine/core/src/query_graph_builder/read/related.rs @@ -1,11 +1,13 @@ use super::*; use crate::{query_document::ParsedField, ReadQuery, RelatedRecordsQuery}; use query_structure::{Model, RelationFieldRef}; +use schema::QuerySchema; pub(crate) fn find_related( field: ParsedField<'_>, parent: RelationFieldRef, model: Model, + query_schema: &QuerySchema, ) -> QueryGraphBuilderResult { let args = extractors::extract_query_args(field.arguments, &model)?; let name = field.name; @@ -14,8 +16,8 @@ pub(crate) fn find_related( let (aggr_fields_pairs, sub_selections) = extractors::extract_nested_rel_aggr_selections(sub_selections); let aggregation_selections = utils::collect_relation_aggr_selections(aggr_fields_pairs, &model)?; let selection_order: Vec = utils::collect_selection_order(&sub_selections); - let selected_fields = utils::collect_selected_fields(&sub_selections, args.distinct.clone(), &model); - let nested = utils::collect_nested_queries(sub_selections, &model)?; + let selected_fields = utils::collect_selected_fields(&sub_selections, args.distinct.clone(), &model, query_schema)?; + let nested = utils::collect_nested_queries(sub_selections, &model, query_schema)?; let parent_field = parent; let selected_fields = utils::merge_relation_selections(selected_fields, Some(parent_field.clone()), &nested); diff --git a/query-engine/core/src/query_graph_builder/read/utils.rs b/query-engine/core/src/query_graph_builder/read/utils.rs index 545393ba3d15..b0f3fdb5da32 100644 --- a/query-engine/core/src/query_graph_builder/read/utils.rs +++ b/query-engine/core/src/query_graph_builder/read/utils.rs @@ -1,8 +1,12 @@ use super::*; use crate::{ArgumentListLookup, FieldPair, ParsedField, ReadQuery}; use connector::RelAggregationSelection; -use query_structure::prelude::*; -use schema::constants::{aggregations::*, args}; +use psl::{datamodel_connector::ConnectorCapability, PreviewFeature}; +use query_structure::{prelude::*, QueryArguments, RelationLoadStrategy}; +use schema::{ + constants::{aggregations::*, args}, + QuerySchema, +}; pub fn collect_selection_order(from: &[FieldPair<'_>]) -> Vec { from.iter() @@ -21,18 +25,19 @@ pub fn collect_selected_fields( from_pairs: &[FieldPair<'_>], distinct: Option, model: &Model, -) -> FieldSelection { + query_schema: &QuerySchema, +) -> QueryGraphBuilderResult { let model_id = model.primary_identifier(); - let selected_fields = pairs_to_selections(model, from_pairs); + let selected_fields = pairs_to_selections(model, from_pairs, query_schema)?; let selection = FieldSelection::new(selected_fields); let selection = model_id.merge(selection); // Distinct fields are always selected because we are processing them in-memory if let Some(distinct) = distinct { - selection.merge(distinct) + Ok(selection.merge(distinct)) } else { - selection + Ok(selection) } } @@ -60,12 +65,20 @@ where .collect() } -fn pairs_to_selections(parent: T, pairs: &[FieldPair<'_>]) -> Vec +fn pairs_to_selections( + parent: T, + pairs: &[FieldPair<'_>], + query_schema: &QuerySchema, +) -> QueryGraphBuilderResult> where T: Into, { + let should_collect_relation_selection = query_schema.has_capability(ConnectorCapability::LateralJoin) + && query_schema.has_feature(PreviewFeature::RelationJoins); + let parent = parent.into(); - pairs + + let selected_fields = pairs .iter() .filter_map(|pair| { parent @@ -73,29 +86,67 @@ where .map(|field| (pair.parsed_field.clone(), field)) }) .flat_map(|field| match field { - (_, Field::Relation(rf)) => rf.scalar_fields().into_iter().map(Into::into).collect(), - (_, Field::Scalar(sf)) => vec![sf.into()], - (pf, Field::Composite(cf)) => vec![extract_composite_selection(pf, cf)], + (pf, Field::Relation(rf)) => { + let mut fields: Vec> = rf + .scalar_fields() + .into_iter() + .map(SelectedField::from) + .map(Ok) + .collect(); + + if should_collect_relation_selection { + fields.push(extract_relation_selection(pf, rf, query_schema)); + } + + fields + } + (_, Field::Scalar(sf)) => vec![Ok(sf.into())], + (pf, Field::Composite(cf)) => vec![extract_composite_selection(pf, cf, query_schema)], }) - .collect() + .collect::, _>>()?; + + Ok(selected_fields) } -fn extract_composite_selection(pf: ParsedField<'_>, cf: CompositeFieldRef) -> SelectedField { +fn extract_composite_selection( + pf: ParsedField<'_>, + cf: CompositeFieldRef, + query_schema: &QuerySchema, +) -> QueryGraphBuilderResult { let object = pf .nested_fields .expect("Invalid composite query shape: Composite field selected without sub-selection."); let typ = cf.typ(); - SelectedField::Composite(CompositeSelection { + Ok(SelectedField::Composite(CompositeSelection { field: cf, - selections: pairs_to_selections(typ, &object.fields), - }) + selections: pairs_to_selections(typ, &object.fields, query_schema)?, + })) +} + +fn extract_relation_selection( + pf: ParsedField<'_>, + rf: RelationFieldRef, + query_schema: &QuerySchema, +) -> QueryGraphBuilderResult { + let object = pf + .nested_fields + .expect("Invalid composite query shape: Composite field selected without sub-selection."); + + let related_model = rf.related_model(); + + Ok(SelectedField::Relation(RelationSelection { + field: rf, + args: extract_query_args(pf.arguments, &related_model)?, + selections: pairs_to_selections(related_model, &object.fields, query_schema)?, + })) } pub(crate) fn collect_nested_queries( from: Vec>, model: &Model, + query_schema: &QuerySchema, ) -> QueryGraphBuilderResult> { from.into_iter() .filter_map(|pair| { @@ -112,7 +163,7 @@ pub(crate) fn collect_nested_queries( let model = rf.related_model(); let parent = rf.clone(); - Some(related::find_related(pair.parsed_field, parent, model)) + Some(related::find_related(pair.parsed_field, parent, model, query_schema)) } } }) @@ -189,3 +240,25 @@ pub fn collect_relation_aggr_selections( Ok(selections) } + +pub(crate) fn get_relation_load_strategy( + args: &QueryArguments, + nested_queries: &[ReadQuery], + aggregation_selections: &[RelAggregationSelection], + query_schema: &QuerySchema, +) -> RelationLoadStrategy { + if query_schema.has_feature(PreviewFeature::RelationJoins) + && query_schema.has_capability(ConnectorCapability::LateralJoin) + && args.cursor.is_none() + && args.distinct.is_none() + && !nested_queries.iter().any(|q| match q { + ReadQuery::RelatedRecordsQuery(q) => q.has_cursor() || q.has_distinct(), + _ => false, + }) + && aggregation_selections.is_empty() + { + RelationLoadStrategy::Join + } else { + RelationLoadStrategy::Query + } +} diff --git a/query-engine/core/src/query_graph_builder/write/create.rs b/query-engine/core/src/query_graph_builder/write/create.rs index 59661c6c16b3..dc6ac8dd8207 100644 --- a/query-engine/core/src/query_graph_builder/write/create.rs +++ b/query-engine/core/src/query_graph_builder/write/create.rs @@ -32,7 +32,7 @@ pub(crate) fn create_record( let create_node = create::create_record_node(graph, query_schema, model.clone(), data_map)?; // Follow-up read query on the write - let read_query = read::find_unique(field, model.clone())?; + let read_query = read::find_unique(field, model.clone(), query_schema)?; let read_node = graph.create_node(Query::Read(read_query)); graph.add_result_node(&read_node); diff --git a/query-engine/core/src/query_graph_builder/write/delete.rs b/query-engine/core/src/query_graph_builder/write/delete.rs index df6a66436022..c6a7c28e2d85 100644 --- a/query-engine/core/src/query_graph_builder/write/delete.rs +++ b/query-engine/core/src/query_graph_builder/write/delete.rs @@ -21,7 +21,7 @@ pub(crate) fn delete_record( let filter = extract_unique_filter(where_arg.value.try_into()?, &model)?; // Prefetch read query for the delete - let mut read_query = read::find_unique(field, model.clone())?; + let mut read_query = read::find_unique(field, model.clone(), query_schema)?; read_query.add_filter(filter.clone()); let read_node = graph.create_node(Query::Read(read_query)); diff --git a/query-engine/core/src/query_graph_builder/write/update.rs b/query-engine/core/src/query_graph_builder/write/update.rs index 001e2b48a96d..5c2054d5d949 100644 --- a/query-engine/core/src/query_graph_builder/write/update.rs +++ b/query-engine/core/src/query_graph_builder/write/update.rs @@ -88,7 +88,7 @@ pub(crate) fn update_record( } else { graph.flag_transactional(); - let read_query = read::find_unique(field, model.clone())?; + let read_query = read::find_unique(field, model.clone(), query_schema)?; let read_node = graph.create_node(Query::Read(read_query)); graph.add_result_node(&read_node); diff --git a/query-engine/core/src/query_graph_builder/write/upsert.rs b/query-engine/core/src/query_graph_builder/write/upsert.rs index 92fcd6d12efd..5cba49c51533 100644 --- a/query-engine/core/src/query_graph_builder/write/upsert.rs +++ b/query-engine/core/src/query_graph_builder/write/upsert.rs @@ -70,7 +70,7 @@ pub(crate) fn upsert_record( ); let filter = extract_unique_filter(where_argument, &model)?; - let read_query = read::find_unique(field.clone(), model.clone())?; + let read_query = read::find_unique(field.clone(), model.clone(), query_schema)?; if can_use_native_upsert { if let ReadQuery::RecordQuery(read) = read_query { diff --git a/query-engine/core/src/query_graph_builder/write/utils.rs b/query-engine/core/src/query_graph_builder/write/utils.rs index 2f2e736aedaf..6feccd912034 100644 --- a/query-engine/core/src/query_graph_builder/write/utils.rs +++ b/query-engine/core/src/query_graph_builder/write/utils.rs @@ -44,6 +44,7 @@ where selection_order: vec![], aggregation_selections: vec![], options: QueryOptions::none(), + relation_load_strategy: query_structure::RelationLoadStrategy::Query, }); Query::Read(read_query) diff --git a/query-engine/query-structure/src/query_arguments.rs b/query-engine/query-structure/src/query_arguments.rs index f9c222d80dbe..4c1ab72673cf 100644 --- a/query-engine/query-structure/src/query_arguments.rs +++ b/query-engine/query-structure/src/query_arguments.rs @@ -12,7 +12,7 @@ use crate::*; /// A query argument struct is always valid over a single model only, meaning that all /// data referenced in a single query argument instance is always refering to data of /// a single model (e.g. the cursor projection, distinct projection, orderby, ...). -#[derive(Clone)] +#[derive(Clone, PartialEq, Eq, Hash)] pub struct QueryArguments { pub model: Model, pub cursor: Option, @@ -25,6 +25,12 @@ pub struct QueryArguments { pub ignore_take: bool, } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum RelationLoadStrategy { + Join, + Query, +} + impl std::fmt::Debug for QueryArguments { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("QueryArguments") diff --git a/query-engine/schema/src/query_schema.rs b/query-engine/schema/src/query_schema.rs index 0324896aea07..3098a96f1597 100644 --- a/query-engine/schema/src/query_schema.rs +++ b/query-engine/schema/src/query_schema.rs @@ -96,7 +96,7 @@ impl QuerySchema { || self.has_capability(ConnectorCapability::FullTextSearchWithIndex)) } - pub(crate) fn has_feature(&self, feature: PreviewFeature) -> bool { + pub fn has_feature(&self, feature: PreviewFeature) -> bool { self.preview_features.contains(feature) } From fb4ada02c2037363c74633eeb8db08c79ac1ee25 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 16:39:17 +0100 Subject: [PATCH 09/38] update json coercion to use RelationSelection + reverse in memory when needed --- .../src/database/operations/coerce.rs | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs index 55673a40cdaf..61ee6389d184 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs @@ -1,46 +1,53 @@ -use connector_interface::RelatedQuery; -use prisma_models::*; -use prisma_value::PrismaValue; +use itertools::{Either, Itertools}; +use query_structure::*; + +use crate::query_arguments_ext::QueryArgumentsExt; // TODO: find better name -pub(crate) fn coerce_record_with_join(record: &mut Record, rq_indexes: Vec<(usize, &RelatedQuery)>) { - for (val_idx, rq) in rq_indexes { +pub(crate) fn coerce_record_with_join(record: &mut Record, rq_indexes: Vec<(usize, &RelationSelection)>) { + for (val_idx, rs) in rq_indexes { let val = record.values.get_mut(val_idx).unwrap(); let json_val: serde_json::Value = serde_json::from_str(&val.as_json().unwrap()).unwrap(); - *val = coerce_json_relation_to_pv(json_val, rq); + *val = coerce_json_relation_to_pv(json_val, rs); } } // TODO: find better name -pub(crate) fn coerce_json_relation_to_pv(value: serde_json::Value, q: &RelatedQuery) -> PrismaValue { +pub(crate) fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &RelationSelection) -> PrismaValue { + let relations = rs.relations().collect_vec(); + match value { // one-to-many - serde_json::Value::Array(values) if q.parent_field.is_list() => PrismaValue::List( - values - .into_iter() - .map(|value| coerce_json_relation_to_pv(value, q)) - .collect(), - ), + serde_json::Value::Array(values) if rs.field.is_list() => { + let iter = values.into_iter().map(|value| coerce_json_relation_to_pv(value, rs)); + + // Reverses order when using negative take. + let iter = match rs.args.needs_reversed_order() { + true => Either::Left(iter.rev()), + false => Either::Right(iter), + }; + + PrismaValue::List(iter.collect()) + } // to-one serde_json::Value::Array(values) => { let coerced = values .into_iter() .next() - .map(|value| coerce_json_relation_to_pv(value, q)); + .map(|value| coerce_json_relation_to_pv(value, rs)); // TODO(HACK): We probably want to update the sql builder instead to not aggregate to-one relations as array - // If the arary is empty, it means there's no relations + // If the arary is empty, it means there's no relations, so we coerce it to if let Some(val) = coerced { val - // else the relation's null } else { PrismaValue::Null } } serde_json::Value::Object(obj) => { let mut map: Vec<(String, PrismaValue)> = Vec::with_capacity(obj.len()); - let related_model = q.parent_field.related_model(); + let related_model = rs.field.related_model(); for (key, value) in obj { match related_model.fields().all().find(|f| f.db_name() == key).unwrap() { @@ -49,17 +56,11 @@ pub(crate) fn coerce_json_relation_to_pv(value: serde_json::Value, q: &RelatedQu } Field::Relation(rf) => { // TODO: optimize this - if let Some(rq) = q - .nested - .as_ref() - .unwrap() - .iter() - .find(|rq| rq.parent_field.name() == rf.name()) - { - map.push((key, coerce_json_relation_to_pv(value, rq))); + if let Some(nested_selection) = relations.iter().find(|rs| rs.field == rf) { + map.push((key, coerce_json_relation_to_pv(value, nested_selection))); } } - _ => unreachable!(), + Field::Composite(_) => unreachable!(), } } From d3e9322cbf557a01422d61869d676225344dee40 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 16:39:43 +0100 Subject: [PATCH 10/38] split get_many_records based on relation load strategy --- .../src/database/operations/read.rs | 66 +++++++++++++------ 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index 71e85ce9baf4..a79878d1137d 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -55,62 +55,89 @@ pub(crate) async fn get_single_record( Ok(record) } +pub(crate) async fn get_many_records( + conn: &dyn Queryable, + model: &Model, + query_arguments: QueryArguments, + selected_fields: &FieldSelection, + aggr_selections: &[RelAggregationSelection], + relation_load_strategy: RelationLoadStrategy, + ctx: &Context<'_>, +) -> crate::Result { + match relation_load_strategy { + RelationLoadStrategy::Join => { + get_many_records_joins(conn, model, query_arguments, selected_fields, aggr_selections, ctx).await + } + RelationLoadStrategy::Query => { + get_many_records_wo_joins( + conn, + model, + query_arguments, + &ModelProjection::from(selected_fields), + aggr_selections, + ctx, + ) + .await + } + } +} + pub(crate) async fn get_many_records_joins( conn: &dyn Queryable, _model: &Model, query_arguments: QueryArguments, - selected_fields: &ModelProjection, - nested: Vec, + selected_fields: &FieldSelection, _aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> crate::Result { - let mut field_names: Vec<_> = selected_fields.db_names().collect(); - field_names.extend(nested.iter().map(|n| n.parent_field.name().to_owned())); - - let mut idents = selected_fields.type_identifiers_with_arities(); - idents.extend(nested.iter().map(|_| (TypeIdentifier::Json, FieldArity::Required))); - + let field_names: Vec<_> = selected_fields.db_names().collect(); + let idents = selected_fields.type_identifiers_with_arities(); let meta = column_metadata::create(field_names.as_slice(), idents.as_slice()); - let rq_indexes = related_queries_indexes(&nested, field_names.as_slice()); + let rs_indexes = relation_selection_indexes(selected_fields.relations().collect(), &field_names); + // dbg!(&rs_indexes); let mut records = ManyRecords::new(field_names.clone()); - let query = query_builder::select::build(query_arguments.clone(), nested.clone(), selected_fields, &[], ctx); + let query = query_builder::select::SelectBuilder::default().build(query_arguments.clone(), selected_fields, ctx); for item in conn.filter(query.into(), meta.as_slice(), ctx).await?.into_iter() { let mut record = Record::from(item); // Coerces json values to prisma values - coerce_record_with_join(&mut record, rq_indexes.clone()); + coerce_record_with_join(&mut record, rs_indexes.clone()); records.push(record) } + // Reverses order when using negative take + if query_arguments.needs_reversed_order() { + records.reverse(); + } + Ok(records) } // TODO: find better name -fn related_queries_indexes<'a>( - related_queries: &'a [RelatedQuery], +fn relation_selection_indexes<'a>( + selections: Vec<&'a RelationSelection>, field_names: &[String], -) -> Vec<(usize, &'a RelatedQuery)> { - let mut output: Vec<(usize, &RelatedQuery)> = Vec::new(); +) -> Vec<(usize, &'a RelationSelection)> { + let mut output: Vec<(usize, &RelationSelection)> = Vec::new(); for (idx, field_name) in field_names.iter().enumerate() { - if let Some(rq) = related_queries.iter().find(|rq| rq.name == *field_name) { - output.push((idx, rq)); + if let Some(rs) = selections.iter().find(|rq| rq.field.name() == *field_name) { + output.push((idx, rs)); } } output } -pub(crate) async fn get_many_records( +pub(crate) async fn get_many_records_wo_joins( conn: &dyn Queryable, model: &Model, mut query_arguments: QueryArguments, selected_fields: &ModelProjection, - nested: Vec, aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> crate::Result { @@ -190,7 +217,6 @@ pub(crate) async fn get_many_records( selected_fields.as_columns(ctx).mark_all_selected(), aggr_selections, query_arguments, - nested, ctx, ); From 3d5ca92cb8d15bac5c434a33f989761b0fcc1aa8 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 16:40:05 +0100 Subject: [PATCH 11/38] update join query builder to better support filters, ordering and pagination --- .../sql-query-connector/src/filter/alias.rs | 6 +- .../sql-query-connector/src/filter/mod.rs | 4 +- .../sql-query-connector/src/filter/visitor.rs | 7 +- .../sql-query-connector/src/ordering.rs | 14 +- .../src/query_builder/read.rs | 20 +- .../src/query_builder/select.rs | 500 ++++++++++-------- query-engine/core/src/response_ir/internal.rs | 6 +- 7 files changed, 319 insertions(+), 238 deletions(-) diff --git a/query-engine/connectors/sql-query-connector/src/filter/alias.rs b/query-engine/connectors/sql-query-connector/src/filter/alias.rs index c7a62bba02ab..af3ad932748a 100644 --- a/query-engine/connectors/sql-query-connector/src/filter/alias.rs +++ b/query-engine/connectors/sql-query-connector/src/filter/alias.rs @@ -16,7 +16,7 @@ pub enum AliasMode { #[derive(Clone, Copy, Debug, Default)] /// Aliasing tool to count the nesting level to help with heavily nested /// self-related queries. -pub(crate) struct Alias { +pub struct Alias { counter: usize, mode: AliasMode, } @@ -49,6 +49,10 @@ impl Alias { AliasMode::Join => format!("j{}", self.counter), } } + + pub fn to_table_string(&self) -> String { + self.to_string(Some(AliasMode::Table)) + } } pub(crate) trait AliasedColumn { diff --git a/query-engine/connectors/sql-query-connector/src/filter/mod.rs b/query-engine/connectors/sql-query-connector/src/filter/mod.rs index b9ae856ef655..573024845b45 100644 --- a/query-engine/connectors/sql-query-connector/src/filter/mod.rs +++ b/query-engine/connectors/sql-query-connector/src/filter/mod.rs @@ -1,9 +1,9 @@ -mod alias; +pub mod alias; mod visitor; use quaint::prelude::*; use query_structure::Filter; -use visitor::*; +pub use visitor::*; use crate::{context::Context, join_utils::AliasedJoin}; diff --git a/query-engine/connectors/sql-query-connector/src/filter/visitor.rs b/query-engine/connectors/sql-query-connector/src/filter/visitor.rs index 1a71cdd824a8..b27ab539e604 100644 --- a/query-engine/connectors/sql-query-connector/src/filter/visitor.rs +++ b/query-engine/connectors/sql-query-connector/src/filter/visitor.rs @@ -27,7 +27,7 @@ pub(crate) trait FilterVisitorExt { } #[derive(Debug, Clone, Default)] -pub(crate) struct FilterVisitor { +pub struct FilterVisitor { /// The last alias that's been rendered. last_alias: Option, /// The parent alias, used when rendering nested filters so that a child filter can refer to its join. @@ -68,6 +68,11 @@ impl FilterVisitor { self.parent_alias } + pub fn set_parent_alias_opt(mut self, alias: Option) -> Self { + self.parent_alias = alias; + self + } + /// A top-level join can be rendered if we're explicitly allowing it or if we're in a nested visitor. fn can_render_join(&self) -> bool { self.with_top_level_joins || self.is_nested diff --git a/query-engine/connectors/sql-query-connector/src/ordering.rs b/query-engine/connectors/sql-query-connector/src/ordering.rs index 310e10ec43d7..2332f28e643b 100644 --- a/query-engine/connectors/sql-query-connector/src/ordering.rs +++ b/query-engine/connectors/sql-query-connector/src/ordering.rs @@ -150,9 +150,14 @@ impl OrderByBuilder { // Unwraps are safe because the SQL connector doesn't yet support any other type of orderBy hop but the relation hop. let mut joins: Vec = vec![]; + let parent_alias = self.parent_alias.clone(); + for (i, hop) in rest_hops.iter().enumerate() { let previous_join = if i > 0 { joins.get(i - 1) } else { None }; - let previous_alias = previous_join.map(|j| j.alias.as_str()); + + let previous_alias = previous_join + .map(|j| j.alias.as_str()) + .or_else(|| parent_alias.as_deref()); let join = compute_one2m_join(hop.as_relation_hop().unwrap(), &self.join_prefix(), previous_alias, ctx); joins.push(join); @@ -192,9 +197,14 @@ impl OrderByBuilder { ) -> (Vec, Column<'static>) { let mut joins: Vec = vec![]; + let parent_alias = self.parent_alias.clone(); + for (i, hop) in order_by.path.iter().enumerate() { let previous_join = if i > 0 { joins.get(i - 1) } else { None }; - let previous_alias = previous_join.map(|j| j.alias.as_str()); + let previous_alias = previous_join + .map(|j| &j.alias) + .or(parent_alias.as_ref()) + .map(|alias| alias.as_str()); let join = compute_one2m_join(hop.as_relation_hop().unwrap(), &self.join_prefix(), previous_alias, ctx); joins.push(join); diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/read.rs b/query-engine/connectors/sql-query-connector/src/query_builder/read.rs index 6291ce0cd165..3aa91288ea90 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/read.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/read.rs @@ -2,19 +2,16 @@ use crate::{ cursor_condition, filter::FilterBuilder, model_extensions::*, nested_aggregations, ordering::OrderByBuilder, sql_trace::SqlTraceComment, Context, }; -use connector_interface::{AggregationSelection, RelAggregationSelection, RelatedQuery}; +use connector_interface::{AggregationSelection, RelAggregationSelection}; use itertools::Itertools; use quaint::ast::*; use query_structure::*; use tracing::Span; -use super::select; - pub(crate) trait SelectDefinition { fn into_select( self, _: &Model, - nested: Vec, aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> (Select<'static>, Vec>); @@ -24,12 +21,11 @@ impl SelectDefinition for Filter { fn into_select( self, model: &Model, - nested: Vec, aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> (Select<'static>, Vec>) { let args = QueryArguments::from((model.clone(), self)); - args.into_select(model, nested, aggr_selections, ctx) + args.into_select(model, aggr_selections, ctx) } } @@ -37,11 +33,10 @@ impl SelectDefinition for &Filter { fn into_select( self, model: &Model, - nested: Vec, aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> (Select<'static>, Vec>) { - self.clone().into_select(model, nested, aggr_selections, ctx) + self.clone().into_select(model, aggr_selections, ctx) } } @@ -49,7 +44,6 @@ impl SelectDefinition for Select<'static> { fn into_select( self, _: &Model, - _: Vec, _: &[RelAggregationSelection], _ctx: &Context<'_>, ) -> (Select<'static>, Vec>) { @@ -61,7 +55,6 @@ impl SelectDefinition for QueryArguments { fn into_select( self, model: &Model, - nested: Vec, aggr_selections: &[RelAggregationSelection], ctx: &Context<'_>, ) -> (Select<'static>, Vec>) { @@ -125,13 +118,12 @@ pub(crate) fn get_records( columns: impl Iterator>, aggr_selections: &[RelAggregationSelection], query: T, - nested: Vec, ctx: &Context<'_>, ) -> Select<'static> where T: SelectDefinition, { - let (select, additional_selection_set) = query.into_select(model, nested, aggr_selections, ctx); + let (select, additional_selection_set) = query.into_select(model, aggr_selections, ctx); let select = columns.fold(select, |acc, col| acc.column(col)); let select = select.append_trace(&Span::current()).add_trace_id(ctx.trace_id); @@ -174,7 +166,7 @@ pub(crate) fn aggregate( ctx: &Context<'_>, ) -> Select<'static> { let columns = extract_columns(model, selections, ctx); - let sub_query = get_records(model, columns.into_iter(), &[], args, Vec::new(), ctx); + let sub_query = get_records(model, columns.into_iter(), &[], args, ctx); let sub_table = Table::from(sub_query).alias("sub"); selections.iter().fold( @@ -231,7 +223,7 @@ pub(crate) fn group_by_aggregate( having: Option, ctx: &Context<'_>, ) -> Select<'static> { - let (base_query, _) = args.into_select(model, Vec::new(), &[], ctx); + let (base_query, _) = args.into_select(model, &[], ctx); let select_query = selections.iter().fold(base_query, |select, next_op| match next_op { AggregationSelection::Field(field) => select.column(field.as_column(ctx).set_is_selected(true)), diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs index b5ccae5f730d..2ca856edbbdb 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs @@ -2,264 +2,322 @@ use std::borrow::Cow; use crate::{ context::Context, - filter::FilterBuilder, + filter::alias::{Alias, AliasMode}, model_extensions::{AsColumn, AsColumns, AsTable, RelationFieldExt}, ordering::OrderByBuilder, }; -use connector_interface::{Filter, QueryArguments, RelAggregationSelection, RelatedQuery}; use itertools::Itertools; -use prisma_models::{ModelProjection, RelationField, ScalarField}; use quaint::prelude::*; +use query_structure::*; pub const JSON_AGG_IDENT: &str = "data"; -pub(crate) fn build( - args: QueryArguments, - nested: Vec, - selection: &ModelProjection, - _aggr_selections: &[RelAggregationSelection], - ctx: &Context<'_>, -) -> Select<'static> { - // SELECT ... FROM Table - let select = Select::from_table(args.model().as_table(ctx)); - - // scalars selection - let select = selection - .scalar_fields() - .fold(select, |acc, sf| acc.column(sf.as_column(ctx))); - - // TODO: check how to select aggregated relations - // Adds relation selections to the top-level query - let select = nested.iter().fold(select, |acc, read| { - let table_name = match read.parent_field.relation().is_many_to_many() { - true => m2m_join_alias_name(&read.parent_field), - false => join_alias_name(&read.parent_field), - }; - - acc.value(Column::from((table_name, JSON_AGG_IDENT)).alias(read.name.to_owned())) - }); - - // Adds joins for relations - let select = with_related_queries(select, nested, ctx); - let select = with_ordering(select, &args, None, ctx); - let select = with_pagination(select, args.take, args.skip); - let select = with_filters(select, args.filter, ctx); - - select +#[derive(Debug, Default)] +pub(crate) struct SelectBuilder { + alias: Alias, } -fn with_related_queries<'a>(input: Select<'a>, related_queries: Vec, ctx: &Context<'_>) -> Select<'a> { - related_queries - .into_iter() - .fold(input, |acc, rq| with_related_query(acc, rq, ctx)) -} - -fn with_related_query<'a>(select: Select<'a>, rq: RelatedQuery, ctx: &Context<'_>) -> Select<'a> { - if rq.parent_field.relation().is_many_to_many() { - let m2m_join = build_m2m_join(rq, ctx); - - // m2m relations need to left join on the relation table first - select.left_join(m2m_join) - } else { - let alias = join_alias_name(&rq.parent_field); +impl SelectBuilder { + pub(crate) fn next_alias(&mut self) -> Alias { + self.alias = self.alias.inc(AliasMode::Table); + self.alias + } - // LEFT JOIN LATERAL () AS ON TRUE - let join_select = Table::from(build_related_query_select(rq, ctx)) - .alias(alias) - .on(ConditionTree::single(true.raw())) - .lateral(); + pub(crate) fn build( + &mut self, + args: QueryArguments, + selected_fields: &FieldSelection, + ctx: &Context<'_>, + ) -> Select<'static> { + let table_alias = self.next_alias(); + + // SELECT ... FROM Table "t1" + let select = Select::from_table(args.model().as_table(ctx).alias(table_alias.to_table_string())); + + // TODO: check how to select aggregated relations + let select = selected_fields + .selections() + .fold(select, |acc, selection| match selection { + SelectedField::Scalar(sf) => acc.column(sf.as_column(ctx).table(table_alias.to_table_string())), + SelectedField::Relation(rs) => { + let table_name = match rs.field.relation().is_many_to_many() { + true => m2m_join_alias_name(&rs.field), + false => join_alias_name(&rs.field), + }; + + acc.value(Column::from((table_name, JSON_AGG_IDENT)).alias(rs.field.name().to_owned())) + } + _ => acc, + }); + + // Adds joins for relations + let select = self.with_related_queries(select, selected_fields.relations(), table_alias, ctx); + let select = self.with_ordering(select, &args, Some(table_alias.to_table_string()), ctx); + let select = self.with_pagination(select, args.take_abs(), args.skip); + let select = self.with_filters(select, args.filter, Some(table_alias), ctx); - select.left_join(join_select) + select } -} -fn build_related_query_select(rq: RelatedQuery, ctx: &Context<'_>) -> Select<'static> { - let mut fields_to_select: Vec = vec![]; - - let mut build_obj_params = ModelProjection::from(rq.selected_fields) - .fields() - .map(|f| match f { - prisma_models::Field::Scalar(sf) => { - (Cow::from(sf.db_name().to_owned()), Expression::from(sf.as_column(ctx))) - } - _ => unreachable!(), - }) - .collect_vec(); - - if let Some(nested_queries) = &rq.nested { - for nested_query in nested_queries { - let table_name = match nested_query.parent_field.relation().is_many_to_many() { - true => m2m_join_alias_name(&nested_query.parent_field), - false => join_alias_name(&nested_query.parent_field), - }; + fn with_related_queries<'a, 'b>( + &mut self, + input: Select<'a>, + relation_selections: impl Iterator, + parent_alias: Alias, + ctx: &Context<'_>, + ) -> Select<'a> { + relation_selections.fold(input, |acc, rs| self.with_related_query(acc, rs, parent_alias, ctx)) + } - build_obj_params.push(( - Cow::from(nested_query.name.to_owned()), - Expression::from(Column::from((table_name, JSON_AGG_IDENT))), - )); + fn with_related_query<'a>( + &mut self, + select: Select<'a>, + rs: &RelationSelection, + parent_alias: Alias, + ctx: &Context<'_>, + ) -> Select<'a> { + if rs.field.relation().is_many_to_many() { + // m2m relations need to left join on the relation table first + let m2m_join = self.build_m2m_join(rs, parent_alias, ctx); + + select.left_join(m2m_join) + } else { + // LEFT JOIN LATERAL () AS ON TRUE + let join_select = Table::from(self.build_related_query_select(rs, parent_alias, ctx)) + .alias(join_alias_name(&rs.field)) + .on(ConditionTree::single(true.raw())) + .lateral(); + + select.left_join(join_select) } } - let inner_alias = join_alias_name(&rq.parent_field.related_field()); - - // SELECT JSON_BUILD_OBJECT() - let inner = Select::from_table(rq.parent_field.related_model().as_table(ctx)) - .value(json_build_object(build_obj_params).alias(JSON_AGG_IDENT)); - - // SELECT - let inner = ModelProjection::from(rq.parent_field.related_field().linking_fields()) - .as_columns(ctx) - .fold(inner, |acc, c| acc.column(c)); + fn build_related_query_select( + &mut self, + rs: &RelationSelection, + parent_alias: Alias, + ctx: &Context<'_>, + ) -> Select<'static> { + let table_alias = self.next_alias(); - let inner = with_join_conditions(inner, &rq.parent_field, ctx); + dbg!(&rs); - let inner = if let Some(nested) = rq.nested { - with_related_queries(inner, nested, ctx) - } else { - inner - }; - - if rq.parent_field.relation().is_many_to_many() { - // SELECT ONLY if it's a m2m table as we need to order by outside of the inner select - let inner = rq - .args - .order_by + let build_obj_params = rs + .selections .iter() - .flat_map(|order_by| match order_by { - prisma_models::OrderBy::Scalar(x) if x.path.is_empty() => vec![x.field.clone()], - prisma_models::OrderBy::Relevance(x) => x.fields.clone(), - _ => Vec::new(), + .filter_map(|f| match f { + SelectedField::Scalar(sf) => Some(( + Cow::from(sf.db_name().to_owned()), + Expression::from(sf.as_column(ctx).table(table_alias.to_table_string())), + )), + SelectedField::Relation(rs) => { + let table_name = match rs.field.relation().is_many_to_many() { + true => m2m_join_alias_name(&rs.field), + false => join_alias_name(&rs.field), + }; + + Some(( + Cow::from(rs.field.name().to_owned()), + Expression::from(Column::from((table_name, JSON_AGG_IDENT))), + )) + } + _ => None, }) - .fold(inner, |acc, sf| acc.column(sf.as_column(ctx))); - - inner - } else { - let inner = with_ordering(inner, &rq.args, None, ctx); - let inner = with_pagination(inner, rq.args.take, rq.args.skip); - let inner = with_filters(inner, rq.args.filter, ctx); - - let inner = Table::from(inner).alias(inner_alias.clone()); - let middle = Select::from_table(inner).column(Column::from((inner_alias.clone(), JSON_AGG_IDENT))); - let outer = Select::from_table(Table::from(middle).alias(format!("{}_1", inner_alias))).value(json_agg()); - - outer + .collect_vec(); + + let inner_alias = join_alias_name(&rs.field.related_field()); + + let related_table = rs + .field + .related_model() + .as_table(ctx) + .alias(table_alias.to_table_string()); + + // SELECT JSON_BUILD_OBJECT() + let inner = Select::from_table(related_table).value(json_build_object(build_obj_params).alias(JSON_AGG_IDENT)); + + // WHERE parent.id = child.parent_id + let inner = self.with_join_conditions(inner, &rs.field, parent_alias, table_alias, ctx); + // LEFT JOIN LATERAL () AS ON TRUE + let inner = self.with_related_queries(inner, rs.relations(), table_alias, ctx); + + let linking_fields = rs.field.related_field().linking_fields(); + + if rs.field.relation().is_many_to_many() { + let order_by_selection = rs + .args + .order_by + .iter() + .flat_map(|order_by| match order_by { + OrderBy::Scalar(x) if x.path.is_empty() => vec![x.field.clone()], + OrderBy::Relevance(x) => x.fields.clone(), + _ => Vec::new(), + }) + .collect_vec(); + let selection = FieldSelection::union(vec![FieldSelection::from(order_by_selection), linking_fields]); + + // SELECT + // SELECT ONLY if it's a m2m table as we need to order by outside of the inner select + let inner = ModelProjection::from(selection) + .as_columns(ctx) + .fold(inner, |acc, c| acc.column(c.table(table_alias.to_table_string()))); + + inner + } else { + // SELECT + let inner = ModelProjection::from(linking_fields) + .as_columns(ctx) + .fold(inner, |acc, c| acc.column(c.table(table_alias.to_table_string()))); + + let inner = self.with_ordering(inner, &rs.args, Some(table_alias.to_table_string()), ctx); + let inner = self.with_pagination(inner, rs.args.take_abs(), rs.args.skip); + let inner = self.with_filters(inner, rs.args.filter.clone(), Some(table_alias), ctx); + + let inner = Table::from(inner).alias(inner_alias.clone()); + let middle = Select::from_table(inner).column(Column::from((inner_alias.clone(), JSON_AGG_IDENT))); + let outer = Select::from_table(Table::from(middle).alias(format!("{}_1", inner_alias))).value(json_agg()); + + outer + } } -} - -fn build_m2m_join<'a>(rq: RelatedQuery, ctx: &Context<'_>) -> JoinData<'a> { - let rf = rq.parent_field.clone(); - let m2m_alias = m2m_join_alias_name(&rf); - - let left_columns = rf.related_field().m2m_columns(ctx); - let right_columns = ModelProjection::from(rf.model().primary_identifier()).as_columns(ctx); - let conditions = left_columns - .into_iter() - .zip(right_columns) - .fold(None::, |acc, (a, b)| match acc { - Some(acc) => Some(acc.and(a.equals(b))), - None => Some(a.equals(b).into()), - }) - .unwrap(); - - let inner = Select::from_table(rf.as_table(ctx)) - .value(Column::from((join_alias_name(&rf), JSON_AGG_IDENT))) - .and_where(conditions); + fn build_m2m_join<'a>(&mut self, rs: &RelationSelection, parent_alias: Alias, ctx: &Context<'_>) -> JoinData<'a> { + let rf = rs.field.clone(); + let m2m_alias = m2m_join_alias_name(&rf); + let m2m_table_alias = self.next_alias(); + + let left_columns = rf.related_field().m2m_columns(ctx); + let right_columns = ModelProjection::from(rf.model().primary_identifier()).as_columns(ctx); + + let conditions = left_columns + .into_iter() + .zip(right_columns) + .fold(None::, |acc, (a, b)| { + let a = a.table(m2m_table_alias.to_table_string()); + let b = b.table(parent_alias.to_table_string()); + let condition = a.equals(b); + + match acc { + Some(acc) => Some(acc.and(condition)), + None => Some(condition.into()), + } + }) + .unwrap(); - let inner = with_ordering(inner, &rq.args, Some(join_alias_name(&rq.parent_field)), ctx); - let inner = with_pagination(inner, rq.args.take, rq.args.skip); - // TODO: avoid clone? - let inner = with_filters(inner, rq.args.filter.clone(), ctx); + let inner = Select::from_table(rf.as_table(ctx).alias(m2m_table_alias.to_table_string())) + .value(Column::from((join_alias_name(&rf), JSON_AGG_IDENT))) + .and_where(conditions); - let join_select = Table::from(build_related_query_select(rq, ctx)) - .alias(join_alias_name(&rf)) - .on(ConditionTree::single(true.raw())) - .lateral(); + let inner = self.with_ordering(inner, &rs.args, Some(join_alias_name(&rs.field)), ctx); + let inner = self.with_pagination(inner, rs.args.take_abs(), rs.args.skip); + // TODO: avoid clone? + let inner = self.with_filters(inner, rs.args.filter.clone(), None, ctx); - let inner = inner.left_join(join_select); + // TODO: parent_alias is likely wrong here + let join_select = Table::from(self.build_related_query_select(rs, m2m_table_alias, ctx)) + .alias(join_alias_name(&rf)) + .on(ConditionTree::single(true.raw())) + .lateral(); - let outer = Select::from_table(Table::from(inner).alias(format!("{}_1", m2m_alias))).value(json_agg()); + let inner = inner.left_join(join_select); - Table::from(outer) - .alias(m2m_alias) - .on(ConditionTree::single(true.raw())) - .lateral() -} + let outer = Select::from_table(Table::from(inner).alias(format!("{}_1", m2m_alias))).value(json_agg()); -fn json_agg() -> Function<'static> { - coalesce(vec![ - json_array_agg(Column::from(JSON_AGG_IDENT)).into(), - Expression::from("[]".raw()), - ]) - .alias(JSON_AGG_IDENT) -} - -/// Builds the lateral join conditions -fn with_join_conditions<'a>(select: Select<'a>, rf: &RelationField, ctx: &Context<'_>) -> Select<'a> { - let join_columns = rf.join_columns(ctx); - // .map(|c| c.opt_table(is_m2m.then(|| m2m_join_alias_name(rf)))); - let related_join_columns = ModelProjection::from(rf.related_field().linking_fields()).as_columns(ctx); - - // WHERE Parent.id = Child.id - let conditions = join_columns - .zip(related_join_columns) - .fold(None::, |acc, (a, b)| match acc { - Some(acc) => Some(acc.and(a.equals(b))), - None => Some(a.equals(b).into()), - }) - .unwrap(); - - select.and_where(conditions) -} + Table::from(outer) + .alias(m2m_alias) + .on(ConditionTree::single(true.raw())) + .lateral() + } -fn with_ordering<'a>( - select: Select<'a>, - args: &QueryArguments, - parent_alias: Option, - ctx: &Context<'_>, -) -> Select<'a> { - let order_by_definitions = OrderByBuilder::default() - .with_parent_alias(parent_alias) - .build(args, ctx); - - let select = order_by_definitions - .iter() - .flat_map(|j| &j.joins) - .fold(select, |acc, join| acc.join(join.clone().data)); - - order_by_definitions - .iter() - .fold(select, |acc, o| acc.order_by(o.order_definition.clone())) -} + /// Builds the lateral join conditions + fn with_join_conditions<'a>( + &mut self, + select: Select<'a>, + rf: &RelationField, + parent_alias: Alias, + child_alias: Alias, + ctx: &Context<'_>, + ) -> Select<'a> { + let join_columns = rf.join_columns(ctx); + let related_join_columns = ModelProjection::from(rf.related_field().linking_fields()).as_columns(ctx); + + // WHERE Parent.id = Child.id + let conditions = join_columns + .zip(related_join_columns) + .fold(None::, |acc, (a, b)| { + let a = a.table(parent_alias.to_table_string()); + let b = b.table(child_alias.to_table_string()); + let condition = a.equals(b); + + match acc { + Some(acc) => Some(acc.and(condition)), + None => Some(condition.into()), + } + }) + .unwrap(); -fn with_pagination<'a>(select: Select<'a>, take: Option, skip: Option) -> Select<'a> { - let select = match take { - Some(take) => select.limit(take as usize), - None => select, - }; + select.and_where(conditions) + } - let select = match skip { - Some(skip) => select.offset(skip as usize), - None => select, - }; + fn with_ordering<'a>( + &mut self, + select: Select<'a>, + args: &QueryArguments, + parent_alias: Option, + ctx: &Context<'_>, + ) -> Select<'a> { + let order_by_definitions = OrderByBuilder::default() + .with_parent_alias(parent_alias) + .build(args, ctx); + + let select = order_by_definitions + .iter() + .flat_map(|j| &j.joins) + .fold(select, |acc, join| acc.join(join.clone().data)); - select -} + order_by_definitions + .iter() + .fold(select, |acc, o| acc.order_by(o.order_definition.clone())) + } -fn with_filters<'a>(select: Select<'a>, filter: Option, ctx: &Context<'_>) -> Select<'a> { - if let Some(filter) = filter { - let (filter, joins) = FilterBuilder::with_top_level_joins().visit_filter(filter, ctx); - let select = select.and_where(filter); + fn with_pagination<'a>(&mut self, select: Select<'a>, take: Option, skip: Option) -> Select<'a> { + let select = match take { + Some(take) => select.limit(take as usize), + None => select, + }; - let select = match joins { - Some(joins) => joins.into_iter().fold(select, |acc, join| acc.join(join.data)), + let select = match skip { + Some(skip) => select.offset(skip as usize), None => select, }; select - } else { - select + } + + fn with_filters<'a>( + &mut self, + select: Select<'a>, + filter: Option, + alias: Option, + ctx: &Context<'_>, + ) -> Select<'a> { + use crate::filter::*; + + if let Some(filter) = filter { + let mut visitor = crate::filter::FilterVisitor::with_top_level_joins().set_parent_alias_opt(alias); + let (filter, joins) = visitor.visit_filter(filter, ctx); + let select = select.and_where(filter); + + let select = match joins { + Some(joins) => joins.into_iter().fold(select, |acc, join| acc.join(join.data)), + None => select, + }; + + select + } else { + select + } } } @@ -270,3 +328,11 @@ fn join_alias_name(rf: &RelationField) -> String { fn m2m_join_alias_name(rf: &RelationField) -> String { format!("{}_{}_m2m", rf.model().name(), rf.name()) } + +fn json_agg() -> Function<'static> { + coalesce(vec![ + json_array_agg(Column::from(JSON_AGG_IDENT)).into(), + Expression::from("[]".raw()), + ]) + .alias(JSON_AGG_IDENT) +} diff --git a/query-engine/core/src/response_ir/internal.rs b/query-engine/core/src/response_ir/internal.rs index fd77628e353c..f5e8417aac4a 100644 --- a/query-engine/core/src/response_ir/internal.rs +++ b/query-engine/core/src/response_ir/internal.rs @@ -394,7 +394,7 @@ fn serialize_objects_with_relation( serialize_relation_selection(rrs, val, inner_typ, query_schema)?, ); } - _ => (), + _ => panic!("unexpected field"), } } @@ -413,6 +413,10 @@ fn serialize_relation_selection( typ: &ObjectType<'_>, query_schema: &QuerySchema, ) -> crate::Result { + if value.is_null() { + return Ok(Item::Value(PrismaValue::Null)); + } + let mut map = Map::new(); // TODO: handle errors From 84141bddb752031007c442b9ca4b9cf00cef1629 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 17:21:55 +0100 Subject: [PATCH 12/38] fix: relation & relevance ordering --- .../queries/order_and_pagination/order_by_dependent.rs | 10 ++++++++-- .../connectors/sql-query-connector/src/ordering.rs | 6 +++++- .../core/src/interpreter/query_interpreters/read.rs | 2 +- .../core/src/query_graph_builder/read/utils.rs | 1 + query-engine/query-structure/src/field_selection.rs | 3 +++ 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/order_by_dependent.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/order_by_dependent.rs index b4c6e7b5ef34..c8f7429451a7 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/order_by_dependent.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/order_by_dependent.rs @@ -248,7 +248,10 @@ mod order_by_dependent { r#"{"data":{"findManyModelA":[{"id":4,"b":null},{"id":3,"b":null},{"id":1,"b":{"c":{"a":{"id":3}}}},{"id":2,"b":{"c":{"a":{"id":4}}}}]}}"#, r#"{"data":{"findManyModelA":[{"id":3,"b":null},{"id":4,"b":null},{"id":1,"b":{"c":{"a":{"id":3}}}},{"id":2,"b":{"c":{"a":{"id":4}}}}]}}"#, ], - _ => vec![r#"{"data":{"findManyModelA":[{"id":1,"b":{"c":{"a":{"id":3}}}},{"id":2,"b":{"c":{"a":{"id":4}}}},{"id":3,"b":null},{"id":4,"b":null}]}}"#] + _ => vec![ + r#"{"data":{"findManyModelA":[{"id":1,"b":{"c":{"a":{"id":3}}}},{"id":2,"b":{"c":{"a":{"id":4}}}},{"id":3,"b":null},{"id":4,"b":null}]}}"#, + r#"{"data":{"findManyModelA":[{"id":1,"b":{"c":{"a":{"id":3}}}},{"id":2,"b":{"c":{"a":{"id":4}}}},{"id":4,"b":null},{"id":3,"b":null}]}}"# + ] ); Ok(()) @@ -280,7 +283,10 @@ mod order_by_dependent { r#"{"data":{"findManyModelA":[{"id":2,"b":{"c":{"a":{"id":4}}}},{"id":1,"b":{"c":{"a":{"id":3}}}},{"id":4,"b":null},{"id":3,"b":null}]}}"#, r#"{"data":{"findManyModelA":[{"id":2,"b":{"c":{"a":{"id":4}}}},{"id":1,"b":{"c":{"a":{"id":3}}}},{"id":3,"b":null},{"id":4,"b":null}]}}"#, ], - _ => vec![r#"{"data":{"findManyModelA":[{"id":3,"b":null},{"id":4,"b":null},{"id":2,"b":{"c":{"a":{"id":4}}}},{"id":1,"b":{"c":{"a":{"id":3}}}}]}}"#] + _ => vec![ + r#"{"data":{"findManyModelA":[{"id":3,"b":null},{"id":4,"b":null},{"id":2,"b":{"c":{"a":{"id":4}}}},{"id":1,"b":{"c":{"a":{"id":3}}}}]}}"#, + r#"{"data":{"findManyModelA":[{"id":4,"b":null},{"id":3,"b":null},{"id":2,"b":{"c":{"a":{"id":4}}}},{"id":1,"b":{"c":{"a":{"id":3}}}}]}}"# + ] ); Ok(()) } diff --git a/query-engine/connectors/sql-query-connector/src/ordering.rs b/query-engine/connectors/sql-query-connector/src/ordering.rs index 2332f28e643b..14e4ca54ad21 100644 --- a/query-engine/connectors/sql-query-connector/src/ordering.rs +++ b/query-engine/connectors/sql-query-connector/src/ordering.rs @@ -76,7 +76,11 @@ impl OrderByBuilder { needs_reversed_order: bool, ctx: &Context<'_>, ) -> OrderByDefinition { - let columns: Vec = order_by.fields.iter().map(|sf| sf.as_column(ctx).into()).collect(); + let columns: Vec = order_by + .fields + .iter() + .map(|sf| sf.as_column(ctx).opt_table(self.parent_alias.clone()).into()) + .collect(); let order_column: Expression = text_search_relevance(&columns, order_by.search.clone()).into(); let order: Option = Some(into_order(&order_by.sort_order, None, needs_reversed_order)); let order_definition: OrderDefinition = (order_column.clone(), order); diff --git a/query-engine/core/src/interpreter/query_interpreters/read.rs b/query-engine/core/src/interpreter/query_interpreters/read.rs index 2f5e71336946..fa58624af792 100644 --- a/query-engine/core/src/interpreter/query_interpreters/read.rs +++ b/query-engine/core/src/interpreter/query_interpreters/read.rs @@ -185,7 +185,7 @@ fn build_relation_record_selection<'a>( selections .map(|rq| RelationRecordSelection { name: rq.field.name().to_owned(), - fields: rq.selections.iter().map(|sf| sf.prisma_name().to_owned()).collect(), + fields: rq.result_fields.clone(), model: rq.field.related_model(), nested: build_relation_record_selection(rq.relations()), }) diff --git a/query-engine/core/src/query_graph_builder/read/utils.rs b/query-engine/core/src/query_graph_builder/read/utils.rs index b0f3fdb5da32..e79c6304f7ba 100644 --- a/query-engine/core/src/query_graph_builder/read/utils.rs +++ b/query-engine/core/src/query_graph_builder/read/utils.rs @@ -139,6 +139,7 @@ fn extract_relation_selection( Ok(SelectedField::Relation(RelationSelection { field: rf, args: extract_query_args(pf.arguments, &related_model)?, + result_fields: collect_selection_order(&object.fields), selections: pairs_to_selections(related_model, &object.fields, query_schema)?, })) } diff --git a/query-engine/query-structure/src/field_selection.rs b/query-engine/query-structure/src/field_selection.rs index 1f2903c094bb..b1538b27e80b 100644 --- a/query-engine/query-structure/src/field_selection.rs +++ b/query-engine/query-structure/src/field_selection.rs @@ -176,6 +176,9 @@ pub enum SelectedField { pub struct RelationSelection { pub field: RelationField, pub args: QueryArguments, + /// Field names that will eventually be serialized + pub result_fields: Vec, + // Fields that will be queried by the connectors pub selections: Vec, } From e80fb7a39b51515957522790c9c0fea9b21aed85 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 17:38:49 +0100 Subject: [PATCH 13/38] fix: bring query parameter exceeded error back --- .../tests/new/regressions/prisma_7434.rs | 6 +-- .../queries/batch/in_selection_batching.rs | 6 +-- .../src/database/operations/read.rs | 49 ++++++++++++------- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/regressions/prisma_7434.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/regressions/prisma_7434.rs index 3471f0c2d72c..e5fa8388d66e 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/regressions/prisma_7434.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/regressions/prisma_7434.rs @@ -11,8 +11,7 @@ mod not_in_batching { assert_error!( runner, "query { findManyTestModel(where: { id: { notIn: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] } }) { id }}", - 2029, - "Parameter limits for this database provider require this query to be split into multiple queries, but the negation filters used prevent the query from being split. Please reduce the used values in the query." + 2029 // QueryParameterLimitExceeded ); Ok(()) @@ -30,8 +29,7 @@ mod not_in_batching_cockroachdb { assert_error!( runner, "query { findManyTestModel(where: { id: { notIn: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] } }) { id }}", - 2029, - "Parameter limits for this database provider require this query to be split into multiple queries, but the negation filters used prevent the query from being split. Please reduce the used values in the query." + 2029 // QueryParameterLimitExceeded ); Ok(()) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs index e2b21fc215ed..1aeac6304ded 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs @@ -91,8 +91,7 @@ mod isb { r#"query { findManyA(where: {id: { in: [5,4,3,2,1,1,1,2,3,4,5,6,7,6,5,4,3,2,1,2,3,4,5,6] }}, orderBy: { b: { as: { _count: asc } } }) { id } }"#, - 2029, - "Your query cannot be split into multiple queries because of the order by aggregation or relevance." + 2029 // QueryParameterLimitExceeded ); Ok(()) @@ -107,8 +106,7 @@ mod isb { r#"query { findManyA(where: {id: { in: [5,4,3,2,1,1,1,2,3,4,5,6,7,6,5,4,3,2,1,2,3,4,5,6] }}, orderBy: { _relevance: { fields: text, search: "something", sort: asc } }) { id } }"#, - 2029, - "Your query cannot be split into multiple queries because of the order by aggregation or relevance." + 2029 // QueryParameterLimitExceeded ); Ok(()) diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index a79878d1137d..94d7787e402c 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -94,10 +94,23 @@ pub(crate) async fn get_many_records_joins( let idents = selected_fields.type_identifiers_with_arities(); let meta = column_metadata::create(field_names.as_slice(), idents.as_slice()); - let rs_indexes = relation_selection_indexes(selected_fields.relations().collect(), &field_names); - // dbg!(&rs_indexes); + let rs_indexes = get_relation_selection_indexes(selected_fields.relations().collect(), &field_names); let mut records = ManyRecords::new(field_names.clone()); + if let Some(0) = query_arguments.take { + return Ok(records); + }; + + match ctx.max_bind_values { + Some(chunk_size) if query_arguments.should_batch(chunk_size) => { + return Err(SqlError::QueryParameterLimitExceeded( + "Join queries cannot be split into multiple queries just yet. If you encounter that issue, please open an issue." + .to_string(), + )); + } + _ => (), + }; + let query = query_builder::select::SelectBuilder::default().build(query_arguments.clone(), selected_fields, ctx); for item in conn.filter(query.into(), meta.as_slice(), ctx).await?.into_iter() { @@ -117,22 +130,6 @@ pub(crate) async fn get_many_records_joins( Ok(records) } -// TODO: find better name -fn relation_selection_indexes<'a>( - selections: Vec<&'a RelationSelection>, - field_names: &[String], -) -> Vec<(usize, &'a RelationSelection)> { - let mut output: Vec<(usize, &RelationSelection)> = Vec::new(); - - for (idx, field_name) in field_names.iter().enumerate() { - if let Some(rs) = selections.iter().find(|rq| rq.field.name() == *field_name) { - output.push((idx, rs)); - } - } - - output -} - pub(crate) async fn get_many_records_wo_joins( conn: &dyn Queryable, model: &Model, @@ -370,3 +367,19 @@ async fn group_by_aggregate( .map(|row| row.into_aggregation_results(&selections)) .collect()) } + +/// Find the indexes of the relation records to traverse a set of records faster when coercing JSON values +fn get_relation_selection_indexes<'a>( + selections: Vec<&'a RelationSelection>, + field_names: &[String], +) -> Vec<(usize, &'a RelationSelection)> { + let mut output: Vec<(usize, &RelationSelection)> = Vec::new(); + + for (idx, field_name) in field_names.iter().enumerate() { + if let Some(rs) = selections.iter().find(|rq| rq.field.name() == *field_name) { + output.push((idx, rs)); + } + } + + output +} \ No newline at end of file From f16fb3dbfb4ec15dc18c888e9093e77ce7529f86 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 17:39:21 +0100 Subject: [PATCH 14/38] fix: avoid joins with nested aggregation selection --- .../src/database/operations/read.rs | 2 +- query-engine/core/src/query_ast/read.rs | 17 +++++++++++++++++ .../core/src/query_graph_builder/read/utils.rs | 4 ++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index 94d7787e402c..b95e5f64da1b 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -382,4 +382,4 @@ fn get_relation_selection_indexes<'a>( } output -} \ No newline at end of file +} diff --git a/query-engine/core/src/query_ast/read.rs b/query-engine/core/src/query_ast/read.rs index c25fc4ab6784..353b2ca6bfb1 100644 --- a/query-engine/core/src/query_ast/read.rs +++ b/query-engine/core/src/query_ast/read.rs @@ -73,6 +73,19 @@ impl ReadQuery { ReadQuery::AggregateRecordsQuery(_) => false, } } + + pub(crate) fn has_aggregation_selections(&self) -> bool { + fn has_aggregations(selections: &[RelAggregationSelection], nested: &[ReadQuery]) -> bool { + !selections.is_empty() || nested.iter().any(|q| q.has_aggregation_selections()) + } + + match self { + ReadQuery::RecordQuery(q) => has_aggregations(&q.aggregation_selections, &q.nested), + ReadQuery::ManyRecordsQuery(q) => has_aggregations(&q.aggregation_selections, &q.nested), + ReadQuery::RelatedRecordsQuery(q) => has_aggregations(&q.aggregation_selections, &q.nested), + ReadQuery::AggregateRecordsQuery(_) => false, + } + } } impl FilteredQuery for ReadQuery { @@ -230,6 +243,10 @@ impl RelatedRecordsQuery { pub fn has_distinct(&self) -> bool { self.args.distinct.is_some() || self.nested.iter().any(|q| q.has_distinct()) } + + pub fn has_aggregation_selections(&self) -> bool { + !self.aggregation_selections.is_empty() || self.nested.iter().any(|q| q.has_aggregation_selections()) + } } #[derive(Debug, Clone)] diff --git a/query-engine/core/src/query_graph_builder/read/utils.rs b/query-engine/core/src/query_graph_builder/read/utils.rs index e79c6304f7ba..25e0f17816eb 100644 --- a/query-engine/core/src/query_graph_builder/read/utils.rs +++ b/query-engine/core/src/query_graph_builder/read/utils.rs @@ -252,11 +252,11 @@ pub(crate) fn get_relation_load_strategy( && query_schema.has_capability(ConnectorCapability::LateralJoin) && args.cursor.is_none() && args.distinct.is_none() + && aggregation_selections.is_empty() && !nested_queries.iter().any(|q| match q { - ReadQuery::RelatedRecordsQuery(q) => q.has_cursor() || q.has_distinct(), + ReadQuery::RelatedRecordsQuery(q) => q.has_cursor() || q.has_distinct() || q.has_aggregation_selections(), _ => false, }) - && aggregation_selections.is_empty() { RelationLoadStrategy::Join } else { From 4b691a697d2faf0084cc02900154484485fee0a3 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 22:43:57 +0100 Subject: [PATCH 15/38] fix aggregated order bys --- query-engine/connectors/sql-query-connector/src/ordering.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/query-engine/connectors/sql-query-connector/src/ordering.rs b/query-engine/connectors/sql-query-connector/src/ordering.rs index 14e4ca54ad21..7c67f0c23482 100644 --- a/query-engine/connectors/sql-query-connector/src/ordering.rs +++ b/query-engine/connectors/sql-query-connector/src/ordering.rs @@ -172,7 +172,10 @@ impl OrderByBuilder { _ => unreachable!("Order by relation aggregation other than count are not supported"), }; - let previous_alias = joins.last().map(|j| j.alias.as_str()); + let previous_alias = joins + .last() + .map(|j| j.alias.as_str()) + .or_else(|| parent_alias.as_deref()); // We perform the aggregation on the last join let last_aggr_join = compute_aggr_join( From bd4f551217b63ed4561d42100dd18fc99e5ea918 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 22:44:10 +0100 Subject: [PATCH 16/38] add comment trace --- .../sql-query-connector/src/query_builder/select.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs index 2ca856edbbdb..3c18aa36d24d 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs @@ -5,11 +5,13 @@ use crate::{ filter::alias::{Alias, AliasMode}, model_extensions::{AsColumn, AsColumns, AsTable, RelationFieldExt}, ordering::OrderByBuilder, + sql_trace::SqlTraceComment, }; use itertools::Itertools; use quaint::prelude::*; use query_structure::*; +use tracing::Span; pub const JSON_AGG_IDENT: &str = "data"; @@ -56,6 +58,7 @@ impl SelectBuilder { let select = self.with_ordering(select, &args, Some(table_alias.to_table_string()), ctx); let select = self.with_pagination(select, args.take_abs(), args.skip); let select = self.with_filters(select, args.filter, Some(table_alias), ctx); + let select = select.append_trace(&Span::current()).add_trace_id(ctx.trace_id); select } @@ -101,8 +104,6 @@ impl SelectBuilder { ) -> Select<'static> { let table_alias = self.next_alias(); - dbg!(&rs); - let build_obj_params = rs .selections .iter() From 6e4341391ba7389f0f5045df9a6c1466a8ab1dfe Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 22:44:20 +0100 Subject: [PATCH 17/38] small cleanup --- .../sql-query-connector/src/database/operations/coerce.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs index 61ee6389d184..a266aae02ffd 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs @@ -60,7 +60,7 @@ pub(crate) fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &Relation map.push((key, coerce_json_relation_to_pv(value, nested_selection))); } } - Field::Composite(_) => unreachable!(), + _ => (), } } From ff68957246bc4ba99bd2e7488a4591cc7e7dc9de Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 27 Nov 2023 22:50:32 +0100 Subject: [PATCH 18/38] clippy fixes --- quaint/src/ast/function.rs | 1 - quaint/src/ast/function/json_array_agg.rs | 2 +- quaint/src/ast/function/json_build_obj.rs | 2 +- .../src/database/operations/coerce.rs | 3 ++- .../src/database/transaction.rs | 2 +- .../sql-query-connector/src/ordering.rs | 9 ++------- .../src/query_builder/select.rs | 15 +++++---------- query-engine/core/src/response_ir/internal.rs | 16 ++++++---------- 8 files changed, 18 insertions(+), 32 deletions(-) diff --git a/quaint/src/ast/function.rs b/quaint/src/ast/function.rs index cbed3a6ed18e..3bcc24c4b072 100644 --- a/quaint/src/ast/function.rs +++ b/quaint/src/ast/function.rs @@ -41,7 +41,6 @@ pub use json_unquote::*; pub use lower::*; pub use maximum::*; pub use minimum::*; -use postgres_types::Json; pub use row_number::*; #[cfg(feature = "postgresql")] pub use row_to_json::*; diff --git a/quaint/src/ast/function/json_array_agg.rs b/quaint/src/ast/function/json_array_agg.rs index b7ffc9f34848..ed7c3fd64221 100644 --- a/quaint/src/ast/function/json_array_agg.rs +++ b/quaint/src/ast/function/json_array_agg.rs @@ -5,7 +5,7 @@ pub struct JsonArrayAgg<'a> { pub(crate) expr: Box>, } -/// This is an internal function used to help construct the JsonArrayBeginsWith Comparable +/// Builds a JSON array out of a list of values. pub fn json_array_agg<'a, E>(expr: E) -> Function<'a> where E: Into>, diff --git a/quaint/src/ast/function/json_build_obj.rs b/quaint/src/ast/function/json_build_obj.rs index 7e2fff404e4e..0578d63e7c56 100644 --- a/quaint/src/ast/function/json_build_obj.rs +++ b/quaint/src/ast/function/json_build_obj.rs @@ -7,7 +7,7 @@ pub struct JsonBuildObject<'a> { pub(crate) exprs: Vec<(Cow<'a, str>, Expression<'a>)>, } -/// This is an internal function used to help construct the JsonArrayBeginsWith Comparable +/// Builds a JSON object out of a list of key-value pairs. pub fn json_build_object<'a>(exprs: Vec<(Cow<'a, str>, Expression<'a>)>) -> Function<'a> { let fun = JsonBuildObject { exprs }; diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs index a266aae02ffd..a8e45ee91c45 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs @@ -7,7 +7,8 @@ use crate::query_arguments_ext::QueryArgumentsExt; pub(crate) fn coerce_record_with_join(record: &mut Record, rq_indexes: Vec<(usize, &RelationSelection)>) { for (val_idx, rs) in rq_indexes { let val = record.values.get_mut(val_idx).unwrap(); - let json_val: serde_json::Value = serde_json::from_str(&val.as_json().unwrap()).unwrap(); + // TODO(perf): Find ways to avoid serializing and deserializing multiple times. + let json_val: serde_json::Value = serde_json::from_str(val.as_json().unwrap()).unwrap(); *val = coerce_json_relation_to_pv(json_val, rs); } diff --git a/query-engine/connectors/sql-query-connector/src/database/transaction.rs b/query-engine/connectors/sql-query-connector/src/database/transaction.rs index 6b5bc668b54d..b69f77848f0d 100644 --- a/query-engine/connectors/sql-query-connector/src/database/transaction.rs +++ b/query-engine/connectors/sql-query-connector/src/database/transaction.rs @@ -101,7 +101,7 @@ impl<'tx> ReadOperations for SqlConnectorTransaction<'tx> { self.inner.as_queryable(), model, query_arguments, - &selected_fields, + selected_fields, aggr_selections, relation_load_strategy, &ctx, diff --git a/query-engine/connectors/sql-query-connector/src/ordering.rs b/query-engine/connectors/sql-query-connector/src/ordering.rs index 7c67f0c23482..ade10aaa7164 100644 --- a/query-engine/connectors/sql-query-connector/src/ordering.rs +++ b/query-engine/connectors/sql-query-connector/src/ordering.rs @@ -159,9 +159,7 @@ impl OrderByBuilder { for (i, hop) in rest_hops.iter().enumerate() { let previous_join = if i > 0 { joins.get(i - 1) } else { None }; - let previous_alias = previous_join - .map(|j| j.alias.as_str()) - .or_else(|| parent_alias.as_deref()); + let previous_alias = previous_join.map(|j| j.alias.as_str()).or(parent_alias.as_deref()); let join = compute_one2m_join(hop.as_relation_hop().unwrap(), &self.join_prefix(), previous_alias, ctx); joins.push(join); @@ -172,10 +170,7 @@ impl OrderByBuilder { _ => unreachable!("Order by relation aggregation other than count are not supported"), }; - let previous_alias = joins - .last() - .map(|j| j.alias.as_str()) - .or_else(|| parent_alias.as_deref()); + let previous_alias = joins.last().map(|j| j.alias.as_str()).or(parent_alias.as_deref()); // We perform the aggregation on the last join let last_aggr_join = compute_aggr_join( diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs index 3c18aa36d24d..19ecbdda732e 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs @@ -58,9 +58,8 @@ impl SelectBuilder { let select = self.with_ordering(select, &args, Some(table_alias.to_table_string()), ctx); let select = self.with_pagination(select, args.take_abs(), args.skip); let select = self.with_filters(select, args.filter, Some(table_alias), ctx); - let select = select.append_trace(&Span::current()).add_trace_id(ctx.trace_id); - select + select.append_trace(&Span::current()).add_trace_id(ctx.trace_id) } fn with_related_queries<'a, 'b>( @@ -160,11 +159,9 @@ impl SelectBuilder { // SELECT // SELECT ONLY if it's a m2m table as we need to order by outside of the inner select - let inner = ModelProjection::from(selection) + ModelProjection::from(selection) .as_columns(ctx) - .fold(inner, |acc, c| acc.column(c.table(table_alias.to_table_string()))); - - inner + .fold(inner, |acc, c| acc.column(c.table(table_alias.to_table_string()))) } else { // SELECT let inner = ModelProjection::from(linking_fields) @@ -310,12 +307,10 @@ impl SelectBuilder { let (filter, joins) = visitor.visit_filter(filter, ctx); let select = select.and_where(filter); - let select = match joins { + match joins { Some(joins) => joins.into_iter().fold(select, |acc, join| acc.join(join.data)), None => select, - }; - - select + } } else { select } diff --git a/query-engine/core/src/response_ir/internal.rs b/query-engine/core/src/response_ir/internal.rs index f5e8417aac4a..4bd8ff5ed102 100644 --- a/query-engine/core/src/response_ir/internal.rs +++ b/query-engine/core/src/response_ir/internal.rs @@ -240,9 +240,7 @@ fn serialize_record_selection_with_relations( query_schema, ), InnerOutputType::Object(obj) => { - // dbg!(&record_selection); - let result = serialize_objects_with_relation(record_selection, obj, query_schema)?; - // dbg!(&result); + let result = serialize_objects_with_relation(record_selection, obj)?; process_object(field, is_list, result, name) } @@ -335,7 +333,6 @@ fn process_object( fn serialize_objects_with_relation( result: RecordSelectionWithRelations, typ: &ObjectType<'_>, - query_schema: &QuerySchema, ) -> crate::Result { let mut object_mapping = UncheckedItemsWithParents::with_capacity(result.records.records.len()); @@ -380,7 +377,7 @@ fn serialize_objects_with_relation( .into_list() .unwrap() .into_iter() - .map(|value| serialize_relation_selection(rrs, value, inner_typ, query_schema)) + .map(|value| serialize_relation_selection(rrs, value, inner_typ)) .collect::>>()?; object.insert(field.name().to_owned(), Item::list(items)); @@ -391,7 +388,7 @@ fn serialize_objects_with_relation( object.insert( field.name().to_owned(), - serialize_relation_selection(rrs, val, inner_typ, query_schema)?, + serialize_relation_selection(rrs, val, inner_typ)?, ); } _ => panic!("unexpected field"), @@ -411,7 +408,6 @@ fn serialize_relation_selection( value: PrismaValue, // parent_id: Option, typ: &ObjectType<'_>, - query_schema: &QuerySchema, ) -> crate::Result { if value.is_null() { return Ok(Item::Value(PrismaValue::Null)); @@ -420,7 +416,7 @@ fn serialize_relation_selection( let mut map = Map::new(); // TODO: handle errors - let mut value_obj: HashMap = HashMap::from_iter(value.into_object().unwrap().into_iter()); + let mut value_obj: HashMap = HashMap::from_iter(value.into_object().unwrap()); let db_field_names = &rrs.fields; let fields: Vec<_> = db_field_names .iter() @@ -443,7 +439,7 @@ fn serialize_relation_selection( .into_list() .unwrap() .into_iter() - .map(|value| serialize_relation_selection(inner_rrs, value, inner_typ, query_schema)) + .map(|value| serialize_relation_selection(inner_rrs, value, inner_typ)) .collect::>>()?; map.insert(field.name().to_owned(), Item::list(items)); @@ -454,7 +450,7 @@ fn serialize_relation_selection( map.insert( field.name().to_owned(), - serialize_relation_selection(inner_rrs, value, inner_typ, query_schema)?, + serialize_relation_selection(inner_rrs, value, inner_typ)?, ); } _ => (), From 02b451ee4addb43640f5512dfb871b7be30a0b2c Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Tue, 28 Nov 2023 15:17:26 +0100 Subject: [PATCH 19/38] temporarily exclude batching tests --- .../tests/queries/batch/in_selection_batching.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs index 1aeac6304ded..1126201865a4 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs @@ -35,7 +35,9 @@ mod isb { } // "batching of IN queries" should "work when having more than the specified amount of items" - #[connector_test] + // TODO(joins): Excluded because we have no support for batched queries with joins. In practice, it should happen under much less circumstances + // TODO(joins): than with the query-based strategy, because we don't issue `WHERE IN (parent_ids)` queries anymore to resolve relations. + #[connector_test(exclude_features("joins"))] async fn in_more_items(runner: Runner) -> TestResult<()> { create_test_data(&runner).await?; @@ -51,7 +53,9 @@ mod isb { } // "ascending ordering of batched IN queries" should "work when having more than the specified amount of items" - #[connector_test] + // TODO(joins): Excluded because we have no support for batched queries with joins. In practice, it should happen under much less circumstances + // TODO(joins): than with the query-based strategy, because we don't issue `WHERE IN (parent_ids)` queries anymore to resolve relations. + #[connector_test(exclude_features("joins"))] async fn asc_in_ordering(runner: Runner) -> TestResult<()> { create_test_data(&runner).await?; @@ -67,7 +71,9 @@ mod isb { } // "ascending ordering of batched IN queries" should "work when having more than the specified amount of items" - #[connector_test] + // TODO(joins): Excluded because we have no support for batched queries with joins. In practice, it should happen under much less circumstances + // TODO(joins): than with the query-based strategy, because we don't issue `WHERE IN (parent_ids)` queries anymore to resolve relations. + #[connector_test(exclude_features("joins"))] async fn desc_in_ordering(runner: Runner) -> TestResult<()> { create_test_data(&runner).await?; From 4ce384b73b9f3def1cec43f447d4bf598c3fbfc9 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Tue, 28 Nov 2023 15:18:04 +0100 Subject: [PATCH 20/38] rename preview feature to "joins" --- psl/psl-core/src/common/preview_features.rs | 4 ++-- query-engine/core/src/query_graph_builder/read/utils.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/psl/psl-core/src/common/preview_features.rs b/psl/psl-core/src/common/preview_features.rs index 318010ebdf9f..6eeac70c1389 100644 --- a/psl/psl-core/src/common/preview_features.rs +++ b/psl/psl-core/src/common/preview_features.rs @@ -76,7 +76,7 @@ features!( TransactionApi, UncheckedScalarInputs, Views, - RelationJoins + Joins ); /// Generator preview features @@ -91,7 +91,7 @@ pub const ALL_PREVIEW_FEATURES: FeatureMap = FeatureMap { | PostgresqlExtensions | Tracing | Views - | RelationJoins + | Joins }), deprecated: enumflags2::make_bitflags!(PreviewFeature::{ AtomicNumberOperations diff --git a/query-engine/core/src/query_graph_builder/read/utils.rs b/query-engine/core/src/query_graph_builder/read/utils.rs index 25e0f17816eb..ea8f00c31818 100644 --- a/query-engine/core/src/query_graph_builder/read/utils.rs +++ b/query-engine/core/src/query_graph_builder/read/utils.rs @@ -74,7 +74,7 @@ where T: Into, { let should_collect_relation_selection = query_schema.has_capability(ConnectorCapability::LateralJoin) - && query_schema.has_feature(PreviewFeature::RelationJoins); + && query_schema.has_feature(PreviewFeature::Joins); let parent = parent.into(); @@ -248,7 +248,7 @@ pub(crate) fn get_relation_load_strategy( aggregation_selections: &[RelAggregationSelection], query_schema: &QuerySchema, ) -> RelationLoadStrategy { - if query_schema.has_feature(PreviewFeature::RelationJoins) + if query_schema.has_feature(PreviewFeature::Joins) && query_schema.has_capability(ConnectorCapability::LateralJoin) && args.cursor.is_none() && args.distinct.is_none() From 159898499b2e5a87956285c8b32e1fda82bf75fe Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Tue, 28 Nov 2023 15:19:17 +0100 Subject: [PATCH 21/38] fix generator error test --- psl/psl/tests/config/generators.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psl/psl/tests/config/generators.rs b/psl/psl/tests/config/generators.rs index f10a9bda3eae..1152ef910c5a 100644 --- a/psl/psl/tests/config/generators.rs +++ b/psl/psl/tests/config/generators.rs @@ -258,7 +258,7 @@ fn nice_error_for_unknown_generator_preview_feature() { .unwrap_err(); let expectation = expect![[r#" - error: The preview feature "foo" is not known. Expected one of: deno, driverAdapters, fullTextIndex, fullTextSearch, metrics, multiSchema, postgresqlExtensions, tracing, views + error: The preview feature "foo" is not known. Expected one of: deno, driverAdapters, fullTextIndex, fullTextSearch, metrics, multiSchema, postgresqlExtensions, tracing, views, joins --> schema.prisma:3  |   2 |  provider = "prisma-client-js" From fd25b2009fac247d28dfc444be17228a9e3769cd Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Tue, 28 Nov 2023 16:04:24 +0100 Subject: [PATCH 22/38] clippy fixes --- query-engine/core/src/response_ir/internal.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/query-engine/core/src/response_ir/internal.rs b/query-engine/core/src/response_ir/internal.rs index 4bd8ff5ed102..c8b5d8915cae 100644 --- a/query-engine/core/src/response_ir/internal.rs +++ b/query-engine/core/src/response_ir/internal.rs @@ -50,7 +50,7 @@ pub(crate) fn serialize_internal( serialize_record_selection(*rs, field, field.field_type(), is_list, query_schema) } QueryResult::RecordSelectionWithRelations(rs) => { - serialize_record_selection_with_relations(*rs, field, field.field_type(), is_list, query_schema) + serialize_record_selection_with_relations(*rs, field, field.field_type(), is_list) } QueryResult::RecordAggregations(ras) => serialize_aggregations(field, ras), QueryResult::Count(c) => { @@ -227,7 +227,6 @@ fn serialize_record_selection_with_relations( field: &OutputField<'_>, typ: &OutputType<'_>, // We additionally pass the type to allow recursing into nested type definitions of a field. is_list: bool, - query_schema: &QuerySchema, ) -> crate::Result { let name = record_selection.name.clone(); @@ -237,7 +236,6 @@ fn serialize_record_selection_with_relations( field, &OutputType::non_list(inner.clone()), true, - query_schema, ), InnerOutputType::Object(obj) => { let result = serialize_objects_with_relation(record_selection, obj)?; From 86ee2f5783d074194ac02afac53adc117b819cda Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Tue, 28 Nov 2023 16:18:48 +0100 Subject: [PATCH 23/38] remove mssql support --- psl/builtin-connectors/src/mssql_datamodel_connector.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/psl/builtin-connectors/src/mssql_datamodel_connector.rs b/psl/builtin-connectors/src/mssql_datamodel_connector.rs index d150deb4fdfc..46647fabe8af 100644 --- a/psl/builtin-connectors/src/mssql_datamodel_connector.rs +++ b/psl/builtin-connectors/src/mssql_datamodel_connector.rs @@ -52,8 +52,7 @@ const CAPABILITIES: ConnectorCapabilities = enumflags2::make_bitflags!(Connector SupportsTxIsolationReadCommitted | SupportsTxIsolationRepeatableRead | SupportsTxIsolationSerializable | - SupportsTxIsolationSnapshot | - LateralJoin + SupportsTxIsolationSnapshot }); pub(crate) struct MsSqlDatamodelConnector; From d5c9986231bcaa189b6bafe5f9cde304eaadf7e3 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Thu, 30 Nov 2023 12:22:25 +0100 Subject: [PATCH 24/38] fix ordering on CRDB & refactor sql builder --- .../tests/queries/filters/one_relation.rs | 5 + .../order_and_pagination/nested_pagination.rs | 32 +- .../src/query_builder/select.rs | 389 +++++++++++------- .../query-structure/src/field_selection.rs | 11 +- 4 files changed, 260 insertions(+), 177 deletions(-) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/one_relation.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/one_relation.rs index cca380f81138..020de727d496 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/one_relation.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/one_relation.rs @@ -183,6 +183,11 @@ mod one_relation { @r###"{"data":{"findManyBlog":[{"name":"blog 1","post":{"title":"post 1","comment":{"text":"comment 1"}}},{"name":"blog 2","post":null},{"name":"blog 3","post":null}]}}"### ); + insta::assert_snapshot!( + run_query!(&runner, r#"query { findManyBlog { name, post(where: { title: "post 1", comment: { is: { text: "comment 1" } } }) { title } }}"#), + @r###"{"data":{"findManyBlog":[{"name":"blog 1","post":{"title":"post 1"}},{"name":"blog 2","post":null},{"name":"blog 3","post":null}]}}"### + ); + Ok(()) } diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/nested_pagination.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/nested_pagination.rs index 27c04241288c..6a67b87d56b1 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/nested_pagination.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/nested_pagination.rs @@ -357,7 +357,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop{t, middles(take: -1, orderBy: { id: asc }){m}} + findManyTop(orderBy: {t: asc}){t, middles(take: -1, orderBy: { id: asc }){m}} }"#), @r###"{"data":{"findManyTop":[{"t":"T1","middles":[{"m":"M13"}]},{"t":"T2","middles":[{"m":"M23"}]},{"t":"T3","middles":[{"m":"M33"}]}]}}"### ); @@ -372,7 +372,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop{t, middles(take: -3, orderBy: { id: asc }) {m}} + findManyTop(orderBy: {t: asc}){t, middles(take: -3, orderBy: { id: asc }) {m}} }"#), @r###"{"data":{"findManyTop":[{"t":"T1","middles":[{"m":"M11"},{"m":"M12"},{"m":"M13"}]},{"t":"T2","middles":[{"m":"M21"},{"m":"M22"},{"m":"M23"}]},{"t":"T3","middles":[{"m":"M31"},{"m":"M32"},{"m":"M33"}]}]}}"### ); @@ -387,7 +387,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop{t, middles(take: -4, orderBy: { id: asc }) {m}} + findManyTop(orderBy: {t: asc}){t, middles(take: -4, orderBy: { id: asc }) {m}} }"#), @r###"{"data":{"findManyTop":[{"t":"T1","middles":[{"m":"M11"},{"m":"M12"},{"m":"M13"}]},{"t":"T2","middles":[{"m":"M21"},{"m":"M22"},{"m":"M23"}]},{"t":"T3","middles":[{"m":"M31"},{"m":"M32"},{"m":"M33"}]}]}}"### ); @@ -402,7 +402,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop{middles{bottoms(take: -1, orderBy: { id: asc }){b}}} + findManyTop{middles(orderBy: {m: asc}){bottoms(take: -1, orderBy: { id: asc }){b}}} }"#), @r###"{"data":{"findManyTop":[{"middles":[{"bottoms":[{"b":"B113"}]},{"bottoms":[{"b":"B123"}]},{"bottoms":[{"b":"B133"}]}]},{"middles":[{"bottoms":[{"b":"B213"}]},{"bottoms":[{"b":"B223"}]},{"bottoms":[{"b":"B233"}]}]},{"middles":[{"bottoms":[{"b":"B313"}]},{"bottoms":[{"b":"B323"}]},{"bottoms":[{"b":"B333"}]}]}]}}"### ); @@ -417,7 +417,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop{middles{bottoms(take: -3, orderBy: { id: asc }){b}}} + findManyTop{middles(orderBy: {m: asc}){bottoms(take: -3, orderBy: { id: asc }){b}}} }"#), @r###"{"data":{"findManyTop":[{"middles":[{"bottoms":[{"b":"B111"},{"b":"B112"},{"b":"B113"}]},{"bottoms":[{"b":"B121"},{"b":"B122"},{"b":"B123"}]},{"bottoms":[{"b":"B131"},{"b":"B132"},{"b":"B133"}]}]},{"middles":[{"bottoms":[{"b":"B211"},{"b":"B212"},{"b":"B213"}]},{"bottoms":[{"b":"B221"},{"b":"B222"},{"b":"B223"}]},{"bottoms":[{"b":"B231"},{"b":"B232"},{"b":"B233"}]}]},{"middles":[{"bottoms":[{"b":"B311"},{"b":"B312"},{"b":"B313"}]},{"bottoms":[{"b":"B321"},{"b":"B322"},{"b":"B323"}]},{"bottoms":[{"b":"B331"},{"b":"B332"},{"b":"B333"}]}]}]}}"### ); @@ -432,7 +432,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop{middles{bottoms(take: -4, orderBy: { id: asc }){b}}} + findManyTop{middles(orderBy: {m: asc}){bottoms(take: -4, orderBy: { id: asc }){b}}} }"#), @r###"{"data":{"findManyTop":[{"middles":[{"bottoms":[{"b":"B111"},{"b":"B112"},{"b":"B113"}]},{"bottoms":[{"b":"B121"},{"b":"B122"},{"b":"B123"}]},{"bottoms":[{"b":"B131"},{"b":"B132"},{"b":"B133"}]}]},{"middles":[{"bottoms":[{"b":"B211"},{"b":"B212"},{"b":"B213"}]},{"bottoms":[{"b":"B221"},{"b":"B222"},{"b":"B223"}]},{"bottoms":[{"b":"B231"},{"b":"B232"},{"b":"B233"}]}]},{"middles":[{"bottoms":[{"b":"B311"},{"b":"B312"},{"b":"B313"}]},{"bottoms":[{"b":"B321"},{"b":"B322"},{"b":"B323"}]},{"bottoms":[{"b":"B331"},{"b":"B332"},{"b":"B333"}]}]}]}}"### ); @@ -451,7 +451,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop(skip: 1, take: 1){t, middles{m}} + findManyTop(skip: 1, take: 1){t, middles(orderBy: { m: asc }){m}} }"#), @r###"{"data":{"findManyTop":[{"t":"T2","middles":[{"m":"M21"},{"m":"M22"},{"m":"M23"}]}]}}"### ); @@ -466,7 +466,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop(skip: 1, take: 3){t, middles{m}} + findManyTop(skip: 1, take: 3){t, middles(orderBy: { m: asc }){m}} }"#), @r###"{"data":{"findManyTop":[{"t":"T2","middles":[{"m":"M21"},{"m":"M22"},{"m":"M23"}]},{"t":"T3","middles":[{"m":"M31"},{"m":"M32"},{"m":"M33"}]}]}}"### ); @@ -511,7 +511,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop(skip: 1, take: -3, orderBy: { id: asc }){t, middles{m}} + findManyTop(skip: 1, take: -3, orderBy: { id: asc }){t, middles(orderBy: { m: asc }){m}} }"#), @r###"{"data":{"findManyTop":[{"t":"T1","middles":[{"m":"M11"},{"m":"M12"},{"m":"M13"}]},{"t":"T2","middles":[{"m":"M21"},{"m":"M22"},{"m":"M23"}]}]}}"### ); @@ -526,7 +526,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop{t, middles(skip: 1, take: -1, orderBy: { id: asc }){m}} + findManyTop(orderBy: { t: asc }){t, middles(skip: 1, take: -1, orderBy: { id: asc }){m}} }"#), @r###"{"data":{"findManyTop":[{"t":"T1","middles":[{"m":"M12"}]},{"t":"T2","middles":[{"m":"M22"}]},{"t":"T3","middles":[{"m":"M32"}]}]}}"### ); @@ -541,7 +541,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop{t, middles(skip: 1, take: -3, orderBy: { id: asc }){m}} + findManyTop(orderBy: { t: asc }){t, middles(skip: 1, take: -3, orderBy: { id: asc }){m}} }"#), @r###"{"data":{"findManyTop":[{"t":"T1","middles":[{"m":"M11"},{"m":"M12"}]},{"t":"T2","middles":[{"m":"M21"},{"m":"M22"}]},{"t":"T3","middles":[{"m":"M31"},{"m":"M32"}]}]}}"### ); @@ -560,7 +560,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop{t, middles(orderBy: { m: desc }, take: 1){m}} + findManyTop(orderBy: { t: asc }){t, middles(orderBy: { m: desc }, take: 1){m}} }"#), @r###"{"data":{"findManyTop":[{"t":"T1","middles":[{"m":"M13"}]},{"t":"T2","middles":[{"m":"M23"}]},{"t":"T3","middles":[{"m":"M33"}]}]}}"### ); @@ -575,7 +575,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyTop{t, middles(orderBy: { m: desc }, take: 3){m}} + findManyTop(orderBy: { t: asc }){t, middles(orderBy: { m: desc }, take: 3){m}} }"#), @r###"{"data":{"findManyTop":[{"t":"T1","middles":[{"m":"M13"},{"m":"M12"},{"m":"M11"}]},{"t":"T2","middles":[{"m":"M23"},{"m":"M22"},{"m":"M21"}]},{"t":"T3","middles":[{"m":"M33"},{"m":"M32"},{"m":"M31"}]}]}}"### ); @@ -885,7 +885,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyModelA { + findManyModelA(orderBy: { id: asc }) { id manyB(skip: 1) { id @@ -909,7 +909,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyModelA { + findManyModelA(orderBy: { id: asc }) { id manyB(skip: 1, take: 2) { id @@ -933,7 +933,7 @@ mod nested_pagination { insta::assert_snapshot!( run_query!(&runner, r#"{ - findManyModelA { + findManyModelA(orderBy: { id: asc }) { id manyB(skip: 1, take: -2, orderBy: { id: asc }) { id diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs index 19ecbdda732e..f53d979f558d 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs @@ -1,17 +1,16 @@ use std::borrow::Cow; +use tracing::Span; use crate::{ context::Context, filter::alias::{Alias, AliasMode}, - model_extensions::{AsColumn, AsColumns, AsTable, RelationFieldExt}, + model_extensions::{AsColumn, AsColumns, AsTable, ColumnIterator, RelationFieldExt}, ordering::OrderByBuilder, sql_trace::SqlTraceComment, }; -use itertools::Itertools; use quaint::prelude::*; use query_structure::*; -use tracing::Span; pub const JSON_AGG_IDENT: &str = "data"; @@ -33,33 +32,19 @@ impl SelectBuilder { ctx: &Context<'_>, ) -> Select<'static> { let table_alias = self.next_alias(); + let table = args.model().as_table(ctx).alias(table_alias.to_table_string()); // SELECT ... FROM Table "t1" - let select = Select::from_table(args.model().as_table(ctx).alias(table_alias.to_table_string())); - - // TODO: check how to select aggregated relations - let select = selected_fields - .selections() - .fold(select, |acc, selection| match selection { - SelectedField::Scalar(sf) => acc.column(sf.as_column(ctx).table(table_alias.to_table_string())), - SelectedField::Relation(rs) => { - let table_name = match rs.field.relation().is_many_to_many() { - true => m2m_join_alias_name(&rs.field), - false => join_alias_name(&rs.field), - }; - - acc.value(Column::from((table_name, JSON_AGG_IDENT)).alias(rs.field.name().to_owned())) - } - _ => acc, - }); + let select = Select::from_table(table) + .with_selection(selected_fields, table_alias, ctx) + .with_ordering(&args, Some(table_alias.to_table_string()), ctx) + .with_pagination(args.take_abs(), args.skip) + .with_filters(args.filter, Some(table_alias), ctx) + .append_trace(&Span::current()) + .add_trace_id(ctx.trace_id); // Adds joins for relations - let select = self.with_related_queries(select, selected_fields.relations(), table_alias, ctx); - let select = self.with_ordering(select, &args, Some(table_alias.to_table_string()), ctx); - let select = self.with_pagination(select, args.take_abs(), args.skip); - let select = self.with_filters(select, args.filter, Some(table_alias), ctx); - - select.append_trace(&Span::current()).add_trace_id(ctx.trace_id) + self.with_related_queries(select, selected_fields.relations(), table_alias, ctx) } fn with_related_queries<'a, 'b>( @@ -85,13 +70,12 @@ impl SelectBuilder { select.left_join(m2m_join) } else { - // LEFT JOIN LATERAL () AS ON TRUE - let join_select = Table::from(self.build_related_query_select(rs, parent_alias, ctx)) - .alias(join_alias_name(&rs.field)) - .on(ConditionTree::single(true.raw())) - .lateral(); + let join_table_alias = join_alias_name(&rs.field); + let join_table = + Table::from(self.build_related_query_select(rs, parent_alias, ctx)).alias(join_table_alias); - select.left_join(join_select) + // LEFT JOIN LATERAL ( ) AS ON TRUE + select.left_join(join_table.on(ConditionTree::single(true.raw())).lateral()) } } @@ -101,82 +85,68 @@ impl SelectBuilder { parent_alias: Alias, ctx: &Context<'_>, ) -> Select<'static> { - let table_alias = self.next_alias(); - - let build_obj_params = rs - .selections - .iter() - .filter_map(|f| match f { - SelectedField::Scalar(sf) => Some(( - Cow::from(sf.db_name().to_owned()), - Expression::from(sf.as_column(ctx).table(table_alias.to_table_string())), - )), - SelectedField::Relation(rs) => { - let table_name = match rs.field.relation().is_many_to_many() { - true => m2m_join_alias_name(&rs.field), - false => join_alias_name(&rs.field), - }; - - Some(( - Cow::from(rs.field.name().to_owned()), - Expression::from(Column::from((table_name, JSON_AGG_IDENT))), - )) - } - _ => None, - }) - .collect_vec(); - - let inner_alias = join_alias_name(&rs.field.related_field()); + let inner_root_table_alias = self.next_alias(); + let root_alias = self.next_alias(); + let inner_alias = self.next_alias(); + let middle_alias = self.next_alias(); let related_table = rs - .field .related_model() .as_table(ctx) - .alias(table_alias.to_table_string()); + .alias(inner_root_table_alias.to_table_string()); - // SELECT JSON_BUILD_OBJECT() - let inner = Select::from_table(related_table).value(json_build_object(build_obj_params).alias(JSON_AGG_IDENT)); + // SELECT * FROM "Table" as WHERE parent.id = child.parent_id + let root = Select::from_table(related_table) + .with_join_conditions(&rs.field, parent_alias, inner_root_table_alias, ctx) + .comment("root select"); + + // SELECT JSON_BUILD_OBJECT() FROM ( ) + let inner = Select::from_table(Table::from(root).alias(root_alias.to_table_string())) + .value(build_json_obj_fn(rs, ctx, root_alias).alias(JSON_AGG_IDENT)); - // WHERE parent.id = child.parent_id - let inner = self.with_join_conditions(inner, &rs.field, parent_alias, table_alias, ctx); // LEFT JOIN LATERAL () AS ON TRUE - let inner = self.with_related_queries(inner, rs.relations(), table_alias, ctx); + let inner = self.with_related_queries(inner, rs.relations(), root_alias, ctx); let linking_fields = rs.field.related_field().linking_fields(); if rs.field.relation().is_many_to_many() { - let order_by_selection = rs - .args - .order_by - .iter() - .flat_map(|order_by| match order_by { - OrderBy::Scalar(x) if x.path.is_empty() => vec![x.field.clone()], - OrderBy::Relevance(x) => x.fields.clone(), - _ => Vec::new(), - }) - .collect_vec(); - let selection = FieldSelection::union(vec![FieldSelection::from(order_by_selection), linking_fields]); - - // SELECT - // SELECT ONLY if it's a m2m table as we need to order by outside of the inner select - ModelProjection::from(selection) - .as_columns(ctx) - .fold(inner, |acc, c| acc.column(c.table(table_alias.to_table_string()))) - } else { - // SELECT - let inner = ModelProjection::from(linking_fields) + let selection: Vec> = FieldSelection::union(vec![order_by_selection(rs), linking_fields]) + .into_projection() .as_columns(ctx) - .fold(inner, |acc, c| acc.column(c.table(table_alias.to_table_string()))); - - let inner = self.with_ordering(inner, &rs.args, Some(table_alias.to_table_string()), ctx); - let inner = self.with_pagination(inner, rs.args.take_abs(), rs.args.skip); - let inner = self.with_filters(inner, rs.args.filter.clone(), Some(table_alias), ctx); - - let inner = Table::from(inner).alias(inner_alias.clone()); - let middle = Select::from_table(inner).column(Column::from((inner_alias.clone(), JSON_AGG_IDENT))); - let outer = Select::from_table(Table::from(middle).alias(format!("{}_1", inner_alias))).value(json_agg()); + .map(|c| c.table(root_alias.to_table_string())) + .collect(); - outer + // SELECT , + inner.with_columns(selection.into()) + } else { + // select ordering, filtering & join fields from child selections to order, filter & join them on the outer query + let inner_selection: Vec> = FieldSelection::union(vec![ + order_by_selection(rs), + filtering_selection(rs), + relation_selection(rs), + ]) + .into_projection() + .as_columns(ctx) + .map(|c| c.table(root_alias.to_table_string())) + .collect(); + + let inner = inner.with_columns(inner_selection.into()).comment("inner select"); + + let middle = Select::from_table(Table::from(inner).alias(inner_alias.to_table_string())) + // SELECT . + .column(Column::from((inner_alias.to_table_string(), JSON_AGG_IDENT))) + // ORDER BY ... + .with_ordering(&rs.args, Some(inner_alias.to_table_string()), ctx) + // WHERE ... + .with_filters(rs.args.filter.clone(), Some(inner_alias), ctx) + // LIMIT $1 OFFSET $2 + .with_pagination(rs.args.take_abs(), rs.args.skip) + .comment("middle select"); + + // SELECT COALESCE(JSON_AGG(), '[]') AS FROM ( ) as + Select::from_table(Table::from(middle).alias(middle_alias.to_table_string())) + .value(json_agg()) + .comment("outer select") } } @@ -188,7 +158,7 @@ impl SelectBuilder { let left_columns = rf.related_field().m2m_columns(ctx); let right_columns = ModelProjection::from(rf.model().primary_identifier()).as_columns(ctx); - let conditions = left_columns + let join_conditions = left_columns .into_iter() .zip(right_columns) .fold(None::, |acc, (a, b)| { @@ -203,22 +173,20 @@ impl SelectBuilder { }) .unwrap(); - let inner = Select::from_table(rf.as_table(ctx).alias(m2m_table_alias.to_table_string())) - .value(Column::from((join_alias_name(&rf), JSON_AGG_IDENT))) - .and_where(conditions); - - let inner = self.with_ordering(inner, &rs.args, Some(join_alias_name(&rs.field)), ctx); - let inner = self.with_pagination(inner, rs.args.take_abs(), rs.args.skip); - // TODO: avoid clone? - let inner = self.with_filters(inner, rs.args.filter.clone(), None, ctx); - - // TODO: parent_alias is likely wrong here - let join_select = Table::from(self.build_related_query_select(rs, m2m_table_alias, ctx)) + let m2m_join_data = Table::from(self.build_related_query_select(rs, m2m_table_alias, ctx)) .alias(join_alias_name(&rf)) .on(ConditionTree::single(true.raw())) .lateral(); - let inner = inner.left_join(join_select); + let child_table = rf.as_table(ctx).alias(m2m_table_alias.to_table_string()); + + let inner = Select::from_table(child_table) + .value(Column::from((join_alias_name(&rf), JSON_AGG_IDENT))) + .left_join(m2m_join_data) // join m2m table + .and_where(join_conditions) // adds join condition to the child table + .with_ordering(&rs.args, Some(join_alias_name(&rs.field)), ctx) // adds ordering stmts + .with_filters(rs.args.filter.clone(), None, ctx) // adds query filters // TODO: avoid clone filter + .with_pagination(rs.args.take_abs(), rs.args.skip); // adds pagination let outer = Select::from_table(Table::from(inner).alias(format!("{}_1", m2m_alias))).value(json_agg()); @@ -227,11 +195,72 @@ impl SelectBuilder { .on(ConditionTree::single(true.raw())) .lateral() } +} - /// Builds the lateral join conditions - fn with_join_conditions<'a>( - &mut self, - select: Select<'a>, +trait SelectBuilderExt<'a> { + fn with_filters(self, filter: Option, parent_alias: Option, ctx: &Context<'_>) -> Select<'a>; + fn with_pagination(self, take: Option, skip: Option) -> Select<'a>; + fn with_ordering(self, args: &QueryArguments, parent_alias: Option, ctx: &Context<'_>) -> Select<'a>; + fn with_join_conditions( + self, + rf: &RelationField, + parent_alias: Alias, + child_alias: Alias, + ctx: &Context<'_>, + ) -> Select<'a>; + fn with_selection(self, selected_fields: &FieldSelection, table_alias: Alias, ctx: &Context<'_>) -> Select<'a>; + fn with_columns(self, columns: ColumnIterator) -> Select<'a>; +} + +impl<'a> SelectBuilderExt<'a> for Select<'a> { + fn with_filters(self, filter: Option, parent_alias: Option, ctx: &Context<'_>) -> Select<'a> { + use crate::filter::*; + + if let Some(filter) = filter { + let mut visitor = crate::filter::FilterVisitor::with_top_level_joins().set_parent_alias_opt(parent_alias); + let (filter, joins) = visitor.visit_filter(filter, ctx); + let select = self.and_where(filter); + + match joins { + Some(joins) => joins.into_iter().fold(select, |acc, join| acc.join(join.data)), + None => select, + } + } else { + self + } + } + + fn with_pagination(self, take: Option, skip: Option) -> Select<'a> { + let select = match take { + Some(take) => self.limit(take as usize), + None => self, + }; + + let select = match skip { + Some(skip) => select.offset(skip as usize), + None => select, + }; + + select + } + + fn with_ordering(self, args: &QueryArguments, parent_alias: Option, ctx: &Context<'_>) -> Select<'a> { + let order_by_definitions = OrderByBuilder::default() + .with_parent_alias(parent_alias) + .build(args, ctx); + + let select = order_by_definitions + .iter() + .flat_map(|j| &j.joins) + .fold(self, |acc, join| acc.join(join.clone().data)); + + order_by_definitions + .iter() + .fold(select, |acc, o| acc.order_by(o.order_definition.clone())) + } + + fn with_join_conditions( + self, rf: &RelationField, parent_alias: Alias, child_alias: Alias, @@ -255,65 +284,105 @@ impl SelectBuilder { }) .unwrap(); - select.and_where(conditions) + self.and_where(conditions) } - fn with_ordering<'a>( - &mut self, - select: Select<'a>, - args: &QueryArguments, - parent_alias: Option, - ctx: &Context<'_>, - ) -> Select<'a> { - let order_by_definitions = OrderByBuilder::default() - .with_parent_alias(parent_alias) - .build(args, ctx); + fn with_selection(self, selected_fields: &FieldSelection, table_alias: Alias, ctx: &Context<'_>) -> Select<'a> { + selected_fields + .selections() + .fold(self, |acc, selection| match selection { + SelectedField::Scalar(sf) => acc.column(sf.as_column(ctx).table(table_alias.to_table_string())), + SelectedField::Relation(rs) => { + let table_name = match rs.field.relation().is_many_to_many() { + true => m2m_join_alias_name(&rs.field), + false => join_alias_name(&rs.field), + }; - let select = order_by_definitions - .iter() - .flat_map(|j| &j.joins) - .fold(select, |acc, join| acc.join(join.clone().data)); + acc.value(Column::from((table_name, JSON_AGG_IDENT)).alias(rs.field.name().to_owned())) + } + _ => acc, + }) + } - order_by_definitions - .iter() - .fold(select, |acc, o| acc.order_by(o.order_definition.clone())) + fn with_columns(self, columns: ColumnIterator) -> Select<'a> { + columns.into_iter().fold(self, |select, col| select.column(col)) } +} - fn with_pagination<'a>(&mut self, select: Select<'a>, take: Option, skip: Option) -> Select<'a> { - let select = match take { - Some(take) => select.limit(take as usize), - None => select, - }; +fn build_json_obj_fn(rs: &RelationSelection, ctx: &Context<'_>, root_alias: Alias) -> Function<'static> { + let build_obj_params = rs + .selections + .iter() + .filter_map(|f| match f { + SelectedField::Scalar(sf) => Some(( + Cow::from(sf.db_name().to_owned()), + Expression::from(sf.as_column(ctx).table(root_alias.to_table_string())), + )), + SelectedField::Relation(rs) => { + let table_name = match rs.field.relation().is_many_to_many() { + true => m2m_join_alias_name(&rs.field), + false => join_alias_name(&rs.field), + }; + + Some(( + Cow::from(rs.field.name().to_owned()), + Expression::from(Column::from((table_name, JSON_AGG_IDENT))), + )) + } + _ => None, + }) + .collect(); - let select = match skip { - Some(skip) => select.offset(skip as usize), - None => select, - }; + json_build_object(build_obj_params) +} - select - } +fn order_by_selection(rs: &RelationSelection) -> FieldSelection { + let selection: Vec<_> = rs + .args + .order_by + .iter() + .flat_map(|order_by| match order_by { + OrderBy::Scalar(x) if x.path.is_empty() => vec![x.field.clone()], + OrderBy::Relevance(x) => x.fields.clone(), + _ => Vec::new(), + }) + .collect(); + + FieldSelection::from(selection) +} - fn with_filters<'a>( - &mut self, - select: Select<'a>, - filter: Option, - alias: Option, - ctx: &Context<'_>, - ) -> Select<'a> { - use crate::filter::*; +fn relation_selection(rs: &RelationSelection) -> FieldSelection { + let relation_fields = rs.relations().flat_map(|rs| join_fields(&rs.field)).collect::>(); - if let Some(filter) = filter { - let mut visitor = crate::filter::FilterVisitor::with_top_level_joins().set_parent_alias_opt(alias); - let (filter, joins) = visitor.visit_filter(filter, ctx); - let select = select.and_where(filter); + FieldSelection::from(relation_fields) +} - match joins { - Some(joins) => joins.into_iter().fold(select, |acc, join| acc.join(join.data)), - None => select, - } - } else { - select - } +fn filtering_selection(rs: &RelationSelection) -> FieldSelection { + if let Some(filter) = &rs.args.filter { + FieldSelection::from(extract_filter_scalars(filter)) + } else { + FieldSelection::default() + } +} + +fn extract_filter_scalars(f: &Filter) -> Vec { + match f { + Filter::And(x) => x.iter().flat_map(extract_filter_scalars).collect(), + Filter::Or(x) => x.iter().flat_map(extract_filter_scalars).collect(), + Filter::Not(x) => x.iter().flat_map(extract_filter_scalars).collect(), + Filter::Scalar(x) => x.scalar_fields().into_iter().map(ToOwned::to_owned).collect(), + Filter::ScalarList(x) => vec![x.field.clone()], + Filter::OneRelationIsNull(x) => join_fields(&x.field), + Filter::Relation(x) => join_fields(&x.field), + _ => Vec::new(), + } +} + +fn join_fields(rf: &RelationField) -> Vec { + if rf.is_inlined_on_enclosing_model() { + rf.scalar_fields() + } else { + rf.related_field().referenced_fields() } } diff --git a/query-engine/query-structure/src/field_selection.rs b/query-engine/query-structure/src/field_selection.rs index b1538b27e80b..c1eda4531530 100644 --- a/query-engine/query-structure/src/field_selection.rs +++ b/query-engine/query-structure/src/field_selection.rs @@ -1,6 +1,7 @@ use crate::{ parent_container::ParentContainer, prisma_value_ext::PrismaValueExtensions, CompositeFieldRef, DomainError, Field, - QueryArguments, RelationField, ScalarField, ScalarFieldRef, SelectionResult, TypeIdentifier, + Model, ModelProjection, QueryArguments, RelationField, ScalarField, ScalarFieldRef, SelectionResult, + TypeIdentifier, }; use itertools::Itertools; use prisma_value::PrismaValue; @@ -161,6 +162,10 @@ impl FieldSelection { _ => None, }) } + + pub fn into_projection(self) -> ModelProjection { + self.into() + } } /// A selected field. Can be contained on a model or composite type. @@ -196,6 +201,10 @@ impl RelationSelection { _ => None, }) } + + pub fn related_model(&self) -> Model { + self.field.related_model() + } } impl SelectedField { From 8988c8c98648a72b2d202cf1aaca5bcd4f49543c Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Thu, 30 Nov 2023 18:53:26 +0100 Subject: [PATCH 25/38] fix crdb tests --- .../tests/new/multi_schema.rs | 2 +- .../nested_multi_order_pagination.rs | 8 ++++---- .../order_by_aggregation.rs | 18 +++++++++--------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/multi_schema.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/multi_schema.rs index db0f020e029c..29c93689f542 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/multi_schema.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/multi_schema.rs @@ -434,7 +434,7 @@ mod multi_schema { insta::assert_snapshot!( run_query!(&runner, r#" query { - findManyCategoriesOnPosts(where: {postId: {gt: 0}}) { + findManyCategoriesOnPosts(orderBy: [{ postId: asc }, { categoryId: asc }], where: {postId: {gt: 0}}) { category { name }, diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/nested_multi_order_pagination.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/nested_multi_order_pagination.rs index cf14f3e8bb45..ccbdc693b555 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/nested_multi_order_pagination.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/nested_multi_order_pagination.rs @@ -46,7 +46,7 @@ mod paging_one2m_stable_order { // Makes: [1 => 2, 2 => 3, 3 => 5] insta::assert_snapshot!( run_query!(&runner, r#"query { - findManyTestModel { + findManyTestModel(orderBy: { id: asc }) { id related(take: 1, orderBy: [{ fieldA: desc }, {fieldB: asc }, { fieldC: asc }, { fieldD: desc }]) { id @@ -73,7 +73,7 @@ mod paging_one2m_stable_order { // Makes: [1 => 1, 2 => 4, 3 => 6] insta::assert_snapshot!( run_query!(&runner, r#"query { - findManyTestModel { + findManyTestModel(orderBy: { id: asc}) { id related(take: -1, orderBy: [{ fieldA: desc }, { fieldB: asc }, { fieldC: asc }, { fieldD: desc }]) { id @@ -185,7 +185,7 @@ mod paging_one2m_unstable_order { run_query!( &runner, r#"query { - findManyTestModel { + findManyTestModel(orderBy: { id: asc }) { id related(take: 1, orderBy: [{ fieldA: desc }, {fieldB: asc }, { fieldC: asc }, { fieldD: desc }]) { id @@ -214,7 +214,7 @@ mod paging_one2m_unstable_order { run_query!( &runner, r#"query { - findManyTestModel { + findManyTestModel(orderBy: { id: asc }) { id related(take: -1, orderBy: [{ fieldA: desc }, { fieldB: asc }, { fieldC: asc }, { fieldD: desc }]) { id diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/order_by_aggregation.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/order_by_aggregation.rs index c21517823309..52e9fcaf8cc4 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/order_by_aggregation.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/order_and_pagination/order_by_aggregation.rs @@ -79,12 +79,12 @@ mod order_by_aggr { run_query!(&runner, r#"{ findManyPost(orderBy: { categories: { _count: asc } }) { title - categories { + categories(orderBy: { name: asc }) { name } } }"#), - @r###"{"data":{"findManyPost":[{"title":"bob_post_1","categories":[{"name":"Finance"}]},{"title":"alice_post_1","categories":[{"name":"News"},{"name":"Society"}]},{"title":"bob_post_2","categories":[{"name":"History"},{"name":"Gaming"},{"name":"Hacking"}]}]}}"### + @r###"{"data":{"findManyPost":[{"title":"bob_post_1","categories":[{"name":"Finance"}]},{"title":"alice_post_1","categories":[{"name":"News"},{"name":"Society"}]},{"title":"bob_post_2","categories":[{"name":"Gaming"},{"name":"Hacking"},{"name":"History"}]}]}}"### ); Ok(()) @@ -98,12 +98,12 @@ mod order_by_aggr { run_query!(&runner, r#"{ findManyPost(orderBy: { categories: { _count: desc } }) { title - categories { + categories(orderBy: { name :asc }) { name } } }"#), - @r###"{"data":{"findManyPost":[{"title":"bob_post_2","categories":[{"name":"History"},{"name":"Gaming"},{"name":"Hacking"}]},{"title":"alice_post_1","categories":[{"name":"News"},{"name":"Society"}]},{"title":"bob_post_1","categories":[{"name":"Finance"}]}]}}"### + @r###"{"data":{"findManyPost":[{"title":"bob_post_2","categories":[{"name":"Gaming"},{"name":"Hacking"},{"name":"History"}]},{"title":"alice_post_1","categories":[{"name":"News"},{"name":"Society"}]},{"title":"bob_post_1","categories":[{"name":"Finance"}]}]}}"### ); Ok(()) @@ -159,12 +159,12 @@ mod order_by_aggr { run_query!(&runner, r#"{ findManyPost(orderBy: [{ categories: { _count: asc } }, { title: asc }]) { title - categories { + categories(orderBy: { name: asc }) { name } } }"#), - @r###"{"data":{"findManyPost":[{"title":"bob_post_1","categories":[{"name":"Finance"}]},{"title":"alice_post_1","categories":[{"name":"News"},{"name":"Society"}]},{"title":"bob_post_2","categories":[{"name":"History"},{"name":"Gaming"},{"name":"Hacking"}]}]}}"### + @r###"{"data":{"findManyPost":[{"title":"bob_post_1","categories":[{"name":"Finance"}]},{"title":"alice_post_1","categories":[{"name":"News"},{"name":"Society"}]},{"title":"bob_post_2","categories":[{"name":"Gaming"},{"name":"Hacking"},{"name":"History"}]}]}}"### ); Ok(()) @@ -181,12 +181,12 @@ mod order_by_aggr { user { name } - categories { + categories(orderBy: { name: asc }) { name } } }"#), - @r###"{"data":{"findManyPost":[{"user":{"name":"Alice"},"categories":[{"name":"News"},{"name":"Society"}]},{"user":{"name":"Bob"},"categories":[{"name":"History"},{"name":"Gaming"},{"name":"Hacking"}]},{"user":{"name":"Bob"},"categories":[{"name":"Finance"}]}]}}"### + @r###"{"data":{"findManyPost":[{"user":{"name":"Alice"},"categories":[{"name":"News"},{"name":"Society"}]},{"user":{"name":"Bob"},"categories":[{"name":"Gaming"},{"name":"Hacking"},{"name":"History"}]},{"user":{"name":"Bob"},"categories":[{"name":"Finance"}]}]}}"### ); Ok(()) @@ -571,7 +571,7 @@ mod order_by_aggr { findManyPost(orderBy: [{ categories: { _count: asc } }, { title: asc }], cursor: { id: 2 }, take: 2) { id title - categories { + categories(orderBy: { name: asc }) { name } } From 9710c410d1bb1f7e7e5fc4f69cc420a72bcdd517 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Thu, 30 Nov 2023 19:17:31 +0100 Subject: [PATCH 26/38] cleanup --- .../src/database/operations/coerce.rs | 8 ++++---- .../src/database/operations/read.rs | 6 +++--- .../src/model_extensions/selection_result.rs | 8 ++++---- .../core/src/query_graph_builder/read/utils.rs | 2 +- query-engine/core/src/response_ir/internal.rs | 9 ++++----- .../query-structure/src/field_selection.rs | 5 +++-- .../src/projections/model_projection.rs | 18 +----------------- .../query-structure/src/selection_result.rs | 2 +- 8 files changed, 21 insertions(+), 37 deletions(-) diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs index a8e45ee91c45..22ec980a0b40 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs @@ -3,8 +3,9 @@ use query_structure::*; use crate::query_arguments_ext::QueryArgumentsExt; -// TODO: find better name -pub(crate) fn coerce_record_with_join(record: &mut Record, rq_indexes: Vec<(usize, &RelationSelection)>) { +/// Coerces relations resolved as JSON to PrismaValues. +/// Note: Some in-memory processing is baked into this function too for performance reasons. +pub(crate) fn coerce_record_with_json_relation(record: &mut Record, rq_indexes: Vec<(usize, &RelationSelection)>) { for (val_idx, rs) in rq_indexes { let val = record.values.get_mut(val_idx).unwrap(); // TODO(perf): Find ways to avoid serializing and deserializing multiple times. @@ -14,8 +15,7 @@ pub(crate) fn coerce_record_with_join(record: &mut Record, rq_indexes: Vec<(usiz } } -// TODO: find better name -pub(crate) fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &RelationSelection) -> PrismaValue { +fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &RelationSelection) -> PrismaValue { let relations = rs.relations().collect_vec(); match value { diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index b95e5f64da1b..dd90ad08476a 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -1,4 +1,4 @@ -use super::coerce::coerce_record_with_join; +use super::coerce::coerce_record_with_json_relation; use crate::{ column_metadata, model_extensions::*, @@ -104,7 +104,7 @@ pub(crate) async fn get_many_records_joins( match ctx.max_bind_values { Some(chunk_size) if query_arguments.should_batch(chunk_size) => { return Err(SqlError::QueryParameterLimitExceeded( - "Join queries cannot be split into multiple queries just yet. If you encounter that issue, please open an issue." + "Joined queries cannot be split into multiple queries just yet. If you encounter this error, please open an issue." .to_string(), )); } @@ -117,7 +117,7 @@ pub(crate) async fn get_many_records_joins( let mut record = Record::from(item); // Coerces json values to prisma values - coerce_record_with_join(&mut record, rs_indexes.clone()); + coerce_record_with_json_relation(&mut record, rs_indexes.clone()); records.push(record) } diff --git a/query-engine/connectors/sql-query-connector/src/model_extensions/selection_result.rs b/query-engine/connectors/sql-query-connector/src/model_extensions/selection_result.rs index 152f15864c0d..9cf94e06d423 100644 --- a/query-engine/connectors/sql-query-connector/src/model_extensions/selection_result.rs +++ b/query-engine/connectors/sql-query-connector/src/model_extensions/selection_result.rs @@ -34,10 +34,10 @@ impl SelectionResultExt for SelectionResult { fn db_values<'a>(&self, ctx: &Context<'_>) -> Vec> { self.pairs .iter() - .map(|(selection, v)| match selection { - SelectedField::Scalar(sf) => sf.value(v.clone(), ctx), - SelectedField::Composite(_cf) => todo!(), - SelectedField::Relation(_) => todo!(), // [Composites] todo + .filter_map(|(selection, v)| match selection { + SelectedField::Scalar(sf) => Some(sf.value(v.clone(), ctx)), + SelectedField::Composite(_) => None, + SelectedField::Relation(_) => None, }) .collect() } diff --git a/query-engine/core/src/query_graph_builder/read/utils.rs b/query-engine/core/src/query_graph_builder/read/utils.rs index ea8f00c31818..95a63db7f8e2 100644 --- a/query-engine/core/src/query_graph_builder/read/utils.rs +++ b/query-engine/core/src/query_graph_builder/read/utils.rs @@ -132,7 +132,7 @@ fn extract_relation_selection( ) -> QueryGraphBuilderResult { let object = pf .nested_fields - .expect("Invalid composite query shape: Composite field selected without sub-selection."); + .expect("Invalid relation query shape: Relation field selected without sub-selection."); let related_model = rf.related_model(); diff --git a/query-engine/core/src/response_ir/internal.rs b/query-engine/core/src/response_ir/internal.rs index c8b5d8915cae..113a113b74d2 100644 --- a/query-engine/core/src/response_ir/internal.rs +++ b/query-engine/core/src/response_ir/internal.rs @@ -240,7 +240,7 @@ fn serialize_record_selection_with_relations( InnerOutputType::Object(obj) => { let result = serialize_objects_with_relation(record_selection, obj)?; - process_object(field, is_list, result, name) + finalize_objects(field, is_list, result, name) } // We always serialize record selections into objects or lists on the top levels. Scalars and enums are handled separately. _ => unreachable!(), @@ -267,15 +267,14 @@ fn serialize_record_selection( InnerOutputType::Object(obj) => { let result = serialize_objects(record_selection, obj, query_schema)?; - process_object(field, is_list, result, name) + finalize_objects(field, is_list, result, name) } _ => unreachable!(), // We always serialize record selections into objects or lists on the top levels. Scalars and enums are handled separately. } } -// TODO: rename function -fn process_object( +fn finalize_objects( field: &OutputField<'_>, is_list: bool, result: IndexMap, Vec>, @@ -413,7 +412,7 @@ fn serialize_relation_selection( let mut map = Map::new(); - // TODO: handle errors + // TODO: better handle errors let mut value_obj: HashMap = HashMap::from_iter(value.into_object().unwrap()); let db_field_names = &rrs.fields; let fields: Vec<_> = db_field_names diff --git a/query-engine/query-structure/src/field_selection.rs b/query-engine/query-structure/src/field_selection.rs index c1eda4531530..5254ccb20cb4 100644 --- a/query-engine/query-structure/src/field_selection.rs +++ b/query-engine/query-structure/src/field_selection.rs @@ -33,7 +33,8 @@ impl FieldSelection { .and_then(|selection| selection.as_composite()) .map(|cs| cs.is_superset_of(other_cs)) .unwrap_or(false), - SelectedField::Relation(_) => todo!(), + // TODO: Relation selections are ignored for now to prevent breaking the existing query-based strategy to resolve relations. + SelectedField::Relation(_) => true, }) } @@ -270,7 +271,7 @@ impl CompositeSelection { .and_then(|selection| selection.as_composite()) .map(|cs| cs.is_superset_of(other_cs)) .unwrap_or(false), - SelectedField::Relation(_) => unreachable!(), + SelectedField::Relation(_) => true, // A composite selection cannot hold relations. }) } diff --git a/query-engine/query-structure/src/projections/model_projection.rs b/query-engine/query-structure/src/projections/model_projection.rs index 9c966bd8a56f..0d1a8f4b5171 100644 --- a/query-engine/query-structure/src/projections/model_projection.rs +++ b/query-engine/query-structure/src/projections/model_projection.rs @@ -1,4 +1,4 @@ -use crate::{Field, FieldSelection, ScalarFieldRef, SelectedField, SelectionResult, TypeIdentifier}; +use crate::{Field, FieldSelection, ScalarFieldRef, SelectedField, TypeIdentifier}; use itertools::Itertools; use psl::schema_ast::ast::FieldArity; @@ -104,19 +104,3 @@ impl IntoIterator for ModelProjection { self.fields.into_iter() } } - -impl From<&SelectionResult> for ModelProjection { - fn from(p: &SelectionResult) -> Self { - let fields = p - .pairs - .iter() - .map(|(field_selection, _)| match field_selection { - SelectedField::Scalar(sf) => sf.clone().into(), - SelectedField::Composite(cf) => cf.field.clone().into(), - SelectedField::Relation(_) => todo!(), - }) - .collect::>(); - - Self::new(fields) - } -} diff --git a/query-engine/query-structure/src/selection_result.rs b/query-engine/query-structure/src/selection_result.rs index 156797bc94ee..6f87ec74f6ce 100644 --- a/query-engine/query-structure/src/selection_result.rs +++ b/query-engine/query-structure/src/selection_result.rs @@ -94,7 +94,7 @@ impl SelectionResult { .filter_map(|(selection, _)| match selection { SelectedField::Scalar(sf) => Some(sf.clone()), SelectedField::Composite(_) => None, - SelectedField::Relation(_) => todo!(), + SelectedField::Relation(_) => None, }) .collect(); From defe57ae7c9b4327ab891adc0cd7419b46c671ba Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Thu, 30 Nov 2023 21:00:17 +0100 Subject: [PATCH 27/38] add scalar coercion tests + handle coercion errors --- Cargo.lock | 1 + .../tests/queries/data_types/mod.rs | 1 + .../queries/data_types/through_relation.rs | 286 ++++++++++++++++++ .../connectors/sql-query-connector/Cargo.toml | 1 + .../src/database/operations/coerce.rs | 129 ++++++-- .../src/database/operations/read.rs | 2 +- query-engine/query-structure/src/field/mod.rs | 5 + 7 files changed, 399 insertions(+), 26 deletions(-) create mode 100644 query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs diff --git a/Cargo.lock b/Cargo.lock index 74f0b840d4f4..93d70a3bae4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4939,6 +4939,7 @@ dependencies = [ "chrono", "cuid", "futures", + "hex", "itertools", "once_cell", "opentelemetry", diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/mod.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/mod.rs index ae1fe75b8837..09ed6668f619 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/mod.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/mod.rs @@ -8,3 +8,4 @@ mod float; mod int; mod json; mod string; +mod through_relation; diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs new file mode 100644 index 000000000000..6b2a6fd6c142 --- /dev/null +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs @@ -0,0 +1,286 @@ +use indoc::indoc; +use query_engine_tests::*; + +#[test_suite] +mod scalar_relations { + fn schema_common() -> String { + let schema = indoc! { + r#"model Parent { + #id(id, Int, @id) + + children Child[] + } + + model Child { + #id(childId, Int, @id) + + parentId Int? + parent Parent? @relation(fields: [parentId], references: [id]) + + string String + int Int + bInt BigInt + float Float + bytes Bytes + bool Boolean + dt DateTime + } + "# + }; + + schema.to_owned() + } + + #[connector_test(schema(schema_common))] + async fn common_types(runner: Runner) -> TestResult<()> { + create_common_children(&runner).await?; + + insta::assert_snapshot!( + run_query!(&runner, r#"{ findManyParent { id children { childId string int bInt float bytes bool dt } } }"#), + @r###"{"data":{"findManyParent":[{"id":1,"children":[{"childId":1,"string":"abc","int":1,"bInt":"1","float":1.5,"bytes":"AQID","bool":false,"dt":"1900-10-10T01:10:10.001Z"},{"childId":2,"string":"def","int":-4234234,"bInt":"14324324234324","float":-2.54367,"bytes":"FDSF","bool":true,"dt":"1999-12-12T21:12:12.121Z"}]}]}}"### + ); + + Ok(()) + } + + fn schema_json() -> String { + let schema = indoc! { + r#"model Parent { + #id(id, Int, @id) + + children Child[] + } + + model Child { + #id(childId, Int, @id) + + parentId Int? + parent Parent? @relation(fields: [parentId], references: [id]) + + json Json + } + "# + }; + + schema.to_owned() + } + + #[connector_test(schema(schema_json), capabilities(Json), exclude(Mysql(5.6)))] + async fn json_type(runner: Runner) -> TestResult<()> { + create_child(&runner, r#"{ childId: 1, json: "1" }"#).await?; + create_child(&runner, r#"{ childId: 2, json: "{}" }"#).await?; + create_child(&runner, r#"{ childId: 3, json: "{\"a\": \"b\"}" }"#).await?; + create_child(&runner, r#"{ childId: 4, json: "[]" }"#).await?; + create_child(&runner, r#"{ childId: 5, json: "[1, -1, true, {\"a\": \"b\"}]" }"#).await?; + create_parent( + &runner, + r#"{ id: 1, children: { connect: [{ childId: 1 }, { childId: 2 }, { childId: 3 }, { childId: 4 }, { childId: 5 }] } }"#, + ) + .await?; + + insta::assert_snapshot!( + run_query!(&runner, r#"{ findManyParent(orderBy: { id: asc }) { id children { childId json } } }"#), + @r###"{"data":{"findManyParent":[{"id":1,"children":[{"childId":1,"json":"1"},{"childId":2,"json":"{}"},{"childId":3,"json":"{\"a\":\"b\"}"},{"childId":4,"json":"[]"},{"childId":5,"json":"[1,-1,true,{\"a\":\"b\"}]"}]}]}}"### + ); + + Ok(()) + } + + fn schema_enum() -> String { + let schema = indoc! { + r#"model Parent { + #id(id, Int, @id) + + children Child[] + } + + model Child { + #id(childId, Int, @id) + + parentId Int? + parent Parent? @relation(fields: [parentId], references: [id]) + + enum Color + } + + enum Color { + Red + Green + Blue + } + "# + }; + + schema.to_owned() + } + + #[connector_test(schema(schema_enum), capabilities(Enums))] + async fn enum_type(runner: Runner) -> TestResult<()> { + create_child(&runner, r#"{ childId: 1, enum: Red }"#).await?; + create_child(&runner, r#"{ childId: 2, enum: Green }"#).await?; + create_child(&runner, r#"{ childId: 3, enum: Blue }"#).await?; + create_parent( + &runner, + r#"{ id: 1, children: { connect: [{ childId: 1 }, { childId: 2 }, { childId: 3 }] } }"#, + ) + .await?; + + insta::assert_snapshot!( + run_query!(&runner, r#"{ findManyParent(orderBy: { id :asc }) { id children { childId enum } } }"#), + @r###"{"data":{"findManyParent":[{"id":1,"children":[{"childId":1,"enum":"Red"},{"childId":2,"enum":"Green"},{"childId":3,"enum":"Blue"}]}]}}"### + ); + + Ok(()) + } + + fn schema_decimal() -> String { + let schema = indoc! { + r#"model Parent { + #id(id, Int, @id) + + children Child[] + } + + model Child { + #id(childId, Int, @id) + + parentId Int? + parent Parent? @relation(fields: [parentId], references: [id]) + + dec Decimal + } + "# + }; + + schema.to_owned() + } + + #[connector_test(schema(schema_decimal), capabilities(DecimalType))] + async fn decimal_type(runner: Runner) -> TestResult<()> { + create_child(&runner, r#"{ childId: 1, dec: "1" }"#).await?; + create_child(&runner, r#"{ childId: 2, dec: "-1" }"#).await?; + create_child(&runner, r#"{ childId: 3, dec: "123.45678910" }"#).await?; + create_parent( + &runner, + r#"{ id: 1, children: { connect: [{ childId: 1 }, { childId: 2 }, { childId: 3 }] } }"#, + ) + .await?; + + insta::assert_snapshot!( + run_query!(&runner, r#"{ findManyParent(orderBy: { id: asc }) { id children { childId dec } } }"#), + @r###"{"data":{"findManyParent":[{"id":1,"children":[{"childId":1,"dec":"1"},{"childId":2,"dec":"-1"},{"childId":3,"dec":"123.4567891"}]}]}}"### + ); + + Ok(()) + } + + fn schema_scalar_lists() -> String { + let schema = indoc! { + r#"model Parent { + #id(id, Int, @id) + + children Child[] + } + + model Child { + #id(childId, Int, @id) + + parentId Int? + parent Parent? @relation(fields: [parentId], references: [id]) + + string String[] + int Int[] + bInt BigInt[] + float Float[] + bytes Bytes[] + bool Boolean[] + dt DateTime[] + } + "# + }; + + schema.to_owned() + } + + #[connector_test(schema(schema_scalar_lists), capabilities(ScalarLists))] + async fn scalar_lists(runner: Runner) -> TestResult<()> { + create_child( + &runner, + r#"{ + childId: 1, + string: ["abc", "def"], + int: [1, -1, 1234567], + bInt: [1, -1, 9223372036854775807, -9223372036854775807], + float: [1.5, -1.5, 1.234567], + bytes: ["AQID", "Qk9OSk9VUg=="], + bool: [false, true], + dt: ["1900-10-10T01:10:10.001Z", "1999-12-12T21:12:12.121Z"], + }"#, + ) + .await?; + create_parent(&runner, r#"{ id: 1, children: { connect: [{ childId: 1 }] } }"#).await?; + + insta::assert_snapshot!( + run_query!(&runner, r#"{ findManyParent { id children { childId string int bInt float bytes bool dt } } }"#), + @r###"{"data":{"findManyParent":[{"id":1,"children":[{"childId":1,"string":["abc","def"],"int":[1,-1,1234567],"bInt":["1","-1","9223372036854775807","-9223372036854775807"],"float":[1.5,-1.5,1.234567],"bytes":["AQID","Qk9OSk9VUg=="],"bool":[false,true],"dt":["1900-10-10T01:10:10.001Z","1999-12-12T21:12:12.121Z"]}]}]}}"### + ); + + Ok(()) + } + + async fn create_common_children(runner: &Runner) -> TestResult<()> { + create_child( + &runner, + r#"{ + childId: 1, + string: "abc", + int: 1, + bInt: 1, + float: 1.5, + bytes: "AQID", + bool: false, + dt: "1900-10-10T01:10:10.001Z", + }"#, + ) + .await?; + + create_child( + &runner, + r#"{ + childId: 2, + string: "def", + int: -4234234, + bInt: 14324324234324, + float: -2.54367, + bytes: "FDSF", + bool: true, + dt: "1999-12-12T21:12:12.121Z", + }"#, + ) + .await?; + + create_parent( + &runner, + r#"{ id: 1, children: { connect: [{ childId: 1 }, { childId: 2 }] } }"#, + ) + .await?; + + Ok(()) + } + + async fn create_child(runner: &Runner, data: &str) -> TestResult<()> { + runner + .query(format!("mutation {{ createOneChild(data: {}) {{ childId }} }}", data)) + .await? + .assert_success(); + Ok(()) + } + + async fn create_parent(runner: &Runner, data: &str) -> TestResult<()> { + runner + .query(format!("mutation {{ createOneParent(data: {}) {{ id }} }}", data)) + .await? + .assert_success(); + Ok(()) + } +} diff --git a/query-engine/connectors/sql-query-connector/Cargo.toml b/query-engine/connectors/sql-query-connector/Cargo.toml index fbe04850164d..9ba23da469c5 100644 --- a/query-engine/connectors/sql-query-connector/Cargo.toml +++ b/query-engine/connectors/sql-query-connector/Cargo.toml @@ -27,6 +27,7 @@ uuid.workspace = true opentelemetry = { version = "0.17", features = ["tokio"] } tracing-opentelemetry = "0.17.3" cuid = { git = "https://github.com/prisma/cuid-rust", branch = "wasm32-support" } +hex = "0.4" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] quaint.workspace = true diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs index 22ec980a0b40..8e8820b29e12 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs @@ -1,21 +1,29 @@ +use std::io; + +use bigdecimal::{BigDecimal, FromPrimitive}; use itertools::{Either, Itertools}; use query_structure::*; -use crate::query_arguments_ext::QueryArgumentsExt; +use crate::{query_arguments_ext::QueryArgumentsExt, SqlError}; /// Coerces relations resolved as JSON to PrismaValues. /// Note: Some in-memory processing is baked into this function too for performance reasons. -pub(crate) fn coerce_record_with_json_relation(record: &mut Record, rq_indexes: Vec<(usize, &RelationSelection)>) { +pub(crate) fn coerce_record_with_json_relation( + record: &mut Record, + rq_indexes: Vec<(usize, &RelationSelection)>, +) -> crate::Result<()> { for (val_idx, rs) in rq_indexes { let val = record.values.get_mut(val_idx).unwrap(); // TODO(perf): Find ways to avoid serializing and deserializing multiple times. let json_val: serde_json::Value = serde_json::from_str(val.as_json().unwrap()).unwrap(); - *val = coerce_json_relation_to_pv(json_val, rs); + *val = coerce_json_relation_to_pv(json_val, rs)?; } + + Ok(()) } -fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &RelationSelection) -> PrismaValue { +fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &RelationSelection) -> crate::Result { let relations = rs.relations().collect_vec(); match value { @@ -29,7 +37,7 @@ fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &RelationSelection) false => Either::Right(iter), }; - PrismaValue::List(iter.collect()) + Ok(PrismaValue::List(iter.collect::>>()?)) } // to-one serde_json::Value::Array(values) => { @@ -43,7 +51,7 @@ fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &RelationSelection) if let Some(val) = coerced { val } else { - PrismaValue::Null + Ok(PrismaValue::Null) } } serde_json::Value::Object(obj) => { @@ -53,44 +61,115 @@ fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &RelationSelection) for (key, value) in obj { match related_model.fields().all().find(|f| f.db_name() == key).unwrap() { Field::Scalar(sf) => { - map.push((key, coerce_json_scalar_to_pv(value, &sf))); + map.push((key, coerce_json_scalar_to_pv(value, &sf)?)); } Field::Relation(rf) => { // TODO: optimize this if let Some(nested_selection) = relations.iter().find(|rs| rs.field == rf) { - map.push((key, coerce_json_relation_to_pv(value, nested_selection))); + map.push((key, coerce_json_relation_to_pv(value, nested_selection)?)); } } _ => (), } } - PrismaValue::Object(map) + Ok(PrismaValue::Object(map)) } _ => unreachable!(), } } -pub(crate) fn coerce_json_scalar_to_pv(value: serde_json::Value, sf: &ScalarField) -> PrismaValue { +pub(crate) fn coerce_json_scalar_to_pv(value: serde_json::Value, sf: &ScalarField) -> crate::Result { + if sf.type_identifier().is_json() { + return Ok(PrismaValue::Json(serde_json::to_string(&value)?)); + } + match value { - serde_json::Value::Null => PrismaValue::Null, - serde_json::Value::Bool(b) => PrismaValue::Boolean(b), + serde_json::Value::Null => Ok(PrismaValue::Null), + serde_json::Value::Bool(b) => Ok(PrismaValue::Boolean(b)), serde_json::Value::Number(n) => match sf.type_identifier() { - TypeIdentifier::Int => PrismaValue::Int(n.as_i64().unwrap()), - TypeIdentifier::BigInt => PrismaValue::BigInt(n.as_i64().unwrap()), - TypeIdentifier::Float => todo!(), - TypeIdentifier::Decimal => todo!(), - _ => unreachable!(), + TypeIdentifier::Int => Ok(PrismaValue::Int(n.as_i64().ok_or_else(|| { + build_conversion_error(&format!("Number({n})"), &format!("{:?}", sf.type_identifier())) + })?)), + TypeIdentifier::BigInt => Ok(PrismaValue::BigInt(n.as_i64().ok_or_else(|| { + build_conversion_error(&format!("Number({n})"), &format!("{:?}", sf.type_identifier())) + })?)), + TypeIdentifier::Float | TypeIdentifier::Decimal => { + let bd = n + .as_f64() + .and_then(BigDecimal::from_f64) + .map(|bd| bd.normalized()) + .ok_or_else(|| { + build_conversion_error(&format!("Number({n})"), &format!("{:?}", sf.type_identifier())) + })?; + + Ok(PrismaValue::Float(bd)) + } + _ => Err(build_conversion_error( + &format!("Number({n})"), + &format!("{:?}", sf.type_identifier()), + )), }, serde_json::Value::String(s) => match sf.type_identifier() { - TypeIdentifier::String => PrismaValue::String(s), - TypeIdentifier::Enum(_) => PrismaValue::Enum(s), - TypeIdentifier::DateTime => PrismaValue::DateTime(parse_datetime(&s).unwrap()), - TypeIdentifier::UUID => PrismaValue::Uuid(uuid::Uuid::parse_str(&s).unwrap()), - TypeIdentifier::Bytes => PrismaValue::Bytes(decode_bytes(&s).unwrap()), - _ => unreachable!(), + TypeIdentifier::String => Ok(PrismaValue::String(s)), + TypeIdentifier::Enum(_) => Ok(PrismaValue::Enum(s)), + TypeIdentifier::DateTime => Ok(PrismaValue::DateTime(parse_datetime(&format!("{s}Z")).map_err( + |err| { + build_conversion_error_with_reason( + &format!("String({s})"), + &format!("{:?}", sf.type_identifier()), + &err.to_string(), + ) + }, + )?)), + TypeIdentifier::UUID => Ok(PrismaValue::Uuid(uuid::Uuid::parse_str(&s).map_err(|err| { + build_conversion_error_with_reason( + &format!("String({s})"), + &format!("{:?}", sf.type_identifier()), + &err.to_string(), + ) + })?)), + TypeIdentifier::Bytes => { + // We skip the first two characters because they are the \x prefix. + let bytes = hex::decode(&s[2..]).map_err(|err| { + build_conversion_error_with_reason( + &format!("String({s})"), + &format!("{:?}", sf.type_identifier()), + &err.to_string(), + ) + })?; + + Ok(PrismaValue::Bytes(bytes)) + } + _ => Err(build_conversion_error( + &format!("String({s})"), + &format!("{:?}", sf.type_identifier()), + )), }, - serde_json::Value::Array(_) => todo!(), - serde_json::Value::Object(_) => todo!(), + serde_json::Value::Array(values) => Ok(PrismaValue::List( + values + .into_iter() + .map(|v| coerce_json_scalar_to_pv(v, sf)) + .collect::>>()?, + )), + serde_json::Value::Object(_) => unreachable!("Objects should be caught by the json catch-all above."), } } + +fn build_conversion_error(from: &str, to: &str) -> SqlError { + let error = io::Error::new( + io::ErrorKind::InvalidData, + format!("Unexpected conversion failure from {from} to {to}."), + ); + + SqlError::ConversionError(error.into()) +} + +fn build_conversion_error_with_reason(from: &str, to: &str, reason: &str) -> SqlError { + let error = io::Error::new( + io::ErrorKind::InvalidData, + format!("Unexpected conversion failure from {from} to {to}. Reason: ${reason}"), + ); + + SqlError::ConversionError(error.into()) +} diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index dd90ad08476a..3a77bf05743f 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -117,7 +117,7 @@ pub(crate) async fn get_many_records_joins( let mut record = Record::from(item); // Coerces json values to prisma values - coerce_record_with_json_relation(&mut record, rs_indexes.clone()); + coerce_record_with_json_relation(&mut record, rs_indexes.clone())?; records.push(record) } diff --git a/query-engine/query-structure/src/field/mod.rs b/query-engine/query-structure/src/field/mod.rs index 45d529c56abf..39e43f186c13 100644 --- a/query-engine/query-structure/src/field/mod.rs +++ b/query-engine/query-structure/src/field/mod.rs @@ -183,6 +183,11 @@ impl TypeIdentifier { pub fn is_enum(&self) -> bool { matches!(self, Self::Enum(..)) } + + /// Returns `true` if the type identifier is [`Json`]. + pub fn is_json(&self) -> bool { + matches!(self, Self::Json) + } } #[derive(Clone, Debug, PartialEq, Eq, Hash)] From 6068b988de6ce4e639fcb532b5cba7f16a81073c Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Fri, 1 Dec 2023 17:12:22 +0100 Subject: [PATCH 28/38] cleanup error message --- .../sql-query-connector/src/database/operations/read.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index 3a77bf05743f..faed596f4fc9 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -104,7 +104,7 @@ pub(crate) async fn get_many_records_joins( match ctx.max_bind_values { Some(chunk_size) if query_arguments.should_batch(chunk_size) => { return Err(SqlError::QueryParameterLimitExceeded( - "Joined queries cannot be split into multiple queries just yet. If you encounter this error, please open an issue." + "Joined queries cannot be split into multiple queries just yet. If you encounter this error, please open an issue" .to_string(), )); } @@ -127,6 +127,8 @@ pub(crate) async fn get_many_records_joins( records.reverse(); } + dbg!(&records); + Ok(records) } From bc60a31f0b25feca4360120f9b061fe0dec3f512 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Fri, 1 Dec 2023 18:27:35 +0100 Subject: [PATCH 29/38] fix driver adapter enum selection --- .../filters/field_reference/enum_filter.rs | 77 +++++++++++++++++++ .../queries/filters/field_reference/mod.rs | 1 + .../src/database/operations/read.rs | 2 - .../src/query_builder/select.rs | 6 +- 4 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/field_reference/enum_filter.rs diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/field_reference/enum_filter.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/field_reference/enum_filter.rs new file mode 100644 index 000000000000..5f8d8182a590 --- /dev/null +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/field_reference/enum_filter.rs @@ -0,0 +1,77 @@ +use query_engine_tests::*; + +#[test_suite(schema(schema))] +mod enum_filter { + use query_engine_tests::run_query; + + fn schema() -> String { + let schema = indoc! { + r#"model TestModel { + #id(id, Int, @id) + + enum TestEnum? + enum2 TestEnum[] + } + + enum TestEnum { + a + b + c + } + "# + }; + + schema.to_owned() + } + + #[connector_test(capabilities(Enums, ScalarLists))] + async fn inclusion_filter(runner: Runner) -> TestResult<()> { + test_data(&runner).await?; + + insta::assert_snapshot!( + run_query!(&runner, r#"query { findManyTestModel(where: { enum: { in: { _ref: "enum2", _container: "TestModel" } } }) { id enum enum2 }}"#), + @r###"{"data":{"findManyTestModel":[{"id":1,"enum":"a","enum2":["a","b"]}]}}"### + ); + + insta::assert_snapshot!( + run_query!(&runner, r#"query { findManyTestModel(where: { enum: { notIn: { _ref: "enum2", _container: "TestModel" } } }) { id }}"#), + @r###"{"data":{"findManyTestModel":[{"id":2}]}}"### + ); + + insta::assert_snapshot!( + run_query!(&runner, r#"query { findManyTestModel(where: { enum: { not: { in: { _ref: "enum2", _container: "TestModel" } } } }) { id }}"#), + @r###"{"data":{"findManyTestModel":[{"id":2}]}}"### + ); + + Ok(()) + } + + pub async fn test_data(runner: &Runner) -> TestResult<()> { + runner + .query(indoc! { r#" + mutation { createOneTestModel(data: { + id: 1, + enum: a, + enum2: [a, b] + }) { id }}"# }) + .await? + .assert_success(); + + runner + .query(indoc! { r#" + mutation { createOneTestModel(data: { + id: 2, + enum: b, + enum2: [a, c] + }) { id }}"# }) + .await? + .assert_success(); + + runner + .query(indoc! { r#"mutation { createOneTestModel(data: { id: 3 }) { id }}"# }) + .await? + .assert_success(); + + Ok(()) + } +} diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/field_reference/mod.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/field_reference/mod.rs index 8cee074b88d7..32a8200484b3 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/field_reference/mod.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/filters/field_reference/mod.rs @@ -7,6 +7,7 @@ pub mod bytes_filter; pub mod composite_filter; pub mod datetime_filter; pub mod decimal_filter; +pub mod enum_filter; pub mod float_filter; pub mod having_filter; pub mod int_filter; diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index faed596f4fc9..5520b78605d7 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -127,8 +127,6 @@ pub(crate) async fn get_many_records_joins( records.reverse(); } - dbg!(&records); - Ok(records) } diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs index f53d979f558d..439f0ef99c36 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs @@ -291,7 +291,11 @@ impl<'a> SelectBuilderExt<'a> for Select<'a> { selected_fields .selections() .fold(self, |acc, selection| match selection { - SelectedField::Scalar(sf) => acc.column(sf.as_column(ctx).table(table_alias.to_table_string())), + SelectedField::Scalar(sf) => acc.column( + sf.as_column(ctx) + .table(table_alias.to_table_string()) + .set_is_selected(true), + ), SelectedField::Relation(rs) => { let table_name = match rs.field.relation().is_many_to_many() { true => m2m_join_alias_name(&rs.field), From a84fe4386fdf0f05bd737b62e5d112ab079c7098 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Fri, 1 Dec 2023 18:28:59 +0100 Subject: [PATCH 30/38] rename preview feature from joins to relationJoins --- psl/psl-core/src/common/preview_features.rs | 4 ++-- psl/psl/tests/config/generators.rs | 2 +- .../tests/queries/batch/in_selection_batching.rs | 6 +++--- query-engine/core/src/query_graph_builder/read/utils.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/psl/psl-core/src/common/preview_features.rs b/psl/psl-core/src/common/preview_features.rs index 6eeac70c1389..318010ebdf9f 100644 --- a/psl/psl-core/src/common/preview_features.rs +++ b/psl/psl-core/src/common/preview_features.rs @@ -76,7 +76,7 @@ features!( TransactionApi, UncheckedScalarInputs, Views, - Joins + RelationJoins ); /// Generator preview features @@ -91,7 +91,7 @@ pub const ALL_PREVIEW_FEATURES: FeatureMap = FeatureMap { | PostgresqlExtensions | Tracing | Views - | Joins + | RelationJoins }), deprecated: enumflags2::make_bitflags!(PreviewFeature::{ AtomicNumberOperations diff --git a/psl/psl/tests/config/generators.rs b/psl/psl/tests/config/generators.rs index 1152ef910c5a..71130fe0ebd1 100644 --- a/psl/psl/tests/config/generators.rs +++ b/psl/psl/tests/config/generators.rs @@ -258,7 +258,7 @@ fn nice_error_for_unknown_generator_preview_feature() { .unwrap_err(); let expectation = expect![[r#" - error: The preview feature "foo" is not known. Expected one of: deno, driverAdapters, fullTextIndex, fullTextSearch, metrics, multiSchema, postgresqlExtensions, tracing, views, joins + error: The preview feature "foo" is not known. Expected one of: deno, driverAdapters, fullTextIndex, fullTextSearch, metrics, multiSchema, postgresqlExtensions, tracing, views, relationJoins[0m --> schema.prisma:3  |   2 |  provider = "prisma-client-js" diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs index 1126201865a4..f5e7face6768 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/batch/in_selection_batching.rs @@ -37,7 +37,7 @@ mod isb { // "batching of IN queries" should "work when having more than the specified amount of items" // TODO(joins): Excluded because we have no support for batched queries with joins. In practice, it should happen under much less circumstances // TODO(joins): than with the query-based strategy, because we don't issue `WHERE IN (parent_ids)` queries anymore to resolve relations. - #[connector_test(exclude_features("joins"))] + #[connector_test(exclude_features("relationJoins"))] async fn in_more_items(runner: Runner) -> TestResult<()> { create_test_data(&runner).await?; @@ -55,7 +55,7 @@ mod isb { // "ascending ordering of batched IN queries" should "work when having more than the specified amount of items" // TODO(joins): Excluded because we have no support for batched queries with joins. In practice, it should happen under much less circumstances // TODO(joins): than with the query-based strategy, because we don't issue `WHERE IN (parent_ids)` queries anymore to resolve relations. - #[connector_test(exclude_features("joins"))] + #[connector_test(exclude_features("relationJoins"))] async fn asc_in_ordering(runner: Runner) -> TestResult<()> { create_test_data(&runner).await?; @@ -73,7 +73,7 @@ mod isb { // "ascending ordering of batched IN queries" should "work when having more than the specified amount of items" // TODO(joins): Excluded because we have no support for batched queries with joins. In practice, it should happen under much less circumstances // TODO(joins): than with the query-based strategy, because we don't issue `WHERE IN (parent_ids)` queries anymore to resolve relations. - #[connector_test(exclude_features("joins"))] + #[connector_test(exclude_features("relationJoins"))] async fn desc_in_ordering(runner: Runner) -> TestResult<()> { create_test_data(&runner).await?; diff --git a/query-engine/core/src/query_graph_builder/read/utils.rs b/query-engine/core/src/query_graph_builder/read/utils.rs index 95a63db7f8e2..8e5f513ddf95 100644 --- a/query-engine/core/src/query_graph_builder/read/utils.rs +++ b/query-engine/core/src/query_graph_builder/read/utils.rs @@ -74,7 +74,7 @@ where T: Into, { let should_collect_relation_selection = query_schema.has_capability(ConnectorCapability::LateralJoin) - && query_schema.has_feature(PreviewFeature::Joins); + && query_schema.has_feature(PreviewFeature::RelationJoins); let parent = parent.into(); @@ -248,7 +248,7 @@ pub(crate) fn get_relation_load_strategy( aggregation_selections: &[RelAggregationSelection], query_schema: &QuerySchema, ) -> RelationLoadStrategy { - if query_schema.has_feature(PreviewFeature::Joins) + if query_schema.has_feature(PreviewFeature::RelationJoins) && query_schema.has_capability(ConnectorCapability::LateralJoin) && args.cursor.is_none() && args.distinct.is_none() From 410139eef35af9df36db3d2a06afe5803bbbe14f Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Fri, 1 Dec 2023 18:40:14 +0100 Subject: [PATCH 31/38] fix unit test --- psl/psl/tests/config/generators.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psl/psl/tests/config/generators.rs b/psl/psl/tests/config/generators.rs index 71130fe0ebd1..35d56347366b 100644 --- a/psl/psl/tests/config/generators.rs +++ b/psl/psl/tests/config/generators.rs @@ -258,7 +258,7 @@ fn nice_error_for_unknown_generator_preview_feature() { .unwrap_err(); let expectation = expect![[r#" - error: The preview feature "foo" is not known. Expected one of: deno, driverAdapters, fullTextIndex, fullTextSearch, metrics, multiSchema, postgresqlExtensions, tracing, views, relationJoins[0m + error: The preview feature "foo" is not known. Expected one of: deno, driverAdapters, fullTextIndex, fullTextSearch, metrics, multiSchema, postgresqlExtensions, tracing, views, relationJoins [0m --> schema.prisma:3  |   2 |  provider = "prisma-client-js" From e04e5f62deedf0cb73737cfe9166206949ec8ba3 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 4 Dec 2023 16:30:49 +0100 Subject: [PATCH 32/38] exclude tests + reframe error message --- .../queries/data_types/through_relation.rs | 17 ++++---- .../nested_update_many_inside_update.rs | 42 +++++++++++++++---- .../src/database/operations/coerce.rs | 4 +- .../src/database/operations/read.rs | 3 +- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs index 6b2a6fd6c142..eddcc3ec38a6 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs @@ -17,13 +17,7 @@ mod scalar_relations { parentId Int? parent Parent? @relation(fields: [parentId], references: [id]) - string String - int Int - bInt BigInt - float Float bytes Bytes - bool Boolean - dt DateTime } "# }; @@ -31,7 +25,8 @@ mod scalar_relations { schema.to_owned() } - #[connector_test(schema(schema_common))] + // TODO: fix https://github.com/prisma/team-orm/issues/684 and unexclude DAs + #[connector_test(schema(schema_common), exclude(Postgres("pg.js", "neon.js")))] async fn common_types(runner: Runner) -> TestResult<()> { create_common_children(&runner).await?; @@ -202,7 +197,13 @@ mod scalar_relations { schema.to_owned() } - #[connector_test(schema(schema_scalar_lists), capabilities(ScalarLists))] + // TODO: fix https://github.com/prisma/team-orm/issues/684 and unexclude DAs + + #[connector_test( + schema(schema_scalar_lists), + capabilities(ScalarLists), + exclude(Postgres("pg.js", "neon.js")) + )] async fn scalar_lists(runner: Runner) -> TestResult<()> { create_child( &runner, diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/already_converted/nested_update_many_inside_update.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/already_converted/nested_update_many_inside_update.rs index 4a42911d9898..12d848195774 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/already_converted/nested_update_many_inside_update.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/already_converted/nested_update_many_inside_update.rs @@ -55,7 +55,11 @@ mod um_inside_update { } // "a PM to C1! relation" should "work" - #[relation_link_test(on_parent = "ToMany", on_child = "ToOneReq")] + #[relation_link_test( + on_parent = "ToMany", + on_child = "ToOneReq", + exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + )] async fn pm_c1_req_should_work(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -89,7 +93,11 @@ mod um_inside_update { } // "a PM to C1 relation " should "work" - #[relation_link_test(on_parent = "ToMany", on_child = "ToOneOpt")] + #[relation_link_test( + on_parent = "ToMany", + on_child = "ToOneOpt", + exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + )] async fn pm_c1_should_work(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -123,7 +131,11 @@ mod um_inside_update { } // "a PM to CM relation " should "work" - #[relation_link_test(on_parent = "ToMany", on_child = "ToMany")] + #[relation_link_test( + on_parent = "ToMany", + on_child = "ToMany", + exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + )] async fn pm_cm_should_work(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -157,7 +169,11 @@ mod um_inside_update { } // "a PM to C1! relation " should "work with several updateManys" - #[relation_link_test(on_parent = "ToMany", on_child = "ToOneReq")] + #[relation_link_test( + on_parent = "ToMany", + on_child = "ToOneReq", + exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + )] async fn pm_c1_req_many_ums(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -197,7 +213,11 @@ mod um_inside_update { } // "a PM to C1! relation " should "work with empty Filter" - #[relation_link_test(on_parent = "ToMany", on_child = "ToOneReq")] + #[relation_link_test( + on_parent = "ToMany", + on_child = "ToOneReq", + exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + )] async fn pm_c1_req_empty_filter(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -233,7 +253,11 @@ mod um_inside_update { } // "a PM to C1! relation " should "not change anything when there is no hit" - #[relation_link_test(on_parent = "ToMany", on_child = "ToOneReq")] + #[relation_link_test( + on_parent = "ToMany", + on_child = "ToOneReq", + exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + )] async fn pm_c1_req_noop_no_hit(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -275,7 +299,11 @@ mod um_inside_update { // optional ordering // "a PM to C1! relation " should "work when multiple filters hit" - #[relation_link_test(on_parent = "ToMany", on_child = "ToOneReq")] + #[relation_link_test( + on_parent = "ToMany", + on_child = "ToOneReq", + exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + )] async fn pm_c1_req_many_filters(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs index 8e8820b29e12..968f1068513a 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs @@ -10,9 +10,9 @@ use crate::{query_arguments_ext::QueryArgumentsExt, SqlError}; /// Note: Some in-memory processing is baked into this function too for performance reasons. pub(crate) fn coerce_record_with_json_relation( record: &mut Record, - rq_indexes: Vec<(usize, &RelationSelection)>, + rs_indexes: Vec<(usize, &RelationSelection)>, ) -> crate::Result<()> { - for (val_idx, rs) in rq_indexes { + for (val_idx, rs) in rs_indexes { let val = record.values.get_mut(val_idx).unwrap(); // TODO(perf): Find ways to avoid serializing and deserializing multiple times. let json_val: serde_json::Value = serde_json::from_str(val.as_json().unwrap()).unwrap(); diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index 5520b78605d7..f2d394152c7c 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -104,8 +104,7 @@ pub(crate) async fn get_many_records_joins( match ctx.max_bind_values { Some(chunk_size) if query_arguments.should_batch(chunk_size) => { return Err(SqlError::QueryParameterLimitExceeded( - "Joined queries cannot be split into multiple queries just yet. If you encounter this error, please open an issue" - .to_string(), + "Joined queries cannot be split into multiple queries.".to_string(), )); } _ => (), From 28d52064d09b36bf5a95538b4739b2f2b00435b5 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 4 Dec 2023 16:35:49 +0100 Subject: [PATCH 33/38] undo some test changes --- .../tests/queries/data_types/through_relation.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs index eddcc3ec38a6..986c1287dd6f 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs @@ -17,7 +17,13 @@ mod scalar_relations { parentId Int? parent Parent? @relation(fields: [parentId], references: [id]) + string String + int Int + bInt BigInt + float Float bytes Bytes + bool Boolean + dt DateTime } "# }; @@ -25,8 +31,11 @@ mod scalar_relations { schema.to_owned() } - // TODO: fix https://github.com/prisma/team-orm/issues/684 and unexclude DAs - #[connector_test(schema(schema_common), exclude(Postgres("pg.js", "neon.js")))] + // TODO: fix https://github.com/prisma/team-orm/issues/684, https://github.com/prisma/team-orm/issues/685 and unexclude DAs + #[connector_test( + schema(schema_common), + exclude(Postgres("pg.js", "neon.js"), Vitess("planetscale.js")) + )] async fn common_types(runner: Runner) -> TestResult<()> { create_common_children(&runner).await?; From c4f05a6f5b8817bc7ab18372a36a152d6b11a1a1 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 4 Dec 2023 16:36:18 +0100 Subject: [PATCH 34/38] rename "data" to "__prisma_data__" --- .../connectors/sql-query-connector/src/query_builder/select.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs index 439f0ef99c36..d2ef3e62b344 100644 --- a/query-engine/connectors/sql-query-connector/src/query_builder/select.rs +++ b/query-engine/connectors/sql-query-connector/src/query_builder/select.rs @@ -12,7 +12,7 @@ use crate::{ use quaint::prelude::*; use query_structure::*; -pub const JSON_AGG_IDENT: &str = "data"; +pub const JSON_AGG_IDENT: &str = "__prisma_data__"; #[derive(Debug, Default)] pub(crate) struct SelectBuilder { From cede9be04526797ca4d566d817c39432b1459987 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 4 Dec 2023 17:57:46 +0100 Subject: [PATCH 35/38] add findUnique support --- .../queries/data_types/through_relation.rs | 30 ++++++++ .../src/interface/connection.rs | 1 + .../src/interface/transaction.rs | 1 + .../query-connector/src/interface.rs | 1 + .../src/database/connection.rs | 4 +- .../src/database/operations/read.rs | 76 +++++++++++++++++-- .../src/database/operations/update.rs | 15 +++- .../src/database/transaction.rs | 4 +- .../interpreter/query_interpreters/read.rs | 15 +++- query-engine/core/src/query_ast/read.rs | 1 + .../core/src/query_graph_builder/read/many.rs | 8 +- .../core/src/query_graph_builder/read/one.rs | 7 +- .../src/query_graph_builder/read/utils.rs | 9 ++- query-engine/core/src/response_ir/internal.rs | 27 ++++--- .../query-structure/src/query_arguments.rs | 5 ++ 15 files changed, 177 insertions(+), 27 deletions(-) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs index 986c1287dd6f..b2af72ab955e 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/data_types/through_relation.rs @@ -44,6 +44,16 @@ mod scalar_relations { @r###"{"data":{"findManyParent":[{"id":1,"children":[{"childId":1,"string":"abc","int":1,"bInt":"1","float":1.5,"bytes":"AQID","bool":false,"dt":"1900-10-10T01:10:10.001Z"},{"childId":2,"string":"def","int":-4234234,"bInt":"14324324234324","float":-2.54367,"bytes":"FDSF","bool":true,"dt":"1999-12-12T21:12:12.121Z"}]}]}}"### ); + insta::assert_snapshot!( + run_query!(&runner, r#"{ findUniqueParent(where: { id: 1 }) { id children { childId string int bInt float bytes bool dt } } }"#), + @r###"{"data":{"findUniqueParent":{"id":1,"children":[{"childId":1,"string":"abc","int":1,"bInt":"1","float":1.5,"bytes":"AQID","bool":false,"dt":"1900-10-10T01:10:10.001Z"},{"childId":2,"string":"def","int":-4234234,"bInt":"14324324234324","float":-2.54367,"bytes":"FDSF","bool":true,"dt":"1999-12-12T21:12:12.121Z"}]}}}"### + ); + + insta::assert_snapshot!( + run_query!(&runner, r#"{ findUniqueParent(where: { id: 2 }) { id children { childId string int bInt float bytes bool dt } } }"#), + @r###"{"data":{"findUniqueParent":null}}"### + ); + Ok(()) } @@ -87,6 +97,11 @@ mod scalar_relations { @r###"{"data":{"findManyParent":[{"id":1,"children":[{"childId":1,"json":"1"},{"childId":2,"json":"{}"},{"childId":3,"json":"{\"a\":\"b\"}"},{"childId":4,"json":"[]"},{"childId":5,"json":"[1,-1,true,{\"a\":\"b\"}]"}]}]}}"### ); + insta::assert_snapshot!( + run_query!(&runner, r#"{ findUniqueParent(where: { id: 1 }) { id children { childId json } } }"#), + @r###"{"data":{"findUniqueParent":{"id":1,"children":[{"childId":1,"json":"1"},{"childId":2,"json":"{}"},{"childId":3,"json":"{\"a\":\"b\"}"},{"childId":4,"json":"[]"},{"childId":5,"json":"[1,-1,true,{\"a\":\"b\"}]"}]}}}"### + ); + Ok(()) } @@ -134,6 +149,11 @@ mod scalar_relations { @r###"{"data":{"findManyParent":[{"id":1,"children":[{"childId":1,"enum":"Red"},{"childId":2,"enum":"Green"},{"childId":3,"enum":"Blue"}]}]}}"### ); + insta::assert_snapshot!( + run_query!(&runner, r#"{ findUniqueParent(where: { id: 1 }) { id children { childId enum } } }"#), + @r###"{"data":{"findUniqueParent":{"id":1,"children":[{"childId":1,"enum":"Red"},{"childId":2,"enum":"Green"},{"childId":3,"enum":"Blue"}]}}}"### + ); + Ok(()) } @@ -175,6 +195,11 @@ mod scalar_relations { @r###"{"data":{"findManyParent":[{"id":1,"children":[{"childId":1,"dec":"1"},{"childId":2,"dec":"-1"},{"childId":3,"dec":"123.4567891"}]}]}}"### ); + insta::assert_snapshot!( + run_query!(&runner, r#"{ findUniqueParent(where: { id: 1 }) { id children { childId dec } } }"#), + @r###"{"data":{"findUniqueParent":{"id":1,"children":[{"childId":1,"dec":"1"},{"childId":2,"dec":"-1"},{"childId":3,"dec":"123.4567891"}]}}}"### + ); + Ok(()) } @@ -235,6 +260,11 @@ mod scalar_relations { @r###"{"data":{"findManyParent":[{"id":1,"children":[{"childId":1,"string":["abc","def"],"int":[1,-1,1234567],"bInt":["1","-1","9223372036854775807","-9223372036854775807"],"float":[1.5,-1.5,1.234567],"bytes":["AQID","Qk9OSk9VUg=="],"bool":[false,true],"dt":["1900-10-10T01:10:10.001Z","1999-12-12T21:12:12.121Z"]}]}]}}"### ); + insta::assert_snapshot!( + run_query!(&runner, r#"{ findUniqueParent(where: { id: 1 }) { id children { childId string int bInt float bytes bool dt } } }"#), + @r###"{"data":{"findUniqueParent":{"id":1,"children":[{"childId":1,"string":["abc","def"],"int":[1,-1,1234567],"bInt":["1","-1","9223372036854775807","-9223372036854775807"],"float":[1.5,-1.5,1.234567],"bytes":["AQID","Qk9OSk9VUg=="],"bool":[false,true],"dt":["1900-10-10T01:10:10.001Z","1999-12-12T21:12:12.121Z"]}]}}}"### + ); + Ok(()) } diff --git a/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs b/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs index 206d0b296d40..9f825241939a 100644 --- a/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs +++ b/query-engine/connectors/mongodb-query-connector/src/interface/connection.rs @@ -190,6 +190,7 @@ impl ReadOperations for MongoDbConnection { filter: &query_structure::Filter, selected_fields: &FieldSelection, aggr_selections: &[RelAggregationSelection], + _relation_load_strategy: RelationLoadStrategy, _trace_id: Option, ) -> connector_interface::Result> { catch(async move { diff --git a/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs b/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs index 107459c4b3b9..6e15d1262123 100644 --- a/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs +++ b/query-engine/connectors/mongodb-query-connector/src/interface/transaction.rs @@ -255,6 +255,7 @@ impl<'conn> ReadOperations for MongoDbTransaction<'conn> { filter: &query_structure::Filter, selected_fields: &FieldSelection, aggr_selections: &[RelAggregationSelection], + _relation_load_strategy: RelationLoadStrategy, _trace_id: Option, ) -> connector_interface::Result> { catch(async move { diff --git a/query-engine/connectors/query-connector/src/interface.rs b/query-engine/connectors/query-connector/src/interface.rs index 167e28340a7d..518f4356d547 100644 --- a/query-engine/connectors/query-connector/src/interface.rs +++ b/query-engine/connectors/query-connector/src/interface.rs @@ -231,6 +231,7 @@ pub trait ReadOperations { filter: &Filter, selected_fields: &FieldSelection, aggregation_selections: &[RelAggregationSelection], + relation_load_strategy: RelationLoadStrategy, trace_id: Option, ) -> crate::Result>; diff --git a/query-engine/connectors/sql-query-connector/src/database/connection.rs b/query-engine/connectors/sql-query-connector/src/database/connection.rs index a987849e8b8d..2bdabe57b2e9 100644 --- a/query-engine/connectors/sql-query-connector/src/database/connection.rs +++ b/query-engine/connectors/sql-query-connector/src/database/connection.rs @@ -87,6 +87,7 @@ where filter: &Filter, selected_fields: &FieldSelection, aggr_selections: &[RelAggregationSelection], + relation_load_strategy: RelationLoadStrategy, trace_id: Option, ) -> connector::Result> { // [Composites] todo: FieldSelection -> ModelProjection conversion @@ -96,8 +97,9 @@ where &self.inner, model, filter, - &selected_fields.into(), + selected_fields, aggr_selections, + relation_load_strategy, &ctx, ) .await diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs index f2d394152c7c..a561c61e6f3c 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/read.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/read.rs @@ -13,6 +13,57 @@ use quaint::ast::*; use query_structure::*; pub(crate) async fn get_single_record( + conn: &dyn Queryable, + model: &Model, + filter: &Filter, + selected_fields: &FieldSelection, + aggr_selections: &[RelAggregationSelection], + relation_load_strategy: RelationLoadStrategy, + ctx: &Context<'_>, +) -> crate::Result> { + match relation_load_strategy { + RelationLoadStrategy::Join => get_single_record_joins(conn, model, filter, selected_fields, ctx).await, + RelationLoadStrategy::Query => { + get_single_record_wo_joins( + conn, + model, + filter, + &ModelProjection::from(selected_fields), + aggr_selections, + ctx, + ) + .await + } + } +} + +pub(crate) async fn get_single_record_joins( + conn: &dyn Queryable, + model: &Model, + filter: &Filter, + selected_fields: &FieldSelection, + ctx: &Context<'_>, +) -> crate::Result> { + let field_names: Vec<_> = selected_fields.db_names().collect(); + let idents = selected_fields.type_identifiers_with_arities(); + let rs_indexes = get_relation_selection_indexes(selected_fields.relations().collect(), &field_names); + + let query = query_builder::select::SelectBuilder::default().build( + QueryArguments::from((model.clone(), filter.clone())), + selected_fields, + ctx, + ); + + let mut record = execute_find_one(conn, query, &idents, &field_names, ctx).await?; + + if let Some(record) = record.as_mut() { + coerce_record_with_json_relation(record, rs_indexes)?; + }; + + Ok(record.map(|record| SingleRecord { record, field_names })) +} + +pub(crate) async fn get_single_record_wo_joins( conn: &dyn Queryable, model: &Model, filter: &Filter, @@ -41,18 +92,31 @@ pub(crate) async fn get_single_record( idents.append(&mut aggr_idents); - let meta = column_metadata::create(field_names.as_slice(), idents.as_slice()); + let record = execute_find_one(conn, query, &idents, &field_names, ctx) + .await? + .map(|record| SingleRecord { record, field_names }); + + Ok(record) +} + +async fn execute_find_one( + conn: &dyn Queryable, + query: Select<'_>, + idents: &[(TypeIdentifier, FieldArity)], + field_names: &[String], + ctx: &Context<'_>, +) -> crate::Result> { + let meta = column_metadata::create(field_names, idents); - let record = (match conn.find(query, meta.as_slice(), ctx).await { + let row = (match conn.find(query, meta.as_slice(), ctx).await { Ok(result) => Ok(Some(result)), Err(_e @ SqlError::RecordNotFoundForWhere(_)) => Ok(None), Err(_e @ SqlError::RecordDoesNotExist) => Ok(None), Err(e) => Err(e), })? - .map(Record::from) - .map(|record| SingleRecord { record, field_names }); + .map(Record::from); - Ok(record) + Ok(row) } pub(crate) async fn get_many_records( @@ -93,8 +157,8 @@ pub(crate) async fn get_many_records_joins( let field_names: Vec<_> = selected_fields.db_names().collect(); let idents = selected_fields.type_identifiers_with_arities(); let meta = column_metadata::create(field_names.as_slice(), idents.as_slice()); - let rs_indexes = get_relation_selection_indexes(selected_fields.relations().collect(), &field_names); + let mut records = ManyRecords::new(field_names.clone()); if let Some(0) = query_arguments.take { diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/update.rs b/query-engine/connectors/sql-query-connector/src/database/operations/update.rs index 617e02455abd..40ca5ce84fc0 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/update.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/update.rs @@ -21,16 +21,25 @@ pub(crate) async fn update_one_with_selection( selected_fields: FieldSelection, ctx: &Context<'_>, ) -> crate::Result> { - let selected_fields = ModelProjection::from(selected_fields); - // If there's nothing to update, just read the record. // TODO(perf): Technically, if the selectors are fulfilling the field selection, there's no need to perform an additional read. if args.args.is_empty() { let filter = build_update_one_filter(record_filter); - return get_single_record(conn, model, &filter, &selected_fields, &[], ctx).await; + return get_single_record( + conn, + model, + &filter, + &selected_fields, + &[], + RelationLoadStrategy::Query, + ctx, + ) + .await; } + let selected_fields = ModelProjection::from(selected_fields); + let cond = FilterBuilder::without_top_level_joins().visit_filter(build_update_one_filter(record_filter), ctx); let update = build_update_and_set_query(model, args, Some(&selected_fields), ctx).so_that(cond); diff --git a/query-engine/connectors/sql-query-connector/src/database/transaction.rs b/query-engine/connectors/sql-query-connector/src/database/transaction.rs index b69f77848f0d..35adddb52ab4 100644 --- a/query-engine/connectors/sql-query-connector/src/database/transaction.rs +++ b/query-engine/connectors/sql-query-connector/src/database/transaction.rs @@ -69,6 +69,7 @@ impl<'tx> ReadOperations for SqlConnectorTransaction<'tx> { filter: &Filter, selected_fields: &FieldSelection, aggr_selections: &[RelAggregationSelection], + relation_load_strategy: RelationLoadStrategy, trace_id: Option, ) -> connector::Result> { catch(self.connection_info.clone(), async move { @@ -77,8 +78,9 @@ impl<'tx> ReadOperations for SqlConnectorTransaction<'tx> { self.inner.as_queryable(), model, filter, - &selected_fields.into(), + selected_fields, aggr_selections, + relation_load_strategy, &ctx, ) .await diff --git a/query-engine/core/src/interpreter/query_interpreters/read.rs b/query-engine/core/src/interpreter/query_interpreters/read.rs index 9583e9fa70ed..7e238774fd8a 100644 --- a/query-engine/core/src/interpreter/query_interpreters/read.rs +++ b/query-engine/core/src/interpreter/query_interpreters/read.rs @@ -39,12 +39,13 @@ fn read_one( &filter, &query.selected_fields, &query.aggregation_selections, + query.relation_load_strategy, trace_id, ) .await?; match scalars { - Some(record) => { + Some(record) if query.relation_load_strategy.is_query() => { let scalars: ManyRecords = record.into(); let (scalars, aggregation_rows) = extract_aggregation_rows_from_scalars(scalars, query.aggregation_selections); @@ -60,6 +61,18 @@ fn read_one( } .into()) } + Some(record) => { + let records: ManyRecords = record.into(); + + Ok(dbg!(RecordSelectionWithRelations { + name: query.name, + model, + fields: query.selection_order, + records, + nested: build_relation_record_selection(query.selected_fields.relations()), + }) + .into()) + } None if query.options.contains(QueryOption::ThrowOnEmpty) => record_not_found(), diff --git a/query-engine/core/src/query_ast/read.rs b/query-engine/core/src/query_ast/read.rs index 353b2ca6bfb1..a01fce4d953a 100644 --- a/query-engine/core/src/query_ast/read.rs +++ b/query-engine/core/src/query_ast/read.rs @@ -203,6 +203,7 @@ pub struct RecordQuery { pub selection_order: Vec, pub aggregation_selections: Vec, pub options: QueryOptions, + pub relation_load_strategy: RelationLoadStrategy, } #[derive(Debug, Clone)] diff --git a/query-engine/core/src/query_graph_builder/read/many.rs b/query-engine/core/src/query_graph_builder/read/many.rs index c0f6ae9f5966..fe8009681c68 100644 --- a/query-engine/core/src/query_graph_builder/read/many.rs +++ b/query-engine/core/src/query_graph_builder/read/many.rs @@ -40,7 +40,13 @@ fn find_many_with_options( let selected_fields = utils::merge_relation_selections(selected_fields, None, &nested); let selected_fields = utils::merge_cursor_fields(selected_fields, &args.cursor); - let relation_load_strategy = get_relation_load_strategy(&args, &nested, &aggregation_selections, query_schema); + let relation_load_strategy = get_relation_load_strategy( + args.cursor.as_ref(), + args.distinct.as_ref(), + &nested, + &aggregation_selections, + query_schema, + ); Ok(ReadQuery::ManyRecordsQuery(ManyRecordsQuery { name, diff --git a/query-engine/core/src/query_graph_builder/read/one.rs b/query-engine/core/src/query_graph_builder/read/one.rs index fa5465324837..dab47a444c5e 100644 --- a/query-engine/core/src/query_graph_builder/read/one.rs +++ b/query-engine/core/src/query_graph_builder/read/one.rs @@ -1,4 +1,4 @@ -use super::*; +use super::{utils::get_relation_load_strategy, *}; use crate::{query_document::*, QueryOption, QueryOptions, ReadQuery, RecordQuery}; use query_structure::Model; use schema::{constants::args, QuerySchema}; @@ -46,6 +46,10 @@ fn find_unique_with_options( let selected_fields = utils::collect_selected_fields(&nested_fields, None, &model, query_schema)?; let nested = utils::collect_nested_queries(nested_fields, &model, query_schema)?; let selected_fields = utils::merge_relation_selections(selected_fields, None, &nested); + let relation_load_strategy = get_relation_load_strategy(None, None, &nested, &aggregation_selections, query_schema); + + dbg!(&selection_order); + dbg!(&selected_fields); Ok(ReadQuery::RecordQuery(RecordQuery { name, @@ -57,5 +61,6 @@ fn find_unique_with_options( selection_order, aggregation_selections, options, + relation_load_strategy, })) } diff --git a/query-engine/core/src/query_graph_builder/read/utils.rs b/query-engine/core/src/query_graph_builder/read/utils.rs index 8e5f513ddf95..fc569b94c839 100644 --- a/query-engine/core/src/query_graph_builder/read/utils.rs +++ b/query-engine/core/src/query_graph_builder/read/utils.rs @@ -2,7 +2,7 @@ use super::*; use crate::{ArgumentListLookup, FieldPair, ParsedField, ReadQuery}; use connector::RelAggregationSelection; use psl::{datamodel_connector::ConnectorCapability, PreviewFeature}; -use query_structure::{prelude::*, QueryArguments, RelationLoadStrategy}; +use query_structure::{prelude::*, RelationLoadStrategy}; use schema::{ constants::{aggregations::*, args}, QuerySchema, @@ -243,15 +243,16 @@ pub fn collect_relation_aggr_selections( } pub(crate) fn get_relation_load_strategy( - args: &QueryArguments, + cursor: Option<&SelectionResult>, + distinct: Option<&FieldSelection>, nested_queries: &[ReadQuery], aggregation_selections: &[RelAggregationSelection], query_schema: &QuerySchema, ) -> RelationLoadStrategy { if query_schema.has_feature(PreviewFeature::RelationJoins) && query_schema.has_capability(ConnectorCapability::LateralJoin) - && args.cursor.is_none() - && args.distinct.is_none() + && cursor.is_none() + && distinct.is_none() && aggregation_selections.is_empty() && !nested_queries.iter().any(|q| match q { ReadQuery::RelatedRecordsQuery(q) => q.has_cursor() || q.has_distinct() || q.has_aggregation_selections(), diff --git a/query-engine/core/src/response_ir/internal.rs b/query-engine/core/src/response_ir/internal.rs index 113a113b74d2..47385692b387 100644 --- a/query-engine/core/src/response_ir/internal.rs +++ b/query-engine/core/src/response_ir/internal.rs @@ -344,7 +344,7 @@ fn serialize_objects_with_relation( // Hack: we convert it to a hashset to support contains with &str as input // because Vec::contains(&str) doesn't work and we don't want to allocate a string record value - let selected_db_field_names: HashSet = result.fields.into_iter().collect(); + let selected_db_field_names: HashSet = result.fields.clone().into_iter().collect(); for record in result.records.records.into_iter() { if !object_mapping.contains_key(&record.parent_id) { @@ -352,7 +352,7 @@ fn serialize_objects_with_relation( } let values = record.values; - let mut object = IndexMap::with_capacity(values.len()); + let mut object = HashMap::with_capacity(values.len()); for (val, field) in values.into_iter().zip(fields.iter()) { // Skip fields that aren't part of the selection set @@ -392,7 +392,9 @@ fn serialize_objects_with_relation( } } - let result = Item::Map(object); + let map = reorder_object_with_selection_order(result.fields.clone(), object); + + let result = Item::Map(map); object_mapping.get_mut(&record.parent_id).unwrap().push(result); } @@ -537,12 +539,7 @@ fn serialize_objects( let mut all_fields = result.fields.clone(); all_fields.append(&mut aggr_fields); - let map = all_fields - .iter() - .fold(Map::with_capacity(all_fields.len()), |mut acc, field_name| { - acc.insert(field_name.to_owned(), object.remove(field_name).unwrap()); - acc - }); + let map = reorder_object_with_selection_order(all_fields, object); object_mapping.get_mut(&record.parent_id).unwrap().push(Item::Map(map)); } @@ -550,6 +547,18 @@ fn serialize_objects( Ok(object_mapping) } +fn reorder_object_with_selection_order( + selection_order: Vec, + mut object: HashMap, +) -> IndexMap { + selection_order + .iter() + .fold(Map::with_capacity(selection_order.len()), |mut acc, field_name| { + acc.insert(field_name.to_owned(), object.remove(field_name).unwrap()); + acc + }) +} + /// Unwraps are safe due to query validation. fn write_nested_items( record_id: &Option, diff --git a/query-engine/query-structure/src/query_arguments.rs b/query-engine/query-structure/src/query_arguments.rs index bdbbbeb2a1c3..35d7336b4673 100644 --- a/query-engine/query-structure/src/query_arguments.rs +++ b/query-engine/query-structure/src/query_arguments.rs @@ -32,6 +32,11 @@ pub enum RelationLoadStrategy { Join, Query, } +impl RelationLoadStrategy { + pub fn is_query(&self) -> bool { + matches!(self, RelationLoadStrategy::Query) + } +} impl std::fmt::Debug for QueryArguments { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { From 91d579b8d1bed77b1f5a2056678085d039c46fc3 Mon Sep 17 00:00:00 2001 From: Alexey Orlenko Date: Mon, 4 Dec 2023 19:50:26 +0100 Subject: [PATCH 36/38] Fix wasm build script The build script had an invalid `sed` command with an extra `''` argument that caused it to fail with ``` sed: can't read s/name = "query_engine_wasm"/name = "query_engine"/g: No such file or directory ``` This is reproducible both on CI and locally for me. Perhaps it was written for BSD sed and doesn't work with GNU sed (so it always fails on Linux and also fails on macOS inside prisma-engines Nix flake but maybe it works on macOS without Nix)? Because of this, a broken package was published from CI. The commit fixes the `sed` command and adds `set -e` so that errors like this would fail CI instead of silently continuing and doing wrong things. --- query-engine/query-engine-wasm/build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/query-engine/query-engine-wasm/build.sh b/query-engine/query-engine-wasm/build.sh index 184e4baf3545..a5b4859e82eb 100755 --- a/query-engine/query-engine-wasm/build.sh +++ b/query-engine/query-engine-wasm/build.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + # Call this script as `./build.sh ` OUT_VERSION="$1" @@ -12,7 +14,7 @@ OUT_NPM_NAME="@prisma/query-engine-wasm" # to avoid conflicts with libquery's `name = "query_engine"` library name declaration. # This little `sed -i` trick below is a hack to publish "@prisma/query-engine-wasm" # with the same binding filenames currently expected by the Prisma Client. -sed -i '' 's/name = "query_engine_wasm"/name = "query_engine"/g' Cargo.toml +sed -i 's/name = "query_engine_wasm"/name = "query_engine"/g' Cargo.toml # use `wasm-pack build --release` on CI only if [[ -z "$BUILDKITE" ]] && [[ -z "$GITHUB_ACTIONS" ]]; then @@ -23,7 +25,7 @@ fi wasm-pack build $BUILD_PROFILE --target $OUT_TARGET -sed -i '' 's/name = "query_engine"/name = "query_engine_wasm"/g' Cargo.toml +sed -i 's/name = "query_engine"/name = "query_engine_wasm"/g' Cargo.toml sleep 1 From 6e804a0f78153eefccadd10078eada66dfd44e87 Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 4 Dec 2023 19:02:41 +0100 Subject: [PATCH 37/38] fix tests and remove dbg --- .../nested_update_many_inside_update.rs | 21 ++++++++++++------- .../nested_atomic_number_ops.rs | 4 ++-- .../nested_update_inside_update.rs | 4 ++-- .../interpreter/query_interpreters/read.rs | 4 ++-- .../core/src/query_graph_builder/read/one.rs | 3 --- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/already_converted/nested_update_many_inside_update.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/already_converted/nested_update_many_inside_update.rs index 12d848195774..05931d16084b 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/already_converted/nested_update_many_inside_update.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/already_converted/nested_update_many_inside_update.rs @@ -55,10 +55,11 @@ mod um_inside_update { } // "a PM to C1! relation" should "work" + // TODO: fix https://github.com/prisma/team-orm/issues/683 and then unexclude DAs #[relation_link_test( on_parent = "ToMany", on_child = "ToOneReq", - exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + exclude(Postgres("pg.js", "neon.js"), Vitess("planetscale.js")) )] async fn pm_c1_req_should_work(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -93,10 +94,11 @@ mod um_inside_update { } // "a PM to C1 relation " should "work" + // TODO: fix https://github.com/prisma/team-orm/issues/683 and then unexclude DAs #[relation_link_test( on_parent = "ToMany", on_child = "ToOneOpt", - exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + exclude(Postgres("pg.js", "neon.js"), Vitess("planetscale.js")) )] async fn pm_c1_should_work(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -131,10 +133,11 @@ mod um_inside_update { } // "a PM to CM relation " should "work" + // TODO: fix https://github.com/prisma/team-orm/issues/683 and then unexclude DAs #[relation_link_test( on_parent = "ToMany", on_child = "ToMany", - exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + exclude(Postgres("pg.js", "neon.js"), Vitess("planetscale.js")) )] async fn pm_cm_should_work(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -169,10 +172,11 @@ mod um_inside_update { } // "a PM to C1! relation " should "work with several updateManys" + // TODO: fix https://github.com/prisma/team-orm/issues/683 and then unexclude DAs #[relation_link_test( on_parent = "ToMany", on_child = "ToOneReq", - exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + exclude(Postgres("pg.js", "neon.js"), Vitess("planetscale.js")) )] async fn pm_c1_req_many_ums(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -213,10 +217,11 @@ mod um_inside_update { } // "a PM to C1! relation " should "work with empty Filter" + // TODO: fix https://github.com/prisma/team-orm/issues/683 and then unexclude DAs #[relation_link_test( on_parent = "ToMany", on_child = "ToOneReq", - exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + exclude(Postgres("pg.js", "neon.js"), Vitess("planetscale.js")) )] async fn pm_c1_req_empty_filter(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -253,10 +258,11 @@ mod um_inside_update { } // "a PM to C1! relation " should "not change anything when there is no hit" + // TODO: fix https://github.com/prisma/team-orm/issues/683 and then unexclude DAs #[relation_link_test( on_parent = "ToMany", on_child = "ToOneReq", - exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + exclude(Postgres("pg.js", "neon.js"), Vitess("planetscale.js")) )] async fn pm_c1_req_noop_no_hit(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; @@ -299,10 +305,11 @@ mod um_inside_update { // optional ordering // "a PM to C1! relation " should "work when multiple filters hit" + // TODO: fix https://github.com/prisma/team-orm/issues/683 and then unexclude DAs #[relation_link_test( on_parent = "ToMany", on_child = "ToOneReq", - exclude(Postgres("pg.js", "neon"), Vitess("planetscale.js")) + exclude(Postgres("pg.js", "neon.js"), Vitess("planetscale.js")) )] async fn pm_c1_req_many_filters(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let parent = setup_data(runner, t).await?; diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/nested_atomic_number_ops.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/nested_atomic_number_ops.rs index c325fccb6d64..014c921705a7 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/nested_atomic_number_ops.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/nested_atomic_number_ops.rs @@ -195,7 +195,7 @@ mod atomic_number_ops { } // "A nested updateOne mutation" should "correctly apply all number operations for Int" - #[connector_test(schema(schema_3), exclude(CockroachDb))] + #[connector_test(schema(schema_3), exclude(CockroachDb, Postgres("pg.js", "neon.js")))] async fn nested_update_int_ops(runner: Runner) -> TestResult<()> { create_test_model(&runner, 1, None, None).await?; create_test_model(&runner, 2, Some(3), None).await?; @@ -324,7 +324,7 @@ mod atomic_number_ops { } // "A nested updateOne mutation" should "correctly apply all number operations for Int" - #[connector_test(schema(schema_3), exclude(MongoDb))] + #[connector_test(schema(schema_3), exclude(MongoDb, Postgres("pg.js", "neon.js")))] async fn nested_update_float_ops(runner: Runner) -> TestResult<()> { create_test_model(&runner, 1, None, None).await?; create_test_model(&runner, 2, None, Some("5.5")).await?; diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/not_using_schema_base/nested_update_inside_update.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/not_using_schema_base/nested_update_inside_update.rs index a543ba7b8f51..1cf206d347a4 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/not_using_schema_base/nested_update_inside_update.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/writes/nested_mutations/not_using_schema_base/nested_update_inside_update.rs @@ -191,7 +191,7 @@ mod update_inside_update { // ---------------------------------- // "A PM to C1 relation relation" should "work" - #[relation_link_test(on_parent = "ToMany", on_child = "ToOneOpt")] + #[relation_link_test(on_parent = "ToMany", on_child = "ToOneOpt", exclude(Postgres("pg.js", "neon.js")))] async fn pm_c1_should_work(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let res = run_query_json!( runner, @@ -384,7 +384,7 @@ mod update_inside_update { // ---------------------------------- // "A PM to CM relation relation" should "work" - #[relation_link_test(on_parent = "ToMany", on_child = "ToMany")] + #[relation_link_test(on_parent = "ToMany", on_child = "ToMany", exclude(Postgres("pg.js", "neon.js")))] async fn pm_cm_should_work(runner: &Runner, t: &DatamodelWithParams) -> TestResult<()> { let res = run_query_json!( runner, diff --git a/query-engine/core/src/interpreter/query_interpreters/read.rs b/query-engine/core/src/interpreter/query_interpreters/read.rs index 7e238774fd8a..8fc0b10d67bf 100644 --- a/query-engine/core/src/interpreter/query_interpreters/read.rs +++ b/query-engine/core/src/interpreter/query_interpreters/read.rs @@ -64,13 +64,13 @@ fn read_one( Some(record) => { let records: ManyRecords = record.into(); - Ok(dbg!(RecordSelectionWithRelations { + Ok(RecordSelectionWithRelations { name: query.name, model, fields: query.selection_order, records, nested: build_relation_record_selection(query.selected_fields.relations()), - }) + } .into()) } diff --git a/query-engine/core/src/query_graph_builder/read/one.rs b/query-engine/core/src/query_graph_builder/read/one.rs index dab47a444c5e..e2b6d2b4b941 100644 --- a/query-engine/core/src/query_graph_builder/read/one.rs +++ b/query-engine/core/src/query_graph_builder/read/one.rs @@ -48,9 +48,6 @@ fn find_unique_with_options( let selected_fields = utils::merge_relation_selections(selected_fields, None, &nested); let relation_load_strategy = get_relation_load_strategy(None, None, &nested, &aggregation_selections, query_schema); - dbg!(&selection_order); - dbg!(&selected_fields); - Ok(ReadQuery::RecordQuery(RecordQuery { name, alias, From 8109c3fa441c3c6797ed19f327d9d667b102ab9a Mon Sep 17 00:00:00 2001 From: Flavian Desverne Date: Mon, 4 Dec 2023 20:20:31 +0100 Subject: [PATCH 38/38] hack: filter null values from joined JSON arrays --- .../tests/queries/simple/m2m.rs | 43 +++++++++++++++++++ .../src/database/operations/coerce.rs | 15 ++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/m2m.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/m2m.rs index 556e5f21f3f8..bf8da606a812 100644 --- a/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/m2m.rs +++ b/query-engine/connector-test-kit-rs/query-engine-tests/tests/queries/simple/m2m.rs @@ -69,6 +69,49 @@ mod m2m { Ok(()) } + fn schema() -> String { + let schema = indoc! { + r#"model Item { + id Int @id @default(autoincrement()) + categories Category[] + createdAt DateTime @default(now()) + updatedAt DateTime? @updatedAt + } + + model Category { + id Int @id @default(autoincrement()) + items Item[] + createdAt DateTime @default(now()) + updatedAt DateTime? @updatedAt + }"# + }; + + schema.to_owned() + } + + // https://github.com/prisma/prisma/issues/16390 + #[connector_test(schema(schema), relation_mode = "prisma", only(Postgres))] + async fn repro_16390(runner: Runner) -> TestResult<()> { + run_query!(&runner, r#"mutation { createOneCategory(data: {}) { id } }"#); + run_query!( + &runner, + r#"mutation { createOneItem(data: { categories: { connect: { id: 1 } } }) { id } }"# + ); + run_query!(&runner, r#"mutation { deleteOneItem(where: { id: 1 }) { id } }"#); + + insta::assert_snapshot!( + run_query!(&runner, r#"{ findUniqueItem(where: { id: 1 }) { id categories { id } } }"#), + @r###"{"data":{"findUniqueItem":null}}"### + ); + + insta::assert_snapshot!( + run_query!(&runner, r#"{ findUniqueCategory(where: { id: 1 }) { id items { id } } }"#), + @r###"{"data":{"findUniqueCategory":{"id":1,"items":[]}}}"### + ); + + Ok(()) + } + async fn test_data(runner: &Runner) -> TestResult<()> { runner .query( diff --git a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs index 968f1068513a..61390bc5fa05 100644 --- a/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs +++ b/query-engine/connectors/sql-query-connector/src/database/operations/coerce.rs @@ -29,7 +29,18 @@ fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &RelationSelection) match value { // one-to-many serde_json::Value::Array(values) if rs.field.is_list() => { - let iter = values.into_iter().map(|value| coerce_json_relation_to_pv(value, rs)); + let iter = values.into_iter().filter_map(|value| { + // FIXME: In the case of m2m relations, the aggregation produces null values if the B side of the m2m table points to a record that doesn't exist. + // FIXME: This only seems to happen because of a bug with `relationMode=prisma`` which doesn't cleanup the relation table properly when deleting records that belongs to a m2m relation. + // FIXME: This hack filters down the null values from the array, but we should fix the root cause instead, if possible. + // FIXME: In theory, the aggregated array should only contain objects, which are the joined rows. + // FIXME: See m2m.rs::repro_16390 for a reproduction. + if value.is_null() && rs.field.relation().is_many_to_many() { + None + } else { + Some(coerce_json_relation_to_pv(value, rs)) + } + }); // Reverses order when using negative take. let iter = match rs.args.needs_reversed_order() { @@ -75,7 +86,7 @@ fn coerce_json_relation_to_pv(value: serde_json::Value, rs: &RelationSelection) Ok(PrismaValue::Object(map)) } - _ => unreachable!(), + x => unreachable!("Unexpected value when deserializing JSON relation data: {x:?}"), } }