From 95bc9c6ca3d70d1dc3a5950aa9edde79e4809214 Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Fri, 30 Jun 2023 10:38:27 +1000 Subject: [PATCH 01/12] Add Select support --- .../QueryableIntegrationFixture.cs | 26 +++++++++++++++++++ .../Advanced/Queryable/QueryTranslator.cs | 12 +++++++++ .../Queryable/SqlExpressionBuilder.cs | 10 +++++-- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs index 56f628ba..fef5beaf 100644 --- a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs +++ b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs @@ -13,6 +13,32 @@ namespace Nevermore.IntegrationTests { public class QueryableIntegrationFixture : FixtureWithRelationalStore { + [Test] + public void Select() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Select(c => c.FirstName) + .ToList(); + + customers.Should().BeEquivalentTo("Alice", "Bob", "Charlie"); + } + [Test] public void WhereEqual() { diff --git a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs index 04eb35c6..100a27f1 100644 --- a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs +++ b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs @@ -59,6 +59,18 @@ protected override Expression VisitMethodCall(MethodCallExpression node) sqlBuilder.Where(new CustomWhereClause((string)constantExpression.Value)); return node; } + case nameof(Enumerable.Select): + { + if (node.Arguments.Count > 2) + { + throw new NotSupportedException("Select does not support custom comparers"); + } + + Visit(node.Arguments[0]); + var fieldName = GetMemberNameFromKeySelectorExpression(node.Arguments[1]); + sqlBuilder.Column(new SelectAllJsonColumnLast(new [] { fieldName })); + return node; + } case nameof(System.Linq.Queryable.Where): { Visit(node.Arguments[0]); diff --git a/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs b/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs index 9bdf7063..349ba402 100644 --- a/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs +++ b/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs @@ -12,6 +12,7 @@ internal class SqlExpressionBuilder { readonly IRelationalStoreConfiguration configuration; + readonly List selectColumns = new(); readonly List orderByFields = new(); readonly List whereClauses = new(); string hint; @@ -29,6 +30,11 @@ public SqlExpressionBuilder(IRelationalStoreConfiguration configuration) public DocumentMap DocumentMap { get; private set; } + public void Column(ISelectColumns selectColumns) + { + this.selectColumns.Add(selectColumns); + } + public void Where(IWhereClause whereClause) { whereClauses.Add(whereClause); @@ -140,10 +146,10 @@ IExpression CreateSelectQuery() { IRowSelection rowSelection = take.HasValue && !skip.HasValue ? new Top(take.Value) : null; var orderBy = orderByFields.Any() ? new OrderBy(orderByFields) : GetDefaultOrderBy(); - var columns = new SelectAllJsonColumnLast(GetDocumentColumns().ToList()); + ISelectColumns columns = selectColumns.Any() ? new AggregateSelectColumns(selectColumns) : new SelectAllJsonColumnLast(GetDocumentColumns().ToList()); var select = new Select( rowSelection ?? new AllRows(), - skip.HasValue ? new AggregateSelectColumns(new ISelectColumns[] { new SelectRowNumber(new Over(orderBy, null), "RowNum"), columns }) : columns, + skip.HasValue ? new AggregateSelectColumns(new [] { new SelectRowNumber(new Over(orderBy, null), "RowNum"), columns }) : columns, from, CreateWhereClause(), null, From f96d304c9f045fc7770c9dfaa4c617601eb178c2 Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Fri, 30 Jun 2023 11:35:33 +1000 Subject: [PATCH 02/12] Add Distinct support --- .../QueryableIntegrationFixture.cs | 29 +++++++++++++++++ .../Advanced/Queryable/QueryTranslator.cs | 6 ++++ .../Queryable/SqlExpressionBuilder.cs | 32 +++++++++++++++++-- .../Nevermore/Querying/AST/IRowSelection.cs | 25 +++++++++++---- 4 files changed, 83 insertions(+), 9 deletions(-) diff --git a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs index fef5beaf..58b83554 100644 --- a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs +++ b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs @@ -953,6 +953,35 @@ public void SkipAndTake() customers.Select(c => c.LastName).Should().BeEquivalentTo("Cherry", "Durian"); } + [Test] + public void Distinct() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Apple" }, + new Customer { FirstName = "Charlie", LastName = "Banana" }, + new Customer { FirstName = "Dan", LastName = "Cherry" }, + new Customer { FirstName = "Erin", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Select(c => c.LastName) + .Distinct() + .ToList(); + + customers.Should().BeEquivalentTo("Apple", "Banana", "Cherry"); + } + [Test] public void Count() { diff --git a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs index 100a27f1..c2d68b11 100644 --- a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs +++ b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs @@ -195,6 +195,12 @@ protected override Expression VisitMethodCall(MethodCallExpression node) sqlBuilder.Skip(skip); return node; } + case nameof(System.Linq.Queryable.Distinct): + { + Visit(node.Arguments[0]); + sqlBuilder.Distinct(); + return node; + } default: throw new NotSupportedException(); } diff --git a/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs b/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs index 349ba402..b9fa301d 100644 --- a/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs +++ b/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs @@ -20,6 +20,7 @@ internal class SqlExpressionBuilder ITableSource from; int? skip; int? take; + bool distinct; QueryType queryType = QueryType.SelectMany; volatile int paramCounter; @@ -107,6 +108,11 @@ public void Hint(string hint) this.hint = hint; } + public void Distinct() + { + distinct = true; + } + public void From(Type documentType) { DocumentMap = configuration.DocumentMaps.Resolve(documentType); @@ -144,11 +150,11 @@ public void From(Type documentType) IExpression CreateSelectQuery() { - IRowSelection rowSelection = take.HasValue && !skip.HasValue ? new Top(take.Value) : null; + var rowSelection = CreateRowSelection(); var orderBy = orderByFields.Any() ? new OrderBy(orderByFields) : GetDefaultOrderBy(); ISelectColumns columns = selectColumns.Any() ? new AggregateSelectColumns(selectColumns) : new SelectAllJsonColumnLast(GetDocumentColumns().ToList()); var select = new Select( - rowSelection ?? new AllRows(), + rowSelection, skip.HasValue ? new AggregateSelectColumns(new [] { new SelectRowNumber(new Over(orderBy, null), "RowNum"), columns }) : columns, from, CreateWhereClause(), @@ -169,7 +175,7 @@ IExpression CreateSelectQuery() } select = new Select( - new AllRows(), + distinct ? new Distinct() : new AllRows(), new SelectAllColumnsWithTableAliasJsonLast("aliased", GetDocumentColumns().ToList()), new SubquerySource(select, "aliased"), new Where(new AndClause(pagingFilters)), @@ -181,6 +187,26 @@ IExpression CreateSelectQuery() return select; } + IRowSelection CreateRowSelection() + { + var selections = new List(); + if (take.HasValue && !skip.HasValue) + { + selections.Add(new Top(take.Value)); + } + else + { + selections.Add(new AllRows()); + } + + if (distinct) + { + selections.Add(new Distinct()); + } + + return new CompositeRowSelection(selections); + } + IExpression CreateExistsQuery() { IRowSelection rowSelection = take.HasValue && !skip.HasValue ? new Top(take.Value) : null; diff --git a/source/Nevermore/Querying/AST/IRowSelection.cs b/source/Nevermore/Querying/AST/IRowSelection.cs index a44257ca..37908367 100644 --- a/source/Nevermore/Querying/AST/IRowSelection.cs +++ b/source/Nevermore/Querying/AST/IRowSelection.cs @@ -1,10 +1,28 @@ -namespace Nevermore.Querying.AST +using System.Collections.Generic; +using System.Linq; + +namespace Nevermore.Querying.AST { public interface IRowSelection { string GenerateSql(); } + public class CompositeRowSelection : IRowSelection + { + readonly IEnumerable rowSelections; + + public CompositeRowSelection(IEnumerable rowSelections) + { + this.rowSelections = rowSelections; + } + + public string GenerateSql() + { + return string.Join(" ", rowSelections.Select(r => r.GenerateSql())); + } + } + public class Top : IRowSelection { readonly int numberOfRows; @@ -26,11 +44,6 @@ public class AllRows : IRowSelection public class Distinct : IRowSelection { - public Distinct() - { - - } - public string GenerateSql() => $"DISTINCT "; public override string ToString() => GenerateSql(); } From 952ef450eea2aaf8b39272a7557d5adad20cd805 Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Fri, 30 Jun 2023 11:51:03 +1000 Subject: [PATCH 03/12] Add DistinctBy support --- .../QueryableIntegrationFixture.cs | 28 +++++++++ .../Advanced/Queryable/QueryTranslator.cs | 12 ++++ .../Queryable/SqlExpressionBuilder.cs | 61 ++++++++++++++++--- .../Nevermore/Querying/AST/ISelectSource.cs | 3 +- 4 files changed, 95 insertions(+), 9 deletions(-) diff --git a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs index 58b83554..e3c642d4 100644 --- a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs +++ b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs @@ -982,6 +982,34 @@ public void Distinct() customers.Should().BeEquivalentTo("Apple", "Banana", "Cherry"); } + [Test] + public void DistinctBy() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Apple" }, + new Customer { FirstName = "Charlie", LastName = "Banana" }, + new Customer { FirstName = "Dan", LastName = "Cherry" }, + new Customer { FirstName = "Erin", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .DistinctBy(c => c.LastName) + .ToList(); + + customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple", "Banana", "Cherry"); + } + [Test] public void Count() { diff --git a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs index c2d68b11..bdaa10a6 100644 --- a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs +++ b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs @@ -201,6 +201,18 @@ protected override Expression VisitMethodCall(MethodCallExpression node) sqlBuilder.Distinct(); return node; } + case "DistinctBy": + { + if (node.Arguments.Count > 2) + { + throw new NotSupportedException("DistinctBy does not support custom comparers"); + } + + Visit(node.Arguments[0]); + var fieldName = GetMemberNameFromKeySelectorExpression(node.Arguments[1]); + sqlBuilder.DistinctBy(new Column(fieldName)); + return node; + } default: throw new NotSupportedException(); } diff --git a/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs b/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs index b9fa301d..318704ca 100644 --- a/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs +++ b/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs @@ -21,6 +21,7 @@ internal class SqlExpressionBuilder int? skip; int? take; bool distinct; + readonly List distinctByColumns = new(); QueryType queryType = QueryType.SelectMany; volatile int paramCounter; @@ -113,6 +114,11 @@ public void Distinct() distinct = true; } + public void DistinctBy(IColumn column) + { + distinctByColumns.Add(column); + } + public void From(Type documentType) { DocumentMap = configuration.DocumentMaps.Resolve(documentType); @@ -137,21 +143,57 @@ public void From(Type documentType) from = new TableSourceWithHint(simpleTableSource, hint); } - var sqlExpression = queryType switch + IExpression sqlExpression; + if (distinctByColumns.Any()) + { + sqlExpression = CreateSelectDistinctByQuery(); + } + else { - QueryType.Exists => CreateExistsQuery(), - QueryType.Count => CreateCountQuery(), - _ => CreateSelectQuery() - }; + sqlExpression = queryType switch + { + QueryType.Exists => CreateExistsQuery(), + QueryType.Count => CreateCountQuery(), + _ => CreateSelectQuery(from) + }; + } return (sqlExpression, parameterValues, queryType); } + IExpression CreateSelectDistinctByQuery() + { + const string distinctByRowNumberFieldName = "distinctby_rownumber"; + const string distinctByRowNumberParameterName = "distinctby_rownumber"; + const string distinctByCteName = "distinctby_cte"; + + whereClauses.Add(new UnaryWhereClause(new WhereFieldReference(distinctByRowNumberFieldName), UnarySqlOperand.Equal, distinctByRowNumberParameterName)); + parameterValues.Add(distinctByRowNumberParameterName, 1); + + var orderBy = GetOrderBy(); + var cteSelect = new Select( + new AllRows(), + new AggregateSelectColumns(new ISelectColumns[] { + new SelectAllSource(), + new SelectRowNumber( + new Over(orderBy, new PartitionBy(distinctByColumns)), + distinctByRowNumberFieldName + ) + }), + from, + new Where(), + null, + null, + new Option(Array.Empty())); - IExpression CreateSelectQuery() + var select = CreateSelectQuery(new SchemalessTableSource(distinctByCteName)); + return new CteSelectSource(cteSelect, distinctByCteName, select); + } + + ISelect CreateSelectQuery(ISelectSource from) { var rowSelection = CreateRowSelection(); - var orderBy = orderByFields.Any() ? new OrderBy(orderByFields) : GetDefaultOrderBy(); + var orderBy = GetOrderBy(); ISelectColumns columns = selectColumns.Any() ? new AggregateSelectColumns(selectColumns) : new SelectAllJsonColumnLast(GetDocumentColumns().ToList()); var select = new Select( rowSelection, @@ -207,6 +249,11 @@ IRowSelection CreateRowSelection() return new CompositeRowSelection(selections); } + OrderBy GetOrderBy() + { + return orderByFields.Any() ? new OrderBy(orderByFields) : GetDefaultOrderBy(); + } + IExpression CreateExistsQuery() { IRowSelection rowSelection = take.HasValue && !skip.HasValue ? new Top(take.Value) : null; diff --git a/source/Nevermore/Querying/AST/ISelectSource.cs b/source/Nevermore/Querying/AST/ISelectSource.cs index 6b1e2415..cef301df 100644 --- a/source/Nevermore/Querying/AST/ISelectSource.cs +++ b/source/Nevermore/Querying/AST/ISelectSource.cs @@ -1,9 +1,8 @@ namespace Nevermore.Querying.AST { - public interface ISelectSource + public interface ISelectSource : IExpression { string Schema { get; } - string GenerateSql(); } public interface IAliasedSelectSource : ISelectSource From b0e4fc4d188eb03a72a08ac75138d27a4eef69f8 Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Fri, 30 Jun 2023 15:02:21 +1000 Subject: [PATCH 04/12] Allow selecting and ordering by json fields --- .../QueryableIntegrationFixture.cs | 164 ++++++++++++++++++ .../Advanced/Queryable/QueryTranslator.cs | 87 +++++++--- source/Nevermore/Querying/AST/IColumn.cs | 28 +++ 3 files changed, 255 insertions(+), 24 deletions(-) diff --git a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs index e3c642d4..31770250 100644 --- a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs +++ b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs @@ -39,6 +39,32 @@ public void Select() customers.Should().BeEquivalentTo("Alice", "Bob", "Charlie"); } + [Test] + public void SelectWithJsonObject() + { + using var t = Store.BeginTransaction(); + + var testMachines = new[] + { + new Machine { Name = "Machine A", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle A" } }, + new Machine { Name = "Machine B", }, + new Machine { Name = "Machine C", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle C" } }, + }; + + foreach (var c in testMachines) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Select(m => m.Endpoint.Name) + .ToList(); + + customers.Should().BeEquivalentTo("Tentacle A", null, "Tentacle C"); + } + [Test] public void WhereEqual() { @@ -1010,6 +1036,32 @@ public void DistinctBy() customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple", "Banana", "Cherry"); } + [Test] + public void DistinctByJson() + { + using var t = Store.BeginTransaction(); + + var testMachines = new[] + { + new Machine { Name = "Machine A", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle A" } }, + new Machine { Name = "Machine B", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle A" } }, + new Machine { Name = "Machine C", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle C" } }, + }; + + foreach (var c in testMachines) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .DistinctBy(m => m.Endpoint.Name) + .ToList(); + + customers.Select(m => m.Endpoint.Name).Should().BeEquivalentTo("Tentacle A", "Tentacle C"); + } + [Test] public void Count() { @@ -1227,6 +1279,32 @@ public void OrderBy() customers.Select(c => c.LastName).Should().BeEquivalentTo("Banana", "Cherry", "Apple"); } + [Test] + public void OrderByJson() + { + using var t = Store.BeginTransaction(); + + var testMachines = new[] + { + new Machine { Name = "Machine A", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle B" } }, + new Machine { Name = "Machine B", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle C" } }, + new Machine { Name = "Machine C", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle A" } }, + }; + + foreach (var c in testMachines) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .OrderBy(m => m.Endpoint.Name) + .ToList(); + + customers.Select(m => m.Endpoint.Name).Should().BeEquivalentTo("Tentacle A", "Tentacle B", "Tentacle C"); + } + [Test] public void OrderByDescending() { @@ -1251,6 +1329,32 @@ public void OrderByDescending() customers.Select(c => c.LastName).Should().BeEquivalentTo("Cherry", "Apple", "Banana"); } + [Test] + public void OrderByDescendingJson() + { + using var t = Store.BeginTransaction(); + + var testMachines = new[] + { + new Machine { Name = "Machine A", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle B" } }, + new Machine { Name = "Machine B", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle C" } }, + new Machine { Name = "Machine C", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle A" } }, + }; + + foreach (var c in testMachines) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .OrderByDescending(m => m.Endpoint.Name) + .ToList(); + + customers.Select(m => m.Endpoint.Name).Should().BeEquivalentTo("Tentacle C", "Tentacle B", "Tentacle A"); + } + [Test] public void OrderByThenBy() { @@ -1278,6 +1382,36 @@ public void OrderByThenBy() customers.Select(c => c.FirstName).Should().BeEquivalentTo("Amanda", "Alice", "Charlie"); } + [Test] + public void OrderByThenByJson() + { + using var t = Store.BeginTransaction(); + + var testMachines = new[] + { + new Machine { Name = "Machine A", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle C" } }, + new Machine { Name = "Machine B", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle E" } }, + new Machine { Name = "Machine C", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle A" } }, + new Machine { Name = "Machine A", Endpoint = new ActiveTentacleEndpoint { Name = "Tentacle F" } }, + new Machine { Name = "Machine B", Endpoint = new ActiveTentacleEndpoint { Name = "Tentacle B" } }, + new Machine { Name = "Machine C", Endpoint = new ActiveTentacleEndpoint { Name = "Tentacle D" } }, + }; + + foreach (var c in testMachines) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .OrderBy(m => m.Endpoint.Type) + .ThenBy(m => m.Endpoint.Name) + .ToList(); + + customers.Select(m => m.Endpoint.Name).Should().BeEquivalentTo("Tentacle B", "Tentacle D", "Tentacle F", "Tentacle A", "Tentacle C", "Tentacle E"); + } + [Test] public void OrderByThenByDescending() { @@ -1305,6 +1439,36 @@ public void OrderByThenByDescending() customers.Select(c => c.FirstName).Should().BeEquivalentTo("Alice", "Amanda", "Charlie"); } + [Test] + public void OrderByThenByDescendingJson() + { + using var t = Store.BeginTransaction(); + + var testMachines = new[] + { + new Machine { Name = "Machine A", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle C" } }, + new Machine { Name = "Machine B", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle E" } }, + new Machine { Name = "Machine C", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle A" } }, + new Machine { Name = "Machine A", Endpoint = new ActiveTentacleEndpoint { Name = "Tentacle F" } }, + new Machine { Name = "Machine B", Endpoint = new ActiveTentacleEndpoint { Name = "Tentacle B" } }, + new Machine { Name = "Machine C", Endpoint = new ActiveTentacleEndpoint { Name = "Tentacle D" } }, + }; + + foreach (var c in testMachines) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .OrderBy(m => m.Endpoint.Type) + .ThenByDescending(m => m.Endpoint.Name) + .ToList(); + + customers.Select(m => m.Endpoint.Name).Should().BeEquivalentTo("Tentacle F", "Tentacle D", "Tentacle B", "Tentacle E", "Tentacle C", "Tentacle A"); + } + [Test] public void Any() { diff --git a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs index bdaa10a6..ecbf0917 100644 --- a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs +++ b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs @@ -7,6 +7,7 @@ using System.Reflection; using Nevermore.Querying.AST; using Nevermore.Util; +using Newtonsoft.Json; namespace Nevermore.Advanced.Queryable { @@ -67,8 +68,8 @@ protected override Expression VisitMethodCall(MethodCallExpression node) } Visit(node.Arguments[0]); - var fieldName = GetMemberNameFromKeySelectorExpression(node.Arguments[1]); - sqlBuilder.Column(new SelectAllJsonColumnLast(new [] { fieldName })); + var column = GetFieldReference(node.Arguments[1]); + sqlBuilder.Column(column); return node; } case nameof(System.Linq.Queryable.Where): @@ -86,8 +87,8 @@ protected override Expression VisitMethodCall(MethodCallExpression node) } Visit(node.Arguments[0]); - var fieldName = GetMemberNameFromKeySelectorExpression(node.Arguments[1]); - sqlBuilder.OrderBy(new OrderByField(new Column(fieldName))); + var column = GetFieldReference(node.Arguments[1]); + sqlBuilder.OrderBy(new OrderByField(column)); return node; } case nameof(System.Linq.Queryable.OrderByDescending): @@ -98,8 +99,8 @@ protected override Expression VisitMethodCall(MethodCallExpression node) } Visit(node.Arguments[0]); - var fieldName = GetMemberNameFromKeySelectorExpression(node.Arguments[1]); - sqlBuilder.OrderBy(new OrderByField(new Column(fieldName), OrderByDirection.Descending)); + var column = GetFieldReference(node.Arguments[1]); + sqlBuilder.OrderBy(new OrderByField(column, OrderByDirection.Descending)); return node; } case nameof(System.Linq.Queryable.ThenBy): @@ -110,8 +111,8 @@ protected override Expression VisitMethodCall(MethodCallExpression node) } Visit(node.Arguments[0]); - var fieldName = GetMemberNameFromKeySelectorExpression(node.Arguments[1]); - sqlBuilder.OrderBy(new OrderByField(new Column(fieldName))); + var column = GetFieldReference(node.Arguments[1]); + sqlBuilder.OrderBy(new OrderByField(column)); return node; } case nameof(System.Linq.Queryable.ThenByDescending): @@ -122,8 +123,8 @@ protected override Expression VisitMethodCall(MethodCallExpression node) } Visit(node.Arguments[0]); - var fieldName = GetMemberNameFromKeySelectorExpression(node.Arguments[1]); - sqlBuilder.OrderBy(new OrderByField(new Column(fieldName), OrderByDirection.Descending)); + var column = GetFieldReference(node.Arguments[1]); + sqlBuilder.OrderBy(new OrderByField(column, OrderByDirection.Descending)); return node; } case nameof(System.Linq.Queryable.First): @@ -209,8 +210,8 @@ protected override Expression VisitMethodCall(MethodCallExpression node) } Visit(node.Arguments[0]); - var fieldName = GetMemberNameFromKeySelectorExpression(node.Arguments[1]); - sqlBuilder.DistinctBy(new Column(fieldName)); + var column = GetFieldReference(node.Arguments[1]); + sqlBuilder.DistinctBy(column); return node; } default: @@ -218,18 +219,6 @@ protected override Expression VisitMethodCall(MethodCallExpression node) } } - static string GetMemberNameFromKeySelectorExpression(Expression expression) - { - var expressionWithoutQuotes = StripQuotes(expression); - - if (expressionWithoutQuotes is LambdaExpression { Body: MemberExpression { NodeType: ExpressionType.MemberAccess } memberExpression }) - { - return memberExpression.Member.Name; - } - - throw new NotSupportedException(); - } - IWhereClause CreateWhereClause(Expression expression, bool invert = false) { if (expression is BinaryExpression binaryExpression) @@ -379,6 +368,56 @@ object GetValueFromExpression(Expression expression, Type propertyType) return result; } + IColumn GetFieldReference(Expression expression) + { + if (expression is not UnaryExpression unaryExpression) + { + throw new NotSupportedException(); + } + + if (unaryExpression.Operand is not LambdaExpression lambdaExpression) + { + throw new NotSupportedException(); + } + + if (lambdaExpression.Body is MemberExpression { Member: PropertyInfo propertyInfo } memberExpression) + { + var hasJsonIgnoreAttribute = propertyInfo.GetCustomAttribute() is not null; + if (hasJsonIgnoreAttribute) + { + throw new NotSupportedException($"Cannot select on a property with the {nameof(JsonIgnoreAttribute)}."); + } + + var documentMap = sqlBuilder.DocumentMap; + var parameterExpression = memberExpression.FindChildOfType(); + var childPropertyExpression = memberExpression.FindChildOfType(); + if (childPropertyExpression == null && documentMap.Type.IsAssignableFrom(parameterExpression.Type)) + { + if (documentMap.IdColumn!.Property.Matches(propertyInfo)) + { + return new Column(documentMap.IdColumn.ColumnName); + } + + var column = documentMap.Columns.Where(c => c.Property is not null).FirstOrDefault(c => c.Property.Matches(propertyInfo)); + if (column is not null) + { + return new Column(column.ColumnName); + } + } + + if (documentMap.HasJsonColumn()) + { + var jsonPath = GetJsonPath(memberExpression); + IColumn fieldReference = propertyInfo.IsScalar() + ? new JsonValueColumn(jsonPath) + : new JsonQueryColumn(jsonPath); + return fieldReference; + } + } + + throw new NotSupportedException(); + } + (IWhereFieldReference, Type) GetFieldReferenceAndType(Expression expression) { if (expression is UnaryExpression unaryExpression) diff --git a/source/Nevermore/Querying/AST/IColumn.cs b/source/Nevermore/Querying/AST/IColumn.cs index a41e2a1f..6cd64a74 100644 --- a/source/Nevermore/Querying/AST/IColumn.cs +++ b/source/Nevermore/Querying/AST/IColumn.cs @@ -54,4 +54,32 @@ public class SelectCountSource : IColumn public string GenerateSql() => "COUNT(*)"; public override string ToString() => GenerateSql(); } + + public class JsonQueryColumn : IColumn + { + readonly string jsonPath; + public bool AggregatesRows => false; + + public JsonQueryColumn(string jsonPath) + { + this.jsonPath = jsonPath; + } + + public string GenerateSql() => $"JSON_QUERY([JSON], '{jsonPath}')"; + public override string ToString() => GenerateSql(); + } + + public class JsonValueColumn : IColumn + { + readonly string jsonPath; + public bool AggregatesRows => false; + + public JsonValueColumn(string jsonPath) + { + this.jsonPath = jsonPath; + } + + public string GenerateSql() => $"JSON_VALUE([JSON], '{jsonPath}')"; + public override string ToString() => GenerateSql(); + } } \ No newline at end of file From f9028ce5ad65ed28a9e4199a15e79abb1b33260c Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Mon, 3 Jul 2023 12:30:58 +1000 Subject: [PATCH 05/12] Add support for Single and SingleOrDefault --- .../QueryableIntegrationFixture.cs | 269 ++++++++++++++++++ .../Queryable/NevermoreQueryableExtensions.cs | 37 +++ .../Advanced/Queryable/QueryProvider.cs | 114 ++++++-- .../Advanced/Queryable/QueryTranslator.cs | 28 +- .../Nevermore/Advanced/Queryable/QueryType.cs | 3 + .../Queryable/SqlExpressionBuilder.cs | 22 +- 6 files changed, 450 insertions(+), 23 deletions(-) diff --git a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs index 31770250..7960c180 100644 --- a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs +++ b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs @@ -798,6 +798,31 @@ public void FirstWithPredicate() customer.LastName.Should().BeEquivalentTo("Apple"); } + [Test] + public void FirstWithNoMatches() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var fn = () => t.Queryable() + .First(c => c.FirstName == "Jim"); + + fn.Should().Throw().WithMessage("Sequence contains no elements"); + } + [Test] public void FirstOrDefault() { @@ -898,6 +923,250 @@ public async Task FirstOrDefaultWithPredicateAsync() customer.Should().BeNull(); } + [Test] + public void Single() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customer = t.Queryable() + .Single(); + + customer.LastName.Should().BeEquivalentTo("Apple"); + } + + [Test] + public void SingleWithPredicate() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customer = t.Queryable() + .Single(c => c.FirstName == "Alice"); + + customer.LastName.Should().BeEquivalentTo("Apple"); + } + + [Test] + public void SingleWithNoMatches() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var fn = () => t.Queryable() + .First(c => c.FirstName == "Jim"); + + fn.Should().Throw().WithMessage("Sequence contains no elements"); + } + + [Test] + public void SingleWithMultipleMatches() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var fn = () => t.Queryable() + .Single(c => c.LastName.Contains("e")); + + fn.Should().Throw().WithMessage("Sequence contains more than one element"); + } + + [Test] + public void SingleOrDefault() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customer = t.Queryable() + .SingleOrDefault(); + + customer.LastName.Should().BeEquivalentTo("Apple"); + } + + [Test] + public async Task SingleOrDefaultAsync() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + await t.CommitAsync(); + + var customer = await t.Queryable() + .SingleOrDefaultAsync(); + + customer.LastName.Should().BeEquivalentTo("Apple"); + } + + [Test] + public void SingleOrDefaultWithPredicate() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customer = t.Queryable() + .SingleOrDefault(c => c.FirstName.EndsWith("y")); + + customer.Should().BeNull(); + } + + [Test] + public async Task SingleOrDefaultWithPredicateAsync() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + await t.CommitAsync(); + + var customer = await t.Queryable() + .SingleOrDefaultAsync(c => c.FirstName.EndsWith("y")); + + customer.Should().BeNull(); + } + + [Test] + public void SingleOrDefaultWithMultipleMatches() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var fn = () => t.Queryable() + .SingleOrDefault(c => c.LastName.Contains("e")); + + fn.Should().ThrowExactly().WithMessage("Sequence contains more than one element"); + } + + [Test] + public async Task SingleOrDefaultAsyncWithMultipleMatches() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + await t.CommitAsync(); + + var fn = async () => await t.Queryable() + .SingleOrDefaultAsync(c => c.LastName.Contains("e")); + + await fn.Should().ThrowExactlyAsync().WithMessage("Sequence contains more than one element."); + } + [Test] public void Skip() { diff --git a/source/Nevermore/Advanced/Queryable/NevermoreQueryableExtensions.cs b/source/Nevermore/Advanced/Queryable/NevermoreQueryableExtensions.cs index c226efdf..c0035a00 100644 --- a/source/Nevermore/Advanced/Queryable/NevermoreQueryableExtensions.cs +++ b/source/Nevermore/Advanced/Queryable/NevermoreQueryableExtensions.cs @@ -14,6 +14,8 @@ public static class NevermoreQueryableExtensions static readonly MethodInfo HintMethodInfo = new Func, string, IQueryable>(Hint).GetMethodInfo().GetGenericMethodDefinition(); static readonly MethodInfo FirstOrDefaultMethodInfo = new Func, object>(System.Linq.Queryable.FirstOrDefault).GetMethodInfo().GetGenericMethodDefinition(); static readonly MethodInfo FirstOrDefaultWithPredicateMethodInfo = new Func, Expression>, object>(System.Linq.Queryable.FirstOrDefault).GetMethodInfo().GetGenericMethodDefinition(); + static readonly MethodInfo SingleOrDefaultMethodInfo = new Func, object>(System.Linq.Queryable.SingleOrDefault).GetMethodInfo().GetGenericMethodDefinition(); + static readonly MethodInfo SingleOrDefaultWithPredicateMethodInfo = new Func, Expression>, object>(System.Linq.Queryable.SingleOrDefault).GetMethodInfo().GetGenericMethodDefinition(); static readonly MethodInfo CountMethodInfo = new Func, int>(System.Linq.Queryable.Count).GetMethodInfo().GetGenericMethodDefinition(); static readonly MethodInfo CountWithPredicateMethodInfo = new Func, Expression>, int>(System.Linq.Queryable.Count).GetMethodInfo().GetGenericMethodDefinition(); static readonly MethodInfo AnyMethodInfo = new Func, bool>(System.Linq.Queryable.Any).GetMethodInfo().GetGenericMethodDefinition(); @@ -174,6 +176,41 @@ public static async Task FirstOrDefaultAsync(this IQueryable SingleOrDefaultAsync(this IQueryable source, CancellationToken cancellationToken = default) + { + if (source == null) + throw new ArgumentNullException(nameof(source)); + + if (source.Provider is IAsyncQueryProvider asyncQueryProvider) + { + var expression = Expression.Call( + null, + SingleOrDefaultMethodInfo.MakeGenericMethod(typeof(TSource)), + source.Expression); + return await asyncQueryProvider.ExecuteAsync(expression, cancellationToken).ConfigureAwait(false); + } + + throw new InvalidOperationException("The query provider does not support async operations."); + } + + public static async Task SingleOrDefaultAsync(this IQueryable source, Expression> predicate, CancellationToken cancellationToken = default) + { + if (source == null) + throw new ArgumentNullException(nameof(source)); + + if (source.Provider is IAsyncQueryProvider asyncQueryProvider) + { + var expression = Expression.Call( + null, + SingleOrDefaultWithPredicateMethodInfo.MakeGenericMethod(typeof(TSource)), + source.Expression, + predicate); + return await asyncQueryProvider.ExecuteAsync(expression, cancellationToken).ConfigureAwait(false); + } + + throw new InvalidOperationException("The query provider does not support async operations."); + } + public static async Task> ToListAsync(this IQueryable source, CancellationToken cancellationToken = default) { if (source == null) diff --git a/source/Nevermore/Advanced/Queryable/QueryProvider.cs b/source/Nevermore/Advanced/Queryable/QueryProvider.cs index c86e59f1..b7b531d2 100644 --- a/source/Nevermore/Advanced/Queryable/QueryProvider.cs +++ b/source/Nevermore/Advanced/Queryable/QueryProvider.cs @@ -59,21 +59,29 @@ public TResult Execute(Expression expression) .Invoke(queryExecutor, new object[] { command }); } - if (queryType == QueryType.SelectSingle) + if (queryType == QueryType.Count || queryType == QueryType.Exists) { - var stream = (IEnumerable)GenericStreamMethod.MakeGenericMethod(expression.Type) + return (TResult)GenericExecuteScalarMethod.MakeGenericMethod(expression.Type) .Invoke(queryExecutor, new object[] { command }); - return (TResult)stream.Cast().FirstOrDefault(); } - return (TResult)GenericExecuteScalarMethod.MakeGenericMethod(expression.Type) + var stream = (IEnumerable)GenericStreamMethod.MakeGenericMethod(expression.Type) .Invoke(queryExecutor, new object[] { command }); - } + return queryType switch + { + QueryType.SelectFirst => (TResult)stream.Cast().First(), + QueryType.SelectFirstOrDefault => (TResult)stream.Cast().FirstOrDefault(), + QueryType.SelectSingle => (TResult)stream.Cast().Single(), + QueryType.SelectSingleOrDefault => (TResult)stream.Cast().SingleOrDefault(), + _ => throw new ArgumentOutOfRangeException() + }; + } public async Task ExecuteAsync(Expression expression, CancellationToken cancellationToken) { var (command, queryType) = Translate(expression); + if (queryType == QueryType.SelectMany) { var sequenceType = expression.Type.GetSequenceType(); @@ -83,24 +91,31 @@ public async Task ExecuteAsync(Expression expression, Cancella return (TResult)await CreateList(asyncStream, sequenceType).ConfigureAwait(false); } - if (queryType == QueryType.SelectSingle) + if (queryType == QueryType.Count || queryType == QueryType.Exists) { - var asyncStream = (IAsyncEnumerable)GenericStreamAsyncMethod.MakeGenericMethod(expression.Type) - .Invoke(queryExecutor, new object[] { command, cancellationToken }); - var firstOrDefaultAsync = await FirstOrDefaultAsync(asyncStream, cancellationToken).ConfigureAwait(false); - - if (firstOrDefaultAsync is not null) return (TResult) firstOrDefaultAsync; - - // TODO: This NEEDS to go away when we turn nullable on in Nevermore - // This method needs to be able to return null for instances like `FirstOrDefaultAsync` - object GetNull() => null; - return (TResult) GetNull(); - + return await ((Task)GenericExecuteScalarAsyncMethod.MakeGenericMethod(expression.Type) + .Invoke(queryExecutor, new object[] { command, cancellationToken })) + .ConfigureAwait(false); } - return await ((Task)GenericExecuteScalarAsyncMethod.MakeGenericMethod(expression.Type) - .Invoke(queryExecutor, new object[] { command, cancellationToken })) - .ConfigureAwait(false); + var stream = (IAsyncEnumerable)GenericStreamAsyncMethod.MakeGenericMethod(expression.Type) + .Invoke(queryExecutor, new object[] { command, cancellationToken }); + + object result = queryType switch + { + QueryType.SelectFirst => (TResult)await FirstAsync(stream, cancellationToken).ConfigureAwait(false), + QueryType.SelectFirstOrDefault => (TResult)await FirstOrDefaultAsync(stream, cancellationToken).ConfigureAwait(false), + QueryType.SelectSingle => (TResult)await SingleAsync(stream, cancellationToken).ConfigureAwait(false), + QueryType.SelectSingleOrDefault => (TResult)await SingleOrDefaultAsync(stream, cancellationToken).ConfigureAwait(false), + _ => throw new ArgumentOutOfRangeException() + }; + + if (result is not null) return (TResult) result; + + // TODO: This NEEDS to go away when we turn nullable on in Nevermore + // This method needs to be able to return null for instances like `FirstOrDefaultAsync` + object GetNull() => null; + return (TResult) GetNull(); } public IAsyncEnumerable StreamAsync(Expression expression, CancellationToken cancellationToken) @@ -147,5 +162,64 @@ static async ValueTask FirstOrDefaultAsync(IAsyncEnumerable enumerable, return default; } + + static async ValueTask FirstAsync(IAsyncEnumerable enumerable, CancellationToken cancellationToken) + { +#pragma warning disable CA2007 + // CA2007 doesn't understand ConfiguredCancelableAsyncEnumerable and incorrectly thinks we need another ConfigureAwait(false) here + await using var enumerator = enumerable.ConfigureAwait(false).WithCancellation(cancellationToken).GetAsyncEnumerator(); +#pragma warning restore CA2007 + + if (!await enumerator.MoveNextAsync()) + { + throw new InvalidOperationException("Sequence contains no elements."); + } + + return enumerator.Current; + } + + static async Task SingleOrDefaultAsync(IAsyncEnumerable enumerable, CancellationToken cancellationToken = default) + { +#pragma warning disable CA2007 + // CA2007 doesn't understand ConfiguredCancelableAsyncEnumerable and incorrectly thinks we need another ConfigureAwait(false) here + await using var enumerator = enumerable.ConfigureAwait(false).WithCancellation(cancellationToken).GetAsyncEnumerator(); +#pragma warning restore CA2007 + + if (!await enumerator.MoveNextAsync()) + { + return default; + } + + var current = enumerator.Current; + + if (await enumerator.MoveNextAsync()) + { + throw new InvalidOperationException("Sequence contains more than one element."); + } + + return current; + } + + static async Task SingleAsync(IAsyncEnumerable enumerable, CancellationToken cancellationToken = default) + { +#pragma warning disable CA2007 + // CA2007 doesn't understand ConfiguredCancelableAsyncEnumerable and incorrectly thinks we need another ConfigureAwait(false) here + await using var enumerator = enumerable.ConfigureAwait(false).WithCancellation(cancellationToken).GetAsyncEnumerator(); +#pragma warning restore CA2007 + + if (!await enumerator.MoveNextAsync()) + { + throw new InvalidOperationException("Sequence contains no elements."); + } + + var current = enumerator.Current; + + if (!await enumerator.MoveNextAsync()) + { + return current; + } + + throw new InvalidOperationException("Sequence contains more than one element."); + } } } \ No newline at end of file diff --git a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs index ecbf0917..89b84bfa 100644 --- a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs +++ b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs @@ -127,7 +127,7 @@ protected override Expression VisitMethodCall(MethodCallExpression node) sqlBuilder.OrderBy(new OrderByField(column, OrderByDirection.Descending)); return node; } - case nameof(System.Linq.Queryable.First): + case nameof(System.Linq.Queryable.Single): { Visit(node.Arguments[0]); if (node.Arguments.Count > 1) @@ -139,6 +139,30 @@ protected override Expression VisitMethodCall(MethodCallExpression node) sqlBuilder.Single(); return node; } + case nameof(System.Linq.Queryable.SingleOrDefault): + { + Visit(node.Arguments[0]); + if (node.Arguments.Count > 1) + { + var expression = (LambdaExpression)StripQuotes(node.Arguments[1]); + sqlBuilder.Where(CreateWhereClause(expression.Body)); + } + + sqlBuilder.SingleOrDefault(); + return node; + } + case nameof(System.Linq.Queryable.First): + { + Visit(node.Arguments[0]); + if (node.Arguments.Count > 1) + { + var expression = (LambdaExpression)StripQuotes(node.Arguments[1]); + sqlBuilder.Where(CreateWhereClause(expression.Body)); + } + + sqlBuilder.First(); + return node; + } case nameof(System.Linq.Queryable.FirstOrDefault): { Visit(node.Arguments[0]); @@ -148,7 +172,7 @@ protected override Expression VisitMethodCall(MethodCallExpression node) sqlBuilder.Where(CreateWhereClause(expression.Body)); } - sqlBuilder.Single(); + sqlBuilder.FirstOrDefault(); return node; } case nameof(System.Linq.Queryable.Any): diff --git a/source/Nevermore/Advanced/Queryable/QueryType.cs b/source/Nevermore/Advanced/Queryable/QueryType.cs index e7349f61..70541418 100644 --- a/source/Nevermore/Advanced/Queryable/QueryType.cs +++ b/source/Nevermore/Advanced/Queryable/QueryType.cs @@ -3,7 +3,10 @@ internal enum QueryType { SelectMany, + SelectFirst, + SelectFirstOrDefault, SelectSingle, + SelectSingleOrDefault, Count, Exists } diff --git a/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs b/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs index 318704ca..e3d3e8b7 100644 --- a/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs +++ b/source/Nevermore/Advanced/Queryable/SqlExpressionBuilder.cs @@ -78,12 +78,32 @@ public void OrderBy(OrderByField field) orderByFields.Add(field); } - public void Single() + public void First() { Take(1); + queryType = QueryType.SelectFirst; + } + + public void FirstOrDefault() + { + Take(1); + queryType = QueryType.SelectFirstOrDefault; + } + + public void Single() + { + // Hack: This is yuck. We're taking two rows here so that we can throw if we get more than one row. + Take(2); queryType = QueryType.SelectSingle; } + public void SingleOrDefault() + { + // Hack: This is yuck. We're taking two rows here so that we can throw if we get more than one row. + Take(2); + queryType = QueryType.SelectSingleOrDefault; + } + public void Exists() { queryType = QueryType.Exists; From 965f347b51e5e5e92c4b9adb89d098ffae31c4bb Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Mon, 3 Jul 2023 15:11:44 +1000 Subject: [PATCH 06/12] Add test for filtering on nested JSON properties --- .../Model/Endpoint.cs | 2 + .../QueryableIntegrationFixture.cs | 52 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/source/Nevermore.IntegrationTests/Model/Endpoint.cs b/source/Nevermore.IntegrationTests/Model/Endpoint.cs index 5cc9de88..9a71923d 100644 --- a/source/Nevermore.IntegrationTests/Model/Endpoint.cs +++ b/source/Nevermore.IntegrationTests/Model/Endpoint.cs @@ -5,5 +5,7 @@ public abstract class Endpoint public string Name { get; set; } public abstract string Type { get; } + + public bool IsEnabled { get; set; } } } \ No newline at end of file diff --git a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs index 7960c180..9a24e171 100644 --- a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs +++ b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs @@ -510,6 +510,58 @@ public void WhereNotUnaryBoolJson() customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple", "Cherry"); } + [Test] + public void WhereUnaryBoolNestedJson() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Machine { Name = "Machine A", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle A", IsEnabled = true} }, + new Machine { Name = "Machine B", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle B", IsEnabled = true } }, + new Machine { Name = "Machine C", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle C", IsEnabled = false } }, + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => c.Endpoint.IsEnabled) + .ToList(); + + customers.Select(c => c.Name).Should().BeEquivalentTo("Machine A", "Machine B"); + } + + [Test] + public void WhereNotUnaryBoolNestedJson() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Machine { Name = "Machine A", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle A", IsEnabled = true} }, + new Machine { Name = "Machine B", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle B", IsEnabled = true } }, + new Machine { Name = "Machine C", Endpoint = new PassiveTentacleEndpoint { Name = "Tentacle C", IsEnabled = false } }, + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => !c.Endpoint.IsEnabled) + .ToList(); + + customers.Select(c => c.Name).Should().BeEquivalentTo("Machine C"); + } + [Test] public void WhereCompositeAnd() { From 75bf33abaaca782c052aa686b0cb029ccf5cd492 Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Mon, 3 Jul 2023 17:34:20 +1000 Subject: [PATCH 07/12] Add tests for parens --- .../QueryableIntegrationFixture.cs | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs index 9a24e171..1884ee4f 100644 --- a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs +++ b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs @@ -800,6 +800,62 @@ public void WhereNotStringContains() customers.Select(c => c.FirstName).Should().BeEquivalentTo("Alice", "Bob"); } + [Test] + public void WhereParens() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Alice", LastName = "Banana" }, + new Customer { FirstName = "Bob", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => (c.FirstName == "Alice" && c.LastName == "Apple") || (c.FirstName == "Bob" && c.LastName == "Banana")) + .ToList(); + + customers.Select(c => c.FirstName).Should().BeEquivalentTo("Alice", "Bob"); + customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple", "Banana"); + } + + [Test] + public void WhereNotParens() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Alice", LastName = "Banana" }, + new Customer { FirstName = "Bob", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => !(c.FirstName == "Alice" && c.LastName == "Apple") || !(c.FirstName == "Bob" && c.LastName == "Banana")) + .ToList(); + + customers.Select(c => c.FirstName).Should().BeEquivalentTo("Alice", "Bob"); + customers.Select(c => c.LastName).Should().BeEquivalentTo("Banana", "Apple"); + } + [Test] public void First() { From 151037722a8c1f5470e8793084fb91af08ddf51c Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Mon, 3 Jul 2023 18:06:55 +1000 Subject: [PATCH 08/12] Support more string methods --- .../QueryableIntegrationFixture.cs | 183 ++++++++++++++++++ .../Advanced/Queryable/QueryTranslator.cs | 35 ++++ .../Querying/AST/IWhereFieldReference.cs | 44 ++++- 3 files changed, 261 insertions(+), 1 deletion(-) diff --git a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs index 1884ee4f..86a78f71 100644 --- a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs +++ b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs @@ -91,6 +91,84 @@ public void WhereEqual() customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple"); } + [Test] + public void WhereEqualLower() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => c.FirstName.ToLower() == "alice") + .ToList(); + + customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple"); + } + + [Test] + public void WhereEqualUpper() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => c.FirstName.ToUpper() == "ALICE") + .ToList(); + + customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple"); + } + + [Test] + public void WhereEqualTrim() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = " Alice ", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => c.FirstName.Trim() == "Alice") + .ToList(); + + customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple"); + } + [Test] public void WhereEqualIdColumn() { @@ -641,6 +719,111 @@ public void WhereContains() customers.Select(c => c.FirstName).Should().BeEquivalentTo("Alice"); } + [Test] + public void WhereLowerContains() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple", Balance = 987.4m }, + new Customer { FirstName = "Bob", LastName = "Banana", Balance = 56.3m }, + new Customer { FirstName = "Charlie", LastName = "Cherry", Balance = 301.4m } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var names = new[] { "apple", "orange", "peach" }; + var customers = t.Queryable() + .Where(c => names.Contains(c.LastName.ToLower())) + .ToList(); + + customers.Select(c => c.FirstName).Should().BeEquivalentTo("Alice"); + } + + [Test] + public void WhereLowerStringContains() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple", Balance = 987.4m }, + new Customer { FirstName = "Bob", LastName = "Banana", Balance = 56.3m }, + new Customer { FirstName = "Charlie", LastName = "Cherry", Balance = 301.4m } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => c.FirstName.ToLower().Contains("a")) + .ToList(); + + customers.Select(c => c.FirstName).Should().BeEquivalentTo("Alice", "Charlie"); + } + + [Test] + public void WhereUpperStringContains() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple", Balance = 987.4m }, + new Customer { FirstName = "Bob", LastName = "Banana", Balance = 56.3m }, + new Customer { FirstName = "Charlie", LastName = "Cherry", Balance = 301.4m } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => c.FirstName.ToUpper().Contains("A")) + .ToList(); + + customers.Select(c => c.FirstName).Should().BeEquivalentTo("Alice", "Charlie"); + } + + [Test] + public void WhereTrimStringContains() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = " Alice ", LastName = "Apple", Balance = 987.4m }, + new Customer { FirstName = " Bob ", LastName = "Banana", Balance = 56.3m }, + new Customer { FirstName = "Charlie ", LastName = "Cherry", Balance = 301.4m } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => c.FirstName.Trim().Contains("e")) + .ToList(); + + customers.Select(c => c.FirstName).Should().BeEquivalentTo(" Alice ", "Charlie "); + } + [Test] public void WhereContainsEmpty() { diff --git a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs index 89b84bfa..b0041287 100644 --- a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs +++ b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs @@ -304,6 +304,12 @@ IWhereClause CreateMethodCallWhere(MethodCallExpression expression, bool invert var (fieldReference, _) = GetFieldReferenceAndType(right); return sqlBuilder.CreateWhere(fieldReference, invert ? ArraySqlOperand.NotIn : ArraySqlOperand.In, values); } + else if (right is MethodCallExpression methodCallExpression) + { + var values = (IEnumerable)GetValueFromExpression(left, typeof(IEnumerable)); + var (fieldReference, _) = GetFieldReferenceAndType(methodCallExpression); + return sqlBuilder.CreateWhere(fieldReference, invert ? ArraySqlOperand.NotIn : ArraySqlOperand.In, values); + } } throw new NotSupportedException(); @@ -476,6 +482,35 @@ IColumn GetFieldReference(Expression expression) } } + if (expression is MethodCallExpression methodCallExpression) + { + var (reference, type) = GetFieldReferenceAndType(methodCallExpression.Object); + if (methodCallExpression.Method.Name == nameof(string.ToLower)) + { + return (new WhereFieldReferenceWithStringFunction(reference, StringFunction.Lower), type); + } + + if (methodCallExpression.Method.Name == nameof(string.ToUpper)) + { + return (new WhereFieldReferenceWithStringFunction(reference, StringFunction.Upper), type); + } + + if (methodCallExpression.Method.Name == nameof(string.Trim)) + { + return (new WhereFieldReferenceWithStringFunction(reference, StringFunction.Trim), type); + } + + if (methodCallExpression.Method.Name == nameof(string.TrimStart)) + { + return (new WhereFieldReferenceWithStringFunction(reference, StringFunction.LeftTrim), type); + } + + if (methodCallExpression.Method.Name == nameof(string.TrimEnd)) + { + return (new WhereFieldReferenceWithStringFunction(reference, StringFunction.RightTrim), type); + } + } + throw new NotSupportedException(); } diff --git a/source/Nevermore/Querying/AST/IWhereFieldReference.cs b/source/Nevermore/Querying/AST/IWhereFieldReference.cs index 408544de..b2116f6d 100644 --- a/source/Nevermore/Querying/AST/IWhereFieldReference.cs +++ b/source/Nevermore/Querying/AST/IWhereFieldReference.cs @@ -1,10 +1,52 @@ -namespace Nevermore.Querying.AST +using System; + +namespace Nevermore.Querying.AST { public interface IWhereFieldReference { string GenerateSql(); } + public enum StringFunction + { + Lower, + Upper, + Trim, + LeftTrim, + RightTrim, + } + + public class WhereFieldReferenceWithStringFunction : IWhereFieldReference + { + readonly IWhereFieldReference inner; + readonly StringFunction function; + + public WhereFieldReferenceWithStringFunction(IWhereFieldReference inner, StringFunction function) + { + this.inner = inner; + this.function = function; + } + + public string GenerateSql() + { + switch (function) + { + case StringFunction.Lower: + return $"LOWER({inner.GenerateSql()})"; + case StringFunction.Upper: + return $"UPPER({inner.GenerateSql()})"; + case StringFunction.Trim: + return $"TRIM({inner.GenerateSql()})"; + case StringFunction.LeftTrim: + return $"LTRIM({inner.GenerateSql()})"; + case StringFunction.RightTrim: + return $"RTRIM({inner.GenerateSql()})"; + default: + throw new ArgumentOutOfRangeException(); + } + } + } + public class WhereFieldReference : IWhereFieldReference { readonly string fieldName; From b1b446a8fe6517727bb5508b4bc0be007319d7bc Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Mon, 3 Jul 2023 18:21:17 +1000 Subject: [PATCH 09/12] Add support for string equals --- .../QueryableIntegrationFixture.cs | 52 +++++++++++++++++++ .../Advanced/Queryable/QueryTranslator.cs | 37 ++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs index 86a78f71..2a28aab2 100644 --- a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs +++ b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs @@ -91,6 +91,58 @@ public void WhereEqual() customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple"); } + [Test] + public void WhereEqualStringEquals() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => c.FirstName.Equals("Alice")) + .ToList(); + + customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple"); + } + + [Test] + public void WhereEqualIgnoreCase() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple" }, + new Customer { FirstName = "Bob", LastName = "Banana" }, + new Customer { FirstName = "Charlie", LastName = "Cherry" } + }; + + foreach (var c in testCustomers) + { + t.Insert(c); + } + + t.Commit(); + + var customers = t.Queryable() + .Where(c => c.FirstName.Equals("alice", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + customers.Select(c => c.LastName).Should().BeEquivalentTo("Apple"); + } + [Test] public void WhereEqualLower() { diff --git a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs index b0041287..3e291fa3 100644 --- a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs +++ b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs @@ -271,7 +271,7 @@ IWhereClause CreateWhereClause(Expression expression, bool invert = false) IWhereClause CreateMethodCallWhere(MethodCallExpression expression, bool invert = false) { - if (expression.Arguments.Count == 1 && expression.Method.DeclaringType == typeof(string)) + if (expression.Method.DeclaringType == typeof(string)) { return CreateStringMethodWhere(expression, invert); } @@ -335,9 +335,44 @@ IWhereClause CreateStringMethodWhere(MethodCallExpression expression, bool inver return sqlBuilder.CreateWhere(fieldReference, invert ? UnarySqlOperand.NotLike : UnarySqlOperand.Like, $"%{value}"); } + if (expression.Method.Name == nameof(string.Equals)) + { + // If one of the IgnoreCase options was specified, then compare in lowercase + if (TryGetFirstArgumentOfType(expression, out StringComparison stringComparison)) + { + switch (stringComparison) + { + case StringComparison.CurrentCultureIgnoreCase: + case StringComparison.InvariantCultureIgnoreCase: + case StringComparison.OrdinalIgnoreCase: + fieldReference = new WhereFieldReferenceWithStringFunction(fieldReference, StringFunction.Lower); + value = value.ToLower(); + break; + } + } + + return sqlBuilder.CreateWhere(fieldReference, invert ? UnarySqlOperand.NotEqual : UnarySqlOperand.Equal, value); + } + throw new NotSupportedException(); } + bool TryGetFirstArgumentOfType(MethodCallExpression expression, out T arg) + { + arg = default; + foreach (var argument in expression.Arguments) + { + if (argument is not ConstantExpression constantExpression || + constantExpression.Type != typeof(T)) + continue; + + arg = (T)constantExpression.Value; + return true; + } + + return false; + } + IWhereClause CreateBinaryWhere(BinaryExpression expression, bool invert = false) { if (expression.NodeType == ExpressionType.AndAlso) From 1d8ecf1bc92e045ae704cfac0cc07374cf48ebf1 Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Wed, 5 Jul 2023 09:42:10 +1000 Subject: [PATCH 10/12] Add Select support --- .../QueryableIntegrationFixture.cs | 78 +++++++++++++-- .../Advanced/Queryable/QueryTranslator.cs | 57 +++++++++-- .../ArbitraryClassReaderStrategy.cs | 96 ++++++++++++++----- source/Nevermore/Querying/AST/IColumn.cs | 16 +++- .../Querying/AST/TypeExtensionMethods.cs | 26 +++++ source/Nevermore/Querying/AST/Where.cs | 21 +--- 6 files changed, 227 insertions(+), 67 deletions(-) create mode 100644 source/Nevermore/Querying/AST/TypeExtensionMethods.cs diff --git a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs index 2a28aab2..b9788095 100644 --- a/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs +++ b/source/Nevermore.IntegrationTests/QueryableIntegrationFixture.cs @@ -14,7 +14,7 @@ namespace Nevermore.IntegrationTests public class QueryableIntegrationFixture : FixtureWithRelationalStore { [Test] - public void Select() + public void SelectColumn() { using var t = Store.BeginTransaction(); @@ -40,7 +40,7 @@ public void Select() } [Test] - public void SelectWithJsonObject() + public async Task SelectJsonField() { using var t = Store.BeginTransaction(); @@ -56,13 +56,79 @@ public void SelectWithJsonObject() t.Insert(c); } - t.Commit(); + await t.CommitAsync(); - var customers = t.Queryable() - .Select(m => m.Endpoint.Name) + var machines = t.Queryable() + .Select(c => c.Endpoint.Name) .ToList(); - customers.Should().BeEquivalentTo("Tentacle A", null, "Tentacle C"); + machines.Should().BeEquivalentTo("Tentacle A", null, "Tentacle C"); + } + + [Test] + public async Task SelectWithParameterizedConstructor() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple", IsEmployee = true }, + new Customer { FirstName = "Bob", LastName = "Banana", IsEmployee = false }, + new Customer { FirstName = "Charlie", LastName = "Cherry", IsEmployee = true } + }; + + await t.InsertManyAsync(testCustomers); + await t.CommitAsync(); + + var customers = await t.Queryable() + .Select(c => new CustomerProjection(c.FirstName, c.IsEmployee)) + .ToListAsync(); + + customers.Should().BeEquivalentTo(new[] + { + new CustomerProjection("Alice", true), + new CustomerProjection("Bob", false), + new CustomerProjection("Charlie", true) + }); + } + + class CustomerProjection + { + public CustomerProjection(string firstName, bool isEmployee) + { + FirstName = firstName; + IsEmployee = isEmployee; + } + + public string FirstName { get; } + public bool IsEmployee { get; } + } + + [Test] + public async Task SelectWithAnonymousType() + { + using var t = Store.BeginTransaction(); + + var testCustomers = new[] + { + new Customer { FirstName = "Alice", LastName = "Apple", IsEmployee = true }, + new Customer { FirstName = "Bob", LastName = "Banana", IsEmployee = false }, + new Customer { FirstName = "Charlie", LastName = "Cherry", IsEmployee = true } + }; + + await t.InsertManyAsync(testCustomers); + await t.CommitAsync(); + + var customers = await t.Queryable() + .Select(c => new { Name = c.FirstName, c.IsEmployee }) + .ToListAsync(); + + customers.Should().BeEquivalentTo(new[] + { + new {Name = "Alice", IsEmployee = true}, + new {Name = "Bob", IsEmployee = false}, + new {Name = "Charlie", IsEmployee = true}, + }); } [Test] diff --git a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs index 3e291fa3..219ecf1e 100644 --- a/source/Nevermore/Advanced/Queryable/QueryTranslator.cs +++ b/source/Nevermore/Advanced/Queryable/QueryTranslator.cs @@ -68,7 +68,31 @@ protected override Expression VisitMethodCall(MethodCallExpression node) } Visit(node.Arguments[0]); - var column = GetFieldReference(node.Arguments[1]); + + var expression = node.Arguments[1]; + if (expression is UnaryExpression { Operand: LambdaExpression { Body: NewExpression newExpression } }) + { + AggregateSelectColumns aggregateColumns; + if (newExpression.Members is not null) + { + var columns = newExpression.Arguments + .Select(GetFieldReference) + .Zip(newExpression.Members, (column, memberInfo) => new AliasedColumn(column, memberInfo.Name)) + .Cast(); + aggregateColumns = new AggregateSelectColumns(columns.ToArray()); + } + else + { + var columns = newExpression.Arguments + .Select(GetFieldReference); + aggregateColumns = new AggregateSelectColumns(columns.ToArray()); + } + + sqlBuilder.Column(aggregateColumns); + return node; + } + + var column = GetFieldReference(expression); sqlBuilder.Column(column); return node; } @@ -433,19 +457,32 @@ object GetValueFromExpression(Expression expression, Type propertyType) return result; } - IColumn GetFieldReference(Expression expression) + bool TryGetMemberExpression(Expression expression, out MemberExpression memberExpression, out PropertyInfo propertyInfo) { - if (expression is not UnaryExpression unaryExpression) + if (expression is MemberExpression { Member: PropertyInfo}) { - throw new NotSupportedException(); + memberExpression = (MemberExpression) expression; + propertyInfo = (PropertyInfo) memberExpression.Member; + return true; } - if (unaryExpression.Operand is not LambdaExpression lambdaExpression) + if (expression is UnaryExpression unaryExpression && + unaryExpression.Operand is LambdaExpression lambdaExpression && + lambdaExpression.Body is MemberExpression { Member: PropertyInfo }) { - throw new NotSupportedException(); + memberExpression = (MemberExpression) lambdaExpression.Body; + propertyInfo = (PropertyInfo) memberExpression.Member; + return true; } - - if (lambdaExpression.Body is MemberExpression { Member: PropertyInfo propertyInfo } memberExpression) + + memberExpression = null; + propertyInfo = null; + return false; + } + + IColumn GetFieldReference(Expression expression) + { + if (TryGetMemberExpression(expression, out MemberExpression memberExpression, out PropertyInfo propertyInfo)) { var hasJsonIgnoreAttribute = propertyInfo.GetCustomAttribute() is not null; if (hasJsonIgnoreAttribute) @@ -474,8 +511,8 @@ IColumn GetFieldReference(Expression expression) { var jsonPath = GetJsonPath(memberExpression); IColumn fieldReference = propertyInfo.IsScalar() - ? new JsonValueColumn(jsonPath) - : new JsonQueryColumn(jsonPath); + ? new JsonValueColumn(jsonPath, propertyInfo.PropertyType) + : new JsonQueryColumn(jsonPath, propertyInfo.PropertyType); return fieldReference; } } diff --git a/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs b/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs index eaace99b..6a14d4b9 100644 --- a/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs +++ b/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs @@ -14,7 +14,7 @@ namespace Nevermore.Advanced.ReaderStrategies.ArbitraryClasses /// This strategy is used for "POCO", or "Plain-Old-CLR-Objects" classes (those that don't have a document map). /// They will be read from the reader automatically. Our requirements are: /// - /// - The class must provide a constructor (declared or not) with no parameters. + /// - The class must provide a constructor (declared or not) with no parameters, or a list of parameters that exactly matches the fields on the reader by type /// - We only bind public, settable properties /// - Column names must match property names, but the casing does not need to match /// - If a column exists in the results, a property must exist on the class @@ -31,10 +31,12 @@ public ArbitraryClassReaderStrategy(RelationalStoreConfiguration configuration) public bool CanRead(Type type) { - return - type.IsClass - && type.GetConstructors().Any(c => c.IsPublic && c.GetParameters().Length == 0) - && !configuration.DocumentMaps.ResolveOptional(type, out _); + return !IsPrimitive(type) && type.IsClass && !configuration.DocumentMaps.ResolveOptional(type, out _); + } + + bool IsPrimitive(Type type) + { + return type.IsPrimitive || type == typeof(string) || type == typeof(decimal); } public Func> CreateReader() @@ -81,18 +83,31 @@ public bool CanRead(Type type) // result.LastName = reader.GetString(1); // return result; // } + // + // -- OR -- + // + // (DbDataReader reader, ArbitraryClassReaderContext context) => + // { + // var result = new Person(reader.GetString(0), reader.GetString(1)); + // return result; + // } CompiledExpression> Compile(IDataRecord record) { // To make it fast - as fast as if we wrote it by hand - we generate and compile C# expression trees for // each property on the class, and one to call the constructor. - - // Find the parameter-less constructor + var constructors = typeof(TRecord).GetConstructors(); - var defaultConstructor = constructors.FirstOrDefault(p => p.GetParameters().Length == 0) ?? throw new InvalidOperationException("When reading query results to a class, the class must provide a default constructor with no parameters."); - - // Create fast setters for all properties on the type - var properties = typeof(TRecord).GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.SetProperty); - + // Find the parameter-less constructor + var defaultConstructor = constructors.FirstOrDefault(c => c.GetParameters().Length == 0); + // Find a parameterized constructor that matches the data reader + var parameterizedConstructor = (from ctor in constructors + let parameters = ctor.GetParameters() + where parameters.Length == record.FieldCount && MatchesTypes(parameters, record) + select ctor).FirstOrDefault(); + var selectedConstructor = parameterizedConstructor + ?? defaultConstructor + ?? throw new InvalidOperationException("No default constructor or constructor that exactly matches the number and types of the reader fields was found."); + var readerArg = Expression.Parameter(typeof(DbDataReader), "reader"); var contextArg = Expression.Parameter(typeof(ArbitraryClassReaderContext), "context"); @@ -101,18 +116,46 @@ CompiledExpression> Compile(IDataReco var resultLocalVariable = Expression.Variable(typeof(TRecord), "result"); locals.Add(resultLocalVariable); - body.Add(Expression.Assign(resultLocalVariable, Expression.New(defaultConstructor))); - + + if (selectedConstructor.GetParameters().Any()) + { + BuildParameterizedConstructorExpression(selectedConstructor, readerArg, body, resultLocalVariable); + } + else + { + BuildParameterlessConstructorExpression(record, body, resultLocalVariable, selectedConstructor, contextArg, readerArg); + } + + // Return it + body.Add(resultLocalVariable); + + var block = Expression.Block( + locals, + body + ); + + var lambda = Expression.Lambda>(block, readerArg, contextArg); + + return ExpressionCompiler.Compile(lambda); + } + + void BuildParameterlessConstructorExpression(IDataRecord record, List body, ParameterExpression resultLocalVariable, ConstructorInfo selectedConstructor, ParameterExpression contextArg, ParameterExpression readerArg) + { + // Create fast setters for all properties on the type + var properties = typeof(TRecord).GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.SetProperty); + + body.Add(Expression.Assign(resultLocalVariable, Expression.New(selectedConstructor))); + var expectedFieldCount = record.FieldCount; for (var i = 0; i < expectedFieldCount; i++) { var name = record.GetName(i); - + var property = properties.FirstOrDefault(p => p.Name == name); if (property != null) { body.Add(Expression.Assign(Expression.Field(contextArg, nameof(ArbitraryClassReaderContext.Column)), Expression.Constant(i))); - + body.Add(Expression.Assign( Expression.Property(resultLocalVariable, property), ExpressionHelper.GetValueFromReaderAsType(readerArg, Expression.Constant(i), property.PropertyType, configuration.TypeHandlers))); @@ -122,18 +165,19 @@ CompiledExpression> Compile(IDataReco throw new Exception($"The query returned a column named '{name}' but no property by that name exists on the target type '{typeof(TRecord).Name}'. When reading to an arbitrary type, all columns must have a matching settable property."); } } + } - // Return it - body.Add(resultLocalVariable); - - var block = Expression.Block( - locals, - body - ); + void BuildParameterizedConstructorExpression(ConstructorInfo selectedConstructor, ParameterExpression readerArg, List body, ParameterExpression resultLocalVariable) + { + var arguments = selectedConstructor + .GetParameters() + .Select((p, i) => ExpressionHelper.GetValueFromReaderAsType(readerArg, Expression.Constant(i), p.ParameterType, configuration.TypeHandlers)); + body.Add(Expression.Assign(resultLocalVariable, Expression.New(selectedConstructor, arguments))); + } - var lambda = Expression.Lambda>(block, readerArg, contextArg); - - return ExpressionCompiler.Compile(lambda); + static bool MatchesTypes(IEnumerable parameters, IDataRecord record) + { + return !parameters.Where((t, i) => t.ParameterType != record.GetFieldType(i)).Any(); } } } \ No newline at end of file diff --git a/source/Nevermore/Querying/AST/IColumn.cs b/source/Nevermore/Querying/AST/IColumn.cs index 6cd64a74..7669dccd 100644 --- a/source/Nevermore/Querying/AST/IColumn.cs +++ b/source/Nevermore/Querying/AST/IColumn.cs @@ -1,4 +1,6 @@ -namespace Nevermore.Querying.AST +using System; + +namespace Nevermore.Querying.AST { public interface IColumn : ISelectColumns { @@ -59,13 +61,15 @@ public class JsonQueryColumn : IColumn { readonly string jsonPath; public bool AggregatesRows => false; + readonly Type elementType; - public JsonQueryColumn(string jsonPath) + public JsonQueryColumn(string jsonPath, Type elementType) { this.jsonPath = jsonPath; + this.elementType = elementType; } - public string GenerateSql() => $"JSON_QUERY([JSON], '{jsonPath}')"; + public string GenerateSql() => $"CAST(JSON_QUERY([JSON], '{jsonPath}') AS {elementType.ToDbType()})"; public override string ToString() => GenerateSql(); } @@ -73,13 +77,15 @@ public class JsonValueColumn : IColumn { readonly string jsonPath; public bool AggregatesRows => false; + readonly Type elementType; - public JsonValueColumn(string jsonPath) + public JsonValueColumn(string jsonPath, Type elementType) { this.jsonPath = jsonPath; + this.elementType = elementType; } - public string GenerateSql() => $"JSON_VALUE([JSON], '{jsonPath}')"; + public string GenerateSql() => $"CAST(JSON_VALUE([JSON], '{jsonPath}') AS {elementType.ToDbType()})"; public override string ToString() => GenerateSql(); } } \ No newline at end of file diff --git a/source/Nevermore/Querying/AST/TypeExtensionMethods.cs b/source/Nevermore/Querying/AST/TypeExtensionMethods.cs new file mode 100644 index 00000000..466b7b1f --- /dev/null +++ b/source/Nevermore/Querying/AST/TypeExtensionMethods.cs @@ -0,0 +1,26 @@ +using System; + +namespace Nevermore.Querying.AST +{ + public static class TypeExtensionMethods + { + public static string ToDbType(this Type type) + { + return Type.GetTypeCode(type) switch + { + TypeCode.String => "nvarchar(max)", + TypeCode.Int16 => "int", + TypeCode.Double => "double", + TypeCode.Boolean => "bit", + TypeCode.Char => "char(max)", + TypeCode.DateTime => "datetime2", + TypeCode.Decimal => "decimal(18,8)", + TypeCode.Int32 => "int", + TypeCode.Int64 => "bigint", + TypeCode.SByte => "tinyint", + TypeCode.Single => "float", + _ => throw new ArgumentOutOfRangeException() + }; + } + } +} \ No newline at end of file diff --git a/source/Nevermore/Querying/AST/Where.cs b/source/Nevermore/Querying/AST/Where.cs index 4f79d13d..a38345f3 100644 --- a/source/Nevermore/Querying/AST/Where.cs +++ b/source/Nevermore/Querying/AST/Where.cs @@ -105,26 +105,7 @@ public JsonArrayWhereClause(string parameterName, ArraySqlOperand operand, strin public string GenerateSql() { - return $"@{parameterName} {GetQueryOperandSql()} (SELECT [Val] FROM OPENJSON([JSON], 'strict {jsonPath}') WITH ([Val] {GetDbType()} '$'))"; - } - - string GetDbType() - { - return Type.GetTypeCode(elementType) switch - { - TypeCode.String => "nvarchar(max)", - TypeCode.Int16 => "int", - TypeCode.Double => "double", - TypeCode.Boolean => "bit", - TypeCode.Char => "char(max)", - TypeCode.DateTime => "datetime2", - TypeCode.Decimal => "decimal(18,8)", - TypeCode.Int32 => "int", - TypeCode.Int64 => "bigint", - TypeCode.SByte => "tinyint", - TypeCode.Single => "float", - _ => throw new ArgumentOutOfRangeException() - }; + return $"@{parameterName} {GetQueryOperandSql()} (SELECT [Val] FROM OPENJSON([JSON], 'strict {jsonPath}') WITH ([Val] {elementType.ToDbType()} '$'))"; } string GetQueryOperandSql() From b2f31ba262f94a95b7a77252bc9d5b2efefed0bd Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Wed, 5 Jul 2023 16:56:54 +1000 Subject: [PATCH 11/12] Ignore arrays in ArbitraryClassReaderStrategy --- .../ArbitraryClasses/ArbitraryClassReaderStrategy.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs b/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs index 6a14d4b9..ea038012 100644 --- a/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs +++ b/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs @@ -36,7 +36,7 @@ public bool CanRead(Type type) bool IsPrimitive(Type type) { - return type.IsPrimitive || type == typeof(string) || type == typeof(decimal); + return type.IsPrimitive || type.IsArray || type == typeof(string) || type == typeof(decimal); } public Func> CreateReader() From efad5fefe4d4af9b22814459a235926ac1d72e7b Mon Sep 17 00:00:00 2001 From: Eoin Motherway <25342760+YuKitsune@users.noreply.github.com> Date: Wed, 5 Jul 2023 16:57:14 +1000 Subject: [PATCH 12/12] Look for non public constructors in ArbitraryClassReaderStrategy --- .../ArbitraryClasses/ArbitraryClassReaderStrategy.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs b/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs index ea038012..9ac1e2d4 100644 --- a/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs +++ b/source/Nevermore/Advanced/ReaderStrategies/ArbitraryClasses/ArbitraryClassReaderStrategy.cs @@ -96,7 +96,7 @@ CompiledExpression> Compile(IDataReco // To make it fast - as fast as if we wrote it by hand - we generate and compile C# expression trees for // each property on the class, and one to call the constructor. - var constructors = typeof(TRecord).GetConstructors(); + var constructors = typeof(TRecord).GetConstructors(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); // Find the parameter-less constructor var defaultConstructor = constructors.FirstOrDefault(c => c.GetParameters().Length == 0); // Find a parameterized constructor that matches the data reader