From 01e651aab2f60fbe18b1816f7b5552a8491d1ac4 Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Mon, 8 May 2023 14:57:34 -0700 Subject: [PATCH 01/16] Infrastructure for performance testing with ODE pipeline. --- .../QueryPerfTest/MetricsSerializer.cs | 107 +++++ ...timisticDirectExecutionPerformanceTests.cs | 394 ++++++++++++++++++ .../QueryStatisticsDatumVisitor.cs | 21 + .../QueryPerfTest/QueryStatisticsMetrics.cs | 10 + 4 files changed, 532 insertions(+) create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs index c73612caa7..e6efc1f3b5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs @@ -176,5 +176,112 @@ public void Serialize(TextWriter textWriter, QueryStatisticsDatumVisitor querySt } textWriter.Flush(); } + + public void ODESerialization(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor, int numberOfIterations, bool rawData) + { + if (rawData) + { + textWriter.WriteLine(); + textWriter.WriteLine(PrintQueryMetrics); + textWriter.Write(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", "Query", "ODE", "RUCharge", "BackendTime", + "TransitTime", "ClientTime", "EndToEndTime")); + textWriter.WriteLine(); + double totalClientTime = 0; + double totalBackendTime = 0; + double totalEndToEndTime = 0; + double totalTransitTime = 0; + double totalRU = 0; + int count = 0; + string prevQuery = ""; + bool prevOde = default; + Guid prevCorrelatedActivityId = default; + + foreach (QueryStatisticsMetrics metrics in queryStatisticsDatumVisitor.QueryMetricsList) + { + if (count == 0) + { + prevCorrelatedActivityId = metrics.CorrelatedActivityId; + } + + double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed; + double backendTime = metrics.TotalQueryExecutionTime; + + if (metrics.CorrelatedActivityId == prevCorrelatedActivityId) + { + totalClientTime += metrics.EndToEndTime - (backendTime + transitTime); + totalBackendTime += backendTime; + totalEndToEndTime += metrics.EndToEndTime; + totalTransitTime += transitTime; + totalRU += metrics.RUCharge; + prevQuery = metrics.Query; + prevOde = metrics.EnableOde; + prevCorrelatedActivityId = metrics.CorrelatedActivityId; + + } + else + { + textWriter.WriteLine(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", + prevQuery, prevOde, totalRU, totalBackendTime, totalTransitTime, totalClientTime, totalEndToEndTime)); + + totalClientTime = metrics.EndToEndTime - (backendTime + transitTime); + totalBackendTime = backendTime; + totalEndToEndTime = metrics.EndToEndTime; + totalTransitTime = transitTime; + totalRU = metrics.RUCharge; + prevCorrelatedActivityId = metrics.CorrelatedActivityId; + prevQuery = metrics.Query; + prevOde = metrics.EnableOde; + } + + count++; + } + + textWriter.WriteLine(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", + prevQuery, prevOde, totalRU, totalBackendTime, totalTransitTime, totalClientTime, totalEndToEndTime)); + } + else + { + textWriter.WriteLine(); + textWriter.WriteLine(PrintQueryMetrics); + textWriter.Write(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime")); + textWriter.WriteLine(); + + string prevQuery = ""; + bool prevOde = default; + double totalEndToEndTime = 0; + double totalRU = 0; + int count = 0; + + foreach (QueryStatisticsMetrics metrics in queryStatisticsDatumVisitor.QueryMetricsList) + { + if (count == 0) + { + prevQuery = metrics.Query; + prevOde = metrics.EnableOde; + } + + if (metrics.Query == prevQuery && metrics.EnableOde == prevOde) + { + totalEndToEndTime += metrics.EndToEndTime; + totalRU += metrics.RUCharge; + } + else + { + textWriter.WriteLine(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", + prevQuery, prevOde, totalRU/numberOfIterations, totalEndToEndTime/numberOfIterations)); + + totalEndToEndTime = metrics.EndToEndTime; + totalRU = metrics.RUCharge; + prevQuery = metrics.Query; + prevOde = metrics.EnableOde; + } + + count++; + } + + textWriter.WriteLine(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", + prevQuery, prevOde, totalRU / numberOfIterations, totalEndToEndTime / numberOfIterations)); + } + } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs new file mode 100644 index 0000000000..350a1362e1 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -0,0 +1,394 @@ +namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Text; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class OptimisticDirectExecutionPerformanceTests + { + private const string RawDataFileName = "OptimisticDirectExecutionPerformanceTestsRawData.csv"; + private const string DiagnosticsDataFileName = "OptimisticDirectExecutionPerformanceTestsAggregatedData.csv"; + private const string TransportKeyValue = "Client Side Request Stats"; + private const string TransportNodeName = "Microsoft.Azure.Documents.ServerStoreModel Transport Request"; + private readonly string RawDataPath = Path.GetFullPath(RawDataFileName); + private readonly string AggregateDataPath = Path.GetFullPath(DiagnosticsDataFileName); + + private readonly QueryStatisticsDatumVisitor queryStatisticsDatumVisitor; + private readonly string endpoint; + private readonly string authKey; + private readonly string cosmosDatabaseId; + private readonly string containerId; + private readonly PartitionKey partitionKeyValue; + private readonly int numberOfIterations; + private readonly int warmupIterations; + private CosmosClient cosmosClient; + private Database database; + private Container container; + + public OptimisticDirectExecutionPerformanceTests() + { + this.queryStatisticsDatumVisitor = new(); + this.endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; + this.authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; + this.cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.CosmosDatabaseId"]; + this.containerId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContainerId"]; + this.numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.NumberOfIterations"]); + this.warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.WarmupIterations"]); + this.partitionKeyValue = new PartitionKey("Andersen"); + } + + [TestMethod] + public async Task PerformanceTestSetup() + { + CosmosClientOptions clientOptions = new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Direct, + }; + + this.cosmosClient = new CosmosClient(this.endpoint, this.authKey, clientOptions); + //await this.CreateDatabaseAsync(); + //await this.CreateContainerAsync(); + //await this.AddItemsToContainerAsync(); + await this.RunAsync(); + } + + private async Task CreateDatabaseAsync() + { + // Create a new database + this.database = await this.cosmosClient.CreateDatabaseIfNotExistsAsync(this.cosmosDatabaseId); + Console.WriteLine("Created Database: {0}\n", this.database.Id); + } + + private async Task CreateContainerAsync() + { + this.container = await this.database.CreateContainerIfNotExistsAsync(this.containerId, partitionKeyPath: "/name"); + Console.WriteLine("Created Container: {0}\n", this.container.Id); + } + + private async Task AddItemsToContainerAsync() + { + Random random = new Random(); + string[] cityOptions = new string[] { "Seattle", "Chicago", "NYC", "SF" }; + + // Create a family object for the Andersen family + foreach (int i in Enumerable.Range(0, 5000)) + { + int numberOfRecipeints = random.Next(1, 4); + List recipientList = new List(); + + for (int j = 0; j < numberOfRecipeints; j++) + { + RecipientList store = new RecipientList() + { + Name = "John", + City = cityOptions[random.Next(0, cityOptions.Length)], + }; + + recipientList.Add(store); + } + + States andersenFamily = new States + { + Id = i.ToString(), + Name = "Andersen", + City = cityOptions[random.Next(0, cityOptions.Length)], + PostalCode = random.Next(0, 1000).ToString(), + Region = "Northwest", + UserDefinedID = i % 10, + RecipientList = recipientList + }; + + try + { + // Read the item to see if it exists. + ItemResponse andersenFamilyResponse = await this.container.ReadItemAsync(andersenFamily.Id, new PartitionKey("Andersen")); + Console.WriteLine("Item in database with id: {0} already exists\n", andersenFamilyResponse.Resource.Id); + } + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + // Create an item in the container representing the Andersen family. Note we provide the value of the partition key for this item, which is "Andersen" + ItemResponse andersenFamilyResponse = await this.container.CreateItemAsync(andersenFamily, new PartitionKey("Andersen")); + + // Note that after creating the item, we can access the body of the item with the Resource property off the ItemResponse. We can also access the RequestCharge property to see the amount of RUs consumed on this request. + Console.WriteLine("Created item in database with id: {0} Operation consumed {1} RUs.\n", andersenFamilyResponse.Resource.Id, andersenFamilyResponse.RequestCharge); + } + + } + } + + private async Task RunAsync() + { + MetricsSerializer metricsSerializer = new MetricsSerializer(); + this.database = this.cosmosClient.GetDatabase(this.cosmosDatabaseId); + this.container = this.database.GetContainer(this.containerId); + string highPrepTimeSumQuery = this.CreateHighPrepTimeSumQuery(); + string highPrepTimeConditionalQuery = this.CreateHighPrepTimeConditionalQuery(); + + Console.WriteLine(this.RawDataPath); + + if (File.Exists(this.RawDataPath)) + { + File.Delete(this.RawDataPath); + } + + if(File.Exists(this.AggregateDataPath)) + { + File.Delete(this.AggregateDataPath); + } + + List odeTestCases = new List() + { + //Simple Query + CreateInput("SELECT * FROM c", this.partitionKeyValue, false, -1, 5000), + CreateInput("SELECT * FROM c", this.partitionKeyValue, true, -1, 5000), + + //TOP + CreateInput("SELECT TOP 1000 c.id FROM c", this.partitionKeyValue, false, -1, 1000), + CreateInput("SELECT TOP 1000 c.id FROM c", this.partitionKeyValue, true, -1, 1000), + + //Filter + CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", this.partitionKeyValue, false, -1, 2557), + CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", this.partitionKeyValue, true, -1, 2557), + + //DISTINCT + Filter + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", this.partitionKeyValue, false, -1, 3), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", this.partitionKeyValue, true, -1, 3), + + CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", this.partitionKeyValue, false, -1, 2), + CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", this.partitionKeyValue, true, -1, 2), + + //JOIN + CreateInput("SELECT root.id " + + "FROM root " + + "JOIN root.id a " + + "JOIN root.id b " + + "JOIN root.id c " + + "WHERE root.id = '1' OR a.id in (1,2,3,4,5,6,7,8,9,10) " + + "OR b.id in (1,2,3,4,5,6,7,8,9,10) " + + "OR c.id in (1,2,3,4,5,6,7,8,9,10)", this.partitionKeyValue, false, -1, 1), + CreateInput("SELECT root.id " + + "FROM root " + + "JOIN root.id a " + + "JOIN root.id b " + + "JOIN root.id c " + + "WHERE root.id = '1' OR a.id in (1,2,3,4,5,6,7,8,9,10) " + + "OR b.id in (1,2,3,4,5,6,7,8,9,10) " + + "OR c.id in (1,2,3,4,5,6,7,8,9,10)", this.partitionKeyValue, true, -1, 1), + + //High Prep Time + CreateInput(highPrepTimeSumQuery, this.partitionKeyValue, false, -1, 5000), + CreateInput(highPrepTimeSumQuery, this.partitionKeyValue, true, -1, 5000), + + CreateInput(highPrepTimeConditionalQuery, this.partitionKeyValue, false, -1, 3770), + CreateInput(highPrepTimeConditionalQuery, this.partitionKeyValue, true, -1, 3770), + + //Order By + CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", this.partitionKeyValue, false, -1, 5000), + CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", this.partitionKeyValue, true, -1, 5000), + + CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", this.partitionKeyValue, false, -1, 5000), + CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", this.partitionKeyValue, true, -1, 5000), + + //Order By + TOP + CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", this.partitionKeyValue, false, -1, 5), + CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", this.partitionKeyValue, true, -1, 5), + + //Order By + DISTINCT + CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", this.partitionKeyValue, false, -1, 5000), + CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", this.partitionKeyValue, true, -1, 5000), + + //Order By + DISTINCT + Filter + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", this.partitionKeyValue, false, -1, 4), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", this.partitionKeyValue, true, -1, 4), + + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", this.partitionKeyValue, false, -1, 6), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", this.partitionKeyValue, true, -1, 6), + + //Group By + CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", this.partitionKeyValue, false, -1, 996), + CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", this.partitionKeyValue, true, -1, 996), + + CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", this.partitionKeyValue, false, -1, 2), + CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", this.partitionKeyValue, true, -1, 2), + + CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", this.partitionKeyValue, false, -1, 4), + CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", this.partitionKeyValue, true, -1, 4), + + //Group By + OFFSET + CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", this.partitionKeyValue, false, -1, 3), + CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", this.partitionKeyValue, true, -1, 3), + + //Group By + TOP + CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", this.partitionKeyValue, false, -1, 25), + CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", this.partitionKeyValue, true, -1, 25), + + //Group By + DISTINCT + CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", this.partitionKeyValue, false, -1, 5000), + CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", this.partitionKeyValue, true, -1, 5000), + + CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", this.partitionKeyValue, false, -1, 996), + CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", this.partitionKeyValue, true, -1, 996), + }; + + foreach (DirectExecutionTestCase testCase in odeTestCases) + { + await this.RunQueryAsync(this.container, testCase); + } + + using (StreamWriter writer = new StreamWriter(new FileStream(this.RawDataPath, FileMode.Append, FileAccess.Write))) + { + metricsSerializer.ODESerialization(writer, this.queryStatisticsDatumVisitor, this.numberOfIterations, true); + } + + using (StreamWriter writer = new StreamWriter(new FileStream(this.AggregateDataPath, FileMode.Append, FileAccess.Write))) + { + metricsSerializer.ODESerialization(writer, this.queryStatisticsDatumVisitor, this.numberOfIterations, false); + } + } + + private async Task RunQueryAsync(Container container, DirectExecutionTestCase queryInput) + { + QueryRequestOptions requestOptions = new QueryRequestOptions() + { + MaxItemCount = queryInput.PageSizeOption, + EnableOptimisticDirectExecution = queryInput.EnableOptimisticDirectExecution, + PartitionKey = queryInput.PartitionKey, + }; + + for (int i = 0; i < this.numberOfIterations; i++) + { + bool isWarmUpIteration = false; + if (i < this.warmupIterations) + { + isWarmUpIteration = true; + } + using (FeedIterator iterator = container.GetItemQueryIterator( + queryText: queryInput.Query, + requestOptions: requestOptions)) + { + await this.GetIteratorResponse(iterator, queryInput, isWarmUpIteration); + } + } + } + + private async Task GetIteratorResponse(FeedIterator feedIterator, DirectExecutionTestCase queryInput, bool isWarmUpIteration) + { + MetricsAccumulator metricsAccumulator = new MetricsAccumulator(); + Documents.ValueStopwatch totalTime = new Documents.ValueStopwatch(); + Documents.ValueStopwatch getTraceTime = new Documents.ValueStopwatch(); + Guid correlatedActivityId = Guid.NewGuid(); + FeedResponse response; + int totalDocumentCount = 0; + + while (feedIterator.HasMoreResults) + { + if (!isWarmUpIteration) + { + totalTime.Start(); + response = await feedIterator.ReadNextAsync(); + getTraceTime.Start(); + if (response.RequestCharge != 0) + { + metricsAccumulator.ReadFromTrace(response, this.queryStatisticsDatumVisitor); + } + + getTraceTime.Stop(); + totalTime.Stop(); + if (response.RequestCharge != 0) + { + this.queryStatisticsDatumVisitor.AddQuery(queryInput.Query); + this.queryStatisticsDatumVisitor.AddEnableOdeFlag(queryInput.EnableOptimisticDirectExecution); + this.queryStatisticsDatumVisitor.AddCorrelatedActivityId(correlatedActivityId); + this.queryStatisticsDatumVisitor.AddRuCharge(response.RequestCharge); + this.queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - getTraceTime.ElapsedMilliseconds); + this.queryStatisticsDatumVisitor.PopulateMetrics(); + } + + totalTime.Reset(); + getTraceTime.Reset(); + } + else + { + response = await feedIterator.ReadNextAsync(); + } + + totalDocumentCount += response.Count; + } + + Assert.AreEqual(queryInput.ExpectedResultCount, totalDocumentCount); + } + + private string CreateHighPrepTimeSumQuery() + { + int exprCount = 9999; + StringBuilder sb = new StringBuilder(); + sb.Append("SELECT r.id FROM root r WHERE "); + for (int i = 0; i < exprCount; i++) + { + sb.Append(i == 0 ? "1" : "+1"); + } + + return sb.Append(" = " + exprCount + " ORDER BY r.id ASC").ToString(); + } + + private string CreateHighPrepTimeConditionalQuery() + { + int exprCount = 999; + StringBuilder sb = new StringBuilder(); + string[] cityOptions = new string[] { "Seattle", "Chicago", "NYC", "SF" }; + + sb.Append("SELECT * FROM root r WHERE "); + for (int nIdx = 0; nIdx < exprCount; nIdx++) + { + if (nIdx > 0) + { + sb.Append(" OR "); + } + + sb.Append("(r.userDefinedId > " + nIdx + " AND r.city = " + "'" + cityOptions[nIdx % cityOptions.Length] + "')"); + } + + return sb.ToString(); + } + + private static DirectExecutionTestCase CreateInput( + string query, + PartitionKey? partitionKey, + bool enableOptimisticDirectExecution, + int pageSizeOption, + int expectedResultCount) + { + return new DirectExecutionTestCase(query, partitionKey, enableOptimisticDirectExecution, pageSizeOption, expectedResultCount); + } + + private readonly struct DirectExecutionTestCase + { + public string Query { get; } + public PartitionKey? PartitionKey { get; } + public bool EnableOptimisticDirectExecution { get; } + public int PageSizeOption { get; } + public int ExpectedResultCount { get; } + + public DirectExecutionTestCase( + string query, + PartitionKey? partitionKey, + bool enableOptimisticDirectExecution, + int pageSizeOption, + int expectedResultCount) + { + this.Query = query; + this.PartitionKey = partitionKey; + this.EnableOptimisticDirectExecution = enableOptimisticDirectExecution; + this.PageSizeOption = pageSizeOption; + this.ExpectedResultCount = expectedResultCount; + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs index 200150875b..1e221be402 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs @@ -1,5 +1,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests { + using System; using System.Collections.Generic; using System.Diagnostics; using Microsoft.Azure.Cosmos.Tracing; @@ -29,6 +30,26 @@ public void AddEndToEndTime(double totalTime) this.queryMetrics.EndToEndTime = totalTime; } + public void AddRuCharge(double ruCharge) + { + this.queryMetrics.RUCharge = ruCharge; + } + + public void AddQuery(string query) + { + this.queryMetrics.Query = query; + } + + public void AddCorrelatedActivityId(Guid activityId) + { + this.queryMetrics.CorrelatedActivityId = activityId; + } + + public void AddEnableOdeFlag(bool enableOde) + { + this.queryMetrics.EnableOde = enableOde; + } + public void AddPocoTime(double time) { this.queryMetrics.PocoTime = time; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs index 70ed146f4c..76daf6a163 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs @@ -1,7 +1,15 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests { + using System; + internal class QueryStatisticsMetrics { + public string Query { get; set; } + + public bool EnableOde { get; set; } + + public Guid CorrelatedActivityId { get; set; } + public double EndToEndTime { get; set; } public double PocoTime { get; set; } @@ -32,6 +40,8 @@ internal class QueryStatisticsMetrics public double Received { get; set; } + public double RUCharge { get; set; } + public double Completed { get; set; } public double BadRequestCreated { get; set; } From 7a2b68c7e7fcde9b557e167fd072890e58f7fbcc Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Wed, 17 May 2023 17:26:14 -0700 Subject: [PATCH 02/16] Resolve comments --- .../QueryPerfTest/MetricsSerializer.cs | 5 ++--- .../OptimisticDirectExecutionPerformanceTests.cs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs index e6efc1f3b5..b617d7d4c6 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs @@ -183,8 +183,7 @@ public void ODESerialization(TextWriter textWriter, QueryStatisticsDatumVisitor { textWriter.WriteLine(); textWriter.WriteLine(PrintQueryMetrics); - textWriter.Write(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", "Query", "ODE", "RUCharge", "BackendTime", - "TransitTime", "ClientTime", "EndToEndTime")); + textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", "Query", "ODE", "RUCharge", "BackendTime", "TransitTime", "ClientTime", "EndToEndTime"); textWriter.WriteLine(); double totalClientTime = 0; double totalBackendTime = 0; @@ -243,7 +242,7 @@ public void ODESerialization(TextWriter textWriter, QueryStatisticsDatumVisitor { textWriter.WriteLine(); textWriter.WriteLine(PrintQueryMetrics); - textWriter.Write(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime")); + textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime"); textWriter.WriteLine(); string prevQuery = ""; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 350a1362e1..29edaf5f5f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -53,9 +53,9 @@ public async Task PerformanceTestSetup() }; this.cosmosClient = new CosmosClient(this.endpoint, this.authKey, clientOptions); - //await this.CreateDatabaseAsync(); - //await this.CreateContainerAsync(); - //await this.AddItemsToContainerAsync(); + await this.CreateDatabaseAsync(); + await this.CreateContainerAsync(); + await this.AddItemsToContainerAsync(); await this.RunAsync(); } @@ -128,8 +128,8 @@ private async Task RunAsync() MetricsSerializer metricsSerializer = new MetricsSerializer(); this.database = this.cosmosClient.GetDatabase(this.cosmosDatabaseId); this.container = this.database.GetContainer(this.containerId); - string highPrepTimeSumQuery = this.CreateHighPrepTimeSumQuery(); - string highPrepTimeConditionalQuery = this.CreateHighPrepTimeConditionalQuery(); + string highPrepTimeSumQuery = CreateHighPrepTimeSumQuery(); + string highPrepTimeConditionalQuery = CreateHighPrepTimeConditionalQuery(); Console.WriteLine(this.RawDataPath); @@ -325,7 +325,7 @@ private async Task GetIteratorResponse(FeedIterator feedIterator, DirectEx Assert.AreEqual(queryInput.ExpectedResultCount, totalDocumentCount); } - private string CreateHighPrepTimeSumQuery() + private static string CreateHighPrepTimeSumQuery() { int exprCount = 9999; StringBuilder sb = new StringBuilder(); @@ -338,7 +338,7 @@ private string CreateHighPrepTimeSumQuery() return sb.Append(" = " + exprCount + " ORDER BY r.id ASC").ToString(); } - private string CreateHighPrepTimeConditionalQuery() + private static string CreateHighPrepTimeConditionalQuery() { int exprCount = 999; StringBuilder sb = new StringBuilder(); @@ -352,7 +352,7 @@ private string CreateHighPrepTimeConditionalQuery() sb.Append(" OR "); } - sb.Append("(r.userDefinedId > " + nIdx + " AND r.city = " + "'" + cityOptions[nIdx % cityOptions.Length] + "')"); + sb.Append($"r.userDefinedId > {nIdx} AND r.city = '{cityOptions[nIdx % cityOptions.Length]}'"); } return sb.ToString(); From b0b59125a1f0c0ff7749139738a864b7f25adbd6 Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Sun, 4 Jun 2023 22:29:58 -0700 Subject: [PATCH 03/16] Removed randomization from data creation process --- .../QueryPerfTest/MetricsSerializer.cs | 165 +++++------ ...timisticDirectExecutionPerformanceTests.cs | 278 +++++++++--------- 2 files changed, 209 insertions(+), 234 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs index b617d7d4c6..e81a1c7d46 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs @@ -177,110 +177,97 @@ public void Serialize(TextWriter textWriter, QueryStatisticsDatumVisitor querySt textWriter.Flush(); } - public void ODESerialization(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor, int numberOfIterations, bool rawData) + public void OdeSerialization(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor, int numberOfIterations, bool rawData) { if (rawData) { - textWriter.WriteLine(); - textWriter.WriteLine(PrintQueryMetrics); - textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", "Query", "ODE", "RUCharge", "BackendTime", "TransitTime", "ClientTime", "EndToEndTime"); - textWriter.WriteLine(); - double totalClientTime = 0; - double totalBackendTime = 0; - double totalEndToEndTime = 0; - double totalTransitTime = 0; - double totalRU = 0; - int count = 0; - string prevQuery = ""; - bool prevOde = default; - Guid prevCorrelatedActivityId = default; - - foreach (QueryStatisticsMetrics metrics in queryStatisticsDatumVisitor.QueryMetricsList) - { - if (count == 0) - { - prevCorrelatedActivityId = metrics.CorrelatedActivityId; - } - - double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed; - double backendTime = metrics.TotalQueryExecutionTime; + OdeRawDataSerialization(textWriter, queryStatisticsDatumVisitor); + } + else + { + OdeProcessedDataSerialization(textWriter, queryStatisticsDatumVisitor, numberOfIterations); + } + } - if (metrics.CorrelatedActivityId == prevCorrelatedActivityId) - { - totalClientTime += metrics.EndToEndTime - (backendTime + transitTime); - totalBackendTime += backendTime; - totalEndToEndTime += metrics.EndToEndTime; - totalTransitTime += transitTime; - totalRU += metrics.RUCharge; - prevQuery = metrics.Query; - prevOde = metrics.EnableOde; - prevCorrelatedActivityId = metrics.CorrelatedActivityId; + private static void OdeRawDataSerialization(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor) + { + textWriter.WriteLine(); + textWriter.WriteLine(PrintQueryMetrics); + textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", "Query", "ODE", "RUCharge", "BackendTime", "TransitTime", "ClientTime", "EndToEndTime"); + textWriter.WriteLine(); + double totalClientTime = 0; + double totalBackendTime = 0; + double totalEndToEndTime = 0; + double totalTransitTime = 0; + double totalRU = 0; + string prevQuery = ""; + bool prevOde = default; + Guid prevCorrelatedActivityId = queryStatisticsDatumVisitor.QueryMetricsList[0].CorrelatedActivityId; - } - else - { - textWriter.WriteLine(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", - prevQuery, prevOde, totalRU, totalBackendTime, totalTransitTime, totalClientTime, totalEndToEndTime)); + foreach (QueryStatisticsMetrics metrics in queryStatisticsDatumVisitor.QueryMetricsList) + { + double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed; + double backendTime = metrics.TotalQueryExecutionTime; - totalClientTime = metrics.EndToEndTime - (backendTime + transitTime); - totalBackendTime = backendTime; - totalEndToEndTime = metrics.EndToEndTime; - totalTransitTime = transitTime; - totalRU = metrics.RUCharge; - prevCorrelatedActivityId = metrics.CorrelatedActivityId; - prevQuery = metrics.Query; - prevOde = metrics.EnableOde; - } + if (metrics.CorrelatedActivityId == prevCorrelatedActivityId) + { + totalClientTime += metrics.EndToEndTime - (backendTime + transitTime); + totalBackendTime += backendTime; + totalEndToEndTime += metrics.EndToEndTime; + totalTransitTime += transitTime; + totalRU += metrics.RUCharge; + prevQuery = metrics.Query; + prevOde = metrics.EnableOde; + prevCorrelatedActivityId = metrics.CorrelatedActivityId; - count++; } - - textWriter.WriteLine(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", - prevQuery, prevOde, totalRU, totalBackendTime, totalTransitTime, totalClientTime, totalEndToEndTime)); + else + { + textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); + totalClientTime = metrics.EndToEndTime - (backendTime + transitTime); + totalBackendTime = backendTime; + totalEndToEndTime = metrics.EndToEndTime; + totalTransitTime = transitTime; + totalRU = metrics.RUCharge; + prevCorrelatedActivityId = metrics.CorrelatedActivityId; + prevQuery = metrics.Query; + prevOde = metrics.EnableOde; + } } - else - { - textWriter.WriteLine(); - textWriter.WriteLine(PrintQueryMetrics); - textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime"); - textWriter.WriteLine(); - string prevQuery = ""; - bool prevOde = default; - double totalEndToEndTime = 0; - double totalRU = 0; - int count = 0; - - foreach (QueryStatisticsMetrics metrics in queryStatisticsDatumVisitor.QueryMetricsList) - { - if (count == 0) - { - prevQuery = metrics.Query; - prevOde = metrics.EnableOde; - } + textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); + } - if (metrics.Query == prevQuery && metrics.EnableOde == prevOde) - { - totalEndToEndTime += metrics.EndToEndTime; - totalRU += metrics.RUCharge; - } - else - { - textWriter.WriteLine(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", - prevQuery, prevOde, totalRU/numberOfIterations, totalEndToEndTime/numberOfIterations)); + private static void OdeProcessedDataSerialization(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor, int numberOfIterations) + { + textWriter.WriteLine(); + textWriter.WriteLine(PrintQueryMetrics); + textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime"); + textWriter.WriteLine(); - totalEndToEndTime = metrics.EndToEndTime; - totalRU = metrics.RUCharge; - prevQuery = metrics.Query; - prevOde = metrics.EnableOde; - } + string prevQuery = queryStatisticsDatumVisitor.QueryMetricsList[0].Query; + bool prevOde = queryStatisticsDatumVisitor.QueryMetricsList[0].EnableOde; + double totalEndToEndTime = 0; + double totalRU = 0; - count++; + foreach (QueryStatisticsMetrics metrics in queryStatisticsDatumVisitor.QueryMetricsList) + { + if (metrics.Query == prevQuery && metrics.EnableOde == prevOde) + { + totalEndToEndTime += metrics.EndToEndTime; + totalRU += metrics.RUCharge; + } + else + { + textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); + totalEndToEndTime = metrics.EndToEndTime; + totalRU = metrics.RUCharge; + prevQuery = metrics.Query; + prevOde = metrics.EnableOde; } - - textWriter.WriteLine(string.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", - prevQuery, prevOde, totalRU / numberOfIterations, totalEndToEndTime / numberOfIterations)); } + + textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 29edaf5f5f..0db3e55bf1 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -15,91 +15,89 @@ public class OptimisticDirectExecutionPerformanceTests { private const string RawDataFileName = "OptimisticDirectExecutionPerformanceTestsRawData.csv"; private const string DiagnosticsDataFileName = "OptimisticDirectExecutionPerformanceTestsAggregatedData.csv"; - private const string TransportKeyValue = "Client Side Request Stats"; - private const string TransportNodeName = "Microsoft.Azure.Documents.ServerStoreModel Transport Request"; private readonly string RawDataPath = Path.GetFullPath(RawDataFileName); private readonly string AggregateDataPath = Path.GetFullPath(DiagnosticsDataFileName); - - private readonly QueryStatisticsDatumVisitor queryStatisticsDatumVisitor; - private readonly string endpoint; - private readonly string authKey; - private readonly string cosmosDatabaseId; - private readonly string containerId; - private readonly PartitionKey partitionKeyValue; - private readonly int numberOfIterations; - private readonly int warmupIterations; - private CosmosClient cosmosClient; - private Database database; - private Container container; - - public OptimisticDirectExecutionPerformanceTests() - { - this.queryStatisticsDatumVisitor = new(); - this.endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; - this.authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; - this.cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.CosmosDatabaseId"]; - this.containerId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContainerId"]; - this.numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.NumberOfIterations"]); - this.warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.WarmupIterations"]); - this.partitionKeyValue = new PartitionKey("Andersen"); - } + private static readonly QueryStatisticsDatumVisitor queryStatisticsDatumVisitor = new QueryStatisticsDatumVisitor(); + private static readonly string endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; + private static readonly string authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; + private static readonly string cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.CosmosDatabaseId"]; + private static readonly string containerId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContainerId"]; + private static readonly PartitionKey partitionKeyValue = new PartitionKey("Andersen"); + private static readonly int numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.NumberOfIterations"]); + private static readonly int warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.WarmupIterations"]); [TestMethod] - public async Task PerformanceTestSetup() + public async Task OptimisticDirectExecutionPerformanceTest() { CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, }; - this.cosmosClient = new CosmosClient(this.endpoint, this.authKey, clientOptions); - await this.CreateDatabaseAsync(); - await this.CreateContainerAsync(); - await this.AddItemsToContainerAsync(); - await this.RunAsync(); + CosmosClient cosmosClient = new CosmosClient(endpoint, authKey, clientOptions); + (Database database, Container container) = await this.TestInitialize(cosmosClient); + await this.RunAsync(container); + await this.TestCleanup(database); } - private async Task CreateDatabaseAsync() + private async Task<(Database, Container)> TestInitialize(CosmosClient cosmosClient) + { + Database database = await this.CreateDatabaseAsync(cosmosClient); + Container container = await this.CreateContainerAsync(database); + await this.AddItemsToContainerAsync(container); + + if (File.Exists(this.RawDataPath)) + { + File.Delete(this.RawDataPath); + } + + if (File.Exists(this.AggregateDataPath)) + { + File.Delete(this.AggregateDataPath); + } + + return (database, container); + } + + private async Task CreateDatabaseAsync(CosmosClient cosmosClient) { // Create a new database - this.database = await this.cosmosClient.CreateDatabaseIfNotExistsAsync(this.cosmosDatabaseId); - Console.WriteLine("Created Database: {0}\n", this.database.Id); + return await cosmosClient.CreateDatabaseIfNotExistsAsync(cosmosDatabaseId); } - private async Task CreateContainerAsync() + private async Task CreateContainerAsync(Database database) { - this.container = await this.database.CreateContainerIfNotExistsAsync(this.containerId, partitionKeyPath: "/name"); - Console.WriteLine("Created Container: {0}\n", this.container.Id); + return await database.CreateContainerIfNotExistsAsync(containerId, partitionKeyPath: "/name"); } - private async Task AddItemsToContainerAsync() + private async Task AddItemsToContainerAsync(Container container) { - Random random = new Random(); + int totalItems = 5000; string[] cityOptions = new string[] { "Seattle", "Chicago", "NYC", "SF" }; // Create a family object for the Andersen family - foreach (int i in Enumerable.Range(0, 5000)) + foreach (int i in Enumerable.Range(0, totalItems)) { - int numberOfRecipeints = random.Next(1, 4); - List recipientList = new List(); + int numberOfRecipeints = cityOptions.Length; + List recipientList = new List(numberOfRecipeints); for (int j = 0; j < numberOfRecipeints; j++) { - RecipientList store = new RecipientList() + RecipientList recipient = new RecipientList() { Name = "John", - City = cityOptions[random.Next(0, cityOptions.Length)], + City = cityOptions[j], }; - recipientList.Add(store); + recipientList.Add(recipient); } States andersenFamily = new States { Id = i.ToString(), - Name = "Andersen", - City = cityOptions[random.Next(0, cityOptions.Length)], - PostalCode = random.Next(0, 1000).ToString(), + Name = i < (totalItems/2) ? "Andersen" : "Smith", + City = cityOptions[i%cityOptions.Length], + PostalCode = (i * 10).ToString(), Region = "Northwest", UserDefinedID = i % 10, RecipientList = recipientList @@ -108,61 +106,46 @@ private async Task AddItemsToContainerAsync() try { // Read the item to see if it exists. - ItemResponse andersenFamilyResponse = await this.container.ReadItemAsync(andersenFamily.Id, new PartitionKey("Andersen")); + ItemResponse andersenFamilyResponse = await container.ReadItemAsync(andersenFamily.Id, new PartitionKey(andersenFamily.Name)); Console.WriteLine("Item in database with id: {0} already exists\n", andersenFamilyResponse.Resource.Id); } catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { // Create an item in the container representing the Andersen family. Note we provide the value of the partition key for this item, which is "Andersen" - ItemResponse andersenFamilyResponse = await this.container.CreateItemAsync(andersenFamily, new PartitionKey("Andersen")); + ItemResponse andersenFamilyResponse = await container.CreateItemAsync(andersenFamily, new PartitionKey(andersenFamily.Name)); // Note that after creating the item, we can access the body of the item with the Resource property off the ItemResponse. We can also access the RequestCharge property to see the amount of RUs consumed on this request. Console.WriteLine("Created item in database with id: {0} Operation consumed {1} RUs.\n", andersenFamilyResponse.Resource.Id, andersenFamilyResponse.RequestCharge); } - } } - private async Task RunAsync() + private async Task RunAsync(Container container) { MetricsSerializer metricsSerializer = new MetricsSerializer(); - this.database = this.cosmosClient.GetDatabase(this.cosmosDatabaseId); - this.container = this.database.GetContainer(this.containerId); string highPrepTimeSumQuery = CreateHighPrepTimeSumQuery(); string highPrepTimeConditionalQuery = CreateHighPrepTimeConditionalQuery(); - Console.WriteLine(this.RawDataPath); - - if (File.Exists(this.RawDataPath)) - { - File.Delete(this.RawDataPath); - } - - if(File.Exists(this.AggregateDataPath)) - { - File.Delete(this.AggregateDataPath); - } - List odeTestCases = new List() { //Simple Query - CreateInput("SELECT * FROM c", this.partitionKeyValue, false, -1, 5000), - CreateInput("SELECT * FROM c", this.partitionKeyValue, true, -1, 5000), + CreateInput("SELECT * FROM c", partitionKeyValue, false, -1, 2500), + CreateInput("SELECT * FROM c", partitionKeyValue, true, -1, 2500), //TOP - CreateInput("SELECT TOP 1000 c.id FROM c", this.partitionKeyValue, false, -1, 1000), - CreateInput("SELECT TOP 1000 c.id FROM c", this.partitionKeyValue, true, -1, 1000), + CreateInput("SELECT TOP 1000 c.id FROM c", partitionKeyValue, false, -1, 1000), + CreateInput("SELECT TOP 1000 c.id FROM c", partitionKeyValue, true, -1, 1000), //Filter - CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", this.partitionKeyValue, false, -1, 2557), - CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", this.partitionKeyValue, true, -1, 2557), + CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", partitionKeyValue, false, -1, 1250), + CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", partitionKeyValue, true, -1, 1250), //DISTINCT + Filter - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", this.partitionKeyValue, false, -1, 3), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", this.partitionKeyValue, true, -1, 3), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", partitionKeyValue, false, -1, 3), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", partitionKeyValue, true, -1, 3), - CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", this.partitionKeyValue, false, -1, 2), - CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", this.partitionKeyValue, true, -1, 2), + CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", partitionKeyValue, false, -1, 2), + CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", partitionKeyValue, true, -1, 2), //JOIN CreateInput("SELECT root.id " + @@ -172,7 +155,7 @@ private async Task RunAsync() "JOIN root.id c " + "WHERE root.id = '1' OR a.id in (1,2,3,4,5,6,7,8,9,10) " + "OR b.id in (1,2,3,4,5,6,7,8,9,10) " + - "OR c.id in (1,2,3,4,5,6,7,8,9,10)", this.partitionKeyValue, false, -1, 1), + "OR c.id in (1,2,3,4,5,6,7,8,9,10)", partitionKeyValue, false, -1, 1), CreateInput("SELECT root.id " + "FROM root " + "JOIN root.id a " + @@ -180,76 +163,76 @@ private async Task RunAsync() "JOIN root.id c " + "WHERE root.id = '1' OR a.id in (1,2,3,4,5,6,7,8,9,10) " + "OR b.id in (1,2,3,4,5,6,7,8,9,10) " + - "OR c.id in (1,2,3,4,5,6,7,8,9,10)", this.partitionKeyValue, true, -1, 1), + "OR c.id in (1,2,3,4,5,6,7,8,9,10)", partitionKeyValue, true, -1, 1), //High Prep Time - CreateInput(highPrepTimeSumQuery, this.partitionKeyValue, false, -1, 5000), - CreateInput(highPrepTimeSumQuery, this.partitionKeyValue, true, -1, 5000), + CreateInput(highPrepTimeSumQuery, partitionKeyValue, false, -1, 2500), + CreateInput(highPrepTimeSumQuery, partitionKeyValue, true, -1, 2500), - CreateInput(highPrepTimeConditionalQuery, this.partitionKeyValue, false, -1, 3770), - CreateInput(highPrepTimeConditionalQuery, this.partitionKeyValue, true, -1, 3770), + CreateInput(highPrepTimeConditionalQuery, partitionKeyValue, false, -1, 1750), + CreateInput(highPrepTimeConditionalQuery, partitionKeyValue, true, -1, 1750), //Order By - CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", this.partitionKeyValue, false, -1, 5000), - CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", this.partitionKeyValue, true, -1, 5000), + CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", partitionKeyValue, false, -1, 2500), + CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", partitionKeyValue, true, -1, 2500), - CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", this.partitionKeyValue, false, -1, 5000), - CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", this.partitionKeyValue, true, -1, 5000), + CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", partitionKeyValue, false, -1, 2500), + CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", partitionKeyValue, true, -1, 2500), //Order By + TOP - CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", this.partitionKeyValue, false, -1, 5), - CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", this.partitionKeyValue, true, -1, 5), + CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", partitionKeyValue, false, -1, 5), + CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", partitionKeyValue, true, -1, 5), //Order By + DISTINCT - CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", this.partitionKeyValue, false, -1, 5000), - CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", this.partitionKeyValue, true, -1, 5000), + CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", partitionKeyValue, false, -1, 2500), + CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", partitionKeyValue, true, -1, 2500), //Order By + DISTINCT + Filter - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", this.partitionKeyValue, false, -1, 4), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", this.partitionKeyValue, true, -1, 4), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", partitionKeyValue, false, -1, 4), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", partitionKeyValue, true, -1, 4), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", this.partitionKeyValue, false, -1, 6), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", this.partitionKeyValue, true, -1, 6), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", partitionKeyValue, false, -1, 6), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", partitionKeyValue, true, -1, 6), //Group By - CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", this.partitionKeyValue, false, -1, 996), - CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", this.partitionKeyValue, true, -1, 996), + CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, false, -1, 2500), + CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, true, -1, 2500), - CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", this.partitionKeyValue, false, -1, 2), - CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", this.partitionKeyValue, true, -1, 2), + CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", partitionKeyValue, false, -1, 2), + CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", partitionKeyValue, true, -1, 2), - CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", this.partitionKeyValue, false, -1, 4), - CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", this.partitionKeyValue, true, -1, 4), + CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", partitionKeyValue, false, -1, 4), + CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", partitionKeyValue, true, -1, 4), //Group By + OFFSET - CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", this.partitionKeyValue, false, -1, 3), - CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", this.partitionKeyValue, true, -1, 3), + CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", partitionKeyValue, false, -1, 3), + CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", partitionKeyValue, true, -1, 3), //Group By + TOP - CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", this.partitionKeyValue, false, -1, 25), - CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", this.partitionKeyValue, true, -1, 25), + CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", partitionKeyValue, false, -1, 25), + CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", partitionKeyValue, true, -1, 25), //Group By + DISTINCT - CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", this.partitionKeyValue, false, -1, 5000), - CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", this.partitionKeyValue, true, -1, 5000), + CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", partitionKeyValue, false, -1, 2500), + CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", partitionKeyValue, true, -1, 2500), - CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", this.partitionKeyValue, false, -1, 996), - CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", this.partitionKeyValue, true, -1, 996), + CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, false, -1, 2500), + CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, true, -1, 2500), }; foreach (DirectExecutionTestCase testCase in odeTestCases) { - await this.RunQueryAsync(this.container, testCase); + await this.RunQueryAsync(container, testCase); } using (StreamWriter writer = new StreamWriter(new FileStream(this.RawDataPath, FileMode.Append, FileAccess.Write))) { - metricsSerializer.ODESerialization(writer, this.queryStatisticsDatumVisitor, this.numberOfIterations, true); + metricsSerializer.OdeSerialization(writer, queryStatisticsDatumVisitor, numberOfIterations, rawData: true); } using (StreamWriter writer = new StreamWriter(new FileStream(this.AggregateDataPath, FileMode.Append, FileAccess.Write))) { - metricsSerializer.ODESerialization(writer, this.queryStatisticsDatumVisitor, this.numberOfIterations, false); + metricsSerializer.OdeSerialization(writer, queryStatisticsDatumVisitor, numberOfIterations, rawData: false); } } @@ -262,23 +245,30 @@ private async Task RunQueryAsync(Container container, DirectExecutionTestCase qu PartitionKey = queryInput.PartitionKey, }; - for (int i = 0; i < this.numberOfIterations; i++) + for (int i = 0; i < numberOfIterations + warmupIterations; i++) { - bool isWarmUpIteration = false; - if (i < this.warmupIterations) - { - isWarmUpIteration = true; - } + bool isWarmUpIteration = i < warmupIterations; using (FeedIterator iterator = container.GetItemQueryIterator( queryText: queryInput.Query, requestOptions: requestOptions)) { - await this.GetIteratorResponse(iterator, queryInput, isWarmUpIteration); + if (isWarmUpIteration) + { + while (iterator.HasMoreResults) + { + await iterator.ReadNextAsync(); + } + } + else + { + await this.GetIteratorResponse(iterator, queryInput); + } + } } } - private async Task GetIteratorResponse(FeedIterator feedIterator, DirectExecutionTestCase queryInput, bool isWarmUpIteration) + private async Task GetIteratorResponse(FeedIterator feedIterator, DirectExecutionTestCase queryInput) { MetricsAccumulator metricsAccumulator = new MetricsAccumulator(); Documents.ValueStopwatch totalTime = new Documents.ValueStopwatch(); @@ -289,36 +279,29 @@ private async Task GetIteratorResponse(FeedIterator feedIterator, DirectEx while (feedIterator.HasMoreResults) { - if (!isWarmUpIteration) + totalTime.Start(); + response = await feedIterator.ReadNextAsync(); + getTraceTime.Start(); + if (response.RequestCharge != 0) { - totalTime.Start(); - response = await feedIterator.ReadNextAsync(); - getTraceTime.Start(); - if (response.RequestCharge != 0) - { - metricsAccumulator.ReadFromTrace(response, this.queryStatisticsDatumVisitor); - } - - getTraceTime.Stop(); - totalTime.Stop(); - if (response.RequestCharge != 0) - { - this.queryStatisticsDatumVisitor.AddQuery(queryInput.Query); - this.queryStatisticsDatumVisitor.AddEnableOdeFlag(queryInput.EnableOptimisticDirectExecution); - this.queryStatisticsDatumVisitor.AddCorrelatedActivityId(correlatedActivityId); - this.queryStatisticsDatumVisitor.AddRuCharge(response.RequestCharge); - this.queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - getTraceTime.ElapsedMilliseconds); - this.queryStatisticsDatumVisitor.PopulateMetrics(); - } - - totalTime.Reset(); - getTraceTime.Reset(); + metricsAccumulator.ReadFromTrace(response, queryStatisticsDatumVisitor); } - else + + getTraceTime.Stop(); + totalTime.Stop(); + if (response.RequestCharge != 0) { - response = await feedIterator.ReadNextAsync(); + queryStatisticsDatumVisitor.AddQuery(queryInput.Query); + queryStatisticsDatumVisitor.AddEnableOdeFlag(queryInput.EnableOptimisticDirectExecution); + queryStatisticsDatumVisitor.AddCorrelatedActivityId(correlatedActivityId); + queryStatisticsDatumVisitor.AddRuCharge(response.RequestCharge); + queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - getTraceTime.ElapsedMilliseconds); + queryStatisticsDatumVisitor.PopulateMetrics(); } + totalTime.Reset(); + getTraceTime.Reset(); + totalDocumentCount += response.Count; } @@ -358,6 +341,11 @@ private static string CreateHighPrepTimeConditionalQuery() return sb.ToString(); } + private async Task TestCleanup(Database database) + { + await database.DeleteAsync(); + } + private static DirectExecutionTestCase CreateInput( string query, PartitionKey? partitionKey, From c8f4327e70172f4b9a55b1ad951cc7944485a7df Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Tue, 18 Jul 2023 16:47:10 -0700 Subject: [PATCH 04/16] Fixed comments --- .../QueryPerfTest/MetricsSerializer.cs | 16 ++------ ...timisticDirectExecutionPerformanceTests.cs | 39 ++++++++++++------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs index e81a1c7d46..06272b89d1 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs @@ -177,19 +177,9 @@ public void Serialize(TextWriter textWriter, QueryStatisticsDatumVisitor querySt textWriter.Flush(); } - public void OdeSerialization(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor, int numberOfIterations, bool rawData) - { - if (rawData) - { - OdeRawDataSerialization(textWriter, queryStatisticsDatumVisitor); - } - else - { - OdeProcessedDataSerialization(textWriter, queryStatisticsDatumVisitor, numberOfIterations); - } - } + - private static void OdeRawDataSerialization(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor) + internal static void SerializeODERawDataQueryMetrics(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor) { textWriter.WriteLine(); textWriter.WriteLine(PrintQueryMetrics); @@ -238,7 +228,7 @@ private static void OdeRawDataSerialization(TextWriter textWriter, QueryStatisti textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); } - private static void OdeProcessedDataSerialization(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor, int numberOfIterations) + internal static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor, int numberOfIterations) { textWriter.WriteLine(); textWriter.WriteLine(PrintQueryMetrics); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 0db3e55bf1..9fc8720baf 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -15,8 +15,8 @@ public class OptimisticDirectExecutionPerformanceTests { private const string RawDataFileName = "OptimisticDirectExecutionPerformanceTestsRawData.csv"; private const string DiagnosticsDataFileName = "OptimisticDirectExecutionPerformanceTestsAggregatedData.csv"; - private readonly string RawDataPath = Path.GetFullPath(RawDataFileName); - private readonly string AggregateDataPath = Path.GetFullPath(DiagnosticsDataFileName); + private static readonly string RawDataPath = Path.GetFullPath(RawDataFileName); + private static readonly string AggregateDataPath = Path.GetFullPath(DiagnosticsDataFileName); private static readonly QueryStatisticsDatumVisitor queryStatisticsDatumVisitor = new QueryStatisticsDatumVisitor(); private static readonly string endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; private static readonly string authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; @@ -35,25 +35,25 @@ public async Task OptimisticDirectExecutionPerformanceTest() }; CosmosClient cosmosClient = new CosmosClient(endpoint, authKey, clientOptions); - (Database database, Container container) = await this.TestInitialize(cosmosClient); + (Database database, Container container) = await this.InitializeTest(cosmosClient); await this.RunAsync(container); await this.TestCleanup(database); } - private async Task<(Database, Container)> TestInitialize(CosmosClient cosmosClient) + private async Task<(Database, Container)> InitializeTest(CosmosClient cosmosClient) { Database database = await this.CreateDatabaseAsync(cosmosClient); Container container = await this.CreateContainerAsync(database); await this.AddItemsToContainerAsync(container); - if (File.Exists(this.RawDataPath)) + if (File.Exists(RawDataPath)) { - File.Delete(this.RawDataPath); + File.Delete(RawDataPath); } - if (File.Exists(this.AggregateDataPath)) + if (File.Exists(AggregateDataPath)) { - File.Delete(this.AggregateDataPath); + File.Delete(AggregateDataPath); } return (database, container); @@ -67,7 +67,7 @@ private async Task CreateDatabaseAsync(CosmosClient cosmosClient) private async Task CreateContainerAsync(Database database) { - return await database.CreateContainerIfNotExistsAsync(containerId, partitionKeyPath: "/name"); + return await database.CreateContainerAsync(containerId, partitionKeyPath: "/name"); } private async Task AddItemsToContainerAsync(Container container) @@ -122,7 +122,6 @@ private async Task AddItemsToContainerAsync(Container container) private async Task RunAsync(Container container) { - MetricsSerializer metricsSerializer = new MetricsSerializer(); string highPrepTimeSumQuery = CreateHighPrepTimeSumQuery(); string highPrepTimeConditionalQuery = CreateHighPrepTimeConditionalQuery(); @@ -225,14 +224,14 @@ private async Task RunAsync(Container container) await this.RunQueryAsync(container, testCase); } - using (StreamWriter writer = new StreamWriter(new FileStream(this.RawDataPath, FileMode.Append, FileAccess.Write))) + using (StreamWriter writer = new StreamWriter(new FileStream(RawDataPath, FileMode.Append, FileAccess.Write))) { - metricsSerializer.OdeSerialization(writer, queryStatisticsDatumVisitor, numberOfIterations, rawData: true); + SerializeODEQueryMetrics(writer, queryStatisticsDatumVisitor, numberOfIterations, rawData: true); } - using (StreamWriter writer = new StreamWriter(new FileStream(this.AggregateDataPath, FileMode.Append, FileAccess.Write))) + using (StreamWriter writer = new StreamWriter(new FileStream(AggregateDataPath, FileMode.Append, FileAccess.Write))) { - metricsSerializer.OdeSerialization(writer, queryStatisticsDatumVisitor, numberOfIterations, rawData: false); + SerializeODEQueryMetrics(writer, queryStatisticsDatumVisitor, numberOfIterations, rawData: false); } } @@ -308,6 +307,18 @@ private async Task GetIteratorResponse(FeedIterator feedIterator, DirectEx Assert.AreEqual(queryInput.ExpectedResultCount, totalDocumentCount); } + private static void SerializeODEQueryMetrics(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor, int numberOfIterations, bool rawData) + { + if (rawData) + { + MetricsSerializer.SerializeODERawDataQueryMetrics(textWriter, queryStatisticsDatumVisitor); + } + else + { + MetricsSerializer.SerializeODEProcessedDataQueryMetrics(textWriter, queryStatisticsDatumVisitor, numberOfIterations); + } + } + private static string CreateHighPrepTimeSumQuery() { int exprCount = 9999; From 24a67d8207b1858db9b87940e783d213eb678f3c Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Fri, 21 Jul 2023 11:34:59 -0700 Subject: [PATCH 05/16] Removed Query and EnableODE from QueryStatisticsMetrics, as they do not relate to query statistics. --- .../QueryPerfTest/MetricsSerializer.cs | 30 +++++++++-------- ...timisticDirectExecutionPerformanceTests.cs | 32 ++++++++++++++----- .../QueryStatisticsDatumVisitor.cs | 10 ------ .../QueryPerfTest/QueryStatisticsMetrics.cs | 4 --- 4 files changed, 40 insertions(+), 36 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs index 06272b89d1..57bd9233d7 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs @@ -179,7 +179,7 @@ public void Serialize(TextWriter textWriter, QueryStatisticsDatumVisitor querySt - internal static void SerializeODERawDataQueryMetrics(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor) + internal static void SerializeODERawDataQueryMetrics(TextWriter textWriter, List customOdeStatsList) { textWriter.WriteLine(); textWriter.WriteLine(PrintQueryMetrics); @@ -192,10 +192,11 @@ internal static void SerializeODERawDataQueryMetrics(TextWriter textWriter, Quer double totalRU = 0; string prevQuery = ""; bool prevOde = default; - Guid prevCorrelatedActivityId = queryStatisticsDatumVisitor.QueryMetricsList[0].CorrelatedActivityId; + Guid prevCorrelatedActivityId = customOdeStatsList[0].QueryStatisticsDatumVisitor.QueryMetricsList[0].CorrelatedActivityId; - foreach (QueryStatisticsMetrics metrics in queryStatisticsDatumVisitor.QueryMetricsList) + foreach (OptimisticDirectExecutionPerformanceTests.CustomOdeStats customOdeStats in customOdeStatsList) { + QueryStatisticsMetrics metrics = customOdeStats.QueryStatisticsDatumVisitor.QueryMetricsList[0]; double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed; double backendTime = metrics.TotalQueryExecutionTime; @@ -206,8 +207,8 @@ internal static void SerializeODERawDataQueryMetrics(TextWriter textWriter, Quer totalEndToEndTime += metrics.EndToEndTime; totalTransitTime += transitTime; totalRU += metrics.RUCharge; - prevQuery = metrics.Query; - prevOde = metrics.EnableOde; + prevQuery = customOdeStats.Query; + prevOde = customOdeStats.EnableOde; prevCorrelatedActivityId = metrics.CorrelatedActivityId; } @@ -220,29 +221,30 @@ internal static void SerializeODERawDataQueryMetrics(TextWriter textWriter, Quer totalTransitTime = transitTime; totalRU = metrics.RUCharge; prevCorrelatedActivityId = metrics.CorrelatedActivityId; - prevQuery = metrics.Query; - prevOde = metrics.EnableOde; + prevQuery = customOdeStats.Query; + prevOde = customOdeStats.EnableOde; } } textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); } - internal static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor, int numberOfIterations) + internal static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter, List customOdeStatsList, int numberOfIterations) { textWriter.WriteLine(); textWriter.WriteLine(PrintQueryMetrics); textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime"); textWriter.WriteLine(); - string prevQuery = queryStatisticsDatumVisitor.QueryMetricsList[0].Query; - bool prevOde = queryStatisticsDatumVisitor.QueryMetricsList[0].EnableOde; + string prevQuery = customOdeStatsList[0].Query; + bool prevOde = customOdeStatsList[0].EnableOde; double totalEndToEndTime = 0; double totalRU = 0; - foreach (QueryStatisticsMetrics metrics in queryStatisticsDatumVisitor.QueryMetricsList) + foreach (OptimisticDirectExecutionPerformanceTests.CustomOdeStats customOdeStats in customOdeStatsList) { - if (metrics.Query == prevQuery && metrics.EnableOde == prevOde) + QueryStatisticsMetrics metrics = customOdeStats.QueryStatisticsDatumVisitor.QueryMetricsList[0]; + if (customOdeStats.Query == prevQuery && customOdeStats.EnableOde == prevOde) { totalEndToEndTime += metrics.EndToEndTime; totalRU += metrics.RUCharge; @@ -252,8 +254,8 @@ internal static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); totalEndToEndTime = metrics.EndToEndTime; totalRU = metrics.RUCharge; - prevQuery = metrics.Query; - prevOde = metrics.EnableOde; + prevQuery = customOdeStats.Query; + prevOde = customOdeStats.EnableOde; } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 9fc8720baf..287ace1029 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -17,7 +17,7 @@ public class OptimisticDirectExecutionPerformanceTests private const string DiagnosticsDataFileName = "OptimisticDirectExecutionPerformanceTestsAggregatedData.csv"; private static readonly string RawDataPath = Path.GetFullPath(RawDataFileName); private static readonly string AggregateDataPath = Path.GetFullPath(DiagnosticsDataFileName); - private static readonly QueryStatisticsDatumVisitor queryStatisticsDatumVisitor = new QueryStatisticsDatumVisitor(); + private static readonly List customOdeStatisticsList = new List(); private static readonly string endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; private static readonly string authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; private static readonly string cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.CosmosDatabaseId"]; @@ -26,6 +26,13 @@ public class OptimisticDirectExecutionPerformanceTests private static readonly int numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.NumberOfIterations"]); private static readonly int warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.WarmupIterations"]); + internal class CustomOdeStats + { + public string Query { get; set; } + public bool EnableOde { get; set; } + public QueryStatisticsDatumVisitor QueryStatisticsDatumVisitor { get; set; } + } + [TestMethod] public async Task OptimisticDirectExecutionPerformanceTest() { @@ -226,12 +233,12 @@ private async Task RunAsync(Container container) using (StreamWriter writer = new StreamWriter(new FileStream(RawDataPath, FileMode.Append, FileAccess.Write))) { - SerializeODEQueryMetrics(writer, queryStatisticsDatumVisitor, numberOfIterations, rawData: true); + SerializeODEQueryMetrics(writer, customOdeStatisticsList, numberOfIterations, rawData: true); } using (StreamWriter writer = new StreamWriter(new FileStream(AggregateDataPath, FileMode.Append, FileAccess.Write))) { - SerializeODEQueryMetrics(writer, queryStatisticsDatumVisitor, numberOfIterations, rawData: false); + SerializeODEQueryMetrics(writer, customOdeStatisticsList, numberOfIterations, rawData: false); } } @@ -275,9 +282,12 @@ private async Task GetIteratorResponse(FeedIterator feedIterator, DirectEx Guid correlatedActivityId = Guid.NewGuid(); FeedResponse response; int totalDocumentCount = 0; + string query; + bool enableOde; while (feedIterator.HasMoreResults) { + QueryStatisticsDatumVisitor queryStatisticsDatumVisitor = new QueryStatisticsDatumVisitor(); totalTime.Start(); response = await feedIterator.ReadNextAsync(); getTraceTime.Start(); @@ -290,12 +300,18 @@ private async Task GetIteratorResponse(FeedIterator feedIterator, DirectEx totalTime.Stop(); if (response.RequestCharge != 0) { - queryStatisticsDatumVisitor.AddQuery(queryInput.Query); - queryStatisticsDatumVisitor.AddEnableOdeFlag(queryInput.EnableOptimisticDirectExecution); + query = queryInput.Query; + enableOde = queryInput.EnableOptimisticDirectExecution; queryStatisticsDatumVisitor.AddCorrelatedActivityId(correlatedActivityId); queryStatisticsDatumVisitor.AddRuCharge(response.RequestCharge); queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - getTraceTime.ElapsedMilliseconds); queryStatisticsDatumVisitor.PopulateMetrics(); + customOdeStatisticsList.Add(new CustomOdeStats + { + Query = query, + EnableOde = enableOde, + QueryStatisticsDatumVisitor = queryStatisticsDatumVisitor + }); } totalTime.Reset(); @@ -307,15 +323,15 @@ private async Task GetIteratorResponse(FeedIterator feedIterator, DirectEx Assert.AreEqual(queryInput.ExpectedResultCount, totalDocumentCount); } - private static void SerializeODEQueryMetrics(TextWriter textWriter, QueryStatisticsDatumVisitor queryStatisticsDatumVisitor, int numberOfIterations, bool rawData) + private static void SerializeODEQueryMetrics(TextWriter textWriter, List customOdeStatisticsList, int numberOfIterations, bool rawData) { if (rawData) { - MetricsSerializer.SerializeODERawDataQueryMetrics(textWriter, queryStatisticsDatumVisitor); + MetricsSerializer.SerializeODERawDataQueryMetrics(textWriter, customOdeStatisticsList); } else { - MetricsSerializer.SerializeODEProcessedDataQueryMetrics(textWriter, queryStatisticsDatumVisitor, numberOfIterations); + MetricsSerializer.SerializeODEProcessedDataQueryMetrics(textWriter, customOdeStatisticsList, numberOfIterations); } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs index 1e221be402..6f6ffbf92c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs @@ -35,21 +35,11 @@ public void AddRuCharge(double ruCharge) this.queryMetrics.RUCharge = ruCharge; } - public void AddQuery(string query) - { - this.queryMetrics.Query = query; - } - public void AddCorrelatedActivityId(Guid activityId) { this.queryMetrics.CorrelatedActivityId = activityId; } - public void AddEnableOdeFlag(bool enableOde) - { - this.queryMetrics.EnableOde = enableOde; - } - public void AddPocoTime(double time) { this.queryMetrics.PocoTime = time; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs index 76daf6a163..d72193fc43 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs @@ -4,10 +4,6 @@ internal class QueryStatisticsMetrics { - public string Query { get; set; } - - public bool EnableOde { get; set; } - public Guid CorrelatedActivityId { get; set; } public double EndToEndTime { get; set; } From 41293fbb1a7a1cc3b11242d982eecba0bc09ea75 Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Sun, 23 Jul 2023 21:46:08 -0700 Subject: [PATCH 06/16] Removed try catch to make CreateItemAsync call always succeed --- ...timisticDirectExecutionPerformanceTests.cs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 287ace1029..8673d076ed 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -109,21 +109,12 @@ private async Task AddItemsToContainerAsync(Container container) UserDefinedID = i % 10, RecipientList = recipientList }; + + // Create an item in the container representing the Andersen family. Note we provide the value of the partition key for this item, which is "Andersen" + ItemResponse andersenFamilyResponse = await container.CreateItemAsync(andersenFamily, new PartitionKey(andersenFamily.Name)); - try - { - // Read the item to see if it exists. - ItemResponse andersenFamilyResponse = await container.ReadItemAsync(andersenFamily.Id, new PartitionKey(andersenFamily.Name)); - Console.WriteLine("Item in database with id: {0} already exists\n", andersenFamilyResponse.Resource.Id); - } - catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) - { - // Create an item in the container representing the Andersen family. Note we provide the value of the partition key for this item, which is "Andersen" - ItemResponse andersenFamilyResponse = await container.CreateItemAsync(andersenFamily, new PartitionKey(andersenFamily.Name)); - - // Note that after creating the item, we can access the body of the item with the Resource property off the ItemResponse. We can also access the RequestCharge property to see the amount of RUs consumed on this request. - Console.WriteLine("Created item in database with id: {0} Operation consumed {1} RUs.\n", andersenFamilyResponse.Resource.Id, andersenFamilyResponse.RequestCharge); - } + // Note that after creating the item, we can access the body of the item with the Resource property off the ItemResponse. We can also access the RequestCharge property to see the amount of RUs consumed on this request. + Console.WriteLine("Created item in database with id: {0} Operation consumed {1} RUs.\n", andersenFamilyResponse.Resource.Id, andersenFamilyResponse.RequestCharge); } } From 6b153795ac0ea5686dadc1c8342451d2e622a4cc Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Mon, 24 Jul 2023 12:28:56 -0700 Subject: [PATCH 07/16] Removed one liner functions --- ...OptimisticDirectExecutionPerformanceTests.cs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 8673d076ed..0fb2bfb11c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -49,8 +49,8 @@ public async Task OptimisticDirectExecutionPerformanceTest() private async Task<(Database, Container)> InitializeTest(CosmosClient cosmosClient) { - Database database = await this.CreateDatabaseAsync(cosmosClient); - Container container = await this.CreateContainerAsync(database); + Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(cosmosDatabaseId); + Container container = await database.CreateContainerAsync(containerId, partitionKeyPath: "/name"); await this.AddItemsToContainerAsync(container); if (File.Exists(RawDataPath)) @@ -65,18 +65,7 @@ public async Task OptimisticDirectExecutionPerformanceTest() return (database, container); } - - private async Task CreateDatabaseAsync(CosmosClient cosmosClient) - { - // Create a new database - return await cosmosClient.CreateDatabaseIfNotExistsAsync(cosmosDatabaseId); - } - - private async Task CreateContainerAsync(Database database) - { - return await database.CreateContainerAsync(containerId, partitionKeyPath: "/name"); - } - + private async Task AddItemsToContainerAsync(Container container) { int totalItems = 5000; From 9851c4f13e94428d231b5734325c6047797c0db2 Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Thu, 27 Jul 2023 23:21:34 -0700 Subject: [PATCH 08/16] Removed code from MetricsSerializer and QueryStatisticsDatumVisitor files --- .../QueryPerfTest/MetricsSerializer.cs | 85 -------- ...timisticDirectExecutionPerformanceTests.cs | 189 +++++++++++++----- .../QueryStatisticsDatumVisitor.cs | 11 - 3 files changed, 141 insertions(+), 144 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs index 57bd9233d7..c73612caa7 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsSerializer.cs @@ -176,90 +176,5 @@ public void Serialize(TextWriter textWriter, QueryStatisticsDatumVisitor querySt } textWriter.Flush(); } - - - - internal static void SerializeODERawDataQueryMetrics(TextWriter textWriter, List customOdeStatsList) - { - textWriter.WriteLine(); - textWriter.WriteLine(PrintQueryMetrics); - textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", "Query", "ODE", "RUCharge", "BackendTime", "TransitTime", "ClientTime", "EndToEndTime"); - textWriter.WriteLine(); - double totalClientTime = 0; - double totalBackendTime = 0; - double totalEndToEndTime = 0; - double totalTransitTime = 0; - double totalRU = 0; - string prevQuery = ""; - bool prevOde = default; - Guid prevCorrelatedActivityId = customOdeStatsList[0].QueryStatisticsDatumVisitor.QueryMetricsList[0].CorrelatedActivityId; - - foreach (OptimisticDirectExecutionPerformanceTests.CustomOdeStats customOdeStats in customOdeStatsList) - { - QueryStatisticsMetrics metrics = customOdeStats.QueryStatisticsDatumVisitor.QueryMetricsList[0]; - double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed; - double backendTime = metrics.TotalQueryExecutionTime; - - if (metrics.CorrelatedActivityId == prevCorrelatedActivityId) - { - totalClientTime += metrics.EndToEndTime - (backendTime + transitTime); - totalBackendTime += backendTime; - totalEndToEndTime += metrics.EndToEndTime; - totalTransitTime += transitTime; - totalRU += metrics.RUCharge; - prevQuery = customOdeStats.Query; - prevOde = customOdeStats.EnableOde; - prevCorrelatedActivityId = metrics.CorrelatedActivityId; - - } - else - { - textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); - totalClientTime = metrics.EndToEndTime - (backendTime + transitTime); - totalBackendTime = backendTime; - totalEndToEndTime = metrics.EndToEndTime; - totalTransitTime = transitTime; - totalRU = metrics.RUCharge; - prevCorrelatedActivityId = metrics.CorrelatedActivityId; - prevQuery = customOdeStats.Query; - prevOde = customOdeStats.EnableOde; - } - } - - textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); - } - - internal static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter, List customOdeStatsList, int numberOfIterations) - { - textWriter.WriteLine(); - textWriter.WriteLine(PrintQueryMetrics); - textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime"); - textWriter.WriteLine(); - - string prevQuery = customOdeStatsList[0].Query; - bool prevOde = customOdeStatsList[0].EnableOde; - double totalEndToEndTime = 0; - double totalRU = 0; - - foreach (OptimisticDirectExecutionPerformanceTests.CustomOdeStats customOdeStats in customOdeStatsList) - { - QueryStatisticsMetrics metrics = customOdeStats.QueryStatisticsDatumVisitor.QueryMetricsList[0]; - if (customOdeStats.Query == prevQuery && customOdeStats.EnableOde == prevOde) - { - totalEndToEndTime += metrics.EndToEndTime; - totalRU += metrics.RUCharge; - } - else - { - textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); - totalEndToEndTime = metrics.EndToEndTime; - totalRU = metrics.RUCharge; - prevQuery = customOdeStats.Query; - prevOde = customOdeStats.EnableOde; - } - } - - textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); - } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 0fb2bfb11c..79a9179a6d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -13,11 +13,12 @@ [TestClass] public class OptimisticDirectExecutionPerformanceTests { + private Container Container; private const string RawDataFileName = "OptimisticDirectExecutionPerformanceTestsRawData.csv"; private const string DiagnosticsDataFileName = "OptimisticDirectExecutionPerformanceTestsAggregatedData.csv"; + private const string PrintQueryMetrics = "QueryMetrics"; private static readonly string RawDataPath = Path.GetFullPath(RawDataFileName); private static readonly string AggregateDataPath = Path.GetFullPath(DiagnosticsDataFileName); - private static readonly List customOdeStatisticsList = new List(); private static readonly string endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; private static readonly string authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; private static readonly string cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.CosmosDatabaseId"]; @@ -26,15 +27,8 @@ public class OptimisticDirectExecutionPerformanceTests private static readonly int numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.NumberOfIterations"]); private static readonly int warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.WarmupIterations"]); - internal class CustomOdeStats - { - public string Query { get; set; } - public bool EnableOde { get; set; } - public QueryStatisticsDatumVisitor QueryStatisticsDatumVisitor { get; set; } - } - - [TestMethod] - public async Task OptimisticDirectExecutionPerformanceTest() + [TestInitialize] + public async Task InitializeTest() { CosmosClientOptions clientOptions = new CosmosClientOptions() { @@ -42,16 +36,9 @@ public async Task OptimisticDirectExecutionPerformanceTest() }; CosmosClient cosmosClient = new CosmosClient(endpoint, authKey, clientOptions); - (Database database, Container container) = await this.InitializeTest(cosmosClient); - await this.RunAsync(container); - await this.TestCleanup(database); - } - - private async Task<(Database, Container)> InitializeTest(CosmosClient cosmosClient) - { - Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(cosmosDatabaseId); - Container container = await database.CreateContainerAsync(containerId, partitionKeyPath: "/name"); - await this.AddItemsToContainerAsync(container); + Database database = await cosmosClient.CreateDatabaseAsync(cosmosDatabaseId); + this.Container = await database.CreateContainerAsync(containerId, partitionKeyPath: "/name"); + await this.AddItemsToContainerAsync(this.Container); if (File.Exists(RawDataPath)) { @@ -62,10 +49,18 @@ public async Task OptimisticDirectExecutionPerformanceTest() { File.Delete(AggregateDataPath); } + } - return (database, container); + [TestCleanup] + public async Task CleanupAsync() + { + if (this.Container != null) + { + await this.Container.Database.DeleteAsync(); + this.Container = null; + } } - + private async Task AddItemsToContainerAsync(Container container) { int totalItems = 5000; @@ -106,11 +101,14 @@ private async Task AddItemsToContainerAsync(Container container) Console.WriteLine("Created item in database with id: {0} Operation consumed {1} RUs.\n", andersenFamilyResponse.Resource.Id, andersenFamilyResponse.RequestCharge); } } - - private async Task RunAsync(Container container) + + [TestMethod] + [Owner("akotalwar")] + public async Task RunAsync() { string highPrepTimeSumQuery = CreateHighPrepTimeSumQuery(); string highPrepTimeConditionalQuery = CreateHighPrepTimeConditionalQuery(); + List globalCustomOdeStatisticsList = new List(); List odeTestCases = new List() { @@ -208,22 +206,24 @@ private async Task RunAsync(Container container) foreach (DirectExecutionTestCase testCase in odeTestCases) { - await this.RunQueryAsync(container, testCase); + List customOdeStats = await this.RunQueryAsync(testCase); + globalCustomOdeStatisticsList.AddRange(customOdeStats); } using (StreamWriter writer = new StreamWriter(new FileStream(RawDataPath, FileMode.Append, FileAccess.Write))) { - SerializeODEQueryMetrics(writer, customOdeStatisticsList, numberOfIterations, rawData: true); + SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, numberOfIterations, rawData: true); } using (StreamWriter writer = new StreamWriter(new FileStream(AggregateDataPath, FileMode.Append, FileAccess.Write))) { - SerializeODEQueryMetrics(writer, customOdeStatisticsList, numberOfIterations, rawData: false); + SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, numberOfIterations, rawData: false); } } - private async Task RunQueryAsync(Container container, DirectExecutionTestCase queryInput) + private async Task> RunQueryAsync(DirectExecutionTestCase queryInput) { + List customOdeStats = new List(); QueryRequestOptions requestOptions = new QueryRequestOptions() { MaxItemCount = queryInput.PageSizeOption, @@ -234,7 +234,7 @@ private async Task RunQueryAsync(Container container, DirectExecutionTestCase qu for (int i = 0; i < numberOfIterations + warmupIterations; i++) { bool isWarmUpIteration = i < warmupIterations; - using (FeedIterator iterator = container.GetItemQueryIterator( + using (FeedIterator iterator = this.Container.GetItemQueryIterator( queryText: queryInput.Query, requestOptions: requestOptions)) { @@ -247,14 +247,16 @@ private async Task RunQueryAsync(Container container, DirectExecutionTestCase qu } else { - await this.GetIteratorResponse(iterator, queryInput); + customOdeStats = await this.GetIteratorStatistics(iterator, queryInput); } } } + + return customOdeStats; } - private async Task GetIteratorResponse(FeedIterator feedIterator, DirectExecutionTestCase queryInput) + private async Task> GetIteratorStatistics(FeedIterator feedIterator, DirectExecutionTestCase queryInput) { MetricsAccumulator metricsAccumulator = new MetricsAccumulator(); Documents.ValueStopwatch totalTime = new Documents.ValueStopwatch(); @@ -264,6 +266,7 @@ private async Task GetIteratorResponse(FeedIterator feedIterator, DirectEx int totalDocumentCount = 0; string query; bool enableOde; + List customOdeStats = new List(); while (feedIterator.HasMoreResults) { @@ -282,15 +285,18 @@ private async Task GetIteratorResponse(FeedIterator feedIterator, DirectEx { query = queryInput.Query; enableOde = queryInput.EnableOptimisticDirectExecution; - queryStatisticsDatumVisitor.AddCorrelatedActivityId(correlatedActivityId); - queryStatisticsDatumVisitor.AddRuCharge(response.RequestCharge); queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - getTraceTime.ElapsedMilliseconds); queryStatisticsDatumVisitor.PopulateMetrics(); - customOdeStatisticsList.Add(new CustomOdeStats + + QueryStatisticsMetrics queryStatistics = queryStatisticsDatumVisitor.QueryMetricsList[0]; + queryStatistics.RUCharge = response.RequestCharge; + queryStatistics.CorrelatedActivityId = correlatedActivityId; + + customOdeStats.Add(new CustomOdeStats { Query = query, EnableOde = enableOde, - QueryStatisticsDatumVisitor = queryStatisticsDatumVisitor + QueryStatisticsMetrics = queryStatistics }); } @@ -301,18 +307,8 @@ private async Task GetIteratorResponse(FeedIterator feedIterator, DirectEx } Assert.AreEqual(queryInput.ExpectedResultCount, totalDocumentCount); - } - private static void SerializeODEQueryMetrics(TextWriter textWriter, List customOdeStatisticsList, int numberOfIterations, bool rawData) - { - if (rawData) - { - MetricsSerializer.SerializeODERawDataQueryMetrics(textWriter, customOdeStatisticsList); - } - else - { - MetricsSerializer.SerializeODEProcessedDataQueryMetrics(textWriter, customOdeStatisticsList, numberOfIterations); - } + return customOdeStats; } private static string CreateHighPrepTimeSumQuery() @@ -348,9 +344,106 @@ private static string CreateHighPrepTimeConditionalQuery() return sb.ToString(); } - private async Task TestCleanup(Database database) + private static void SerializeODEQueryMetrics(TextWriter textWriter, List customOdeStatisticsList, int numberOfIterations, bool rawData) { - await database.DeleteAsync(); + if (rawData) + { + SerializeODERawDataQueryMetrics(textWriter, customOdeStatisticsList); + } + else + { + SerializeODEProcessedDataQueryMetrics(textWriter, customOdeStatisticsList, numberOfIterations); + } + } + + private static void SerializeODERawDataQueryMetrics(TextWriter textWriter, List customOdeStatsList) + { + textWriter.WriteLine(); + textWriter.WriteLine(PrintQueryMetrics); + textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", "Query", "ODE", "RUCharge", "BackendTime", "TransitTime", "ClientTime", "EndToEndTime"); + textWriter.WriteLine(); + double totalClientTime = 0; + double totalBackendTime = 0; + double totalEndToEndTime = 0; + double totalTransitTime = 0; + double totalRU = 0; + string prevQuery = ""; + bool prevOde = default; + Guid prevCorrelatedActivityId = customOdeStatsList[0].QueryStatisticsMetrics.CorrelatedActivityId; + + foreach (OptimisticDirectExecutionPerformanceTests.CustomOdeStats customOdeStats in customOdeStatsList) + { + QueryStatisticsMetrics metrics = customOdeStats.QueryStatisticsMetrics; + double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed; + double backendTime = metrics.TotalQueryExecutionTime; + + if (metrics.CorrelatedActivityId == prevCorrelatedActivityId) + { + totalClientTime += metrics.EndToEndTime - (backendTime + transitTime); + totalBackendTime += backendTime; + totalEndToEndTime += metrics.EndToEndTime; + totalTransitTime += transitTime; + totalRU += metrics.RUCharge; + prevQuery = customOdeStats.Query; + prevOde = customOdeStats.EnableOde; + prevCorrelatedActivityId = metrics.CorrelatedActivityId; + + } + else + { + textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); + totalClientTime = metrics.EndToEndTime - (backendTime + transitTime); + totalBackendTime = backendTime; + totalEndToEndTime = metrics.EndToEndTime; + totalTransitTime = transitTime; + totalRU = metrics.RUCharge; + prevCorrelatedActivityId = metrics.CorrelatedActivityId; + prevQuery = customOdeStats.Query; + prevOde = customOdeStats.EnableOde; + } + } + + textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); + } + + private static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter, List customOdeStatsList, int numberOfIterations) + { + textWriter.WriteLine(); + textWriter.WriteLine(PrintQueryMetrics); + textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime"); + textWriter.WriteLine(); + + string prevQuery = customOdeStatsList[0].Query; + bool prevOde = customOdeStatsList[0].EnableOde; + double totalEndToEndTime = 0; + double totalRU = 0; + + foreach (OptimisticDirectExecutionPerformanceTests.CustomOdeStats customOdeStats in customOdeStatsList) + { + QueryStatisticsMetrics metrics = customOdeStats.QueryStatisticsMetrics; + if (customOdeStats.Query == prevQuery && customOdeStats.EnableOde == prevOde) + { + totalEndToEndTime += metrics.EndToEndTime; + totalRU += metrics.RUCharge; + } + else + { + textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); + totalEndToEndTime = metrics.EndToEndTime; + totalRU = metrics.RUCharge; + prevQuery = customOdeStats.Query; + prevOde = customOdeStats.EnableOde; + } + } + + textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); + } + + private class CustomOdeStats + { + public string Query { get; set; } + public bool EnableOde { get; set; } + public QueryStatisticsMetrics QueryStatisticsMetrics { get; set; } } private static DirectExecutionTestCase CreateInput( diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs index 6f6ffbf92c..200150875b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsDatumVisitor.cs @@ -1,6 +1,5 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests { - using System; using System.Collections.Generic; using System.Diagnostics; using Microsoft.Azure.Cosmos.Tracing; @@ -30,16 +29,6 @@ public void AddEndToEndTime(double totalTime) this.queryMetrics.EndToEndTime = totalTime; } - public void AddRuCharge(double ruCharge) - { - this.queryMetrics.RUCharge = ruCharge; - } - - public void AddCorrelatedActivityId(Guid activityId) - { - this.queryMetrics.CorrelatedActivityId = activityId; - } - public void AddPocoTime(double time) { this.queryMetrics.PocoTime = time; From b0c9c3533357f1a911ea6464aefb6d023a7a2b02 Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Tue, 1 Aug 2023 21:42:34 -0700 Subject: [PATCH 09/16] Fixed comments --- .../ContentSerializationPerformanceTests.cs | 18 +- ...timisticDirectExecutionPerformanceTests.cs | 270 +++++++++--------- .../settings.json | 18 +- 3 files changed, 146 insertions(+), 160 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs index 759e63b345..8ff5660c6f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs @@ -33,15 +33,15 @@ public ContentSerializationPerformanceTests() this.queryStatisticsDatumVisitor = new(); this.endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; this.authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; - this.cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.CosmosDatabaseId"]; - this.containerId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContainerId"]; - this.contentSerialization = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContentSerialization"]; - this.query = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.Query"]; - this.numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.NumberOfIterations"]); - this.warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.WarmupIterations"]); - this.MaxConcurrency = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.MaxConcurrency"]); - this.MaxItemCount = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.MaxItemCount"]); - this.useStronglyTypedIterator = bool.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.UseStronglyTypedIterator"]); + this.cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.CosmosDatabaseId"]; + this.containerId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.ContainerId"]; + this.contentSerialization = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.ContentSerialization"]; + this.query = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.Query"]; + this.numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.NumberOfIterations"]); + this.warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.WarmupIterations"]); + this.MaxConcurrency = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.MaxConcurrency"]); + this.MaxItemCount = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.MaxItemCount"]); + this.useStronglyTypedIterator = bool.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.UseStronglyTypedIterator"]); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 79a9179a6d..1489e970ac 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; - using System.Net; using System.Text; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; @@ -15,17 +14,17 @@ public class OptimisticDirectExecutionPerformanceTests { private Container Container; private const string RawDataFileName = "OptimisticDirectExecutionPerformanceTestsRawData.csv"; - private const string DiagnosticsDataFileName = "OptimisticDirectExecutionPerformanceTestsAggregatedData.csv"; + private const string AggregateDataFileName = "OptimisticDirectExecutionPerformanceTestsAggregatedData.csv"; private const string PrintQueryMetrics = "QueryMetrics"; private static readonly string RawDataPath = Path.GetFullPath(RawDataFileName); - private static readonly string AggregateDataPath = Path.GetFullPath(DiagnosticsDataFileName); - private static readonly string endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; - private static readonly string authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; - private static readonly string cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.CosmosDatabaseId"]; - private static readonly string containerId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContainerId"]; - private static readonly PartitionKey partitionKeyValue = new PartitionKey("Andersen"); - private static readonly int numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.NumberOfIterations"]); - private static readonly int warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.WarmupIterations"]); + private static readonly string AggregateDataPath = Path.GetFullPath(AggregateDataFileName); + private static readonly string Endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; + private static readonly string AuthKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; + private static readonly string CosmosDatabaseId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.CosmosDatabaseId"]; + private static readonly string ContainerId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.ContainerId"]; + private static readonly PartitionKey PartitionKeyValue = new PartitionKey("Andersen"); + private static readonly int NumberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.NumberOfIterations"]); + private static readonly int WarmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.WarmupIterations"]); [TestInitialize] public async Task InitializeTest() @@ -35,9 +34,9 @@ public async Task InitializeTest() ConnectionMode = ConnectionMode.Direct, }; - CosmosClient cosmosClient = new CosmosClient(endpoint, authKey, clientOptions); - Database database = await cosmosClient.CreateDatabaseAsync(cosmosDatabaseId); - this.Container = await database.CreateContainerAsync(containerId, partitionKeyPath: "/name"); + CosmosClient cosmosClient = new CosmosClient(Endpoint, AuthKey, clientOptions); + Database database = await cosmosClient.CreateDatabaseAsync(CosmosDatabaseId); + this.Container = await database.CreateContainerAsync(ContainerId, partitionKeyPath: "/name"); await this.AddItemsToContainerAsync(this.Container); if (File.Exists(RawDataPath)) @@ -65,24 +64,23 @@ private async Task AddItemsToContainerAsync(Container container) { int totalItems = 5000; string[] cityOptions = new string[] { "Seattle", "Chicago", "NYC", "SF" }; + int numberOfRecipeints = cityOptions.Length; + List recipientList = new List(numberOfRecipeints); - // Create a family object for the Andersen family - foreach (int i in Enumerable.Range(0, totalItems)) + for (int j = 0; j < numberOfRecipeints; j++) { - int numberOfRecipeints = cityOptions.Length; - List recipientList = new List(numberOfRecipeints); - - for (int j = 0; j < numberOfRecipeints; j++) + RecipientList recipient = new RecipientList() { - RecipientList recipient = new RecipientList() - { - Name = "John", - City = cityOptions[j], - }; + Name = "John", + City = cityOptions[j], + }; - recipientList.Add(recipient); - } + recipientList.Add(recipient); + } + // Create a family object for the Andersen family + foreach (int i in Enumerable.Range(0, totalItems)) + { States andersenFamily = new States { Id = i.ToString(), @@ -94,10 +92,7 @@ private async Task AddItemsToContainerAsync(Container container) RecipientList = recipientList }; - // Create an item in the container representing the Andersen family. Note we provide the value of the partition key for this item, which is "Andersen" ItemResponse andersenFamilyResponse = await container.CreateItemAsync(andersenFamily, new PartitionKey(andersenFamily.Name)); - - // Note that after creating the item, we can access the body of the item with the Resource property off the ItemResponse. We can also access the RequestCharge property to see the amount of RUs consumed on this request. Console.WriteLine("Created item in database with id: {0} Operation consumed {1} RUs.\n", andersenFamilyResponse.Resource.Id, andersenFamilyResponse.RequestCharge); } } @@ -108,28 +103,28 @@ public async Task RunAsync() { string highPrepTimeSumQuery = CreateHighPrepTimeSumQuery(); string highPrepTimeConditionalQuery = CreateHighPrepTimeConditionalQuery(); - List globalCustomOdeStatisticsList = new List(); + List> globalCustomOdeStatisticsList = new List>(); List odeTestCases = new List() { //Simple Query - CreateInput("SELECT * FROM c", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT * FROM c", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT * FROM c", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT * FROM c", PartitionKeyValue, true, -1, 2500), //TOP - CreateInput("SELECT TOP 1000 c.id FROM c", partitionKeyValue, false, -1, 1000), - CreateInput("SELECT TOP 1000 c.id FROM c", partitionKeyValue, true, -1, 1000), + CreateInput("SELECT TOP 1000 c.id FROM c", PartitionKeyValue, false, -1, 1000), + CreateInput("SELECT TOP 1000 c.id FROM c", PartitionKeyValue, true, -1, 1000), //Filter - CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", partitionKeyValue, false, -1, 1250), - CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", partitionKeyValue, true, -1, 1250), + CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", PartitionKeyValue, false, -1, 1250), + CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", PartitionKeyValue, true, -1, 1250), //DISTINCT + Filter - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", partitionKeyValue, false, -1, 3), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", partitionKeyValue, true, -1, 3), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", PartitionKeyValue, false, -1, 3), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", PartitionKeyValue, true, -1, 3), - CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", partitionKeyValue, false, -1, 2), - CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", partitionKeyValue, true, -1, 2), + CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", PartitionKeyValue, false, -1, 2), + CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", PartitionKeyValue, true, -1, 2), //JOIN CreateInput("SELECT root.id " + @@ -139,7 +134,7 @@ public async Task RunAsync() "JOIN root.id c " + "WHERE root.id = '1' OR a.id in (1,2,3,4,5,6,7,8,9,10) " + "OR b.id in (1,2,3,4,5,6,7,8,9,10) " + - "OR c.id in (1,2,3,4,5,6,7,8,9,10)", partitionKeyValue, false, -1, 1), + "OR c.id in (1,2,3,4,5,6,7,8,9,10)", PartitionKeyValue, false, -1, 1), CreateInput("SELECT root.id " + "FROM root " + "JOIN root.id a " + @@ -147,83 +142,82 @@ public async Task RunAsync() "JOIN root.id c " + "WHERE root.id = '1' OR a.id in (1,2,3,4,5,6,7,8,9,10) " + "OR b.id in (1,2,3,4,5,6,7,8,9,10) " + - "OR c.id in (1,2,3,4,5,6,7,8,9,10)", partitionKeyValue, true, -1, 1), + "OR c.id in (1,2,3,4,5,6,7,8,9,10)", PartitionKeyValue, true, -1, 1), //High Prep Time - CreateInput(highPrepTimeSumQuery, partitionKeyValue, false, -1, 2500), - CreateInput(highPrepTimeSumQuery, partitionKeyValue, true, -1, 2500), + CreateInput(highPrepTimeSumQuery, PartitionKeyValue, false, -1, 2500), + CreateInput(highPrepTimeSumQuery, PartitionKeyValue, true, -1, 2500), - CreateInput(highPrepTimeConditionalQuery, partitionKeyValue, false, -1, 1750), - CreateInput(highPrepTimeConditionalQuery, partitionKeyValue, true, -1, 1750), + CreateInput(highPrepTimeConditionalQuery, PartitionKeyValue, false, -1, 1750), + CreateInput(highPrepTimeConditionalQuery, PartitionKeyValue, true, -1, 1750), //Order By - CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", PartitionKeyValue, true, -1, 2500), - CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", PartitionKeyValue, true, -1, 2500), //Order By + TOP - CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", partitionKeyValue, false, -1, 5), - CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", partitionKeyValue, true, -1, 5), + CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", PartitionKeyValue, false, -1, 5), + CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", PartitionKeyValue, true, -1, 5), //Order By + DISTINCT - CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", PartitionKeyValue, true, -1, 2500), //Order By + DISTINCT + Filter - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", partitionKeyValue, false, -1, 4), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", partitionKeyValue, true, -1, 4), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", PartitionKeyValue, false, -1, 4), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", PartitionKeyValue, true, -1, 4), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", partitionKeyValue, false, -1, 6), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", partitionKeyValue, true, -1, 6), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", PartitionKeyValue, false, -1, 6), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", PartitionKeyValue, true, -1, 6), //Group By - CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, true, -1, 2500), - CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", partitionKeyValue, false, -1, 2), - CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", partitionKeyValue, true, -1, 2), + CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", PartitionKeyValue, false, -1, 2), + CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", PartitionKeyValue, true, -1, 2), - CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", partitionKeyValue, false, -1, 4), - CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", partitionKeyValue, true, -1, 4), + CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", PartitionKeyValue, false, -1, 4), + CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", PartitionKeyValue, true, -1, 4), //Group By + OFFSET - CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", partitionKeyValue, false, -1, 3), - CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", partitionKeyValue, true, -1, 3), + CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", PartitionKeyValue, false, -1, 3), + CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", PartitionKeyValue, true, -1, 3), //Group By + TOP - CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", partitionKeyValue, false, -1, 25), - CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", partitionKeyValue, true, -1, 25), + CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", PartitionKeyValue, false, -1, 25), + CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", PartitionKeyValue, true, -1, 25), //Group By + DISTINCT - CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", PartitionKeyValue, true, -1, 2500), - CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, true, -1, 2500), }; foreach (DirectExecutionTestCase testCase in odeTestCases) { - List customOdeStats = await this.RunQueryAsync(testCase); - globalCustomOdeStatisticsList.AddRange(customOdeStats); + globalCustomOdeStatisticsList.AddRange(await this.RunQueryAsync(testCase)); } using (StreamWriter writer = new StreamWriter(new FileStream(RawDataPath, FileMode.Append, FileAccess.Write))) { - SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, numberOfIterations, rawData: true); + SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, NumberOfIterations, rawData: true); } using (StreamWriter writer = new StreamWriter(new FileStream(AggregateDataPath, FileMode.Append, FileAccess.Write))) { - SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, numberOfIterations, rawData: false); + SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, NumberOfIterations, rawData: false); } } - private async Task> RunQueryAsync(DirectExecutionTestCase queryInput) + private async Task>> RunQueryAsync(DirectExecutionTestCase queryInput) { - List customOdeStats = new List(); + List> odeQueryStatisticsList = new List>(); QueryRequestOptions requestOptions = new QueryRequestOptions() { MaxItemCount = queryInput.PageSizeOption, @@ -231,9 +225,9 @@ private async Task> RunQueryAsync(DirectExecutionTestCase q PartitionKey = queryInput.PartitionKey, }; - for (int i = 0; i < numberOfIterations + warmupIterations; i++) + for (int i = 0; i < NumberOfIterations + WarmupIterations; i++) { - bool isWarmUpIteration = i < warmupIterations; + bool isWarmUpIteration = i < WarmupIterations; using (FeedIterator iterator = this.Container.GetItemQueryIterator( queryText: queryInput.Query, requestOptions: requestOptions)) @@ -247,52 +241,53 @@ private async Task> RunQueryAsync(DirectExecutionTestCase q } else { - customOdeStats = await this.GetIteratorStatistics(iterator, queryInput); + odeQueryStatisticsList.Add(await this.GetIteratorStatistics(iterator, queryInput)); } - } } - return customOdeStats; + return odeQueryStatisticsList; } - private async Task> GetIteratorStatistics(FeedIterator feedIterator, DirectExecutionTestCase queryInput) + private async Task> GetIteratorStatistics(FeedIterator feedIterator, DirectExecutionTestCase queryInput) { MetricsAccumulator metricsAccumulator = new MetricsAccumulator(); - Documents.ValueStopwatch totalTime = new Documents.ValueStopwatch(); - Documents.ValueStopwatch getTraceTime = new Documents.ValueStopwatch(); Guid correlatedActivityId = Guid.NewGuid(); FeedResponse response; int totalDocumentCount = 0; string query; bool enableOde; - List customOdeStats = new List(); + List odeQueryStatisticsList = new List(); while (feedIterator.HasMoreResults) { QueryStatisticsDatumVisitor queryStatisticsDatumVisitor = new QueryStatisticsDatumVisitor(); + System.Diagnostics.Stopwatch totalTime = new System.Diagnostics.Stopwatch(); + System.Diagnostics.Stopwatch traceTime = new System.Diagnostics.Stopwatch(); + totalTime.Start(); response = await feedIterator.ReadNextAsync(); - getTraceTime.Start(); + traceTime.Start(); if (response.RequestCharge != 0) { metricsAccumulator.ReadFromTrace(response, queryStatisticsDatumVisitor); } - getTraceTime.Stop(); + traceTime.Stop(); totalTime.Stop(); if (response.RequestCharge != 0) { query = queryInput.Query; enableOde = queryInput.EnableOptimisticDirectExecution; - queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - getTraceTime.ElapsedMilliseconds); + queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - traceTime.ElapsedMilliseconds); queryStatisticsDatumVisitor.PopulateMetrics(); QueryStatisticsMetrics queryStatistics = queryStatisticsDatumVisitor.QueryMetricsList[0]; queryStatistics.RUCharge = response.RequestCharge; queryStatistics.CorrelatedActivityId = correlatedActivityId; - customOdeStats.Add(new CustomOdeStats + // Each roundtrip is a new item in the list + odeQueryStatisticsList.Add(new OdeQueryStatistics { Query = query, EnableOde = enableOde, @@ -300,15 +295,12 @@ private async Task> GetIteratorStatistics(FeedIterator customOdeStatisticsList, int numberOfIterations, bool rawData) + private static void SerializeODEQueryMetrics(TextWriter textWriter, List> customOdeStatisticsList, int numberOfIterations, bool rawData) { if (rawData) { @@ -356,90 +348,84 @@ private static void SerializeODEQueryMetrics(TextWriter textWriter, List customOdeStatsList) + private static void SerializeODERawDataQueryMetrics(TextWriter textWriter, List> globalOdeQueryStatisticsList) { textWriter.WriteLine(); textWriter.WriteLine(PrintQueryMetrics); textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", "Query", "ODE", "RUCharge", "BackendTime", "TransitTime", "ClientTime", "EndToEndTime"); textWriter.WriteLine(); - double totalClientTime = 0; - double totalBackendTime = 0; - double totalEndToEndTime = 0; - double totalTransitTime = 0; - double totalRU = 0; - string prevQuery = ""; - bool prevOde = default; - Guid prevCorrelatedActivityId = customOdeStatsList[0].QueryStatisticsMetrics.CorrelatedActivityId; - foreach (OptimisticDirectExecutionPerformanceTests.CustomOdeStats customOdeStats in customOdeStatsList) + foreach (List queryStatisticsList in globalOdeQueryStatisticsList) { - QueryStatisticsMetrics metrics = customOdeStats.QueryStatisticsMetrics; - double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed; - double backendTime = metrics.TotalQueryExecutionTime; - - if (metrics.CorrelatedActivityId == prevCorrelatedActivityId) + double totalClientTime = 0; + double totalBackendTime = 0; + double totalEndToEndTime = 0; + double totalTransitTime = 0; + double totalRU = 0; + string query = ""; + bool ode = false; + + foreach (OdeQueryStatistics queryStatistics in queryStatisticsList) { + QueryStatisticsMetrics metrics = queryStatistics.QueryStatisticsMetrics; + double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed; + double backendTime = metrics.TotalQueryExecutionTime; + totalClientTime += metrics.EndToEndTime - (backendTime + transitTime); totalBackendTime += backendTime; totalEndToEndTime += metrics.EndToEndTime; totalTransitTime += transitTime; totalRU += metrics.RUCharge; - prevQuery = customOdeStats.Query; - prevOde = customOdeStats.EnableOde; - prevCorrelatedActivityId = metrics.CorrelatedActivityId; - - } - else - { - textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); - totalClientTime = metrics.EndToEndTime - (backendTime + transitTime); - totalBackendTime = backendTime; - totalEndToEndTime = metrics.EndToEndTime; - totalTransitTime = transitTime; - totalRU = metrics.RUCharge; - prevCorrelatedActivityId = metrics.CorrelatedActivityId; - prevQuery = customOdeStats.Query; - prevOde = customOdeStats.EnableOde; + query = queryStatistics.Query; + ode = queryStatistics.EnableOde; } - } - textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); + textWriter.WriteLine($"{query},{ode},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); + } } - private static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter, List customOdeStatsList, int numberOfIterations) + private static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter, List> globalOdeQueryStatisticsList, int numberOfIterations) { textWriter.WriteLine(); textWriter.WriteLine(PrintQueryMetrics); textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime"); textWriter.WriteLine(); - string prevQuery = customOdeStatsList[0].Query; - bool prevOde = customOdeStatsList[0].EnableOde; + string prevQuery = globalOdeQueryStatisticsList[0][0].Query; + bool prevOde = globalOdeQueryStatisticsList[0][0].EnableOde; double totalEndToEndTime = 0; double totalRU = 0; - foreach (OptimisticDirectExecutionPerformanceTests.CustomOdeStats customOdeStats in customOdeStatsList) + foreach (List odeQueryStatisticsList in globalOdeQueryStatisticsList) { - QueryStatisticsMetrics metrics = customOdeStats.QueryStatisticsMetrics; - if (customOdeStats.Query == prevQuery && customOdeStats.EnableOde == prevOde) + if (odeQueryStatisticsList[0].Query == prevQuery && odeQueryStatisticsList[0].EnableOde == prevOde) { - totalEndToEndTime += metrics.EndToEndTime; - totalRU += metrics.RUCharge; + foreach (OdeQueryStatistics odeQueryStatistics in odeQueryStatisticsList) + { + QueryStatisticsMetrics metrics = odeQueryStatistics.QueryStatisticsMetrics; + totalEndToEndTime += metrics.EndToEndTime; + totalRU += metrics.RUCharge; + } } else { textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); - totalEndToEndTime = metrics.EndToEndTime; - totalRU = metrics.RUCharge; - prevQuery = customOdeStats.Query; - prevOde = customOdeStats.EnableOde; + + foreach (OdeQueryStatistics odeQueryStatistics in odeQueryStatisticsList) + { + QueryStatisticsMetrics metrics = odeQueryStatistics.QueryStatisticsMetrics; + totalEndToEndTime = metrics.EndToEndTime; + totalRU = metrics.RUCharge; + prevQuery = odeQueryStatistics.Query; + prevOde = odeQueryStatistics.EnableOde; + } } } textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); } - private class CustomOdeStats + private class OdeQueryStatistics { public string Query { get; set; } public bool EnableOde { get; set; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json index 51ecc8a707..611ba8c677 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json @@ -11,14 +11,14 @@ "MasterKey": "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", "ServerStalenessIntervalInSeconds": "5", "MasterStalenessIntervalInSeconds": "1", - "ContentSerializationPerformanceTests.CosmosDatabaseId": "db", - "ContentSerializationPerformanceTests.ContainerId": "container", - "ContentSerializationPerformanceTests.ContentSerialization": "JsonText", - "ContentSerializationPerformanceTests.Query": "SELECT TOP 1 c.region FROM c", - "ContentSerializationPerformanceTests.MaxConcurrency": "-1", - "ContentSerializationPerformanceTests.MaxItemCount": "-1", - "ContentSerializationPerformanceTests.NumberOfIterations": "1", - "ContentSerializationPerformanceTests.WarmupIterations": "0", - "ContentSerializationPerformanceTests.UseStronglyTypedIterator": "true" + "QueryPerformanceTests.CosmosDatabaseId": "db", + "QueryPerformanceTests.ContainerId": "container", + "QueryPerformanceTests.ContentSerialization": "JsonText", + "QueryPerformanceTests.Query": "SELECT TOP 1 c.region FROM c", + "QueryPerformanceTests.MaxConcurrency": "-1", + "QueryPerformanceTests.MaxItemCount": "-1", + "QueryPerformanceTests.NumberOfIterations": "1", + "QueryPerformanceTests.WarmupIterations": "0", + "QueryPerformanceTests.UseStronglyTypedIterator": "true" } } \ No newline at end of file From 0d0328695b57ef3e460dc2cb360c97618d107baf Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Wed, 2 Aug 2023 13:51:38 -0700 Subject: [PATCH 10/16] Removed request Charge check --- ...timisticDirectExecutionPerformanceTests.cs | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 1489e970ac..195e770703 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -275,25 +275,23 @@ private async Task> GetIteratorStatistics(FeedIterat traceTime.Stop(); totalTime.Stop(); - if (response.RequestCharge != 0) - { - query = queryInput.Query; - enableOde = queryInput.EnableOptimisticDirectExecution; - queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - traceTime.ElapsedMilliseconds); - queryStatisticsDatumVisitor.PopulateMetrics(); - QueryStatisticsMetrics queryStatistics = queryStatisticsDatumVisitor.QueryMetricsList[0]; - queryStatistics.RUCharge = response.RequestCharge; - queryStatistics.CorrelatedActivityId = correlatedActivityId; + query = queryInput.Query; + enableOde = queryInput.EnableOptimisticDirectExecution; + queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - traceTime.ElapsedMilliseconds); + queryStatisticsDatumVisitor.PopulateMetrics(); - // Each roundtrip is a new item in the list - odeQueryStatisticsList.Add(new OdeQueryStatistics - { - Query = query, - EnableOde = enableOde, - QueryStatisticsMetrics = queryStatistics - }); - } + QueryStatisticsMetrics queryStatistics = queryStatisticsDatumVisitor.QueryMetricsList[0]; + queryStatistics.RUCharge = response.RequestCharge; + queryStatistics.CorrelatedActivityId = correlatedActivityId; + + // Each roundtrip is a new item in the list + odeQueryStatisticsList.Add(new OdeQueryStatistics + { + Query = query, + EnableOde = enableOde, + QueryStatisticsMetrics = queryStatistics + }); totalDocumentCount += response.Count; } From 3e4923d908cea984f6992acd64d1c757bb305ceb Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Fri, 4 Aug 2023 14:10:34 -0700 Subject: [PATCH 11/16] Bug in Debug Assert --- .../QueryPerfTest/MetricsAccumulator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs index 56fc16b683..b9a1efee0c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs @@ -76,7 +76,7 @@ public void ReadFromTrace(FeedResponse Response, QueryStatisticsDatumVisit { if (metrics.Item2 != null) { - Debug.Assert(metrics.Item1 == null, "'Get Cosmos Element Response' is null"); + Debug.Assert(metrics.Item1 != null, "'Get Cosmos Element Response' is null"); queryStatisticsDatumVisitor.AddGetCosmosElementResponseTime(metrics.Item1.Duration.TotalMilliseconds); foreach (KeyValuePair kvp in metrics.Item2.Data) { From e1c9b9055e286d6bdf89760307a03fe065758cfc Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Fri, 4 Aug 2023 14:23:01 -0700 Subject: [PATCH 12/16] Test --- .../QueryPerfTest/MetricsAccumulator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs index b9a1efee0c..56fc16b683 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs @@ -76,7 +76,7 @@ public void ReadFromTrace(FeedResponse Response, QueryStatisticsDatumVisit { if (metrics.Item2 != null) { - Debug.Assert(metrics.Item1 != null, "'Get Cosmos Element Response' is null"); + Debug.Assert(metrics.Item1 == null, "'Get Cosmos Element Response' is null"); queryStatisticsDatumVisitor.AddGetCosmosElementResponseTime(metrics.Item1.Duration.TotalMilliseconds); foreach (KeyValuePair kvp in metrics.Item2.Data) { From 7d2c00945f417e35c5db7b60b6e2786c942590cd Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Fri, 4 Aug 2023 14:23:57 -0700 Subject: [PATCH 13/16] Bug in debug assert fix --- .../QueryPerfTest/MetricsAccumulator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs index 56fc16b683..b9a1efee0c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs @@ -76,7 +76,7 @@ public void ReadFromTrace(FeedResponse Response, QueryStatisticsDatumVisit { if (metrics.Item2 != null) { - Debug.Assert(metrics.Item1 == null, "'Get Cosmos Element Response' is null"); + Debug.Assert(metrics.Item1 != null, "'Get Cosmos Element Response' is null"); queryStatisticsDatumVisitor.AddGetCosmosElementResponseTime(metrics.Item1.Duration.TotalMilliseconds); foreach (KeyValuePair kvp in metrics.Item2.Data) { From c5cfef27fdc35d4e982ffc1f7516af800141097b Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Fri, 4 Aug 2023 16:12:20 -0700 Subject: [PATCH 14/16] Fixed second bug in Metrics Accumalator class --- .../QueryPerfTest/MetricsAccumulator.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs index b9a1efee0c..a7e6032001 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs @@ -53,18 +53,16 @@ public void ReadFromTrace(FeedResponse Response, QueryStatisticsDatumVisit { if (storeResponse.StoreResult.StatusCode == StatusCodes.Ok) { - backendAndClientMetrics.Add(Tuple.Create(retrieveCosmosElementTraces[k], backendMetrics[j], transitMetrics[i])); + backendAndClientMetrics.Add(Tuple.Create(retrieveCosmosElementTraces[k], backendMetrics[j], node)); j++; k++; } else { //We add null values to the tuple since status codes other than Ok will not have data for 'Query Metrics' and 'Get Cosmos Element Response' - backendAndClientMetrics.Add(Tuple.Create(null, null, transitMetrics[i])); + backendAndClientMetrics.Add(Tuple.Create(null, null, node)); } } - - i++; } Debug.Assert(i == transitMetrics.Count, "All 'transit metrics' must be grouped."); From 050a14be1041194233bef330d091c66c480d50a2 Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Sat, 5 Aug 2023 09:54:51 -0700 Subject: [PATCH 15/16] Added ignore flag to ode perf tests so that they do not run on every loop build --- .../QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 195e770703..80dada1abf 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -97,6 +97,7 @@ private async Task AddItemsToContainerAsync(Container container) } } + [Ignore] [TestMethod] [Owner("akotalwar")] public async Task RunAsync() From cba35e2d262757faa52f44f5782edc13c6222694 Mon Sep 17 00:00:00 2001 From: Aditya Kotalwar Date: Sat, 5 Aug 2023 09:56:36 -0700 Subject: [PATCH 16/16] Added comment explaining the Ignore flag. --- .../QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 80dada1abf..756357e611 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -97,6 +97,8 @@ private async Task AddItemsToContainerAsync(Container container) } } + //Set Ode perf tests to ignore so that they dont run on every loop build. + //Ignore flag can be removed when checking for Ode performance. [Ignore] [TestMethod] [Owner("akotalwar")]