diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs index 759e63b345..8ff5660c6f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs @@ -33,15 +33,15 @@ public ContentSerializationPerformanceTests() this.queryStatisticsDatumVisitor = new(); this.endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; this.authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; - this.cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.CosmosDatabaseId"]; - this.containerId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContainerId"]; - this.contentSerialization = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContentSerialization"]; - this.query = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.Query"]; - this.numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.NumberOfIterations"]); - this.warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.WarmupIterations"]); - this.MaxConcurrency = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.MaxConcurrency"]); - this.MaxItemCount = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.MaxItemCount"]); - this.useStronglyTypedIterator = bool.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.UseStronglyTypedIterator"]); + this.cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.CosmosDatabaseId"]; + this.containerId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.ContainerId"]; + this.contentSerialization = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.ContentSerialization"]; + this.query = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.Query"]; + this.numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.NumberOfIterations"]); + this.warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.WarmupIterations"]); + this.MaxConcurrency = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.MaxConcurrency"]); + this.MaxItemCount = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.MaxItemCount"]); + this.useStronglyTypedIterator = bool.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.UseStronglyTypedIterator"]); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs index 79a9179a6d..1489e970ac 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; - using System.Net; using System.Text; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; @@ -15,17 +14,17 @@ public class OptimisticDirectExecutionPerformanceTests { private Container Container; private const string RawDataFileName = "OptimisticDirectExecutionPerformanceTestsRawData.csv"; - private const string DiagnosticsDataFileName = "OptimisticDirectExecutionPerformanceTestsAggregatedData.csv"; + private const string AggregateDataFileName = "OptimisticDirectExecutionPerformanceTestsAggregatedData.csv"; private const string PrintQueryMetrics = "QueryMetrics"; private static readonly string RawDataPath = Path.GetFullPath(RawDataFileName); - private static readonly string AggregateDataPath = Path.GetFullPath(DiagnosticsDataFileName); - private static readonly string endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; - private static readonly string authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; - private static readonly string cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.CosmosDatabaseId"]; - private static readonly string containerId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContainerId"]; - private static readonly PartitionKey partitionKeyValue = new PartitionKey("Andersen"); - private static readonly int numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.NumberOfIterations"]); - private static readonly int warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.WarmupIterations"]); + private static readonly string AggregateDataPath = Path.GetFullPath(AggregateDataFileName); + private static readonly string Endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; + private static readonly string AuthKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; + private static readonly string CosmosDatabaseId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.CosmosDatabaseId"]; + private static readonly string ContainerId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.ContainerId"]; + private static readonly PartitionKey PartitionKeyValue = new PartitionKey("Andersen"); + private static readonly int NumberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.NumberOfIterations"]); + private static readonly int WarmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.WarmupIterations"]); [TestInitialize] public async Task InitializeTest() @@ -35,9 +34,9 @@ public async Task InitializeTest() ConnectionMode = ConnectionMode.Direct, }; - CosmosClient cosmosClient = new CosmosClient(endpoint, authKey, clientOptions); - Database database = await cosmosClient.CreateDatabaseAsync(cosmosDatabaseId); - this.Container = await database.CreateContainerAsync(containerId, partitionKeyPath: "/name"); + CosmosClient cosmosClient = new CosmosClient(Endpoint, AuthKey, clientOptions); + Database database = await cosmosClient.CreateDatabaseAsync(CosmosDatabaseId); + this.Container = await database.CreateContainerAsync(ContainerId, partitionKeyPath: "/name"); await this.AddItemsToContainerAsync(this.Container); if (File.Exists(RawDataPath)) @@ -65,24 +64,23 @@ private async Task AddItemsToContainerAsync(Container container) { int totalItems = 5000; string[] cityOptions = new string[] { "Seattle", "Chicago", "NYC", "SF" }; + int numberOfRecipeints = cityOptions.Length; + List recipientList = new List(numberOfRecipeints); - // Create a family object for the Andersen family - foreach (int i in Enumerable.Range(0, totalItems)) + for (int j = 0; j < numberOfRecipeints; j++) { - int numberOfRecipeints = cityOptions.Length; - List recipientList = new List(numberOfRecipeints); - - for (int j = 0; j < numberOfRecipeints; j++) + RecipientList recipient = new RecipientList() { - RecipientList recipient = new RecipientList() - { - Name = "John", - City = cityOptions[j], - }; + Name = "John", + City = cityOptions[j], + }; - recipientList.Add(recipient); - } + recipientList.Add(recipient); + } + // Create a family object for the Andersen family + foreach (int i in Enumerable.Range(0, totalItems)) + { States andersenFamily = new States { Id = i.ToString(), @@ -94,10 +92,7 @@ private async Task AddItemsToContainerAsync(Container container) RecipientList = recipientList }; - // Create an item in the container representing the Andersen family. Note we provide the value of the partition key for this item, which is "Andersen" ItemResponse andersenFamilyResponse = await container.CreateItemAsync(andersenFamily, new PartitionKey(andersenFamily.Name)); - - // Note that after creating the item, we can access the body of the item with the Resource property off the ItemResponse. We can also access the RequestCharge property to see the amount of RUs consumed on this request. Console.WriteLine("Created item in database with id: {0} Operation consumed {1} RUs.\n", andersenFamilyResponse.Resource.Id, andersenFamilyResponse.RequestCharge); } } @@ -108,28 +103,28 @@ public async Task RunAsync() { string highPrepTimeSumQuery = CreateHighPrepTimeSumQuery(); string highPrepTimeConditionalQuery = CreateHighPrepTimeConditionalQuery(); - List globalCustomOdeStatisticsList = new List(); + List> globalCustomOdeStatisticsList = new List>(); List odeTestCases = new List() { //Simple Query - CreateInput("SELECT * FROM c", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT * FROM c", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT * FROM c", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT * FROM c", PartitionKeyValue, true, -1, 2500), //TOP - CreateInput("SELECT TOP 1000 c.id FROM c", partitionKeyValue, false, -1, 1000), - CreateInput("SELECT TOP 1000 c.id FROM c", partitionKeyValue, true, -1, 1000), + CreateInput("SELECT TOP 1000 c.id FROM c", PartitionKeyValue, false, -1, 1000), + CreateInput("SELECT TOP 1000 c.id FROM c", PartitionKeyValue, true, -1, 1000), //Filter - CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", partitionKeyValue, false, -1, 1250), - CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", partitionKeyValue, true, -1, 1250), + CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", PartitionKeyValue, false, -1, 1250), + CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", PartitionKeyValue, true, -1, 1250), //DISTINCT + Filter - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", partitionKeyValue, false, -1, 3), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", partitionKeyValue, true, -1, 3), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", PartitionKeyValue, false, -1, 3), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", PartitionKeyValue, true, -1, 3), - CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", partitionKeyValue, false, -1, 2), - CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", partitionKeyValue, true, -1, 2), + CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", PartitionKeyValue, false, -1, 2), + CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", PartitionKeyValue, true, -1, 2), //JOIN CreateInput("SELECT root.id " + @@ -139,7 +134,7 @@ public async Task RunAsync() "JOIN root.id c " + "WHERE root.id = '1' OR a.id in (1,2,3,4,5,6,7,8,9,10) " + "OR b.id in (1,2,3,4,5,6,7,8,9,10) " + - "OR c.id in (1,2,3,4,5,6,7,8,9,10)", partitionKeyValue, false, -1, 1), + "OR c.id in (1,2,3,4,5,6,7,8,9,10)", PartitionKeyValue, false, -1, 1), CreateInput("SELECT root.id " + "FROM root " + "JOIN root.id a " + @@ -147,83 +142,82 @@ public async Task RunAsync() "JOIN root.id c " + "WHERE root.id = '1' OR a.id in (1,2,3,4,5,6,7,8,9,10) " + "OR b.id in (1,2,3,4,5,6,7,8,9,10) " + - "OR c.id in (1,2,3,4,5,6,7,8,9,10)", partitionKeyValue, true, -1, 1), + "OR c.id in (1,2,3,4,5,6,7,8,9,10)", PartitionKeyValue, true, -1, 1), //High Prep Time - CreateInput(highPrepTimeSumQuery, partitionKeyValue, false, -1, 2500), - CreateInput(highPrepTimeSumQuery, partitionKeyValue, true, -1, 2500), + CreateInput(highPrepTimeSumQuery, PartitionKeyValue, false, -1, 2500), + CreateInput(highPrepTimeSumQuery, PartitionKeyValue, true, -1, 2500), - CreateInput(highPrepTimeConditionalQuery, partitionKeyValue, false, -1, 1750), - CreateInput(highPrepTimeConditionalQuery, partitionKeyValue, true, -1, 1750), + CreateInput(highPrepTimeConditionalQuery, PartitionKeyValue, false, -1, 1750), + CreateInput(highPrepTimeConditionalQuery, PartitionKeyValue, true, -1, 1750), //Order By - CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", PartitionKeyValue, true, -1, 2500), - CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", PartitionKeyValue, true, -1, 2500), //Order By + TOP - CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", partitionKeyValue, false, -1, 5), - CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", partitionKeyValue, true, -1, 5), + CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", PartitionKeyValue, false, -1, 5), + CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", PartitionKeyValue, true, -1, 5), //Order By + DISTINCT - CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", PartitionKeyValue, true, -1, 2500), //Order By + DISTINCT + Filter - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", partitionKeyValue, false, -1, 4), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", partitionKeyValue, true, -1, 4), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", PartitionKeyValue, false, -1, 4), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", PartitionKeyValue, true, -1, 4), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", partitionKeyValue, false, -1, 6), - CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", partitionKeyValue, true, -1, 6), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", PartitionKeyValue, false, -1, 6), + CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", PartitionKeyValue, true, -1, 6), //Group By - CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, true, -1, 2500), - CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", partitionKeyValue, false, -1, 2), - CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", partitionKeyValue, true, -1, 2), + CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", PartitionKeyValue, false, -1, 2), + CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", PartitionKeyValue, true, -1, 2), - CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", partitionKeyValue, false, -1, 4), - CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", partitionKeyValue, true, -1, 4), + CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", PartitionKeyValue, false, -1, 4), + CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", PartitionKeyValue, true, -1, 4), //Group By + OFFSET - CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", partitionKeyValue, false, -1, 3), - CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", partitionKeyValue, true, -1, 3), + CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", PartitionKeyValue, false, -1, 3), + CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", PartitionKeyValue, true, -1, 3), //Group By + TOP - CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", partitionKeyValue, false, -1, 25), - CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", partitionKeyValue, true, -1, 25), + CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", PartitionKeyValue, false, -1, 25), + CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", PartitionKeyValue, true, -1, 25), //Group By + DISTINCT - CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", PartitionKeyValue, true, -1, 2500), - CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, false, -1, 2500), - CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", partitionKeyValue, true, -1, 2500), + CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, false, -1, 2500), + CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, true, -1, 2500), }; foreach (DirectExecutionTestCase testCase in odeTestCases) { - List customOdeStats = await this.RunQueryAsync(testCase); - globalCustomOdeStatisticsList.AddRange(customOdeStats); + globalCustomOdeStatisticsList.AddRange(await this.RunQueryAsync(testCase)); } using (StreamWriter writer = new StreamWriter(new FileStream(RawDataPath, FileMode.Append, FileAccess.Write))) { - SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, numberOfIterations, rawData: true); + SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, NumberOfIterations, rawData: true); } using (StreamWriter writer = new StreamWriter(new FileStream(AggregateDataPath, FileMode.Append, FileAccess.Write))) { - SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, numberOfIterations, rawData: false); + SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, NumberOfIterations, rawData: false); } } - private async Task> RunQueryAsync(DirectExecutionTestCase queryInput) + private async Task>> RunQueryAsync(DirectExecutionTestCase queryInput) { - List customOdeStats = new List(); + List> odeQueryStatisticsList = new List>(); QueryRequestOptions requestOptions = new QueryRequestOptions() { MaxItemCount = queryInput.PageSizeOption, @@ -231,9 +225,9 @@ private async Task> RunQueryAsync(DirectExecutionTestCase q PartitionKey = queryInput.PartitionKey, }; - for (int i = 0; i < numberOfIterations + warmupIterations; i++) + for (int i = 0; i < NumberOfIterations + WarmupIterations; i++) { - bool isWarmUpIteration = i < warmupIterations; + bool isWarmUpIteration = i < WarmupIterations; using (FeedIterator iterator = this.Container.GetItemQueryIterator( queryText: queryInput.Query, requestOptions: requestOptions)) @@ -247,52 +241,53 @@ private async Task> RunQueryAsync(DirectExecutionTestCase q } else { - customOdeStats = await this.GetIteratorStatistics(iterator, queryInput); + odeQueryStatisticsList.Add(await this.GetIteratorStatistics(iterator, queryInput)); } - } } - return customOdeStats; + return odeQueryStatisticsList; } - private async Task> GetIteratorStatistics(FeedIterator feedIterator, DirectExecutionTestCase queryInput) + private async Task> GetIteratorStatistics(FeedIterator feedIterator, DirectExecutionTestCase queryInput) { MetricsAccumulator metricsAccumulator = new MetricsAccumulator(); - Documents.ValueStopwatch totalTime = new Documents.ValueStopwatch(); - Documents.ValueStopwatch getTraceTime = new Documents.ValueStopwatch(); Guid correlatedActivityId = Guid.NewGuid(); FeedResponse response; int totalDocumentCount = 0; string query; bool enableOde; - List customOdeStats = new List(); + List odeQueryStatisticsList = new List(); while (feedIterator.HasMoreResults) { QueryStatisticsDatumVisitor queryStatisticsDatumVisitor = new QueryStatisticsDatumVisitor(); + System.Diagnostics.Stopwatch totalTime = new System.Diagnostics.Stopwatch(); + System.Diagnostics.Stopwatch traceTime = new System.Diagnostics.Stopwatch(); + totalTime.Start(); response = await feedIterator.ReadNextAsync(); - getTraceTime.Start(); + traceTime.Start(); if (response.RequestCharge != 0) { metricsAccumulator.ReadFromTrace(response, queryStatisticsDatumVisitor); } - getTraceTime.Stop(); + traceTime.Stop(); totalTime.Stop(); if (response.RequestCharge != 0) { query = queryInput.Query; enableOde = queryInput.EnableOptimisticDirectExecution; - queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - getTraceTime.ElapsedMilliseconds); + queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - traceTime.ElapsedMilliseconds); queryStatisticsDatumVisitor.PopulateMetrics(); QueryStatisticsMetrics queryStatistics = queryStatisticsDatumVisitor.QueryMetricsList[0]; queryStatistics.RUCharge = response.RequestCharge; queryStatistics.CorrelatedActivityId = correlatedActivityId; - customOdeStats.Add(new CustomOdeStats + // Each roundtrip is a new item in the list + odeQueryStatisticsList.Add(new OdeQueryStatistics { Query = query, EnableOde = enableOde, @@ -300,15 +295,12 @@ private async Task> GetIteratorStatistics(FeedIterator customOdeStatisticsList, int numberOfIterations, bool rawData) + private static void SerializeODEQueryMetrics(TextWriter textWriter, List> customOdeStatisticsList, int numberOfIterations, bool rawData) { if (rawData) { @@ -356,90 +348,84 @@ private static void SerializeODEQueryMetrics(TextWriter textWriter, List customOdeStatsList) + private static void SerializeODERawDataQueryMetrics(TextWriter textWriter, List> globalOdeQueryStatisticsList) { textWriter.WriteLine(); textWriter.WriteLine(PrintQueryMetrics); textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", "Query", "ODE", "RUCharge", "BackendTime", "TransitTime", "ClientTime", "EndToEndTime"); textWriter.WriteLine(); - double totalClientTime = 0; - double totalBackendTime = 0; - double totalEndToEndTime = 0; - double totalTransitTime = 0; - double totalRU = 0; - string prevQuery = ""; - bool prevOde = default; - Guid prevCorrelatedActivityId = customOdeStatsList[0].QueryStatisticsMetrics.CorrelatedActivityId; - foreach (OptimisticDirectExecutionPerformanceTests.CustomOdeStats customOdeStats in customOdeStatsList) + foreach (List queryStatisticsList in globalOdeQueryStatisticsList) { - QueryStatisticsMetrics metrics = customOdeStats.QueryStatisticsMetrics; - double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed; - double backendTime = metrics.TotalQueryExecutionTime; - - if (metrics.CorrelatedActivityId == prevCorrelatedActivityId) + double totalClientTime = 0; + double totalBackendTime = 0; + double totalEndToEndTime = 0; + double totalTransitTime = 0; + double totalRU = 0; + string query = ""; + bool ode = false; + + foreach (OdeQueryStatistics queryStatistics in queryStatisticsList) { + QueryStatisticsMetrics metrics = queryStatistics.QueryStatisticsMetrics; + double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed; + double backendTime = metrics.TotalQueryExecutionTime; + totalClientTime += metrics.EndToEndTime - (backendTime + transitTime); totalBackendTime += backendTime; totalEndToEndTime += metrics.EndToEndTime; totalTransitTime += transitTime; totalRU += metrics.RUCharge; - prevQuery = customOdeStats.Query; - prevOde = customOdeStats.EnableOde; - prevCorrelatedActivityId = metrics.CorrelatedActivityId; - - } - else - { - textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); - totalClientTime = metrics.EndToEndTime - (backendTime + transitTime); - totalBackendTime = backendTime; - totalEndToEndTime = metrics.EndToEndTime; - totalTransitTime = transitTime; - totalRU = metrics.RUCharge; - prevCorrelatedActivityId = metrics.CorrelatedActivityId; - prevQuery = customOdeStats.Query; - prevOde = customOdeStats.EnableOde; + query = queryStatistics.Query; + ode = queryStatistics.EnableOde; } - } - textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); + textWriter.WriteLine($"{query},{ode},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}"); + } } - private static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter, List customOdeStatsList, int numberOfIterations) + private static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter, List> globalOdeQueryStatisticsList, int numberOfIterations) { textWriter.WriteLine(); textWriter.WriteLine(PrintQueryMetrics); textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime"); textWriter.WriteLine(); - string prevQuery = customOdeStatsList[0].Query; - bool prevOde = customOdeStatsList[0].EnableOde; + string prevQuery = globalOdeQueryStatisticsList[0][0].Query; + bool prevOde = globalOdeQueryStatisticsList[0][0].EnableOde; double totalEndToEndTime = 0; double totalRU = 0; - foreach (OptimisticDirectExecutionPerformanceTests.CustomOdeStats customOdeStats in customOdeStatsList) + foreach (List odeQueryStatisticsList in globalOdeQueryStatisticsList) { - QueryStatisticsMetrics metrics = customOdeStats.QueryStatisticsMetrics; - if (customOdeStats.Query == prevQuery && customOdeStats.EnableOde == prevOde) + if (odeQueryStatisticsList[0].Query == prevQuery && odeQueryStatisticsList[0].EnableOde == prevOde) { - totalEndToEndTime += metrics.EndToEndTime; - totalRU += metrics.RUCharge; + foreach (OdeQueryStatistics odeQueryStatistics in odeQueryStatisticsList) + { + QueryStatisticsMetrics metrics = odeQueryStatistics.QueryStatisticsMetrics; + totalEndToEndTime += metrics.EndToEndTime; + totalRU += metrics.RUCharge; + } } else { textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); - totalEndToEndTime = metrics.EndToEndTime; - totalRU = metrics.RUCharge; - prevQuery = customOdeStats.Query; - prevOde = customOdeStats.EnableOde; + + foreach (OdeQueryStatistics odeQueryStatistics in odeQueryStatisticsList) + { + QueryStatisticsMetrics metrics = odeQueryStatistics.QueryStatisticsMetrics; + totalEndToEndTime = metrics.EndToEndTime; + totalRU = metrics.RUCharge; + prevQuery = odeQueryStatistics.Query; + prevOde = odeQueryStatistics.EnableOde; + } } } textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}"); } - private class CustomOdeStats + private class OdeQueryStatistics { public string Query { get; set; } public bool EnableOde { get; set; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json index 51ecc8a707..611ba8c677 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json @@ -11,14 +11,14 @@ "MasterKey": "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", "ServerStalenessIntervalInSeconds": "5", "MasterStalenessIntervalInSeconds": "1", - "ContentSerializationPerformanceTests.CosmosDatabaseId": "db", - "ContentSerializationPerformanceTests.ContainerId": "container", - "ContentSerializationPerformanceTests.ContentSerialization": "JsonText", - "ContentSerializationPerformanceTests.Query": "SELECT TOP 1 c.region FROM c", - "ContentSerializationPerformanceTests.MaxConcurrency": "-1", - "ContentSerializationPerformanceTests.MaxItemCount": "-1", - "ContentSerializationPerformanceTests.NumberOfIterations": "1", - "ContentSerializationPerformanceTests.WarmupIterations": "0", - "ContentSerializationPerformanceTests.UseStronglyTypedIterator": "true" + "QueryPerformanceTests.CosmosDatabaseId": "db", + "QueryPerformanceTests.ContainerId": "container", + "QueryPerformanceTests.ContentSerialization": "JsonText", + "QueryPerformanceTests.Query": "SELECT TOP 1 c.region FROM c", + "QueryPerformanceTests.MaxConcurrency": "-1", + "QueryPerformanceTests.MaxItemCount": "-1", + "QueryPerformanceTests.NumberOfIterations": "1", + "QueryPerformanceTests.WarmupIterations": "0", + "QueryPerformanceTests.UseStronglyTypedIterator": "true" } } \ No newline at end of file