From fa51c2c521a418f724a61aaef1a6bd4fca2ef835 Mon Sep 17 00:00:00 2001 From: David Jimenez Barrantes Date: Mon, 7 Oct 2024 11:19:09 -0600 Subject: [PATCH] [APIPUB-76] - Adds Sonar Analyzer (#80) * Add sonar analyzer Treat warnings as errors * Add editorconfig files * Fix many SonarLint warnings * Fixes * fixes some warnings. * Fixes format warnings * Trying to fix tests. * More warnings fixed. * More warnings fixed. * Removes unnecessary file --------- Co-authored-by: Stephen A. Fuqua --- .editorconfig | 17 ++ src/.editorconfig | 143 ++++++++++ src/EdFi.Tools.ApiPublisher.Cli/Program.cs | 10 +- ...stemManagerChangeVersionProcessedWriter.cs | 10 +- ...mManagerNamedApiConnectionDetailsReader.cs | 6 +- .../Plugin.cs | 2 +- .../Modules/PluginModule.cs | 2 +- ...JsonFileNamedApiConnectionDetailsReader.cs | 6 +- .../PlainTextNamedConnectionConfiguration.cs | 2 +- .../PlaintextChangeVersionProcessedWriter.cs | 4 +- .../ConfigurationBuilderExtensions.cs | 2 +- .../Modules/PluginModule.cs | 4 +- .../Plugin.cs | 2 +- ...nfigurationChangeVersionProcessedWriter.cs | 6 +- ...gurationNamedApiConnectionDetailsReader.cs | 8 +- .../PostgreSqlConfigurationProvider.cs | 6 +- .../PostgreSqlConfigurationSource.cs | 4 +- .../PostgreSqlConfigurationValuesProvider.cs | 6 +- .../PostgresConfiguration.cs | 4 +- .../ConfigurationBuilderExtensions.cs | 2 +- .../Modules/PluginModule.cs | 2 +- .../Plugin.cs | 2 +- .../SqlServerConfiguration.cs | 4 +- ...nfigurationChangeVersionProcessedWriter.cs | 4 +- ...gurationNamedApiConnectionDetailsReader.cs | 2 +- .../SqlServerConfigurationProvider.cs | 4 +- .../SqlServerConfigurationSource.cs | 4 +- .../ApiClientManagement/EdFiApiClient.cs | 230 +++++++++++----- .../EdFiApiClientProvider.cs | 6 +- .../Configuration/ApiConnectionDetails.cs | 47 +--- ...ConnectionsConfigurationBuilderEnhancer.cs | 30 +-- .../INamedApiConnectionDetailsReader.cs | 2 +- .../ApiSourceResourceItemProvider.cs | 91 +------ ....Tools.ApiPublisher.Connections.Api.csproj | 14 +- .../Helpers/RequestHelpers.cs | 10 +- ...dFiApiGraphMLDependencyMetadataProvider.cs | 17 +- .../EdFiApiVersionMetadataProviderBase.cs | 11 +- .../Modules/EdFiApiAsSourceModule.cs | 12 +- .../Modules/EdFiApiAsTargetModule.cs | 10 +- .../Modules/PluginModule.cs | 2 +- .../Capabilities/EdFiApiSourceCapabilities.cs | 20 +- .../EdFiApiSourceTotalCountProvider.cs | 147 ++++++---- .../EdFiApiSourceIsolationApplicator.cs | 29 +- ...EdFiApiStreamResourcePageMessageHandler.cs | 45 ++-- ...PagingStreamResourcePageMessageProducer.cs | 13 +- ...PagingStreamResourcePageMessageProducer.cs | 18 +- ...PagingStreamResourcePageMessageProducer.cs | 26 +- ...FiApiSourceCurrentChangeVersionProvider.cs | 12 +- ...hangeResourceKeyProcessingBlocksFactory.cs | 78 +++--- .../DeleteResourceProcessingBlocksFactory.cs | 85 +++--- .../PostResourceProcessingBlocksFactory.cs | 250 +++++++++--------- .../UpsertPublishingStageInitiator.cs | 4 +- .../Target/Messages/ChangeKeyMessage.cs | 4 +- .../Target/Messages/DeleteItemMessage.cs | 2 +- .../Messages/GetItemForDeletionMessage.cs | 6 +- .../Messages/GetItemForKeyChangeMessage.cs | 8 +- .../Target/Messages/PostItemMessage.cs | 6 +- .../Configuration/SqliteConnectionDetails.cs | 2 +- ...ngOperationMetadataFinalizationActivity.cs | 14 +- .../SourceSqliteVersionMetadataProvider.cs | 2 +- .../Modules/SqliteAsSourceModule.cs | 8 +- .../Modules/SqliteAsTargetModule.cs | 4 +- .../Plugin.cs | 2 +- .../SqliteSourceTotalCountProvider.cs | 6 +- .../SqliteStreamResourcePageMessageHandler.cs | 4 +- ...SqliteStreamResourcePageMessageProducer.cs | 14 +- ...qliteSourceCurrentChangeVersionProvider.cs | 2 +- .../SqlLiteProcessingBlocksFactoryBase.cs | 8 +- .../Target/Messages/ResourceJsonMessage.cs | 6 +- .../ApiClientManagement/HttpClients.cs | 2 +- .../AppSettingsConfigurationProvider.cs | 8 +- .../ChangeProcessorConfiguration.cs | 2 +- .../Configuration/ConfigurationHelper.cs | 4 +- .../Configuration/ConfigurationStoreHelper.cs | 2 +- .../Configuration/ConnectionConfiguration.cs | 6 +- .../IConfigurationBuilderEnhancer.cs | 2 +- .../IAppSettingsConfigurationProvider.cs | 2 +- ...geProcessorRuntimeConfigurationProvider.cs | 2 +- .../Configuration/ISourceConnectionDetails.cs | 10 +- .../Configuration/PollyRateLimiter.cs | 14 +- .../Configuration/Serilog/TextFormatter.cs | 6 +- ...llbackGraphMLDependencyMetadataProvider.cs | 2 +- .../IResourceDependencyProvider.cs | 2 +- .../ResourceDependencyProvider.cs | 2 +- .../Extensions/HttpStatusCodeExtensions.cs | 2 +- .../Extensions/IDictionaryExtensions.cs | 2 +- .../Extensions/NullableBooleanExtensions.cs | 2 +- .../Extensions/StringExtensions.cs | 10 +- .../Helpers/ApiRequestHelper.cs | 4 +- .../Helpers/ExponentialBackOffHelper.cs | 8 +- .../Helpers/ResourcePathHelper.cs | 2 +- .../Helpers/Version.cs | 6 +- .../Helpers/VersionHelper.cs | 4 +- .../Inflection/CompositeTermInflector.cs | 20 +- .../Inflection/Inflector.cs | 22 +- .../Metadata/CurrentChangeVersionCollector.cs | 2 +- .../IPublishingOperationMetadataCollector.cs | 2 +- .../PublishingOperationMetadataCollector.cs | 4 +- .../Metadata/ResourceItemCountCollector.cs | 4 +- .../SourceEdFiVersionMetadataCollector.cs | 2 +- .../TargetEdFiVersionMetadataCollector.cs | 2 +- .../Modules/CoreModule.cs | 12 +- .../Modules/NodeJsRemediationsModule.cs | 2 +- .../Plugin/IPlugin.cs | 4 +- .../Processing/Blocks/CreateBlocksRequest.cs | 8 +- .../Blocks/PublishErrorsBlocksFactory.cs | 10 +- .../Blocks/StreamResourceBlockFactory.cs | 13 +- .../Blocks/StreamResourcePagesBlockFactory.cs | 18 +- .../Processing/ChangeProcessor.cs | 172 ++++++------ .../Processing/ChangeWindow.cs | 6 +- .../Processing/EdFiApiConstants.cs | 10 +- .../IChangeVersionProcessedWriter.cs | 2 +- .../Processing/IErrorPublisher.cs | 2 +- .../Processing/Messages/ErrorItemMessage.cs | 10 +- .../Messages/StreamResourceMessage.cs | 6 +- .../Messages/StreamResourcePageMessage.cs | 10 +- .../Processing/SerilogErrorPublisher.cs | 12 +- .../Processing/StreamingPagesItem.cs | 2 +- .../Processing/StreamingResourceProcessor.cs | 4 +- .../Registration/AssemblyLoaderHelper.cs | 138 +--------- .../AutofacRegistrationExtensions.cs | 6 +- .../Versioning/EdFiVersionsChecker.cs | 18 +- .../Serilog/TextFormatterTests.cs | 12 +- .../HttpRequestMessageExtensions.cs | 6 +- .../Extensions/ObjectExtensions.cs | 6 +- .../FakeResponse.cs | 7 +- .../Helpers/TestHelpers.cs | 20 +- .../HttpClientHandlerFakeBridge.cs | 4 +- .../IFakeHttpRequestHandler.cs | 2 +- .../JavascriptHosting/ScriptExecutionTests.cs | 64 ++--- .../MockRateLimitingMethod.cs | 6 +- .../MockRequests.cs | 48 ++-- .../Models/FakeKey.cs | 8 +- .../Models/GenericResource.cs | 6 +- .../Models/KeyChange.cs | 2 +- .../Models/Snapshot.cs | 8 +- .../ExcludeExtensionResourcesTests.cs | 8 +- .../Processing/ExcludeOnlyResourcesTests.cs | 16 +- .../Processing/ExcludeResourcesTests.cs | 14 +- .../Processing/IgnoreIsolationTests.cs | 8 +- .../Processing/IncludeOnlyResources.cs | 14 +- .../Processing/IncludeResourcesTests.cs | 18 +- .../Processing/KeyChangesTests.cs | 36 +-- .../Processing/PostRetryTests.cs | 16 +- .../Processing/ProfileApplicationTests.cs | 38 +-- .../Processing/RateLimitingTests.cs | 2 +- .../Processing/RemediationIntegrationTests.cs | 40 +-- .../Processing/RemediationTests.cs | 30 +-- .../Processing/ReversePagingTests.cs | 2 +- .../SchoolYearSpecificClientTests.cs | 34 +-- ...vedDependencyOnPrimaryRelationshipTests.cs | 40 +-- .../Resources/TestData.cs | 2 +- .../Serialization/GuidConverter.cs | 4 +- .../Iso8601UtcDateOnlyConverter.cs | 2 +- .../TestFixtureAsyncBase.cs | 2 +- 155 files changed, 1409 insertions(+), 1320 deletions(-) create mode 100644 .editorconfig create mode 100644 src/.editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..e8f5fb3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +# http://editorconfig.org +root = true + +[*] +indent_style=space +indent_size=4 +tab_width=2 +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +max_line_length = 110 + +[*.{yml,json}] +indent_size=2 + +[*.md] +trim_trailing_whitespace = false diff --git a/src/.editorconfig b/src/.editorconfig new file mode 100644 index 0000000..fc738fa --- /dev/null +++ b/src/.editorconfig @@ -0,0 +1,143 @@ +# References +# https://docs.microsoft.com/en-us/visualstudio/code-quality/use-roslyn-analyzers?view=vs-2022 +# https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/language-rules +# https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/naming-rules +# https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ +# https://rules.sonarsource.com/csharp +# https://editorconfig.org/ + +root = true + +[**/*] +# Sensible Defaults +file_header_template = SPDX-License-Identifier: Apache-2.0\nLicensed to the Ed-Fi Alliance under one or more agreements.\nThe Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.\nSee the LICENSE and NOTICES files in the project root for more information. +indent_style=space +indent_size=4 +tab_width=2 +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +max_line_length = 110 + +[**/*.md] +trim_trailing_whitespace = false + +[**/*.{cs,cshtml,csx}] +indent_style = space +indent_size = 4 +tab_width = 4 +end_of_line = lf +csharp_new_line_before_members_in_object_initializers = false +csharp_preferred_modifier_order = private, public, protected, internal, new, abstract, virtual, sealed, override, static, readonly, extern, unsafe, volatile, async:suggestion + +dotnet_diagnostic.IDE0073.severity=error # File header template required + +# Formatting rules +dotnet_diagnostic.IDE0055.severity=warning +csharp_new_line_before_open_brace = all # Allman style +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true +csharp_indent_case_contents = true +csharp_indent_switch_labels = true +csharp_indent_labels = no_change +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents_when_block = true +csharp_space_after_cast = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_between_parentheses = none +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_after_comma = true +csharp_space_before_comma = false +csharp_space_after_dot = false +csharp_space_before_dot = false +csharp_space_after_semicolon_in_for_statement = true +csharp_space_before_semicolon_in_for_statement = false +csharp_space_around_declaration_statements = false +csharp_space_before_open_square_brackets = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_square_brackets = false +csharp_preserve_single_line_blocks = true +csharp_preserve_single_line_statements = false + +# Naming rules +dotnet_naming_rule.local_constants_rule.severity = warning +dotnet_naming_rule.local_constants_rule.style = upper_camel_case_style +dotnet_naming_rule.local_constants_rule.symbols = local_constants_symbols +dotnet_naming_rule.private_constants_rule.import_to_resharper = as_predefined +dotnet_naming_rule.private_constants_rule.severity = warning +dotnet_naming_rule.private_constants_rule.style = upper_camel_case_style +dotnet_naming_rule.private_constants_rule.symbols = private_constants_symbols +dotnet_naming_rule.private_static_readonly_rule.import_to_resharper = as_predefined +dotnet_naming_rule.private_static_readonly_rule.severity = warning +dotnet_naming_rule.private_static_readonly_rule.style = lower_camel_case_style +dotnet_naming_rule.private_static_readonly_rule.symbols = private_static_readonly_symbols +dotnet_naming_style.lower_camel_case_style.capitalization = camel_case +dotnet_naming_style.lower_camel_case_style.required_prefix = _ +dotnet_naming_style.upper_camel_case_style.capitalization = pascal_case +dotnet_naming_symbols.local_constants_symbols.applicable_accessibilities = * +dotnet_naming_symbols.local_constants_symbols.applicable_kinds = local +dotnet_naming_symbols.local_constants_symbols.required_modifiers = const +dotnet_naming_symbols.private_constants_symbols.applicable_accessibilities = private +dotnet_naming_symbols.private_constants_symbols.applicable_kinds = field +dotnet_naming_symbols.private_constants_symbols.required_modifiers = const +dotnet_naming_symbols.private_static_readonly_symbols.applicable_accessibilities = private +dotnet_naming_symbols.private_static_readonly_symbols.applicable_kinds = field +dotnet_naming_symbols.private_static_readonly_symbols.required_modifiers = static,readonly + +# Misc style +dotnet_style_parentheses_in_arithmetic_binary_operators = never_if_unnecessary:none +dotnet_style_parentheses_in_other_binary_operators = never_if_unnecessary:none +dotnet_style_parentheses_in_relational_binary_operators = never_if_unnecessary:none +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion +dotnet_style_qualification_for_event = false:suggestion +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning +dotnet_style_operator_placement_when_wrapping = beginning_of_line +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = false + +# We *like* var +dotnet_diagnostic.IDE0008.severity=none # IDE0008: Use explicit type +csharp_style_var_elsewhere = false:none +csharp_style_var_for_built_in_types = false:suggestion + +# Using statements +csharp_using_directive_placement=outside_namespace:warning +dotnet_diagnostic.IDE0065.severity=warning # placement of using statements +dotnet_diagnostic.IDE0005.severity=suggestion # Remove unnecessary using directives + +# Lower the priority +dotnet_diagnostic.S4136.severity=suggestion # Method overloads should be grouped together +dotnet_diagnostic.S1135.severity=suggestion # Complete TODO comments +dotnet_diagnostic.S112.severity=suggestion # 'System.Exception' should not be thrown by user code. +dotnet_diagnostic.S3247.severity=suggestion # Remove redundant cast - getting false positives + +# Allow empty records for discriminated union types +dotnet_diagnostic.S2094.severity=none # S2094: Classes should not be empty + +[**/tests/**/*.cs] +# Allow our strange test class naming convention +dotnet_naming_symbols.amt_tests.applicable_kinds = class,method +dotnet_naming_symbols.amt_tests.word_separator = "_" +dotnet_naming_symbols.amt_tests.capitalization = first_word_upper +dotnet_diagnostic.IDE1006.severity=none +dotnet_diagnostic.S101.severity=none + +# SonarLint doesn't understand implied assertions from FakeItEasy +dotnet_diagnostic.S2699.severity=none # S2699: Tests should include assertions diff --git a/src/EdFi.Tools.ApiPublisher.Cli/Program.cs b/src/EdFi.Tools.ApiPublisher.Cli/Program.cs index 72abe34..1694812 100644 --- a/src/EdFi.Tools.ApiPublisher.Cli/Program.cs +++ b/src/EdFi.Tools.ApiPublisher.Cli/Program.cs @@ -13,6 +13,7 @@ using EdFi.Tools.ApiPublisher.Core.Registration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Polly.RateLimit; using Serilog; using System; using System.Collections.Generic; @@ -22,8 +23,8 @@ using System.Threading.Tasks; namespace EdFi.Tools.ApiPublisher.Cli -{ - internal class Program +{ + internal class Program { private static readonly ILogger _logger = Log.ForContext(typeof(Program)); @@ -174,6 +175,11 @@ private static async Task Main(string[] args) _logger.Information($"Processing complete."); return 0; } + //catch (RateLimitRejectedException ex) + //{ + // _logger.Fatal(ex, ex.Message); + // return -1; + //} catch (Exception ex) { _logger.Error($"Processing failed: {string.Join(" ", GetExceptionMessages(ex))}"); diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/AwsSystemManagerChangeVersionProcessedWriter.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/AwsSystemManagerChangeVersionProcessedWriter.cs index 6c3e12b..4cdccab 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/AwsSystemManagerChangeVersionProcessedWriter.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/AwsSystemManagerChangeVersionProcessedWriter.cs @@ -16,10 +16,10 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.Aws { - public class AwsSystemManagerChangeVersionProcessedWriter : IChangeVersionProcessedWriter + public class AwsSystemManagerChangeVersionProcessedWriter : IChangeVersionProcessedWriter { private readonly ILogger _logger = Log.ForContext(typeof(AwsSystemManagerChangeVersionProcessedWriter)); - + public async Task SetProcessedChangeVersionAsync( string sourceConnectionName, string targetConnectionName, @@ -34,7 +34,7 @@ public async Task SetProcessedChangeVersionAsync( // Assign the new "LastChangeVersionProcessed" value currentParameter[targetConnectionName] = changeVersion; - + // Serialize the parameter's values string newParameterJson = currentParameter.ToString(Formatting.None); @@ -51,7 +51,7 @@ public async Task SetProcessedChangeVersionAsync( var response = await amazonSimpleSystemsManagement.PutParameterAsync(putRequest) .ConfigureAwait(false); - if ((int) response.HttpStatusCode >= 400) + if ((int)response.HttpStatusCode >= 400) { throw new Exception( $"Failed to write updated change version of {changeVersion} for source connection '{sourceConnectionName}' to target connection '{targetConnectionName}' (AWS response status: {response.HttpStatusCode})."); @@ -63,7 +63,7 @@ private async Task GetParameterValueAsync( string sourceConnectionName) { string parameterName = $"{ConfigurationStoreHelper.Key(sourceConnectionName)}/lastChangeVersionsProcessed"; - + var getRequest = new GetParameterRequest { Name = parameterName, diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/AwsSystemManagerNamedApiConnectionDetailsReader.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/AwsSystemManagerNamedApiConnectionDetailsReader.cs index c403d41..042b03c 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/AwsSystemManagerNamedApiConnectionDetailsReader.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/AwsSystemManagerNamedApiConnectionDetailsReader.cs @@ -9,7 +9,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.Aws { - public class AwsSystemManagerNamedApiConnectionDetailsReader : INamedApiConnectionDetailsReader + public class AwsSystemManagerNamedApiConnectionDetailsReader : INamedApiConnectionDetailsReader { public ApiConnectionDetails GetNamedApiConnectionDetails( string apiConnectionName, @@ -21,13 +21,13 @@ public ApiConnectionDetails GetNamedApiConnectionDetails( var config = new ConfigurationBuilder() .AddSystemsManager(ConfigurationStoreHelper.Key(apiConnectionName), awsOptions) .Build(); - + // Read the connection details from the configuration values var connectionDetails = config.Get(); // Assign the connection name connectionDetails.Name = apiConnectionName; - + return connectionDetails; } } diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/Plugin.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/Plugin.cs index c45b167..6b1b323 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/Plugin.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Aws/Plugin.cs @@ -14,7 +14,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.Aws; public class Plugin : IPlugin { private const string ConfigurationProviderName = "awsParameterStore"; - + public void ApplyConfiguration(string[] args, IConfigurationBuilder configBuilder) { // Nothing to do diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/Modules/PluginModule.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/Modules/PluginModule.cs index ad4e2a7..c2927bd 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/Modules/PluginModule.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/Modules/PluginModule.cs @@ -9,7 +9,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext.Modules { - public class PluginModule : Module + public class PluginModule : Module { protected override void Load(ContainerBuilder builder) { diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlainTextJsonFileNamedApiConnectionDetailsReader.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlainTextJsonFileNamedApiConnectionDetailsReader.cs index 270686b..c1afb24 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlainTextJsonFileNamedApiConnectionDetailsReader.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlainTextJsonFileNamedApiConnectionDetailsReader.cs @@ -8,7 +8,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext { - public class PlainTextJsonFileNamedApiConnectionDetailsReader : INamedApiConnectionDetailsReader + public class PlainTextJsonFileNamedApiConnectionDetailsReader : INamedApiConnectionDetailsReader { public ApiConnectionDetails GetNamedApiConnectionDetails( string apiConnectionName, @@ -18,9 +18,9 @@ public ApiConnectionDetails GetNamedApiConnectionDetails( var config = new ConfigurationBuilder() .AddJsonFile("plainTextNamedConnections.json") .Build(); - + var connections = config.Get(); - + return connections.Connections? .Where(details => details.Name != null) .FirstOrDefault(details => details.Name!.Equals(apiConnectionName, StringComparison.OrdinalIgnoreCase)) diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlainTextNamedConnectionConfiguration.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlainTextNamedConnectionConfiguration.cs index ae8f1f4..27ce6fd 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlainTextNamedConnectionConfiguration.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlainTextNamedConnectionConfiguration.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext { - internal class PlainTextNamedConnectionConfiguration + internal class PlainTextNamedConnectionConfiguration { public ApiConnectionDetails[] Connections { get; set; } } diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlaintextChangeVersionProcessedWriter.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlaintextChangeVersionProcessedWriter.cs index ec97da5..cf50ba0 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlaintextChangeVersionProcessedWriter.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext/PlaintextChangeVersionProcessedWriter.cs @@ -9,10 +9,10 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.Plaintext { - public class PlaintextChangeVersionProcessedWriter : IChangeVersionProcessedWriter + public class PlaintextChangeVersionProcessedWriter : IChangeVersionProcessedWriter { private readonly ILogger _logger = Log.Logger.ForContext(typeof(PlaintextChangeVersionProcessedWriter)); - + public Task SetProcessedChangeVersionAsync( string sourceConnectionName, string targetConnectionName, diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/ConfigurationBuilderExtensions.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/ConfigurationBuilderExtensions.cs index 97d0ff4..c527efb 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/ConfigurationBuilderExtensions.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/ConfigurationBuilderExtensions.cs @@ -8,7 +8,7 @@ // ReSharper disable once CheckNamespace namespace Microsoft.Extensions.Configuration { - public static class ConfigurationBuilderExtensions + public static class ConfigurationBuilderExtensions { public static IConfigurationBuilder AddConfigurationStoreForPostgreSql( this IConfigurationBuilder builder, diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/Modules/PluginModule.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/Modules/PluginModule.cs index b389327..b8d4b09 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/Modules/PluginModule.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/Modules/PluginModule.cs @@ -9,14 +9,14 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql.Modules { - public class PluginModule : Module + public class PluginModule : Module { protected override void Load(ContainerBuilder builder) { builder.RegisterType() .As() .SingleInstance(); - + builder.RegisterType() .As() .SingleInstance(); diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/Plugin.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/Plugin.cs index 5666af0..f9c9325 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/Plugin.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/Plugin.cs @@ -14,7 +14,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql; public class Plugin : IPlugin { private const string ConfigurationProviderName = "postgreSql"; - + public void ApplyConfiguration(string[] args, IConfigurationBuilder configBuilder) { // Nothing to do diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationChangeVersionProcessedWriter.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationChangeVersionProcessedWriter.cs index cf41172..7444bf3 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationChangeVersionProcessedWriter.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationChangeVersionProcessedWriter.cs @@ -14,7 +14,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql { - public class PostgreSqlConfigurationChangeVersionProcessedWriter : IChangeVersionProcessedWriter + public class PostgreSqlConfigurationChangeVersionProcessedWriter : IChangeVersionProcessedWriter { public async Task SetProcessedChangeVersionAsync( string sourceConnectionName, @@ -25,7 +25,7 @@ public async Task SetProcessedChangeVersionAsync( var postgresConfiguration = configurationStoreSection.Get().PostgreSql; // Make sure Postgres configuration has encryption key provided - if (string.IsNullOrWhiteSpace(postgresConfiguration?.EncryptionPassword)) + if (string.IsNullOrWhiteSpace(postgresConfiguration?.EncryptionPassword)) { throw new Exception("The PostgreSQL Configuration Store encryption key for storing API keys and secrets was not provided."); } @@ -37,7 +37,7 @@ public async Task SetProcessedChangeVersionAsync( postgresConfiguration.ConnectionString, postgresConfiguration.EncryptionPassword, ConfigurationStoreHelper.Key(sourceConnectionName)); - + var currentParameter = new JObject(); if (configurationValues.TryGetValue("lastChangeVersionsProcessed", out string changeVersionsJson)) diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationNamedApiConnectionDetailsReader.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationNamedApiConnectionDetailsReader.cs index ca10490..10ce0a5 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationNamedApiConnectionDetailsReader.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationNamedApiConnectionDetailsReader.cs @@ -10,19 +10,19 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql { - public class PostgreSqlConfigurationNamedApiConnectionDetailsReader : INamedApiConnectionDetailsReader + public class PostgreSqlConfigurationNamedApiConnectionDetailsReader : INamedApiConnectionDetailsReader { public ApiConnectionDetails GetNamedApiConnectionDetails( string apiConnectionName, IConfigurationSection configurationStoreSection) { var postgresConfiguration = configurationStoreSection.Get().PostgreSql; - - if (string.IsNullOrWhiteSpace(postgresConfiguration?.EncryptionPassword)) + + if (string.IsNullOrWhiteSpace(postgresConfiguration?.EncryptionPassword)) { throw new Exception("The PostgreSQL Configuration Store encryption key for storing API keys and secrets was not provided."); } - + // Load named connection information from PostgreSQL configuration store var config = new ConfigurationBuilder() .AddConfigurationStoreForPostgreSql( diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationProvider.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationProvider.cs index a24a3f5..416382a 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationProvider.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql { - public class PostgreSqlConfigurationProvider : ConfigurationProvider + public class PostgreSqlConfigurationProvider : ConfigurationProvider { private readonly PostgreSqlConfigurationSource _postgreSqlConfigurationSource; @@ -20,8 +20,8 @@ public override void Load() { var settings = new PostgreSqlConfigurationValuesProvider() .GetConfigurationValues( - _postgreSqlConfigurationSource.ConnectionString, - _postgreSqlConfigurationSource.EncryptionPassword, + _postgreSqlConfigurationSource.ConnectionString, + _postgreSqlConfigurationSource.EncryptionPassword, _postgreSqlConfigurationSource.ConfigurationKeyPrefix); Data = settings; diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationSource.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationSource.cs index c0c05ce..36d1a7c 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationSource.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationSource.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql { - public class PostgreSqlConfigurationSource : IConfigurationSource + public class PostgreSqlConfigurationSource : IConfigurationSource { public string ConfigurationKeyPrefix { get; } public string ConnectionString { get; } @@ -20,7 +20,7 @@ public PostgreSqlConfigurationSource(string configurationKeyPrefix, string conne ConnectionString = connectionString; EncryptionPassword = encryptionPassword; } - + public IConfigurationProvider Build(IConfigurationBuilder builder) { return new PostgreSqlConfigurationProvider(this); diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationValuesProvider.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationValuesProvider.cs index 2967e11..2dd15e0 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationValuesProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgreSqlConfigurationValuesProvider.cs @@ -10,11 +10,11 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql { - public class PostgreSqlConfigurationValuesProvider + public class PostgreSqlConfigurationValuesProvider { public IDictionary GetConfigurationValues( - string connectionString, - string encryptionPassword, + string connectionString, + string encryptionPassword, string configurationKeyPrefix) { using var conn = new NpgsqlConnection(connectionString); diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgresConfiguration.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgresConfiguration.cs index d184cfc..7aa2e97 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgresConfiguration.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql/PostgresConfiguration.cs @@ -5,12 +5,12 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.PostgreSql { - public class PostgresConfigurationStore + public class PostgresConfigurationStore { public PostgresConfiguration PostgreSql { get; set; } } - public class PostgresConfiguration + public class PostgresConfiguration { public string ConnectionString { get; set; } public string EncryptionPassword { get; set; } diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/ConfigurationBuilderExtensions.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/ConfigurationBuilderExtensions.cs index cee3586..f3d863c 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/ConfigurationBuilderExtensions.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/ConfigurationBuilderExtensions.cs @@ -8,7 +8,7 @@ // ReSharper disable once CheckNamespace namespace Microsoft.Extensions.Configuration { - public static class ConfigurationBuilderExtensions + public static class ConfigurationBuilderExtensions { public static IConfigurationBuilder AddConfigurationStoreForSqlServer( this IConfigurationBuilder builder, diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/Modules/PluginModule.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/Modules/PluginModule.cs index 6980d4c..91a4fa0 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/Modules/PluginModule.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/Modules/PluginModule.cs @@ -9,7 +9,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer.Modules { - public class PluginModule : Module + public class PluginModule : Module { protected override void Load(ContainerBuilder builder) { diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/Plugin.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/Plugin.cs index 24c5378..da42427 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/Plugin.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/Plugin.cs @@ -14,7 +14,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer; public class Plugin : IPlugin { private const string ConfigurationProviderName = "sqlServer"; - + public void ApplyConfiguration(string[] args, IConfigurationBuilder configBuilder) { // Nothing to do diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfiguration.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfiguration.cs index 43ff1ba..3ac3a93 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfiguration.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfiguration.cs @@ -5,11 +5,11 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer { - public class SqlServerConfigurationStore + public class SqlServerConfigurationStore { public SqlServerConfiguration SqlServer { get; set; } } - + public class SqlServerConfiguration { public string ConnectionString { get; set; } diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationChangeVersionProcessedWriter.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationChangeVersionProcessedWriter.cs index 519f152..df992c3 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationChangeVersionProcessedWriter.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationChangeVersionProcessedWriter.cs @@ -15,7 +15,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer { - public class SqlServerConfigurationChangeVersionProcessedWriter : IChangeVersionProcessedWriter + public class SqlServerConfigurationChangeVersionProcessedWriter : IChangeVersionProcessedWriter { public async Task SetProcessedChangeVersionAsync( string sourceConnectionName, @@ -44,7 +44,7 @@ public async Task SetProcessedChangeVersionAsync( using (var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SingleRow).ConfigureAwait(false)) { var currentParameter = new JObject(); - + if (await reader.ReadAsync().ConfigureAwait(false)) { string changeVersionsJson = reader.GetString("ConfigurationValue"); diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationNamedApiConnectionDetailsReader.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationNamedApiConnectionDetailsReader.cs index 4d8416a..ead47d9 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationNamedApiConnectionDetailsReader.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationNamedApiConnectionDetailsReader.cs @@ -9,7 +9,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer { - public class SqlServerConfigurationNamedApiConnectionDetailsReader : INamedApiConnectionDetailsReader + public class SqlServerConfigurationNamedApiConnectionDetailsReader : INamedApiConnectionDetailsReader { public ApiConnectionDetails GetNamedApiConnectionDetails( string apiConnectionName, diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationProvider.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationProvider.cs index d499024..a7156a8 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationProvider.cs @@ -11,7 +11,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer { - public class SqlServerConfigurationProvider : ConfigurationProvider + public class SqlServerConfigurationProvider : ConfigurationProvider { private readonly SqlServerConfigurationSource _sqlServerConfigurationSource; @@ -51,7 +51,7 @@ public override void Load() { key = key.Substring(_sqlServerConfigurationSource.ConfigurationKey.Length); } - + settings.Add(key, value); } diff --git a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationSource.cs b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationSource.cs index dc594d2..816987f 100644 --- a/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationSource.cs +++ b/src/EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer/SqlServerConfigurationSource.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.ConfigurationStore.SqlServer { - public class SqlServerConfigurationSource : IConfigurationSource + public class SqlServerConfigurationSource : IConfigurationSource { public string ConfigurationKey { get; } public string ConnectionString { get; } @@ -18,7 +18,7 @@ public SqlServerConfigurationSource(string configurationKey, string connectionSt ConfigurationKey = configurationKey.TrimEnd('/') + '/'; ConnectionString = connectionString; } - + public IConfigurationProvider Build(IConfigurationBuilder builder) { return new SqlServerConfigurationProvider(this); diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/ApiClientManagement/EdFiApiClient.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/ApiClientManagement/EdFiApiClient.cs index e66b3b5..ead21b2 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/ApiClientManagement/EdFiApiClient.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/ApiClientManagement/EdFiApiClient.cs @@ -3,26 +3,26 @@ // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. -using EdFi.Tools.ApiPublisher.Connections.Api.Configuration; -using EdFi.Tools.ApiPublisher.Core.Extensions; -using EdFi.Tools.ApiPublisher.Core.Processing; -using Newtonsoft.Json.Linq; -using Serilog; -using Serilog.Events; using System.Diagnostics; using System.Net.Http.Headers; using System.Reflection; using System.Runtime.Versioning; using System.Text; using System.Web; +using EdFi.Tools.ApiPublisher.Connections.Api.Configuration; +using EdFi.Tools.ApiPublisher.Core.Extensions; +using EdFi.Tools.ApiPublisher.Core.Processing; +using Newtonsoft.Json.Linq; +using Serilog; +using Serilog.Events; namespace EdFi.Tools.ApiPublisher.Connections.Api.ApiClientManagement { - public class EdFiApiClient : IDisposable + public class EdFiApiClient : IDisposable { private readonly string _name; private readonly ILogger _logger = Log.ForContext(typeof(EdFiApiClient)); - + private readonly HttpClient _httpClient; private readonly Timer _bearerTokenRefreshTimer; private readonly HttpClient _tokenRefreshHttpClient; @@ -35,31 +35,39 @@ public EdFiApiClient( ApiConnectionDetails apiConnectionDetails, int bearerTokenRefreshMinutes, bool ignoreSslErrors, - HttpClientHandler httpClientHandler = null) + HttpClientHandler httpClientHandler = null + ) { - ConnectionDetails = apiConnectionDetails ?? throw new ArgumentNullException(nameof(apiConnectionDetails)); + ConnectionDetails = + apiConnectionDetails ?? throw new ArgumentNullException(nameof(apiConnectionDetails)); _name = name; - string apiUrl = apiConnectionDetails.Url ?? throw new Exception("URL for API connection '{name}' was not assigned."); - - _dataManagementApiSegment - = new Lazy( - () => ConnectionDetails.SchoolYear == null - ? EdFiApiConstants.DataManagementApiSegment - : $"{EdFiApiConstants.DataManagementApiSegment}/{ConnectionDetails.SchoolYear}"); + string apiUrl = + apiConnectionDetails.Url + ?? throw new InvalidOperationException("URL for API connection '{name}' was not assigned."); + + _dataManagementApiSegment = new Lazy( + () => + ConnectionDetails.SchoolYear == null + ? EdFiApiConstants.DataManagementApiSegment + : $"{EdFiApiConstants.DataManagementApiSegment}/{ConnectionDetails.SchoolYear}" + ); - _changeQueriesApiSegment - = new Lazy( - () => ConnectionDetails.SchoolYear == null - ? EdFiApiConstants.ChangeQueriesApiSegment - : $"{EdFiApiConstants.ChangeQueriesApiSegment}/{ConnectionDetails.SchoolYear}"); + _changeQueriesApiSegment = new Lazy( + () => + ConnectionDetails.SchoolYear == null + ? EdFiApiConstants.ChangeQueriesApiSegment + : $"{EdFiApiConstants.ChangeQueriesApiSegment}/{ConnectionDetails.SchoolYear}" + ); httpClientHandler ??= new HttpClientHandler(); if (ignoreSslErrors) { +#pragma warning disable S4830 // Server certificates should be verified during SSL/TLS connections httpClientHandler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; +#pragma warning restore S4830 // Server certificates should be verified during SSL/TLS connections } _httpClient = new HttpClient(httpClientHandler) @@ -79,14 +87,16 @@ public EdFiApiClient( // Get initial bearer token for Ed-Fi ODS API RefreshBearerToken(true); - + // Refresh the bearer tokens periodically - _bearerTokenRefreshTimer = new Timer(RefreshBearerToken, + _bearerTokenRefreshTimer = new Timer( + RefreshBearerToken, false, TimeSpan.FromMinutes(bearerTokenRefreshMinutes), - TimeSpan.FromMinutes(bearerTokenRefreshMinutes)); + TimeSpan.FromMinutes(bearerTokenRefreshMinutes) + ); - void AddProductInfoToRequestHeader(HttpClient httpClient) + static void AddProductInfoToRequestHeader(HttpClient httpClient) { var assembly = Assembly.GetExecutingAssembly(); var fileVersion = FileVersionInfo.GetVersionInfo(assembly.Location); @@ -94,15 +104,19 @@ void AddProductInfoToRequestHeader(HttpClient httpClient) var productInfo = new ProductInfoHeaderValue("Ed-Fi-API-Publisher", version); var targetFrameWorkAttributes = assembly.CustomAttributes.Where(attribute => - attribute.AttributeType.Name == nameof(TargetFrameworkAttribute)); + attribute.AttributeType.Name == nameof(TargetFrameworkAttribute) + ); var customAttribute = targetFrameWorkAttributes.FirstOrDefault(); var customAttributeValue = customAttribute?.NamedArguments.FirstOrDefault(); if (customAttributeValue != null) { - var dotnetVersionValues = customAttributeValue?.TypedValue.Value.ToString().Split(' '); + var dotnetVersionValues = ((CustomAttributeNamedArgument)customAttributeValue).TypedValue.Value.ToString().Split(' '); if (dotnetVersionValues.Length > 0) { - var dotnetInfo = new ProductInfoHeaderValue(dotnetVersionValues[0], dotnetVersionValues[1]); + var dotnetInfo = new ProductInfoHeaderValue( + dotnetVersionValues[0], + dotnetVersionValues[1] + ); httpClient.DefaultRequestHeaders.UserAgent.Add(dotnetInfo); } } @@ -112,46 +126,76 @@ void AddProductInfoToRequestHeader(HttpClient httpClient) public HttpClient HttpClient => _httpClient; - private async Task GetBearerTokenAsync(HttpClient httpClient, string key, string secret, string scope) + private async Task GetBearerTokenAsync( + HttpClient httpClient, + string key, + string secret, + string scope + ) { if (_logger.IsEnabled(LogEventLevel.Debug)) - _logger.Debug($"Getting bearer token for {_name} API client with key {key.Substring(0, 3)}..."); - + _logger.Debug( + "Getting bearer token for {Name} API client with key {Key}...", + _name, + key[..3] + ); + var authRequest = new HttpRequestMessage(HttpMethod.Post, "oauth/token"); string encodedKeyAndSecret = Base64Encode($"{key}:{secret}"); - string bodyContent = "grant_type=client_credentials" - + (string.IsNullOrEmpty(scope) - ? null - : $"&scope={HttpUtility.UrlEncode(scope)}"); - - authRequest.Content = new StringContent(bodyContent, - Encoding.UTF8, "application/x-www-form-urlencoded"); + string bodyContent = + "grant_type=client_credentials" + + (string.IsNullOrEmpty(scope) ? null : $"&scope={HttpUtility.UrlEncode(scope)}"); - authRequest.Headers.Authorization = - new AuthenticationHeaderValue( - "Basic", - encodedKeyAndSecret); + authRequest.Content = new StringContent( + bodyContent, + Encoding.UTF8, + "application/x-www-form-urlencoded" + ); + + authRequest.Headers.Authorization = new AuthenticationHeaderValue("Basic", encodedKeyAndSecret); if (_logger.IsEnabled(LogEventLevel.Debug)) { if (string.IsNullOrEmpty(scope)) { - _logger.Debug($"Sending token request for {_name.ToLower()} API client to '{authRequest.Method} {authRequest.RequestUri}'..."); + _logger.Debug( + "Sending token request for {Name} API client to '{Method} {Uri}'...", + _name.ToLower(), + authRequest.Method, + authRequest.RequestUri + ); } else { - _logger.Debug($"Sending token request for {_name.ToLower()} API client to '{authRequest.Method} {authRequest.RequestUri}' with scope '{scope}'..."); + _logger.Debug( + "Sending token request for {Name} API client to '{Method} {Uri}' with scope '{Scope}'...", + _name.ToLower(), + authRequest.Method, + authRequest.RequestUri, + scope + ); } } var authResponseMessage = await httpClient.SendAsync(authRequest).ConfigureAwait(false); - string authResponseContent = await authResponseMessage.Content.ReadAsStringAsync().ConfigureAwait(false); + string authResponseContent = await authResponseMessage + .Content.ReadAsStringAsync() + .ConfigureAwait(false); if (!authResponseMessage.IsSuccessStatusCode) { - _logger.Error($"Authentication of {_name.ToLower()} API client against '{authRequest.RequestUri}' failed. {authRequest.Method} request returned status {authResponseMessage.StatusCode}:{Environment.NewLine}{authResponseContent}"); - throw new Exception($"Authentication failed for {_name.ToLower()} API client."); + _logger.Error( + "Authentication of {Name} API client against '{Uri}' failed. {Method} request returned status {StatusCode}:\r{Content}", + _name.ToLower(), + authRequest.RequestUri, + authRequest.Method, + authResponseMessage.StatusCode, + authResponseContent + ); + throw new InvalidOperationException( + $"Authentication failed for {_name.ToLower()} API client." + ); } var authResponseObject = JObject.Parse(authResponseContent); @@ -160,85 +204,127 @@ private async Task GetBearerTokenAsync(HttpClient httpClient, string key { if (scope != authResponseObject["scope"]?.Value()) { - throw new Exception($"Authentication was successful for {_name.ToLower()} API client but the requested scope of '{scope}' was not honored by the host. Remove the 'scope' parameter from the connection information for this API endpoint to proceed with an unscoped access token."); + throw new InvalidOperationException( + $"Authentication was successful for {_name.ToLower()} API client but the requested scope of '{scope}' was not honored by the host. Remove the 'scope' parameter from the connection information for this API endpoint to proceed with an unscoped access token." + ); } if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug($"Token request for {_name.ToLower()} API client with scope '{scope}' was returned by server."); + _logger.Debug( + "Token request for {Name} API client with scope '{Scope}' was returned by server.", + _name.ToLower(), + scope + ); } } - + string bearerToken = authResponseObject["access_token"].Value(); return bearerToken; } - private static string Base64Encode(string plainText) + private static string Base64Encode(string plainText) { var plainTextBytes = Encoding.UTF8.GetBytes(plainText); return Convert.ToBase64String(plainTextBytes); } - + private void RefreshBearerToken(object state) { try { - bool isInitializing = ((bool?) state).GetValueOrDefault(); - + bool isInitializing = ((bool?)state).GetValueOrDefault(); + if (isInitializing) { - _logger.Information($"Retrieving initial bearer token for {_name.ToLower()} API client."); + _logger.Information( + "Retrieving initial bearer token for {Name} API client.", + _name.ToLower() + ); } else { - _logger.Information($"Refreshing bearer token for {_name.ToLower()} API client."); + _logger.Information("Refreshing bearer token for {Name} API client.", _name.ToLower()); } try { - var bearerToken = GetBearerTokenAsync(_tokenRefreshHttpClient, ConnectionDetails.Key, ConnectionDetails.Secret, ConnectionDetails.Scope) - .ConfigureAwait(false).GetAwaiter().GetResult(); - - HttpClient.DefaultRequestHeaders.Authorization = - AuthenticationHeaderValue.Parse($"Bearer {bearerToken}"); + var bearerToken = GetBearerTokenAsync( + _tokenRefreshHttpClient, + ConnectionDetails.Key, + ConnectionDetails.Secret, + ConnectionDetails.Scope + ) + .ConfigureAwait(false) + .GetAwaiter() + .GetResult(); + + HttpClient.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue.Parse( + $"Bearer {bearerToken}" + ); if (isInitializing) { - _logger.Information($"Bearer token retrieved successfully for {_name.ToLower()} API client."); + _logger.Information( + "Bearer token retrieved successfully for {Name} API client.", + _name.ToLower() + ); } else { - _logger.Information($"Bearer token refreshed successfully for {_name.ToLower()} API client."); + _logger.Information( + "Bearer token refreshed successfully for {Name} API client.", + _name.ToLower() + ); } } catch (Exception ex) { if (isInitializing) { - throw new Exception($"Unable to obtain initial bearer token for {_name.ToLower()} API client.", ex); + throw new InvalidOperationException( + $"Unable to obtain initial bearer token for {_name.ToLower()} API client.", + ex + ); } - - _logger.Error($"Refresh of bearer token failed for {_name.ToLower()} API client. Token may expire soon resulting in 401 responses.{Environment.NewLine}{ex}"); + + _logger.Error( + ex, + "Refresh of bearer token failed for {Name} API client. Token may expire soon resulting in 401 responses.", + _name.ToLower() + ); } } catch (Exception ex) { - _logger.Error($"An unhandled exception occurred during bearer token refresh. Token expiration may occur. {ex}"); + _logger.Error( + ex, + "An unhandled exception occurred during bearer token refresh. Token expiration may occur." + ); } } public ApiConnectionDetails ConnectionDetails { get; } public string DataManagementApiSegment => _dataManagementApiSegment.Value; - + public string ChangeQueriesApiSegment => _changeQueriesApiSegment.Value; public void Dispose() { - _httpClient?.Dispose(); - _bearerTokenRefreshTimer?.Dispose(); - _tokenRefreshHttpClient?.Dispose(); + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + _httpClient?.Dispose(); + _bearerTokenRefreshTimer?.Dispose(); + _tokenRefreshHttpClient?.Dispose(); + } } } } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/ApiClientManagement/EdFiApiClientProvider.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/ApiClientManagement/EdFiApiClientProvider.cs index ed86f5f..42804e0 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/ApiClientManagement/EdFiApiClientProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/ApiClientManagement/EdFiApiClientProvider.cs @@ -12,18 +12,18 @@ public class EdFiApiClientProvider : ISourceEdFiApiClientProvider, ITargetEdFiAp private readonly Lazy _apiClient; private readonly ILogger _logger = Log.ForContext(typeof(EdFiApiClientProvider)); - + public EdFiApiClientProvider(Lazy apiClient) { _apiClient = apiClient; } - + public EdFiApiClient GetApiClient() { if (!_apiClient.IsValueCreated) { // Establish connection to API - _logger.Information($"Initializing API client '{_apiClient.Value.ConnectionDetails.Name}'..."); + _logger.Information("Initializing API client '{ConnectionDetailsName}'...", _apiClient.Value.ConnectionDetails.Name); } return _apiClient.Value; diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/ApiConnectionDetails.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/ApiConnectionDetails.cs index a1dbee5..5ca56de 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/ApiConnectionDetails.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/ApiConnectionDetails.cs @@ -14,49 +14,6 @@ public class ApiConnectionDetails : SourceConnectionDetailsBase, ISourceConnecti public string Scope { get; set; } public int? SchoolYear { get; set; } - [Obsolete( - "The 'Resources' configuration setting has been replaced by 'Include'. Adjust your connection configuration appropriately and try again.")] - public string Resources - { - get => null; - set - { - if (!string.IsNullOrWhiteSpace(value)) - { - throw new Exception( - "The 'Connections:Source:Resources' configuration setting has been replaced by 'Connections:Source:Include'. Adjust your connection configuration appropriately and try again."); - } - } - } - - [Obsolete( - "The 'ExcludeResources' configuration setting has been replaced by 'Exclude'. Adjust your connection configuration appropriately and try again.")] - public string ExcludeResources - { - get => null; - set - { - if (!string.IsNullOrWhiteSpace(value)) - { - throw new Exception( - "The 'Connections:Source:ExcludeResources' configuration setting has been replaced by 'Connections:Source:Exclude'. Adjust your connection configuration appropriately and try again."); - } - } - } - - [Obsolete("The 'SkipResources' configuration setting has been replaced by 'ExcludeOnly'. Adjust your connection configuration appropriately and try again.")] - public string SkipResources - { - get => null; - set - { - if (!string.IsNullOrWhiteSpace(value)) - { - throw new Exception("The 'Connections:Source:SkipResources' configuration setting has been replaced by 'Connections:Source:ExcludeOnly'. Adjust your connection configuration appropriately and try again."); - } - } - } - public bool? TreatForbiddenPostAsWarning { get; set; } public string ProfileName { get; set; } @@ -72,12 +29,12 @@ public IEnumerable MissingConfigurationValues() { yield return "Url"; } - + if (Key == null) { yield return "Key"; } - + if (Secret == null) { yield return "Secret"; diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/Enhancers/EdFiApiConnectionsConfigurationBuilderEnhancer.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/Enhancers/EdFiApiConnectionsConfigurationBuilderEnhancer.cs index c63aed5..cafd791 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/Enhancers/EdFiApiConnectionsConfigurationBuilderEnhancer.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/Enhancers/EdFiApiConnectionsConfigurationBuilderEnhancer.cs @@ -10,7 +10,7 @@ namespace EdFi.Tools.ApiPublisher.Connections.Api.Configuration.Enhancers { - public class EdFiApiConnectionsConfigurationBuilderEnhancer : IConfigurationBuilderEnhancer + public class EdFiApiConnectionsConfigurationBuilderEnhancer : IConfigurationBuilderEnhancer { private readonly ILogger _logger = Log.Logger.ForContext(typeof(EdFiApiConnectionsConfigurationBuilderEnhancer)); private readonly INamedApiConnectionDetailsReader _namedApiConnectionDetailsReader; @@ -23,7 +23,7 @@ public EdFiApiConnectionsConfigurationBuilderEnhancer(INamedApiConnectionDetails public void Enhance(IConfigurationRoot initialConfiguration, IConfigurationBuilder configurationBuilder) { var connectionsConfiguration = initialConfiguration.GetSection("Connections"); - + var sourceConnectionConfiguration = connectionsConfiguration.GetSection("Source"); var sourceConnectionDetails = sourceConnectionConfiguration.Get(); @@ -70,31 +70,25 @@ IEnumerable> GetEnhancedConnectionConfigurationValu ApiConnectionDetails connection, ConnectionRole connectionType) { - // Get additional named configuration values for source/target, if necessary - // if (!connection.IsFullyDefined()) - // { - _logger.Debug($"{connectionType} connection details are not fully defined."); - - if (string.IsNullOrEmpty(connection.Name)) - { - throw new ArgumentException( + _logger.Debug("{ConnectionType} connection details are not fully defined.", connectionType); + + if (string.IsNullOrEmpty(connection.Name)) + { + throw new ArgumentException( $"{connectionType} connection details were not available and no connection name was supplied."); - } + } - var configurationValues = CreateNamedConnectionConfigurationValues(connection.Name, connectionType).ToArray(); + var configurationValues = CreateNamedConnectionConfigurationValues(connection.Name, connectionType).ToArray(); - return configurationValues; - // } - // - // return Enumerable.Empty>(); + return configurationValues; } IEnumerable> CreateNamedConnectionConfigurationValues( string apiConnectionName, ConnectionRole connectionRole) { - _logger.Debug( - $"Obtaining {connectionRole.ToString().ToLower()} API connection details for connection '{apiConnectionName}' using '{_namedApiConnectionDetailsReader.GetType().Name}'."); + _logger.Debug("Obtaining {ConnectionRole} API connection details for connection '{ApiConnectionName}' using '{NamedApiConnectionDetailsReaderName}'.", + connectionRole.ToString().ToLower(), apiConnectionName, _namedApiConnectionDetailsReader.GetType().Name); var namedApiConnectionDetails = _namedApiConnectionDetailsReader.GetNamedApiConnectionDetails(apiConnectionName, configurationStoreSection); diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/INamedApiConnectionDetailsReader.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/INamedApiConnectionDetailsReader.cs index c12a091..5b30cef 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/INamedApiConnectionDetailsReader.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Configuration/INamedApiConnectionDetailsReader.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.Connections.Api.Configuration { - public interface INamedApiConnectionDetailsReader + public interface INamedApiConnectionDetailsReader { ApiConnectionDetails GetNamedApiConnectionDetails( string apiConnectionName, diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/DependencyResolution/ApiSourceResourceItemProvider.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/DependencyResolution/ApiSourceResourceItemProvider.cs index 52f0847..72d98ce 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/DependencyResolution/ApiSourceResourceItemProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/DependencyResolution/ApiSourceResourceItemProvider.cs @@ -21,9 +21,9 @@ public class ApiSourceResourceItemProvider : ISourceResourceItemProvider private readonly ISourceEdFiApiClientProvider _sourceEdFiApiClientProvider; private readonly IRateLimiting _rateLimiter; private readonly Options _options; - + private readonly ILogger _logger = Log.ForContext(typeof(ApiSourceResourceItemProvider)); - + public ApiSourceResourceItemProvider(ISourceEdFiApiClientProvider sourceEdFiApiClientProvider, Options options, IRateLimiting rateLimiter = null) { _sourceEdFiApiClientProvider = sourceEdFiApiClientProvider; @@ -51,8 +51,8 @@ public ApiSourceResourceItemProvider(ISourceEdFiApiClientProvider sourceEdFiApiC getByIdDelay, (result, ts, retryAttempt, ctx) => { - _logger.Warning( - $"Retrying GET for resource item '{resourceItemUrl}' from source failed with status '{result.Result.StatusCode}'. Retrying... (retry #{retryAttempt} of {_options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay)"); + _logger.Warning("Retrying GET for resource item '{ResourceItemUrl}' from source failed with status '{StatusCode}'. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay)", + resourceItemUrl, result.Result.StatusCode, retryAttempt, _options.MaxRetryAttempts, ts.TotalSeconds); }); IAsyncPolicy policy = isRateLimitingEnabled ? Policy.WrapAsync(_rateLimiter?.GetRateLimitingPolicy(), retryPolicy) : retryPolicy; try @@ -62,13 +62,10 @@ public ApiSourceResourceItemProvider(ISourceEdFiApiClientProvider sourceEdFiApiC { getByIdAttempts++; - if (getByIdAttempts > 1) + if (getByIdAttempts > 1 && _logger.IsEnabled(LogEventLevel.Debug)) { - if (_logger.IsEnabled(LogEventLevel.Debug)) - { - _logger.Debug( - $"GET for missing dependency '{resourceItemUrl}' reference from source attempt #{getByIdAttempts}."); - } + _logger.Debug("GET for missing dependency '{ResourceItemUrl}' reference from source attempt #{GetByIdAttempts}.", + resourceItemUrl, getByIdAttempts); } return sourceEdFiApiClient.HttpClient.GetAsync( @@ -91,83 +88,19 @@ public ApiSourceResourceItemProvider(ISourceEdFiApiClientProvider sourceEdFiApiC if (getByIdResponse.StatusCode == HttpStatusCode.OK) { return (true, responseContent); - - // string missingItemJson = await getByIdResponse.Content.ReadAsStringAsync(); - // - // return missingItemJson; - - /* - var missingItemDelay = Backoff.ExponentialBackoff( - TimeSpan.FromMilliseconds(_options.RetryStartingDelayMilliseconds), - _options.MaxRetryAttempts); - - if (_logger.IsEnabled(LogEventLevel.Debug)) - { - _logger.Debug( - $"{resourceUrl}: Attempting to POST missing '{referencedResourceName}' reference to the target."); - } - - // Post the resource to target now - var missingItemPostResponse = await Policy - .HandleResult(r => r.StatusCode.IsPotentiallyTransientFailure()) - .WaitAndRetryAsync( - missingItemDelay, - (result, ts, retryAttempt, ctx) => - { - _logger.Warning( - $"{resourceUrl}: Retrying POST for missing '{referencedResourceName}' reference against target failed with status '{result.Result.StatusCode}'. Retrying... (retry #{retryAttempt} of {_options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay)"); - }) - .ExecuteAsync( - (ctx, ct) => - { - getByIdAttempts++; - - if (getByIdAttempts > 1) - { - if (_logger.IsEnabled(LogEventLevel.Debug)) - { - _logger.Debug( - $"{resourceUrl}: GET for missing '{referencedResourceName}' reference from source attempt #{getByIdAttempts}."); - } - } - - return targetEdFiApiClient.HttpClient.PostAsync( - $"{targetEdFiApiClient.DataManagementApiSegment}{missingDependencyResourcePath}", - new StringContent( - missingItem.ToString(Formatting.None), - Encoding.UTF8, - "application/json"), - ct); - }, - new Context(), - CancellationToken.None); - - if (!missingItemPostResponse.IsSuccessStatusCode) - { - string responseContent = - await getByIdResponse.Content.ReadAsStringAsync().ConfigureAwait(false); - - _logger.Error( - $"{resourceUrl}: POST of missing '{referencedResourceName}' reference to the target returned status '{missingItemPostResponse.StatusCode}': {responseContent}."); - } - else - { - _logger.Information( - $"{resourceUrl}: POST of missing '{referencedResourceName}' reference to the target returned status '{missingItemPostResponse.StatusCode}'."); - } - */ } else { - _logger.Warning( - $"GET request from source API for '{resourceItemUrl}' reference failed with status '{getByIdResponse.StatusCode}': {responseContent}"); + _logger.Warning("GET request from source API for '{ResourceItemUrl}' reference failed with status '{StatusCode}': {ResponseContent}", + resourceItemUrl, getByIdResponse.StatusCode, responseContent); return (false, null); } } - catch (RateLimitRejectedException) + catch (RateLimitRejectedException ex) { - _logger.Fatal($"{sourceEdFiApiClient.DataManagementApiSegment}{resourceItemUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal(ex, "{DataManagementApiSegment}{ResourceItemUrl}: Rate limit exceeded. Please try again later.", + sourceEdFiApiClient.DataManagementApiSegment, resourceItemUrl); return (false, null); } //---------------------------------------------------------------------------------------------- diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/EdFi.Tools.ApiPublisher.Connections.Api.csproj b/src/EdFi.Tools.ApiPublisher.Connections.Api/EdFi.Tools.ApiPublisher.Connections.Api.csproj index e897712..8308cd0 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/EdFi.Tools.ApiPublisher.Connections.Api.csproj +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/EdFi.Tools.ApiPublisher.Connections.Api.csproj @@ -1,14 +1,24 @@ - + net8.0 enable + true + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + - \ No newline at end of file + diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Helpers/RequestHelpers.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Helpers/RequestHelpers.cs index 95ed3bb..da34bc3 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Helpers/RequestHelpers.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Helpers/RequestHelpers.cs @@ -15,8 +15,8 @@ namespace EdFi.Tools.ApiPublisher.Connections.Api.Helpers; public static class RequestHelpers { private static readonly char[] _pathSeparatorChars = { '/' }; - private static readonly ConcurrentDictionary _writableContentTypeByResourceUrl = new (); - private static readonly ConcurrentDictionary _readableContentTypeByResourceUrl = new (); + private static readonly ConcurrentDictionary _writableContentTypeByResourceUrl = new(); + private static readonly ConcurrentDictionary _readableContentTypeByResourceUrl = new(); /// /// Sends a POST request (applying a Profile content type via the Content-Type header, if appropriate). @@ -60,7 +60,7 @@ public static async Task SendPostRequestAsync( new StringContent(requestBodyJson, Encoding.UTF8, "application/json"), ct); } - + /// /// Sends a GET request (applying an applied Profile content type via the Accept header, if appropriate). /// @@ -102,7 +102,7 @@ public static async Task SendGetRequestAsync( private static bool ShouldApplyProfileContentType(string requestUri) { var uri = new Uri(requestUri); - + // Don't apply Profiles to deletes requests if (uri.LocalPath.EndsWith("/deletes")) { @@ -114,7 +114,7 @@ private static bool ShouldApplyProfileContentType(string requestUri) { return false; } - + // Don't apply Profiles to descriptors requests if (uri.LocalPath.EndsWith("Descriptors")) { diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Metadata/Dependencies/EdFiApiGraphMLDependencyMetadataProvider.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Metadata/Dependencies/EdFiApiGraphMLDependencyMetadataProvider.cs index 0c80a69..ce8f39c 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Metadata/Dependencies/EdFiApiGraphMLDependencyMetadataProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Metadata/Dependencies/EdFiApiGraphMLDependencyMetadataProvider.cs @@ -17,7 +17,7 @@ namespace EdFi.Tools.ApiPublisher.Connections.Api.Metadata.Dependencies; public class EdFiApiGraphMLDependencyMetadataProvider : IGraphMLDependencyMetadataProvider { private readonly IEdFiApiClientProvider _edFiApiClientProvider; - + private readonly ILogger _logger = Log.ForContext(typeof(EdFiApiGraphMLDependencyMetadataProvider)); public EdFiApiGraphMLDependencyMetadataProvider(IEdFiApiClientProvider edFiApiClientProvider) @@ -28,12 +28,13 @@ public EdFiApiGraphMLDependencyMetadataProvider(IEdFiApiClientProvider edFiApiCl public async Task<(XElement, XNamespace)> GetDependencyMetadataAsync() { var edFiApiClient = _edFiApiClientProvider.GetApiClient(); - + string dependenciesRequestUri = $"metadata/{edFiApiClient.DataManagementApiSegment}/dependencies"; // Get the resource dependencies from the target - _logger.Information($"Getting dependencies from API at {edFiApiClient.HttpClient.BaseAddress}{dependenciesRequestUri}..."); - + _logger.Information("Getting dependencies from API at {BaseAddress}{DependenciesRequestUri}...", + edFiApiClient.HttpClient.BaseAddress, dependenciesRequestUri); + var dependencyRequest = new HttpRequestMessage(HttpMethod.Get, dependenciesRequestUri); dependencyRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/graphml")); var dependencyResponse = await edFiApiClient.HttpClient.SendAsync(dependencyRequest).ConfigureAwait(false); @@ -42,10 +43,11 @@ public EdFiApiGraphMLDependencyMetadataProvider(IEdFiApiClientProvider edFiApiCl if (!dependencyResponse.IsSuccessStatusCode) { - _logger.Error($"Ed-Fi ODS API request for dependencies to '{dependencyRequest.RequestUri}' returned '{dependencyResponse.StatusCode}' with content:{Environment.NewLine}{dependencyResponseContent}"); + _logger.Error("Ed-Fi ODS API request for dependencies to '{RequestUri}' returned '{StatusCode}' with content:{NewLine}{DependencyResponseContent}", + dependencyRequest.RequestUri, dependencyResponse.StatusCode, Environment.NewLine, dependencyResponseContent); throw new Exception("Resource dependencies could not be obtained."); } - + XNamespace ns = "http://graphml.graphdrawing.org/xmlns"; try @@ -55,7 +57,8 @@ public EdFiApiGraphMLDependencyMetadataProvider(IEdFiApiClientProvider edFiApiCl } catch (Exception ex) { - _logger.Error($"Unable to parse dependency response as GraphML: {dependencyResponseContent}{Environment.NewLine}{ex}"); + _logger.Error(ex, "Unable to parse dependency response as GraphML: {DependencyResponseContent}{NewLine}{Ex}", + dependencyResponseContent, Environment.NewLine, ex); throw new Exception("Resource dependencies could not be obtained."); } } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Metadata/Versioning/EdFiApiVersionMetadataProviderBase.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Metadata/Versioning/EdFiApiVersionMetadataProviderBase.cs index 19acfbd..b32c261 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Metadata/Versioning/EdFiApiVersionMetadataProviderBase.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Metadata/Versioning/EdFiApiVersionMetadataProviderBase.cs @@ -16,12 +16,12 @@ public class EdFiApiVersionMetadataProviderBase private readonly IEdFiApiClientProvider _edFiApiClientProvider; private readonly ILogger _logger; - + protected EdFiApiVersionMetadataProviderBase(string role, IEdFiApiClientProvider edFiApiClientProvider) { _role = role; _edFiApiClientProvider = edFiApiClientProvider; - + _logger = Log.ForContext(GetType()); } @@ -33,11 +33,11 @@ public async Task GetVersionMetadata() { throw new Exception($"{_role} API at '{_edFiApiClientProvider.GetApiClient().HttpClient.BaseAddress}' returned status code '{versionResponse.Result.StatusCode}' for request for version information."); } - + string responseJson = await versionResponse.Result.Content.ReadAsStringAsync().ConfigureAwait(false); return GetVersionObject(responseJson); - + JObject GetVersionObject(string versionJson) { JObject versionObject; @@ -45,7 +45,8 @@ JObject GetVersionObject(string versionJson) try { versionObject = JObject.Parse(versionJson); - _logger.Information($"{_role} version information: {versionObject.ToString(Formatting.Indented)}"); + _logger.Information("{Role} version information: {VersionObject}", + _role, versionObject.ToString(Formatting.Indented)); } catch (Exception) { diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs index 9a6475b..1c554fb 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsSourceModule.cs @@ -36,11 +36,11 @@ public EdFiApiAsSourceModule(IConfigurationRoot finalConfiguration) { _finalConfiguration = finalConfiguration; } - + protected override void Load(ContainerBuilder builder) { var options = _finalConfiguration.Get().Options; - + // Initialize source/target API clients var connectionsConfiguration = _finalConfiguration.GetSection("Connections"); var sourceConnectionConfiguration = connectionsConfiguration.GetSection("Source"); @@ -59,7 +59,7 @@ protected override void Load(ContainerBuilder builder) builder.RegisterInstance(new EdFiApiClientProvider(sourceEdFiApiClient)) .As() .SingleInstance(); - + // Available ChangeVersions for Source API builder.RegisterType() .As() @@ -108,19 +108,19 @@ protected override void Load(ContainerBuilder builder) .As() .SingleInstance(); } - + // Register handler to perform page-based requests against a Source API builder.RegisterType() .As() .WithParameter("rateLimiter", rateLimiter) .SingleInstance(); - + // Register Data Source Total Count provider for Source API builder.RegisterType() .As() .WithParameter("rateLimiter", rateLimiter) .SingleInstance(); - + // API dependency metadata from Ed-Fi ODS API (using Source API) if (options.UseSourceDependencyMetadata) { diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsTargetModule.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsTargetModule.cs index ab9dddf..f48f960 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsTargetModule.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/EdFiApiAsTargetModule.cs @@ -28,11 +28,11 @@ public EdFiApiAsTargetModule(IConfigurationRoot finalConfiguration) { _finalConfiguration = finalConfiguration; } - + protected override void Load(ContainerBuilder builder) { var options = _finalConfiguration.Get().Options; - + // Initialize source/target API clients var connectionsConfiguration = _finalConfiguration.GetSection("Connections"); var targetConnectionConfiguration = connectionsConfiguration.GetSection("Target"); @@ -40,7 +40,7 @@ protected override void Load(ContainerBuilder builder) var rateLimiter = new PollyRateLimiter(options); builder.RegisterInstance(targetApiConnectionDetails).As(); - + var targetEdFiApiClient = new Lazy( () => new EdFiApiClient( "Target", @@ -51,7 +51,7 @@ protected override void Load(ContainerBuilder builder) builder.RegisterInstance(new EdFiApiClientProvider(targetEdFiApiClient)) .As() .SingleInstance(); - + // Version metadata for a Target API builder.RegisterType() .As() @@ -84,7 +84,7 @@ protected override void Load(ContainerBuilder builder) .As>() .WithParameter("rateLimiter", rateLimiter) .SingleInstance(); - + // Register the processing stage initiators builder.RegisterType().Keyed(PublishingStage.KeyChanges); builder.RegisterType().Keyed(PublishingStage.Upserts); diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/PluginModule.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/PluginModule.cs index d460c35..57bc48e 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/PluginModule.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Modules/PluginModule.cs @@ -18,7 +18,7 @@ protected override void Load(ContainerBuilder builder) { builder.RegisterType() .Named(Plugin.ApiConnectionType); - + builder.RegisterType() .As() .SingleInstance(); diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Capabilities/EdFiApiSourceCapabilities.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Capabilities/EdFiApiSourceCapabilities.cs index 95a6d4a..8c01a79 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Capabilities/EdFiApiSourceCapabilities.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Capabilities/EdFiApiSourceCapabilities.cs @@ -15,29 +15,30 @@ public class EdFiApiSourceCapabilities : ISourceCapabilities private readonly ISourceEdFiApiClientProvider _sourceEdFiApiClientProvider; private readonly ILogger _logger = Log.ForContext(typeof(EdFiApiSourceCapabilities)); - + public EdFiApiSourceCapabilities(ISourceEdFiApiClientProvider sourceEdFiApiClientProvider) { _sourceEdFiApiClientProvider = sourceEdFiApiClientProvider; } - + public async Task SupportsKeyChangesAsync(string probeResourceKey) { var edFiApiClient = _sourceEdFiApiClientProvider.GetApiClient(); - + string probeUrl = $"{edFiApiClient.DataManagementApiSegment}{probeResourceKey}{EdFiApiConstants.KeyChangesPathSuffix}"; - _logger.Debug($"Probing source API for key changes support at '{probeUrl}'."); + _logger.Debug("Probing source API for key changes support at '{ProbeUrl}'.", probeUrl); var probeResponse = await edFiApiClient.HttpClient.GetAsync($"{probeUrl}?limit=1").ConfigureAwait(false); if (probeResponse.IsSuccessStatusCode) { - _logger.Debug($"Probe response status was '{probeResponse.StatusCode}'."); + _logger.Debug("Probe response status was '{StatusCode}'.", probeResponse.StatusCode); return true; } - _logger.Warning($"Request to Source API for the '{EdFiApiConstants.KeyChangesPathSuffix}' child resource was unsuccessful (response status was '{probeResponse.StatusCode}'). Key change processing cannot be performed."); + _logger.Warning("Request to Source API for the '{KeyChangesPathSuffix}' child resource was unsuccessful (response status was '{StatusCode}'). Key change processing cannot be performed.", + EdFiApiConstants.KeyChangesPathSuffix, probeResponse.StatusCode); return false; } @@ -49,17 +50,18 @@ public async Task SupportsDeletesAsync(string probeResourceKey) // Probe for deletes support string probeUrl = $"{edFiApiClient.DataManagementApiSegment}{probeResourceKey}{EdFiApiConstants.DeletesPathSuffix}"; - _logger.Debug($"Probing source API for deletes support at '{probeUrl}'."); + _logger.Debug("Probing source API for deletes support at '{ProbeUrl}'.", probeUrl); var probeResponse = await edFiApiClient.HttpClient.GetAsync($"{probeUrl}?limit=1").ConfigureAwait(false); if (probeResponse.IsSuccessStatusCode) { - _logger.Debug($"Probe response status was '{probeResponse.StatusCode}'."); + _logger.Debug("Probe response status was '{StatusCode}'.", probeResponse.StatusCode); return true; } - _logger.Warning($"Request to Source API for the '{EdFiApiConstants.DeletesPathSuffix}' child resource was unsuccessful (response status was '{probeResponse.StatusCode}'). Delete processing cannot be performed."); + _logger.Warning("Request to Source API for the '{DeletesPathSuffix}' child resource was unsuccessful (response status was '{StatusCode}'). Delete processing cannot be performed.", + EdFiApiConstants.DeletesPathSuffix, probeResponse.StatusCode); return false; } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Counting/EdFiApiSourceTotalCountProvider.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Counting/EdFiApiSourceTotalCountProvider.cs index f85871c..ee2a7a1 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Counting/EdFiApiSourceTotalCountProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Counting/EdFiApiSourceTotalCountProvider.cs @@ -3,6 +3,8 @@ // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. +using System.Net; +using System.Threading.Tasks.Dataflow; using EdFi.Tools.ApiPublisher.Connections.Api.ApiClientManagement; using EdFi.Tools.ApiPublisher.Connections.Api.Helpers; using EdFi.Tools.ApiPublisher.Core.Configuration; @@ -14,12 +16,8 @@ using Polly; using Polly.Contrib.WaitAndRetry; using Polly.RateLimit; -using Polly.RateLimiting; -using Polly.Retry; using Serilog; using Serilog.Events; -using System.Net; -using System.Threading.Tasks.Dataflow; namespace EdFi.Tools.ApiPublisher.Connections.Api.Processing.Source.Counting; @@ -31,83 +29,130 @@ public class EdFiApiSourceTotalCountProvider : ISourceTotalCountProvider private readonly IRateLimiting _rateLimiter; - public EdFiApiSourceTotalCountProvider(ISourceEdFiApiClientProvider sourceEdFiApiClientProvider, IRateLimiting rateLimiter =null) + public EdFiApiSourceTotalCountProvider( + ISourceEdFiApiClientProvider sourceEdFiApiClientProvider, + IRateLimiting rateLimiter = null + ) { _sourceEdFiApiClientProvider = sourceEdFiApiClientProvider; _rateLimiter = rateLimiter; } - public async Task<(bool, long)> TryGetTotalCountAsync(string resourceUrl, Options options, ChangeWindow changeWindow, ITargetBlock errorHandlingBlock, CancellationToken cancellationToken) + public async Task<(bool, long)> TryGetTotalCountAsync( + string resourceUrl, + Options options, + ChangeWindow changeWindow, + ITargetBlock errorHandlingBlock, + CancellationToken cancellationToken + ) { var edFiApiClient = _sourceEdFiApiClientProvider.GetApiClient(); // Source-specific: Ed-Fi ODS API - string changeWindowQueryStringParameters = ApiRequestHelper.GetChangeWindowQueryStringParameters(changeWindow); + string changeWindowQueryStringParameters = ApiRequestHelper.GetChangeWindowQueryStringParameters( + changeWindow + ); var delay = Backoff.ExponentialBackoff( TimeSpan.FromMilliseconds(options.RetryStartingDelayMilliseconds), - options.MaxRetryAttempts); + options.MaxRetryAttempts + ); int attempt = 0; + // Rate Limit bool isRateLimitingEnabled = options.EnableRateLimit; - + var retryPolicy = Policy .HandleResult(r => r.StatusCode.IsPotentiallyTransientFailure()) - .WaitAndRetryAsync(delay, (result, ts, retryAttempt, ctx) => - { - _logger.Warning( - $"{resourceUrl}: Getting item count from source failed with status '{result.Result.StatusCode}'. Retrying... (retry #{retryAttempt} of {options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay)"); - }); - IAsyncPolicy policy = isRateLimitingEnabled ? Policy.WrapAsync(_rateLimiter?.GetRateLimitingPolicy(), retryPolicy) : retryPolicy; + .WaitAndRetryAsync( + delay, + (result, ts, retryAttempt, ctx) => + { + _logger.Warning( + "{Url}: Getting item count from source failed with status '{StatusCode}'. Retrying... (retry #{RetryAttempt} of {MaxNumber} with {Seconds:N1}s delay)", + resourceUrl, + result.Result.StatusCode, + retryAttempt, + options.MaxRetryAttempts, + ts.TotalSeconds + ); + } + ); + IAsyncPolicy policy = isRateLimitingEnabled + ? Policy.WrapAsync(_rateLimiter?.GetRateLimitingPolicy(), retryPolicy) + : retryPolicy; try { - var apiResponse = await policy - .ExecuteAsync(async (ctx, ct) => + var apiResponse = await policy.ExecuteAsync( + async (ctx, ct) => { attempt++; if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug($"{resourceUrl}): Getting item count from source (attempt #{attempt})..."); + _logger.Debug( + "{Url}): Getting item count from source (attempt #{Attempt})...", + resourceUrl, + attempt + ); } string requestUri = $"{edFiApiClient.DataManagementApiSegment}{resourceUrl}?offset=0&limit=1&totalCount=true{changeWindowQueryStringParameters}"; - return await RequestHelpers.SendGetRequestAsync(edFiApiClient, resourceUrl, requestUri, ct); - }, new Context(), cancellationToken); + + return await RequestHelpers.SendGetRequestAsync( + edFiApiClient, + resourceUrl, + requestUri, + ct + ); + }, + new Context(), + cancellationToken + ); string responseContent = null; if (!apiResponse.IsSuccessStatusCode) { _logger.Error( - $"{resourceUrl}: Count request returned {apiResponse.StatusCode}{Environment.NewLine}{responseContent}"); + "{Url}: Count request returned {StatusCode}\r{Content}", + resourceUrl, + apiResponse.StatusCode, + responseContent + ); await HandleResourceCountRequestErrorAsync(resourceUrl, errorHandlingBlock, apiResponse) .ConfigureAwait(false); // Allow processing to continue with no additional work on this resource - return (false, 0); // Enumerable.Empty>(); + return (false, 0); } // Try to get the count header from the response if (!apiResponse.Headers.TryGetValues("total-count", out IEnumerable headerValues)) { _logger.Warning( - $"{resourceUrl}: Unable to obtain total count because Total-Count header was not returned by the source API -- skipping item processing, but overall processing will fail."); + "{Url}: Unable to obtain total count because Total-Count header was not returned by the source API -- skipping item processing, but overall processing will fail.", + resourceUrl + ); // Publish an error for the resource. Feature is not supported. await HandleResourceCountRequestErrorAsync(resourceUrl, errorHandlingBlock, apiResponse) .ConfigureAwait(false); // Allow processing to continue as best it can with no additional work on this resource - return (false, 0); // Enumerable.Empty>(); + return (false, 0); } string totalCountHeaderValue = headerValues.First(); - _logger.Debug($"{resourceUrl}: Total count header value = {totalCountHeaderValue}"); + _logger.Debug( + "{Url}: Total count header value = {TotalCount}", + resourceUrl, + totalCountHeaderValue + ); try { @@ -115,11 +160,15 @@ await HandleResourceCountRequestErrorAsync(resourceUrl, errorHandlingBlock, apiR return (true, totalCount); } - catch (Exception) + catch (Exception ex) { // Publish an error for the resource to allow processing to continue, but to force failure. _logger.Error( - $"{resourceUrl}: Unable to convert Total-Count header value of '{totalCountHeaderValue}' returned by the source API to an integer."); + ex, + "{Url}: Unable to convert Total-Count header value of '{TotalCount}' returned by the source API to an integer.", + resourceUrl, + totalCountHeaderValue + ); errorHandlingBlock.Post( new ErrorItemMessage @@ -128,15 +177,16 @@ await HandleResourceCountRequestErrorAsync(resourceUrl, errorHandlingBlock, apiR Method = HttpMethod.Get.ToString(), ResponseStatus = apiResponse.StatusCode, ResponseContent = $"Total-Count: {totalCountHeaderValue}", - }); + } + ); // Allow processing to continue without performing additional work on this resource. return (false, 0); } } - catch (RateLimitRejectedException) + catch (RateLimitRejectedException ex) { - _logger.Fatal($"{edFiApiClient.DataManagementApiSegment}{resourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal(ex, "{Segment}{Url}: Rate limit exceeded. Please try again later.", edFiApiClient.DataManagementApiSegment, resourceUrl); return (false, 0); } } @@ -144,37 +194,42 @@ await HandleResourceCountRequestErrorAsync(resourceUrl, errorHandlingBlock, apiR private async Task HandleResourceCountRequestErrorAsync( string resourceUrl, ITargetBlock errorHandlingBlock, - HttpResponseMessage apiResponse) + HttpResponseMessage apiResponse + ) { string responseContent = await apiResponse.Content.ReadAsStringAsync().ConfigureAwait(false); // Was this an authorization failure? - if (apiResponse.StatusCode == HttpStatusCode.Forbidden) + var authFailure = apiResponse.StatusCode == HttpStatusCode.Forbidden; + var isDescriptor = ResourcePathHelper.IsDescriptor(resourceUrl); + if (authFailure && isDescriptor) { - // Is this a descriptor resource? - if (ResourcePathHelper.IsDescriptor(resourceUrl)) - { - // Being denied read access to descriptors is potentially problematic, but is not considered - // to be breaking in its own right for change processing. We'll fail downstream - // POSTs if descriptors haven't been initialized correctly on the target. - _logger.Warning( - $"{resourceUrl}: {apiResponse.StatusCode} - Unable to obtain total count for descriptor due to authorization failure. Descriptor values will not be published to the target, but processing will continue.{Environment.NewLine}Response content: {responseContent}"); - - return; - } + // Being denied read access to descriptors is potentially problematic, but is not considered + // to be breaking in its own right for change processing. We'll fail downstream + // POSTs if descriptors haven't been initialized correctly on the target. + _logger.Warning( + "{Url}: {StatusCode} - Unable to obtain total count for descriptor due to authorization failure. Descriptor values will not be published to the target, but processing will continue.\rResponse content: {Content}", + resourceUrl, apiResponse.StatusCode, responseContent + ); + + return; } _logger.Error( - $"{resourceUrl}: {apiResponse.StatusCode} - Unable to obtain total count due to request failure. This resource will not be processed. Downstream failures are possible.{Environment.NewLine}Response content: {responseContent}"); + "{Url}: {StatusCode} - Unable to obtain total count due to request failure. This resource will not be processed. Downstream failures are possible.\rResponse content: {Content}", + resourceUrl, apiResponse.StatusCode, responseContent + ); // Publish an error for the resource to allow processing to continue, but to force failure. errorHandlingBlock.Post( new ErrorItemMessage { - ResourceUrl = $"{_sourceEdFiApiClientProvider.GetApiClient().DataManagementApiSegment}{resourceUrl}", + ResourceUrl = + $"{_sourceEdFiApiClientProvider.GetApiClient().DataManagementApiSegment}{resourceUrl}", Method = HttpMethod.Get.ToString(), ResponseStatus = apiResponse.StatusCode, ResponseContent = await apiResponse.Content.ReadAsStringAsync().ConfigureAwait(false), - }); + } + ); } } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Isolation/EdFiApiSourceIsolationApplicator.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Isolation/EdFiApiSourceIsolationApplicator.cs index d42a090..a013c0c 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Isolation/EdFiApiSourceIsolationApplicator.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Isolation/EdFiApiSourceIsolationApplicator.cs @@ -8,6 +8,7 @@ using EdFi.Tools.ApiPublisher.Core.Isolation; using Newtonsoft.Json.Linq; using Serilog; +using System.Globalization; using System.Net; using Version = EdFi.Tools.ApiPublisher.Core.Helpers.Version; @@ -18,7 +19,7 @@ public class EdFiApiSourceIsolationApplicator : ISourceIsolationApplicator private readonly ISourceEdFiApiClientProvider _sourceEdFiApiClientProvider; private readonly ILogger _logger = Log.ForContext(typeof(EdFiApiSourceIsolationApplicator)); - + public EdFiApiSourceIsolationApplicator(ISourceEdFiApiClientProvider sourceEdFiApiClientProvider) { _sourceEdFiApiClientProvider = sourceEdFiApiClientProvider; @@ -28,7 +29,8 @@ public async Task ApplySourceSnapshotIdentifierAsync(Version sourceApiVersion) { var sourceApiClient = _sourceEdFiApiClientProvider.GetApiClient(); var sourceApiConnectionDetails = sourceApiClient.ConnectionDetails; - if (sourceApiVersion.Major >= 7) { + if (sourceApiVersion.Major >= 7) + { sourceApiClient.HttpClient.DefaultRequestHeaders.Add("Use-Snapshot", "true"); } else @@ -47,7 +49,7 @@ public async Task ApplySourceSnapshotIdentifierAsync(Version sourceApiVersion) // Configure source HTTP client to add the snapshot identifier header to every request against the source API sourceApiClient.HttpClient.DefaultRequestHeaders.Add("Snapshot-Identifier", snapshotIdentifier); - } + } } private async Task GetSourceSnapshotIdentifierAsync(EdFiApiClient sourceApiClient, Version sourceApiVersion) @@ -68,16 +70,16 @@ private async Task GetSourceSnapshotIdentifierAsync(EdFiApiClient source if (snapshotsResponse.StatusCode == HttpStatusCode.NotFound) { - _logger.Warning( - $"Source API at '{sourceApiClient.HttpClient.BaseAddress}' does not support the necessary isolation for reliable API publishing. Errors may occur, or some data may not be published without causing failures."); + _logger.Warning("Source API at '{BaseAddress}' does not support the necessary isolation for reliable API publishing. Errors may occur, or some data may not be published without causing failures.", + sourceApiClient.HttpClient.BaseAddress); return null; } if (snapshotsResponse.StatusCode == HttpStatusCode.Forbidden) { - _logger.Warning( - $"The API publisher does not have permissions to access the source API's 'snapshots' resource at '{sourceApiClient.HttpClient.BaseAddress}{snapshotsRelativePath}'. Make sure that the source API is using a correctly configured claim set for your API Publisher's API client."); + _logger.Warning("The API publisher does not have permissions to access the source API's 'snapshots' resource at '{BaseAddress}{SnapshotsRelativePath}'. Make sure that the source API is using a correctly configured claim set for your API Publisher's API client.", + sourceApiClient.HttpClient.BaseAddress, snapshotsRelativePath); return null; } @@ -98,8 +100,8 @@ private async Task GetSourceSnapshotIdentifierAsync(EdFiApiClient source if (!snapshotResponseArray.Any()) { // No snapshots available. - _logger.Warning( - $"Snapshots are supported, but no snapshots are available from source API at '{sourceApiClient.HttpClient.BaseAddress}{snapshotsRelativePath}'."); + _logger.Warning("Snapshots are supported, but no snapshots are available from source API at '{BaseAddress}{SnapshotsRelativePath}'.", + sourceApiClient.HttpClient.BaseAddress, snapshotsRelativePath); return null; } @@ -110,7 +112,7 @@ private async Task GetSourceSnapshotIdentifierAsync(EdFiApiClient source string snapshotIdentifier = jt["snapshotIdentifier"].Value(); string snapshotDateTimeText = jt["snapshotDateTime"].Value(); - if (!DateTime.TryParse(snapshotDateTimeText, out var snapshotDateTimeValue)) + if (!DateTime.TryParse(snapshotDateTimeText, new CultureInfo("en-US"), out var snapshotDateTimeValue)) { snapshotDateTimeValue = DateTime.MinValue; } @@ -125,15 +127,16 @@ private async Task GetSourceSnapshotIdentifierAsync(EdFiApiClient source .OrderByDescending(x => x.SnapshotDateTime) .First(); - _logger.Information($"Using snapshot identifier '{snapshot.SnapshotIdentifier}' created at '{snapshot.SnapshotDateTime}'."); + _logger.Information("Using snapshot identifier '{SnapshotIdentifier}' created at '{SnapshotDateTime}'.", + snapshot.SnapshotIdentifier, snapshot.SnapshotDateTime); return snapshot.SnapshotIdentifier; } string errorResponseText = await snapshotsResponse.Content.ReadAsStringAsync().ConfigureAwait(false); - _logger.Error( - $"Unable to get snapshot identifier from API at '{sourceApiClient.HttpClient.BaseAddress}{snapshotsRelativePath}'. Request for available snapshots returned status '{snapshotsResponse.StatusCode}' with message body: {errorResponseText}"); + _logger.Error("Unable to get snapshot identifier from API at '{BaseAddress}{SnapshotsRelativePath}'. Request for available snapshots returned status '{StatusCode}' with message body: {ErrorResponseText}", + sourceApiClient.HttpClient.BaseAddress, snapshotsRelativePath, snapshotsResponse.StatusCode, errorResponseText); return null; } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs index 54e5301..3cf6c93 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs @@ -31,7 +31,7 @@ public class EdFiApiStreamResourcePageMessageHandler : IStreamResourcePageMessag private readonly IRateLimiting _rateLimiter; public EdFiApiStreamResourcePageMessageHandler( - ISourceEdFiApiClientProvider sourceEdFiApiClientProvider, IRateLimiting rateLimiter =null) + ISourceEdFiApiClientProvider sourceEdFiApiClientProvider, IRateLimiting rateLimiter = null) { _sourceEdFiApiClientProvider = sourceEdFiApiClientProvider; _rateLimiter = rateLimiter; @@ -58,14 +58,17 @@ public async Task> HandleStreamResourcePageAsyn if (message.CancellationSource.IsCancellationRequested) { _logger.Debug( - $"{message.ResourceUrl}: Cancellation requested while processing page of source items starting at offset {offset}."); + "{MessageResourceUrl}: Cancellation requested while processing page of source items starting at offset {Offset}.", + message.ResourceUrl, offset); return Enumerable.Empty(); } if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug($"{message.ResourceUrl}: Retrieving page items {offset} to {offset + limit - 1}."); + _logger.Debug( + "{MessageResourceUrl}: Retrieving page items {Offset} to {OffsetLimitMinus1}.", + message.ResourceUrl, offset, offset + limit - 1); } var delay = Backoff.ExponentialBackoff( @@ -75,15 +78,15 @@ public async Task> HandleStreamResourcePageAsyn int attempts = 0; // Rate Limit bool isRateLimitingEnabled = options.EnableRateLimit; - + var retryPolicy = Policy .HandleResult(r => r.StatusCode.IsPotentiallyTransientFailure()) .WaitAndRetryAsync( delay, (result, ts, retryAttempt, ctx) => { - _logger.Warning( - $"{message.ResourceUrl}: Retrying GET page items {offset} to {offset + limit - 1} from source failed with status '{result.Result.StatusCode}'. Retrying... (retry #{retryAttempt} of {options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay)"); + _logger.Warning("{ResourceUrl}: Retrying GET page items {Offset} to {OffsetPlusLimitMinus1} from source failed with status '{StatusCode}'. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay)", + message.ResourceUrl, offset, offset + limit - 1, result.Result.StatusCode, retryAttempt, options.MaxRetryAttempts, ts.TotalSeconds); }); IAsyncPolicy policy = isRateLimitingEnabled ? Policy.WrapAsync(_rateLimiter?.GetRateLimitingPolicy(), retryPolicy) : retryPolicy; try @@ -93,13 +96,10 @@ public async Task> HandleStreamResourcePageAsyn { attempts++; - if (attempts > 1) + if (attempts > 1 && _logger.IsEnabled(LogEventLevel.Debug)) { - if (_logger.IsEnabled(LogEventLevel.Debug)) - { - _logger.Debug( - $"{message.ResourceUrl}: GET page items {offset} to {offset + limit - 1} from source attempt #{attempts}."); - } + _logger.Debug("{ResourceUrl}: GET page items {Offset} to {OffsetPlusLimitMinus1} from source attempt #{Attempts}.", + message.ResourceUrl, offset, offset + limit - 1, attempts); } // Possible seam for getting a page of data (here, using Ed-Fi ODS API w/ offset/limit paging strategy) @@ -136,7 +136,8 @@ public async Task> HandleStreamResourcePageAsyn // Publish the failure errorHandlingBlock.Post(error); - _logger.Error($"{message.ResourceUrl}: GET page items failed with response status '{apiResponse.StatusCode}'."); + _logger.Error("{ResourceUrl}: GET page items failed with response status '{StatusCode}'.", + message.ResourceUrl, apiResponse.StatusCode); break; } @@ -144,8 +145,8 @@ public async Task> HandleStreamResourcePageAsyn // Success if (_logger.IsEnabled(LogEventLevel.Information) && attempts > 1) { - _logger.Information( - $"{message.ResourceUrl}: GET page items {offset} to {offset + limit - 1} attempt #{attempts} returned {apiResponse.StatusCode}."); + _logger.Information("{ResourceUrl}: GET page items {Offset} to {OffsetPlusLimitMinus1} attempt #{Attempts} returned {StatusCode}.", + message.ResourceUrl, offset, offset + limit - 1, attempts, apiResponse.StatusCode); } // Transform the page content to item actions @@ -170,8 +171,8 @@ public async Task> HandleStreamResourcePageAsyn // Publish the failure errorHandlingBlock.Post(error); - _logger.Error( - $"{message.ResourceUrl}: JSON parsing of source page data failed: {ex}{Environment.NewLine}{responseContent}"); + _logger.Error(ex, "{ResourceUrl}: JSON parsing of source page data failed: {Ex}{NewLine}{ResponseContent}", + message.ResourceUrl, ex, Environment.NewLine, responseContent); break; } @@ -183,7 +184,8 @@ public async Task> HandleStreamResourcePageAsyn { if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug($"{message.ResourceUrl}: Final page was full. Attempting to retrieve more data."); + _logger.Debug("{ResourceUrl}: Final page was full. Attempting to retrieve more data.", + message.ResourceUrl); } // Looks like there could be more data @@ -197,9 +199,10 @@ public async Task> HandleStreamResourcePageAsyn break; } } - catch (RateLimitRejectedException) + catch (RateLimitRejectedException ex) { - _logger.Fatal($"{message.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal(ex, "{ResourceUrl}: Rate limit exceeded. Please try again later.", + message.ResourceUrl); } break; } @@ -209,7 +212,7 @@ public async Task> HandleStreamResourcePageAsyn } catch (Exception ex) { - _logger.Error($"{message.ResourceUrl}: {ex}"); + _logger.Error(ex, "{ResourceUrl}: {Ex}", message.ResourceUrl, ex); // An error occurred while parsing the JSON var error = new ErrorItemMessage diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs index 836a54d..b69910e 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionPagingStreamResourcePageMessageProducer.cs @@ -10,6 +10,7 @@ using EdFi.Tools.ApiPublisher.Core.Processing.Messages; using Serilog; using System.Threading.Tasks.Dataflow; +using static System.Runtime.InteropServices.JavaScript.JSType; namespace EdFi.Tools.ApiPublisher.Connections.Api.Processing.Source.MessageProducers; @@ -32,12 +33,12 @@ public async Task>> P { if (message.ChangeWindow?.MaxChangeVersion != default(long) && message.ChangeWindow?.MaxChangeVersion != null) { - _logger.Information( - $"{message.ResourceUrl}: Retrieving total count of items in change versions {message.ChangeWindow.MinChangeVersion} to {message.ChangeWindow.MaxChangeVersion}."); + _logger.Information("{ResourceUrl}: Retrieving total count of items in change versions {MinChangeVersion} to {MaxChangeVersion}.", + message.ResourceUrl, message.ChangeWindow.MinChangeVersion, message.ChangeWindow.MaxChangeVersion); } else { - _logger.Information($"{message.ResourceUrl}: Retrieving total count of items."); + _logger.Information("{ResourceUrl}: Retrieving total count of items.", message.ResourceUrl); } // Get total count of items in source resource for change window (if applicable) @@ -54,7 +55,7 @@ public async Task>> P return Enumerable.Empty>(); } - _logger.Information($"{message.ResourceUrl}: Total count = {totalCount}"); + _logger.Information("{ResourceUrl}: Total count = {TotalCount}", message.ResourceUrl, totalCount); int limit = message.PageSize; @@ -129,9 +130,9 @@ public async Task>> P if (pageMessages.Any()) { // Page-strategy specific context - pageMessages.Last().IsFinalPage = true; + pageMessages[pageMessages.Count - 1].IsFinalPage = true; } return pageMessages; } -} \ No newline at end of file +} diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs index e90c518..d85b13c 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer.cs @@ -32,12 +32,12 @@ public async Task>> P { if (message.ChangeWindow?.MaxChangeVersion != default(long) && message.ChangeWindow?.MaxChangeVersion != null) { - _logger.Information( - $"{message.ResourceUrl}: Retrieving total count of items in change versions {message.ChangeWindow.MinChangeVersion} to {message.ChangeWindow.MaxChangeVersion}."); + _logger.Information("{ResourceUrl}: Retrieving total count of items in change versions {MinChangeVersion} to {MaxChangeVersion}.", + message.ResourceUrl, message.ChangeWindow.MinChangeVersion, message.ChangeWindow.MaxChangeVersion); } else - { - _logger.Information($"{message.ResourceUrl}: Retrieving total count of items."); + { + _logger.Information("{ResourceUrl}: Retrieving total count of items.", message.ResourceUrl); } // Get total count of items in source resource for change window (if applicable) @@ -54,12 +54,12 @@ public async Task>> P return Enumerable.Empty>(); } - _logger.Information($"{message.ResourceUrl}: Total count = {totalCount}"); + _logger.Information("{ResourceUrl}: Total count = {TotalCount}", message.ResourceUrl, totalCount); int limit = message.PageSize; var pageMessages = new List>(); - + if (totalCount > 0) { var noOfPartitions = Math.Ceiling((decimal)(message.ChangeWindow.MaxChangeVersion - message.ChangeWindow.MinChangeVersion) @@ -72,7 +72,7 @@ public async Task>> P { long changeVersionWindowEndValue = (changeVersionWindowStartValue > 0 ? changeVersionWindowStartValue - 1 : changeVersionWindowStartValue) + options.ChangeVersionPagingWindowSize; - + if (changeVersionWindowEndValue > message.ChangeWindow.MaxChangeVersion) { changeVersionWindowEndValue = message.ChangeWindow.MaxChangeVersion; @@ -106,7 +106,7 @@ public async Task>> P } int limitOnWindow = totalCountOnWindow < limit ? (int)totalCountOnWindow : limit; - while ((offsetOnWindow >= 0 || isLastOne == true) && totalCountOnWindow > 0 && limitOnWindow > 0) + while (totalCountOnWindow > 0 && limitOnWindow > 0) { var pageMessage = new StreamResourcePageMessage { @@ -145,7 +145,7 @@ public async Task>> P if (pageMessages.Any()) { // Page-strategy specific context - pageMessages.Last().IsFinalPage = true; + pageMessages[pageMessages.Count - 1].IsFinalPage = true; } return pageMessages; diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer.cs index 965a69e..62c2076 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageProducers/EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer.cs @@ -15,29 +15,29 @@ namespace EdFi.Tools.ApiPublisher.Connections.Api.Processing.Source.MessageProdu public class EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer : IStreamResourcePageMessageProducer { private readonly ISourceTotalCountProvider _sourceTotalCountProvider; - + private readonly ILogger _logger = Log.ForContext(typeof(EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer)); - + public EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer(ISourceTotalCountProvider sourceTotalCountProvider) { _sourceTotalCountProvider = sourceTotalCountProvider; } - + public async Task>> ProduceMessagesAsync( - StreamResourceMessage message, + StreamResourceMessage message, Options options, - ITargetBlock errorHandlingBlock, + ITargetBlock errorHandlingBlock, Func, string, IEnumerable> createProcessDataMessages, CancellationToken cancellationToken) { if (message.ChangeWindow?.MaxChangeVersion != default(long) && message.ChangeWindow?.MaxChangeVersion != null) { - _logger.Information( - $"{message.ResourceUrl}: Retrieving total count of items in change versions {message.ChangeWindow.MinChangeVersion} to {message.ChangeWindow.MaxChangeVersion}."); + _logger.Information("{ResourceUrl}: Retrieving total count of items in change versions {MinChangeVersion} to {MaxChangeVersion}.", + message.ResourceUrl, message.ChangeWindow.MinChangeVersion, message.ChangeWindow.MaxChangeVersion); } else { - _logger.Information($"{message.ResourceUrl}: Retrieving total count of items."); + _logger.Information("{ResourceUrl}: Retrieving total count of items.", message.ResourceUrl); } // Get total count of items in source resource for change window (if applicable) @@ -47,14 +47,14 @@ public async Task>> P message.ChangeWindow, errorHandlingBlock, cancellationToken); - + if (!totalCountSuccess) { // Allow processing to continue without performing additional work on this resource. return Enumerable.Empty>(); } - _logger.Information($"{message.ResourceUrl}: Total count = {totalCount}"); + _logger.Information("{ResourceUrl}: Total count = {TotalCount}", message.ResourceUrl, totalCount); long offset = 0; int limit = message.PageSize; @@ -72,14 +72,14 @@ public async Task>> P // Page-strategy specific context Limit = limit, Offset = offset, - + // Source Ed-Fi ODS API processing context (shared) // EdFiApiClient = message.EdFiApiClient, // Global processing context ChangeWindow = message.ChangeWindow, CreateProcessDataMessages = createProcessDataMessages, - + CancellationSource = message.CancellationSource, }; @@ -92,7 +92,7 @@ public async Task>> P if (pageMessages.Any()) { // Page-strategy specific context - pageMessages.Last().IsFinalPage = true; + pageMessages[pageMessages.Count - 1].IsFinalPage = true; } return pageMessages; diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Versioning/EdFiApiSourceCurrentChangeVersionProvider.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Versioning/EdFiApiSourceCurrentChangeVersionProvider.cs index 3bc9384..ff8a793 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Versioning/EdFiApiSourceCurrentChangeVersionProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Versioning/EdFiApiSourceCurrentChangeVersionProvider.cs @@ -20,11 +20,11 @@ public EdFiApiSourceCurrentChangeVersionProvider(ISourceEdFiApiClientProvider so { _sourceEdFiApiClientProvider = sourceEdFiApiClientProvider; } - + public async Task GetCurrentChangeVersionAsync() { var sourceApiClient = _sourceEdFiApiClientProvider.GetApiClient(); - + // Get current source version information string availableChangeVersionsRelativePath = $"{sourceApiClient.ChangeQueriesApiSegment}/availableChangeVersions"; @@ -33,16 +33,16 @@ public EdFiApiSourceCurrentChangeVersionProvider(ISourceEdFiApiClientProvider so if (!versionResponse.IsSuccessStatusCode) { - _logger.Warning( - $"Unable to get current change version from source API at '{sourceApiClient.HttpClient.BaseAddress}{availableChangeVersionsRelativePath}' (response status: {versionResponse.StatusCode}). Full synchronization will always be performed against this source, and any concurrent changes made against the source may cause change processing to produce unreliable results."); + _logger.Warning("Unable to get current change version from source API at '{BaseAddress}{AvailableChangeVersionsRelativePath}' (response status: {StatusCode}). Full synchronization will always be performed against this source, and any concurrent changes made against the source may cause change processing to produce unreliable results.", + sourceApiClient.HttpClient.BaseAddress, availableChangeVersionsRelativePath, versionResponse.StatusCode); return null; } string versionResponseText = await versionResponse.Content.ReadAsStringAsync().ConfigureAwait(false); - _logger.Debug( - $"Available change versions request from {sourceApiClient.HttpClient.BaseAddress}{availableChangeVersionsRelativePath} returned {versionResponse.StatusCode}: {versionResponseText}"); + _logger.Debug("Available change versions request from {BaseAddress}{AvailableChangeVersionsRelativePath} returned {StatusCode}: {VersionResponseText}", + sourceApiClient.HttpClient.BaseAddress, availableChangeVersionsRelativePath, versionResponse.StatusCode, versionResponseText); try { diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/ChangeResourceKeyProcessingBlocksFactory.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/ChangeResourceKeyProcessingBlocksFactory.cs index c2220e1..11cff72 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/ChangeResourceKeyProcessingBlocksFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/ChangeResourceKeyProcessingBlocksFactory.cs @@ -90,7 +90,7 @@ private TransformManyBlock CreateG int attempts = 0; // Rate Limit bool isRateLimitingEnabled = options.EnableRateLimit; - + var retryPolicy = Policy .Handle() .OrResult(r => r.StatusCode.IsPotentiallyTransientFailure()) @@ -98,12 +98,13 @@ private TransformManyBlock CreateG { if (result.Exception != null) { - _logger.Warning($"{message.ResourceUrl} (source id: {sourceId}): GET by key on resource failed with an exception. Retrying... (retry #{retryAttempt} of {options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay){Environment.NewLine}{result.Exception}"); + _logger.Warning(result.Exception, "{ResourceUrl} (source id: {SourceId}): GET by key on resource failed with an exception. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay){NewLine}{Exception}", + message.ResourceUrl, sourceId, retryAttempt, options.MaxRetryAttempts, ts.TotalSeconds, Environment.NewLine, result.Exception); } else { - _logger.Warning( - $"{message.ResourceUrl} (source id: {sourceId}): GET by key on resource failed with status '{result.Result.StatusCode}'. Retrying... (retry #{retryAttempt} of {options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay)"); + _logger.Warning("{ResourceUrl} (source id: {SourceId}): GET by key on resource failed with status '{StatusCode}'. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay)", + message.ResourceUrl, sourceId, result.Result.StatusCode, retryAttempt, options.MaxRetryAttempts, ts.TotalSeconds); } }); @@ -113,12 +114,10 @@ private TransformManyBlock CreateG { attempts++; - if (attempts > 1) + if (attempts > 1 && _logger.IsEnabled(LogEventLevel.Debug)) { - if (_logger.IsEnabled(LogEventLevel.Debug)) - { - _logger.Debug($"{message.ResourceUrl} (source id: {message.SourceId}): GET by key on target attempt #{attempts} ({queryString})."); - } + _logger.Debug("{ResourceUrl} (source id: {SourceId}): GET by key on target attempt #{Attempts} ({QueryString}).", + message.ResourceUrl, message.SourceId, attempts, queryString); } return targetApiClient.HttpClient.GetAsync($"{targetApiClient.DataManagementApiSegment}{message.ResourceUrl}?{queryString}", ct); @@ -135,8 +134,8 @@ private TransformManyBlock CreateG // Failure if (!apiResponse.IsSuccessStatusCode) { - _logger.Error( - $"{message.ResourceUrl} (source id: {sourceId}): GET by key returned {apiResponse.StatusCode}{Environment.NewLine}{responseContent}"); + _logger.Error("{ResourceUrl} (source id: {SourceId}): GET by key returned {StatusCode}{NewLine}{ResponseContent}", + message.ResourceUrl, sourceId, apiResponse.StatusCode, Environment.NewLine, responseContent); var error = new ErrorItemMessage { @@ -144,7 +143,7 @@ private TransformManyBlock CreateG ResourceUrl = $"{message.ResourceUrl}?{queryString}", Id = sourceId, Body = null, - ResponseStatus = apiResponse?.StatusCode, + ResponseStatus = apiResponse.StatusCode, ResponseContent = responseContent }; @@ -158,13 +157,14 @@ private TransformManyBlock CreateG // Success if (_logger.IsEnabled(LogEventLevel.Information) && attempts > 1) { - _logger.Information( - $"{message.ResourceUrl} (source id: {sourceId}): GET by key attempt #{attempts} returned {apiResponse.StatusCode}."); + _logger.Information("{ResourceUrl} (source id: {SourceId}): GET by key attempt #{Attempts} returned {StatusCode}.", + message.ResourceUrl, sourceId, attempts, apiResponse.StatusCode); } if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug($"{message.ResourceUrl} (source id: {sourceId}): GET by key returned {apiResponse.StatusCode}"); + _logger.Debug("{ResourceUrl} (source id: {SourceId}): GET by key returned {StatusCode}", + message.ResourceUrl, sourceId, apiResponse.StatusCode); } var getByKeyResults = JArray.Parse(responseContent); @@ -174,7 +174,8 @@ private TransformManyBlock CreateG { if (_logger.IsEnabled(LogEventLevel.Warning)) { - _logger.Warning($"{message.ResourceUrl} (source id: {sourceId}): GET by key for key change returned no results on target API ({queryString})."); + _logger.Warning("{ResourceUrl} (source id: {SourceId}): GET by key for key change returned no results on target API ({QueryString}).", + message.ResourceUrl, sourceId, queryString); } // No key changes to process @@ -215,7 +216,8 @@ private TransformManyBlock CreateG { if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug($"{message.ResourceUrl} (source id: {message.SourceId}): Assigning new value for '{candidateProperty.Name}' as '{newValueProperty.Value}'..."); + _logger.Debug("{ResourceUrl} (source id: {SourceId}): Assigning new value for '{CandidatePropertyName}' as '{NewValuePropertyValue}'...", + message.ResourceUrl, message.SourceId, candidateProperty.Name, newValueProperty.Value); } candidateProperty.Value = newValueProperty.Value; @@ -233,16 +235,19 @@ private TransformManyBlock CreateG } }; } - catch (RateLimitRejectedException) +#pragma warning disable S2139 + catch (RateLimitRejectedException ex) { - _logger.Fatal($"{message.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal(ex, "{ResourceUrl}: Rate limit exceeded. Please try again later.", message.ResourceUrl); throw; } catch (Exception ex) { - _logger.Error($"{message.ResourceUrl} (source id: {sourceId}): An unhandled exception occurred in the block created by '{nameof(CreateGetItemForKeyChangeBlock)}': {ex}"); + _logger.Error(ex, "{ResourceUrl} (source id: {SourceId}): An unhandled exception occurred in the block created by '{CreateGetItemForKeyChangeBlock}': {Ex}", + message.ResourceUrl, sourceId, nameof(CreateGetItemForKeyChangeBlock), ex); throw; } +#pragma warning restore S2139 }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = options.MaxDegreeOfParallelismForPostResourceItem @@ -299,12 +304,13 @@ private TransformManyBlock CreateChangeKeyBl { if (result.Exception != null) { - _logger.Warning($"{msg.ResourceUrl} (source id: {sourceId}): Key change attempt #{attempt} threw an exception: {result.Exception}"); + _logger.Warning(result.Exception, "{ResourceUrl} (source id: {SourceId}): Key change attempt #{Attempt} threw an exception: {Exception}", + msg.ResourceUrl, sourceId, attempt, result.Exception); } else { - _logger.Warning( - $"{msg.ResourceUrl} (source id: {id}): Select by key on target resource failed with status '{result.Result.StatusCode}'. Retrying... (retry #{retryAttempt} of {options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay)"); + _logger.Warning(result.Exception, "{ResourceUrl} (source id: {Id}): Select by key on target resource failed with status '{StatusCode}'. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay)", + msg.ResourceUrl, id, result.Result.StatusCode, retryAttempt, options.MaxRetryAttempts, ts.TotalSeconds); } }); @@ -314,12 +320,10 @@ private TransformManyBlock CreateChangeKeyBl { attempt++; - if (attempt > 1) + if (attempt > 1 && _logger.IsEnabled(LogEventLevel.Debug)) { - if (_logger.IsEnabled(LogEventLevel.Debug)) - { - _logger.Debug($"{msg.ResourceUrl} (source id: {sourceId}): PUT request to update key (attempt #{attempt}."); - } + _logger.Debug("{ResourceUrl} (source id: {SourceId}): PUT request to update key (attempt #{Attempt}.", + msg.ResourceUrl, sourceId, attempt); } return targetApiClient.HttpClient.PutAsync( @@ -334,7 +338,8 @@ private TransformManyBlock CreateChangeKeyBl string responseContent = await apiResponse.Content.ReadAsStringAsync().ConfigureAwait(false); _logger.Error( - $"{msg.ResourceUrl} (source id: {sourceId}): PUT returned {apiResponse.StatusCode}{Environment.NewLine}{responseContent}"); + "{ResourceUrl} (source id: {SourceId}): PUT returned {StatusCode}{NewLine}{ResponseContent}", + msg.ResourceUrl, sourceId, apiResponse.StatusCode, Environment.NewLine, responseContent); // Publish the failure var error = new ErrorItemMessage @@ -353,28 +358,31 @@ private TransformManyBlock CreateChangeKeyBl // Success if (_logger.IsEnabled(LogEventLevel.Information) && attempt > 1) { - _logger.Information( - $"{msg.ResourceUrl} (source id: {sourceId}): PUT attempt #{attempt} returned {apiResponse.StatusCode}."); + _logger.Information("{ResourceUrl} (source id: {SourceId}): PUT attempt #{Attempt} returned {StatusCode}.", + msg.ResourceUrl, sourceId, attempt, apiResponse.StatusCode); } if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug($"{msg.ResourceUrl} (source id: {sourceId}): PUT returned {apiResponse.StatusCode}"); + _logger.Debug("{ResourceUrl} (source id: {SourceId}): PUT returned {StatusCode}", + msg.ResourceUrl, sourceId, apiResponse.StatusCode); } // Success - no errors to publish return Enumerable.Empty(); } - catch (RateLimitRejectedException) +#pragma warning disable S2139 + catch (RateLimitRejectedException ex) { - _logger.Fatal($"{msg.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal(ex, "{ResourceUrl}: Rate limit exceeded. Please try again later.", msg.ResourceUrl); throw; } catch (Exception ex) { - _logger.Error($"{msg.ResourceUrl} (source id: {sourceId}): An unhandled exception occurred in the ChangeResourceKey block: {ex}"); + _logger.Error(ex, "{ResourceUrl} (source id: {SourceId}): An unhandled exception occurred in the ChangeResourceKey block: {Ex}", msg.ResourceUrl, sourceId, ex); throw; } +#pragma warning restore S2139 }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = options.MaxDegreeOfParallelismForPostResourceItem diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/DeleteResourceProcessingBlocksFactory.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/DeleteResourceProcessingBlocksFactory.cs index 3f54170..61e7e87 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/DeleteResourceProcessingBlocksFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/DeleteResourceProcessingBlocksFactory.cs @@ -35,7 +35,7 @@ public class DeleteResourceProcessingBlocksFactory : IProcessingBlocksFactory _rateLimiter; private static readonly ILogger _logger = Log.Logger.ForContext(typeof(DeleteResourceProcessingBlocksFactory)); - public DeleteResourceProcessingBlocksFactory(ITargetEdFiApiClientProvider targetEdFiApiClientProvider, IRateLimiting rateLimiter =null) + public DeleteResourceProcessingBlocksFactory(ITargetEdFiApiClientProvider targetEdFiApiClientProvider, IRateLimiting rateLimiter = null) { _targetEdFiApiClientProvider = targetEdFiApiClientProvider; _rateLimiter = rateLimiter; @@ -89,7 +89,7 @@ private TransformManyBlock CreateG int attempts = 0; // Rate Limit bool isRateLimitingEnabled = options.EnableRateLimit; - + var retryPolicy = Policy .Handle() .OrResult(r => r.StatusCode.IsPotentiallyTransientFailure()) @@ -97,12 +97,13 @@ private TransformManyBlock CreateG { if (result.Exception != null) { - _logger.Warning($"{msg.ResourceUrl} (source id: {id}): GET by key for deletion of target resource attempt #{attempts}): {result.Exception}"); + _logger.Warning(result.Exception, "{ResourceUrl} (source id: {Id}): GET by key for deletion of target resource attempt #{Attempts}): {Exception}", + msg.ResourceUrl, id, attempts, result.Exception); } else { - _logger.Warning( - $"{msg.ResourceUrl} (source id: {id}): GET by key for deletion of target resource failed with status '{result.Result.StatusCode}'. Retrying... (retry #{retryAttempt} of {options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay)"); + _logger.Warning("{ResourceUrl} (source id: {Id}): GET by key for deletion of target resource failed with status '{StatusCode}'. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay)", + msg.ResourceUrl, id, result.Result.StatusCode, retryAttempt, options.MaxRetryAttempts, ts.TotalSeconds); } }); @@ -111,12 +112,10 @@ private TransformManyBlock CreateG { attempts++; - if (attempts > 1) + if (attempts > 1 && _logger.IsEnabled(LogEventLevel.Debug)) { - if (_logger.IsEnabled(LogEventLevel.Debug)) - { - _logger.Debug($"{msg.ResourceUrl} (source id: {msg.Id}): GET by key for deletion of target resource (attempt #{attempts}) using '{queryString}'..."); - } + _logger.Debug("{ResourceUrl} (source id: {Id}): GET by key for deletion of target resource (attempt #{Attempts}) using '{QueryString}'...", + msg.ResourceUrl, msg.Id, attempts, queryString); } return targetApiClient.HttpClient.GetAsync($"{targetApiClient.DataManagementApiSegment}{msg.ResourceUrl}?{queryString}", ct); @@ -128,8 +127,8 @@ private TransformManyBlock CreateG if (!apiResponse.IsSuccessStatusCode) { - _logger.Error( - $"{msg.ResourceUrl} (source id: {id}): GET by key returned {apiResponse.StatusCode}{Environment.NewLine}{responseContent}"); + _logger.Error("{ResourceUrl} (source id: {Id}): GET by key returned {StatusCode}{NewLine}{ResponseContent}", + msg.ResourceUrl, id, apiResponse.StatusCode, Environment.NewLine, responseContent); var error = new ErrorItemMessage { @@ -153,13 +152,13 @@ private TransformManyBlock CreateG // Log a message if this was successful after a retry. if (_logger.IsEnabled(LogEventLevel.Information) && attempts > 1) { - _logger.Information( - $"{msg.ResourceUrl} (source id: {id}): GET by key attempt #{attempts} returned {apiResponse.StatusCode}."); + _logger.Information("{ResourceUrl} (source id: {Id}): GET by key attempt #{Attempts} returned {StatusCode}.", + msg.ResourceUrl, id, attempts, apiResponse.StatusCode); } if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug($"{msg.ResourceUrl} (source id: {id}): GET by key returned {apiResponse.StatusCode}"); + _logger.Debug("{ResourceUrl} (source id: {Id}): GET by key returned {StatusCode}", msg.ResourceUrl, id, apiResponse.StatusCode); } var getByKeyResults = JArray.Parse(responseContent); @@ -169,7 +168,8 @@ private TransformManyBlock CreateG { if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug($"{msg.ResourceUrl} (source id: {msg.Id}): GET by key for deletion returned no results on target API ({queryString})."); + _logger.Debug("{ResourceUrl} (source id: {Id}): GET by key for deletion returned no results on target API ({QueryString}).", + msg.ResourceUrl, msg.Id, queryString); } return Enumerable.Empty(); @@ -185,16 +185,18 @@ private TransformManyBlock CreateG } }; } - catch (RateLimitRejectedException) +#pragma warning disable S2139 + catch (RateLimitRejectedException ex) { - _logger.Fatal($"{msg.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal(ex, "{ResourceUrl}: Rate limit exceeded. Please try again later.", msg.ResourceUrl); throw; } catch (Exception ex) { - _logger.Error($"{msg.ResourceUrl} (source id: {id}): An unhandled exception occurred in the GetItemForDeletion block: {ex}"); + _logger.Error(ex, "{ResourceUrl} (source id: {Id}): An unhandled exception occurred in the GetItemForDeletion block: {Ex}", msg.ResourceUrl, id, ex); throw; } +#pragma warning restore S2139 }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = options.MaxDegreeOfParallelismForPostResourceItem @@ -243,7 +245,7 @@ private TransformManyBlock CreateDeleteReso int attempts = 0; // Rate Limit bool isRateLimitingEnabled = options.EnableRateLimit; - + var retryPolicy = Policy .Handle() .OrResult(r => @@ -252,12 +254,13 @@ private TransformManyBlock CreateDeleteReso { if (result.Exception != null) { - _logger.Warning($"{msg.ResourceUrl} (source id: {sourceId}): Delete resource attempt #{attempts} threw an exception: {result.Exception}"); + _logger.Warning(result.Exception, "{ResourceUrl} (source id: {SourceId}): Delete resource attempt #{Attempts} threw an exception: {Exception}", + msg.ResourceUrl, sourceId, attempts, result.Exception); } else { - _logger.Warning( - $"{msg.ResourceUrl} (source id: {sourceId}): Delete resource failed with status '{result.Result.StatusCode}'. Retrying... (retry #{retryAttempt} of {options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay)"); + _logger.Warning(result.Exception, "{ResourceUrl} (source id: {SourceId}): Delete resource failed with status '{StatusCode}'. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay)", + msg.ResourceUrl, sourceId, result.Result.StatusCode, retryAttempt, options.MaxRetryAttempts, ts.TotalSeconds); } }); IAsyncPolicy policy = isRateLimitingEnabled ? Policy.WrapAsync(_rateLimiter?.GetRateLimitingPolicy(), retryPolicy) : retryPolicy; @@ -265,12 +268,10 @@ private TransformManyBlock CreateDeleteReso { attempts++; - if (attempts > 1) + if (attempts > 1 && _logger.IsEnabled(LogEventLevel.Debug)) { - if (_logger.IsEnabled(LogEventLevel.Debug)) - { - _logger.Debug($"{msg.ResourceUrl} (source id: {sourceId}): DELETE request (attempt #{attempts}."); - } + _logger.Debug("{ResourceUrl} (source id: {SourceId}): DELETE request (attempt #{Attempts}.", + msg.ResourceUrl, sourceId, attempts); } return targetApiClient.HttpClient.DeleteAsync($"{targetApiClient.DataManagementApiSegment}{msg.ResourceUrl}/{id}", ct); @@ -281,8 +282,8 @@ private TransformManyBlock CreateDeleteReso { string responseContent = await apiResponse.Content.ReadAsStringAsync().ConfigureAwait(false); - _logger.Error( - $"{msg.ResourceUrl} (source id: {sourceId}): DELETE returned {apiResponse.StatusCode}{Environment.NewLine}{responseContent}"); + _logger.Error("{ResourceUrl} (source id: {SourceId}): DELETE returned {StatusCode}{NewLine}{ResponseContent}", + msg.ResourceUrl, sourceId, apiResponse.StatusCode, Environment.NewLine, responseContent); // Publish the failure var error = new ErrorItemMessage @@ -301,28 +302,31 @@ private TransformManyBlock CreateDeleteReso // Success if (_logger.IsEnabled(LogEventLevel.Information) && attempts > 1) { - _logger.Information( - $"{msg.ResourceUrl} (source id: {sourceId}): DELETE attempt #{attempts} returned {apiResponse.StatusCode}."); + _logger.Information("{ResourceUrl} (source id: {SourceId}): DELETE attempt #{Attempts} returned {StatusCode}.", + msg.ResourceUrl, sourceId, attempts, apiResponse.StatusCode); } if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug($"{msg.ResourceUrl} (source id: {sourceId}): DELETE returned {apiResponse.StatusCode}"); + _logger.Debug("{ResourceUrl} (source id: {SourceId}): DELETE returned {StatusCode}", msg.ResourceUrl, sourceId, apiResponse.StatusCode); } // Success - no errors to publish return Enumerable.Empty(); } - catch (RateLimitRejectedException) +#pragma warning disable S2139 + catch (RateLimitRejectedException ex) { - _logger.Fatal($"{msg.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal(ex, "{ResourceUrl}: Rate limit exceeded. Please try again later.", msg.ResourceUrl); throw; } catch (Exception ex) { - _logger.Error($"{msg.ResourceUrl} (source id: {sourceId}): An unhandled exception occurred in the DeleteResource block: {ex}"); + _logger.Error(ex, "{ResourceUrl} (source id: {SourceId}): An unhandled exception occurred in the DeleteResource block: {Ex}", + msg.ResourceUrl, sourceId, ex); throw; } +#pragma warning restore S2139 }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = options.MaxDegreeOfParallelismForPostResourceItem @@ -342,18 +346,11 @@ public IEnumerable CreateProcessDataMessages(StreamRe if (message.CancellationSource.IsCancellationRequested) { _logger.Debug( - $"{message.ResourceUrl}: Cancellation requested during item '{nameof(GetItemForDeletionMessage)}' creation."); + "{ResourceUrl}: Cancellation requested during item '{NameofGetItemForDeletionMessage}' creation.", message.ResourceUrl, nameof(GetItemForDeletionMessage)); yield break; } - // // Add the item to the buffer for processing into the target API - // if (_logger.IsEnabled(LogEventLevel.Debug)) - // { - // _logger.Debug( - // $"{message.ResourceUrl}: Adding individual action message of type '{nameof(GetItemForDeletionMessage)}' for item '{item["id"]?.Value() ?? "unknown"}'..."); - // } - var itemMessage = CreateItemActionMessage(item); if (itemMessage == null) diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/PostResourceProcessingBlocksFactory.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/PostResourceProcessingBlocksFactory.cs index d00a9a8..a772920 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/PostResourceProcessingBlocksFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/PostResourceProcessingBlocksFactory.cs @@ -48,7 +48,7 @@ public PostResourceProcessingBlocksFactory( ISourceConnectionDetails sourceConnectionDetails, ISourceCapabilities sourceCapabilities, ISourceResourceItemProvider sourceResourceItemProvider, - IRateLimiting rateLimiter =null + IRateLimiting rateLimiter = null ) { _nodeJsService = nodeJsService; @@ -60,18 +60,16 @@ public PostResourceProcessingBlocksFactory( // Ensure that the API connections are configured correctly with regards to Profiles // If we have no Profile applied to the target... ensure that the source also has no profile specified (to prevent accidental data loss on POST) - if (string.IsNullOrEmpty(targetEdFiApiClientProvider.GetApiClient().ConnectionDetails.ProfileName)) - { + if (string.IsNullOrEmpty(targetEdFiApiClientProvider.GetApiClient().ConnectionDetails.ProfileName) + // If the source is an API - if (sourceConnectionDetails is ApiConnectionDetails sourceApiConnectionDetails) - { - // If the source API is not using a Profile, prevent processing by throwing an exception - if (!string.IsNullOrEmpty(sourceApiConnectionDetails.ProfileName)) - { - throw new Exception( - "The source API connection has a ProfileName specified, but the target API connection does not. POST requests against a target API without the Profile-based context of the source data can lead to accidental data loss."); - } - } + && (sourceConnectionDetails is ApiConnectionDetails sourceApiConnectionDetails) + + // If the source API is not using a Profile, prevent processing by throwing an exception + && (!string.IsNullOrEmpty(sourceApiConnectionDetails.ProfileName))) + { + throw new Exception( + "The source API connection has a ProfileName specified, but the target API connection does not. POST requests against a target API without the Profile-based context of the source data can lead to accidental data loss."); } } @@ -139,8 +137,8 @@ private async Task> HandlePostItemMessage( { if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug( - $"{postItemMessage.ResourceUrl} (source id: {id}): Processing PostItemMessage (with up to {options.MaxRetryAttempts} retries)."); + _logger.Debug("{ResourceUrl} (source id: {Id}): Processing PostItemMessage (with up to {MaxRetryAttempts} retries).", + postItemMessage.ResourceUrl, id, options.MaxRetryAttempts); } // Remove attributes not usable between API instances @@ -151,7 +149,8 @@ private async Task> HandlePostItemMessage( // For descriptors, also strip the surrogate id if (postItemMessage.ResourceUrl.EndsWith("Descriptors")) { - string descriptorBaseName = postItemMessage.ResourceUrl.Split('/').Last().TrimEnd('s'); + string[] descriptorBaseNameSplit = postItemMessage.ResourceUrl.Split('/'); + string descriptorBaseName = descriptorBaseNameSplit[descriptorBaseNameSplit.Length - 1].TrimEnd('s'); string descriptorIdPropertyName = $"{descriptorBaseName}Id"; postItemMessage.Item.Remove(descriptorIdPropertyName); @@ -167,7 +166,6 @@ private async Task> HandlePostItemMessage( var retryPolicy = Policy.Handle() .OrResult( r => - // Descriptor Conflicts are not to be retried (r.StatusCode == HttpStatusCode.Conflict && !postItemMessage.ResourceUrl.EndsWith("Descriptors", StringComparison.OrdinalIgnoreCase)) @@ -209,8 +207,8 @@ private async Task> HandlePostItemMessage( remediationResult.ModifiedRequestBody, new JsonSerializerOptions { WriteIndented = true }); - _logger.Debug( - $"{postItemMessage.ResourceUrl} (source id: {id}): Remediation plan provided a modified request body: {modifiedRequestBodyJson} "); + _logger.Debug("{ResourceUrl} (source id: {Id}): Remediation plan provided a modified request body: {ModifiedRequestBodyJson} ", + postItemMessage.ResourceUrl, id, modifiedRequestBodyJson); } ctx["ModifiedRequestBody"] = remediationResult.ModifiedRequestBody; @@ -219,15 +217,15 @@ private async Task> HandlePostItemMessage( if (result.Exception != null) { - _logger.Warning( - $"{postItemMessage.ResourceUrl} (source id: {id}): POST attempt #{attempts} failed with an exception. Retrying... (retry #{retryAttempt} of {options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay):{Environment.NewLine}{result.Exception}"); + _logger.Warning(result.Exception, "{ResourceUrl} (source id: {Id}): POST attempt #{Attempts} failed with an exception. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay):{NewLine}{Exception}", + postItemMessage.ResourceUrl, id, attempts, retryAttempt, options.MaxRetryAttempts, ts.TotalSeconds, Environment.NewLine, result.Exception); } else { string responseContent = await result.Result.Content.ReadAsStringAsync().ConfigureAwait(false); - _logger.Warning( - $"{postItemMessage.ResourceUrl} (source id: {id}): POST attempt #{attempts} failed with status '{result.Result.StatusCode}'. Retrying... (retry #{retryAttempt} of {options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay):{Environment.NewLine}{responseContent}"); + _logger.Warning("{ResourceUrl} (source id: {Id}): POST attempt #{Attempts} failed with status '{StatusCode}'. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay):{NewLine}{ResponseContent}", + postItemMessage.ResourceUrl, id, attempts, result.Result.StatusCode, retryAttempt, options.MaxRetryAttempts, ts.TotalSeconds, Environment.NewLine, responseContent); } }); IAsyncPolicy policy = isRateLimitingEnabled ? Policy.WrapAsync(_rateLimiter?.GetRateLimitingPolicy(), retryPolicy) : retryPolicy; @@ -240,11 +238,11 @@ private async Task> HandlePostItemMessage( { if (attempts > 1) { - _logger.Debug($"{postItemMessage.ResourceUrl} (source id: {id}): POST attempt #{attempts}."); + _logger.Debug("{ResourceUrl} (source id: {Id}): POST attempt #{Attempts}.", postItemMessage.ResourceUrl, id, attempts); } else { - _logger.Debug($"{postItemMessage.ResourceUrl} (source id: {id}): Sending POST request."); + _logger.Debug("{ResourceUrl} (source id: {Id}): Sending POST request.", postItemMessage.ResourceUrl, id); } } @@ -253,8 +251,7 @@ private async Task> HandlePostItemMessage( if (ctx.TryGetValue("ModifiedRequestBody", out dynamic modifiedRequestBody)) { - _logger.Information( - $"{postItemMessage.ResourceUrl} (source id: {id}): Applying modified request body from remediation plan..."); + _logger.Information("{ResourceUrl} (source id: {Id}): Applying modified request body from remediation plan...", postItemMessage.ResourceUrl, id); requestBodyJson = JsonSerializer.Serialize(modifiedRequestBody); } @@ -276,14 +273,14 @@ private async Task> HandlePostItemMessage( { if (!_sourceCapabilities.SupportsGetItemById) { - _logger.Warning( - $"{postItemMessage.ResourceUrl}: Reference '{missingDependencyDetails!.ReferenceName}' to resource '{missingDependencyDetails.ReferencedResourceName}' could not be automatically resolved because the source connection does not support retrieving items by id."); + _logger.Warning("{ResourceUrl}: Reference '{ReferenceName}' to resource '{ReferencedResourceName}' could not be automatically resolved because the source connection does not support retrieving items by id.", + postItemMessage.ResourceUrl, missingDependencyDetails!.ReferenceName, missingDependencyDetails.ReferencedResourceName); return response; } - _logger.Information( - $"{postItemMessage.ResourceUrl}: Attempting to retrieve missing '{missingDependencyDetails.ReferencedResourceName}' reference based on 'authorizationFailureHandling' metadata in apiPublisherSettings.json."); + _logger.Information("{ResourceUrl}: Attempting to retrieve missing '{ReferencedResourceName}' reference based on 'authorizationFailureHandling' metadata in apiPublisherSettings.json.", + postItemMessage.ResourceUrl, missingDependencyDetails.ReferencedResourceName); var (missingDependencyItemRetrieved, missingItemJson) = await _sourceResourceItemProvider.TryGetResourceItemAsync(missingDependencyDetails.SourceDependencyItemUrl); @@ -326,8 +323,8 @@ await HandlePostItemMessage( { if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug( - $"{postItemMessage.ResourceUrl} (source id: {id}): POST returned {HttpStatusCode.Conflict}, but for descriptors this means the value is already present."); + _logger.Debug("{ResourceUrl} (source id: {Id}): POST returned {Conflict}, but for descriptors this means the value is already present.", + postItemMessage.ResourceUrl, id, HttpStatusCode.Conflict); } return Enumerable.Empty(); @@ -335,24 +332,23 @@ await HandlePostItemMessage( // Gracefully handle authorization errors by using the retry action delegate // (if present) to post the message to the retry "resource" queue - if (apiResponse.StatusCode == HttpStatusCode.Forbidden) - { + if (apiResponse.StatusCode == HttpStatusCode.Forbidden + // Determine if current resource has an authorization retry queue - if (postItemMessage.PostAuthorizationFailureRetry != null) + && postItemMessage.PostAuthorizationFailureRetry != null) + { + if (_logger.IsEnabled(LogEventLevel.Debug)) { - if (_logger.IsEnabled(LogEventLevel.Debug)) - { - _logger.Debug( - $"{postItemMessage.ResourceUrl} (source id: {id}): Authorization failed -- deferring for retry after pertinent associations are processed."); - } + _logger.Debug("{ResourceUrl} (source id: {Id}): Authorization failed -- deferring for retry after pertinent associations are processed.", + postItemMessage.ResourceUrl, id); + } - // Re-add the identifier, and pass the message along to the "retry" resource (after associations have been processed) - postItemMessage.Item.Add("id", idToken); - postItemMessage.PostAuthorizationFailureRetry(postItemMessage); + // Re-add the identifier, and pass the message along to the "retry" resource (after associations have been processed) + postItemMessage.Item.Add("id", idToken); + postItemMessage.PostAuthorizationFailureRetry(postItemMessage); - // Deferring for retry - no errors to publish - return Enumerable.Empty(); - } + // Deferring for retry - no errors to publish + return Enumerable.Empty(); } string responseContent = await apiResponse.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -363,8 +359,8 @@ await HandlePostItemMessage( && targetEdFiApiClient.ConnectionDetails?.TreatForbiddenPostAsWarning == true) { // Warn and ignore all future data for this resource - _logger.Warning( - $"{postItemMessage.ResourceUrl} (source id: {id}): Authorization failed on POST of resource with no authorization failure handling defined. Remaining resource items will be ignored. Response status: {apiResponse.StatusCode}{Environment.NewLine}{responseContent}"); + _logger.Warning("{ResourceUrl} (source id: {Id}): Authorization failed on POST of resource with no authorization failure handling defined. Remaining resource items will be ignored. Response status: {StatusCode}{NewLine}{ResponseContent}", + postItemMessage.ResourceUrl, id, apiResponse.StatusCode, Environment.NewLine, responseContent); ignoredResourceByUrl.TryAdd(postItemMessage.ResourceUrl, true); @@ -372,8 +368,8 @@ await HandlePostItemMessage( } // Error is final, log it and indicate failure for processing - _logger.Error( - $"{postItemMessage.ResourceUrl} (source id: {id}): POST attempt #{attempts} failed with status '{apiResponse.StatusCode}':{Environment.NewLine}{responseContent}"); + _logger.Error("{ResourceUrl} (source id: {Id}): POST attempt #{Attempts} failed with status '{StatusCode}':{NewLine}{ResponseContent}", + postItemMessage.ResourceUrl, id, attempts, apiResponse.StatusCode, Environment.NewLine, responseContent); // Publish the failed data var error = new ErrorItemMessage @@ -394,8 +390,8 @@ await HandlePostItemMessage( { if (_logger.IsEnabled(LogEventLevel.Information)) { - _logger.Information( - $"{postItemMessage.ResourceUrl} (source id: {id}): POST attempt #{attempts} returned {apiResponse.StatusCode}."); + _logger.Information("{ResourceUrl} (source id: {Id}): POST attempt #{Attempts} returned {StatusCode}.", + postItemMessage.ResourceUrl, id, attempts, apiResponse.StatusCode); } } else @@ -403,26 +399,26 @@ await HandlePostItemMessage( // Ensure a log entry when POST succeeds on first attempt and DEBUG logging is enabled if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug( - $"{postItemMessage.ResourceUrl} (source id: {id}): POST attempt #{attempts} returned {apiResponse.StatusCode}."); + _logger.Debug("{ResourceUrl} (source id: {Id}): POST attempt #{Attempts} returned {StatusCode}.", + postItemMessage.ResourceUrl, id, attempts, apiResponse.StatusCode); } } // Success - no errors to publish return Enumerable.Empty(); } - catch (RateLimitRejectedException) +#pragma warning disable S2139 + catch (RateLimitRejectedException ex) { - _logger.Fatal($"{postItemMessage.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal(ex, "{ResourceUrl}: Rate limit exceeded. Please try again later.", postItemMessage.ResourceUrl); return Enumerable.Empty(); } catch (Exception ex) { - _logger.Error( - $"{postItemMessage.ResourceUrl} (source id: {id}): An unhandled exception occurred in the PostResource block: {ex}"); - + _logger.Error(ex, "{ResourceUrl} (source id: {Id}): An unhandled exception occurred in the PostResource block: {Ex}", postItemMessage.ResourceUrl, id, ex); throw; } +#pragma warning restore S2139 finally { // Drop reference to JObject so it can be GC'd. @@ -449,17 +445,16 @@ string GetResponseMessageText(HttpResponseMessage response) bool IsBadRequestForUnresolvedReferenceOfPrimaryRelationship(HttpResponseMessage postItemResponse, PostItemMessage msg) { // If response is a Bad Request, check for need to explicitly fetch dependencies - if (postItemResponse.StatusCode == HttpStatusCode.BadRequest) - { + if (postItemResponse.StatusCode == HttpStatusCode.BadRequest && + // If resource is a "primary relationship" configured in authorization failure handling - if (missingDependencyByResourcePath.TryGetValue(msg.ResourceUrl, out string missingDependencyResourcePath)) - { - string responseMessageText = GetResponseMessageText(postItemResponse); + missingDependencyByResourcePath.TryGetValue(msg.ResourceUrl, out string missingDependencyResourcePath)) + { + string responseMessageText = GetResponseMessageText(postItemResponse); - if (responseMessageText?.Contains("reference could not be resolved.") == true) - { - return true; - } + if (responseMessageText?.Contains("reference could not be resolved.") == true) + { + return true; } } @@ -491,57 +486,59 @@ async Task GetResponseMessageTextAsync(HttpResponseMessage response) { // If response is a Bad Request (which is the API's error response for missing Staff/Student/Parent), check for need to explicitly fetch dependencies // NOTE: If support is expanded for other missing dependencies, the response code from the API (currently) will be a 409 Conflict status. - if (postItemResponse.StatusCode == HttpStatusCode.BadRequest) - { + if (postItemResponse.StatusCode == HttpStatusCode.BadRequest && + // If resource is a "primary relationship" configured in authorization failure handling - if (missingDependencyByResourcePath.TryGetValue(msg.ResourceUrl, out string missingDependencyResourcePath)) + missingDependencyByResourcePath.TryGetValue(msg.ResourceUrl, out string missingDependencyResourcePath)) + { + string responseMessageText = await GetResponseMessageTextAsync(postItemResponse); + + if (responseMessageText?.Contains("reference could not be resolved.") == true) { - string responseMessageText = await GetResponseMessageTextAsync(postItemResponse); + // Infer reference name from message. This is a bit fragile, but no other choice here. + var referenceNameMatch = Regex.Match( + responseMessageText, + @"(?\w+) reference could not be resolved."); - if (responseMessageText?.Contains("reference could not be resolved.") == true) + if (referenceNameMatch.Success) { - // Infer reference name from message. This is a bit fragile, but no other choice here. - var referenceNameMatch = Regex.Match( - responseMessageText, - @"(?\w+) reference could not be resolved."); - - if (referenceNameMatch.Success) - { - string referencedResourceName = referenceNameMatch.Groups["ReferencedResourceName"].Value; - string referenceName = referencedResourceName.ToCamelCase() + "Reference"; + string referencedResourceName = referenceNameMatch.Groups["ReferencedResourceName"].Value; + string referenceName = referencedResourceName.ToCamelCase() + "Reference"; - // Get the missing reference's source URL - string dependencyItemUrl = msg.Item.SelectToken($"{referenceName}.link.href")?.Value(); + // Get the missing reference's source URL + string dependencyItemUrl = msg.Item.SelectToken($"{referenceName}.link.href")?.Value(); - if (dependencyItemUrl == null) - { - _logger.Warning($"{msg.ResourceUrl}: Unable to extract href to '{referenceName}' from JSON body for obtaining missing dependency."); - return (false, null); - } + if (dependencyItemUrl == null) + { + _logger.Warning("{ResourceUrl}: Unable to extract href to '{ReferenceName}' from JSON body for obtaining missing dependency.", + msg.ResourceUrl, referenceName); + return (false, null); + } - // URL is expected to be of the format of - var parts = dependencyItemUrl.Split('/'); + // URL is expected to be of the format of + var parts = dependencyItemUrl.Split('/'); - if (parts.Length < 3) - { - _logger.Warning($"{msg.ResourceUrl}: Unable to identify missing dependency resource URL from the supplied dependency item URL '{dependencyItemUrl}'."); - } + if (parts.Length < 3) + { + _logger.Warning("{ResourceUrl}: Unable to identify missing dependency resource URL from the supplied dependency item URL '{DependencyItemUrl}'.", + msg.ResourceUrl, dependencyItemUrl); + } - try - { - return (true, new MissingDependencyDetails() - { - DependentResourceUrl = msg.ResourceUrl, - DependencyResourceUrl = $"/{parts[^3]}/{parts[^2]}", - ReferenceName = referenceName, - ReferencedResourceName = referencedResourceName, - SourceDependencyItemUrl = dependencyItemUrl, - }); - } - catch (Exception) + try + { + return (true, new MissingDependencyDetails() { - _logger.Warning($"{msg.ResourceUrl}: Unable to identify missing dependency resource URL from the supplied dependency item URL '{dependencyItemUrl}'."); - } + DependentResourceUrl = msg.ResourceUrl, + DependencyResourceUrl = $"/{parts[^3]}/{parts[^2]}", + ReferenceName = referenceName, + ReferencedResourceName = referencedResourceName, + SourceDependencyItemUrl = dependencyItemUrl, + }); + } + catch (Exception ex) + { + _logger.Warning(ex, "{ResourceUrl}: Unable to identify missing dependency resource URL from the supplied dependency item URL '{DependencyItemUrl}'.", + msg.ResourceUrl, dependencyItemUrl); } } } @@ -554,7 +551,7 @@ async Task GetResponseMessageTextAsync(HttpResponseMessage response) /// /// Contains details about a missing dependency. /// - private record MissingDependencyDetails + private sealed record MissingDependencyDetails { /// /// The relative resource URL of the resource being processed for which a missing dependency was identified. @@ -594,8 +591,7 @@ public IEnumerable CreateProcessDataMessages(StreamResourcePage // Stop processing individual items if cancellation has been requested if (message.CancellationSource.IsCancellationRequested) { - _logger.Debug( - $"{message.ResourceUrl}: Cancellation requested during item '{nameof(PostItemMessage)}' creation."); + _logger.Debug("{ResourceUrl}: Cancellation requested during item '{NameofPostItemMessage}' creation.", message.ResourceUrl, nameof(PostItemMessage)); yield break; } @@ -603,8 +599,8 @@ public IEnumerable CreateProcessDataMessages(StreamResourcePage // Add the item to the buffer for processing into the target API if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug( - $"{message.ResourceUrl}: Adding individual action message of type '{nameof(PostItemMessage)}' for item '{item["id"]?.Value() ?? "unknown"}'..."); + _logger.Debug("{ResourceUrl}: Adding individual action message of type '{NameofPostItemMessage}' for item '{ItemId}'...", + message.ResourceUrl, nameof(PostItemMessage), item["id"]?.Value() ?? "unknown"); } yield return itemMessage; @@ -667,8 +663,8 @@ private async Task TryRemediateFailureAsync( { if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug( - $"{resourceUrl} (source id: {sourceId}): Remediation plan for '{responseMessage.StatusCode}' did not return any additional remediation requests."); + _logger.Debug("{ResourceUrl} (source id: {SourceId}): Remediation plan for '{StatusCode}' did not return any additional remediation requests.", + resourceUrl, sourceId, responseMessage.StatusCode); } return remediationResult; @@ -681,8 +677,8 @@ private async Task TryRemediateFailureAsync( if (_logger.IsEnabled(LogEventLevel.Debug)) { - _logger.Debug( - $"{resourceUrl} (source id: {sourceId}): Remediating request with POST request to '{remediationRequest.resource}' on target API: {remediationRequestBodyJson}"); + _logger.Debug("{ResourceUrl} (source id: {SourceId}): Remediating request with POST request to '{RemediationRequestResource}' on target API: {RemediationRequestBodyJson}", + resourceUrl, sourceId, remediationRequest.resource, remediationRequestBodyJson); } var remediationResponse = await RequestHelpers.SendPostRequestAsync( @@ -694,13 +690,13 @@ private async Task TryRemediateFailureAsync( if (remediationResponse.IsSuccessStatusCode) { - _logger.Information( - $"{resourceUrl} (source id: {sourceId}): Remediation for retry attempt {retryAttempt} with POST request to '{remediationRequest.resource}' on target API succeeded with status '{remediationResponse.StatusCode}'."); + _logger.Information("{ResourceUrl} (source id: {SourceId}): Remediation for retry attempt {RetryAttempt} with POST request to '{RemediationRequestResource}' on target API succeeded with status '{RemediationResponseStatusCode}'.", + resourceUrl, sourceId, retryAttempt, remediationRequest.resource, remediationResponse.StatusCode); } else { - _logger.Warning( - $"{resourceUrl} (source id: {sourceId}): Remediation for retry attempt {retryAttempt} with POST request to '{remediationRequest.resource}' on target API failed with status '{remediationResponse.StatusCode}'."); + _logger.Warning("{ResourceUrl} (source id: {SourceId}): Remediation for retry attempt {RetryAttempt} with POST request to '{RemediationRequestResource}' on target API failed with status '{RemediationResponseStatusCode}'.", + resourceUrl, sourceId, retryAttempt, remediationRequest.resource, remediationResponse.StatusCode); } } @@ -710,12 +706,12 @@ private async Task TryRemediateFailureAsync( { if (!ex.Message.Contains("has no export named")) { - _logger.Warning($"{resourceUrl} (source id: {sourceId}): Error occurred during remediation invocation: {ex}"); + _logger.Warning(ex, "{ResourceUrl} (source id: {SourceId}): Error occurred during remediation invocation: {Ex}", resourceUrl, sourceId, ex); } else { - _logger.Debug( - $"{resourceUrl} (source id: {sourceId}): No remediation found for status code '{responseMessage.StatusCode}'."); + _logger.Debug(ex, "{ResourceUrl} (source id: {SourceId}): No remediation found for status code '{ResponseMessageStatusCode}'.", + resourceUrl, sourceId, responseMessage.StatusCode); } return RemediationResult.NotFound; @@ -725,8 +721,8 @@ private async Task TryRemediateFailureAsync( public class RemediationResult { - public static RemediationResult Found = new(true); - public static RemediationResult NotFound = new(false); + public static RemediationResult Found { get; set; } = new(true); + public static RemediationResult NotFound { get; set; } = new(false); private RemediationResult(bool foundRemediation) { diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Initiators/UpsertPublishingStageInitiator.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Initiators/UpsertPublishingStageInitiator.cs index ed733ba..1b7a577 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Initiators/UpsertPublishingStageInitiator.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Initiators/UpsertPublishingStageInitiator.cs @@ -18,13 +18,13 @@ public UpsertPublishingStageInitiator(IStreamingResourceProcessor streamingResou _streamingResourceProcessor = streamingResourceProcessor; _processingBlocksFactory = processingBlocksFactory; } - + public IDictionary Start(ProcessingContext processingContext, CancellationToken cancellationToken) { return _streamingResourceProcessor.Start( _processingBlocksFactory.CreateProcessingBlocks, _processingBlocksFactory.CreateProcessDataMessages, - processingContext, + processingContext, cancellationToken); } } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/ChangeKeyMessage.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/ChangeKeyMessage.cs index 1d5e2d3..f781987 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/ChangeKeyMessage.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/ChangeKeyMessage.cs @@ -5,7 +5,7 @@ namespace EdFi.Tools.ApiPublisher.Connections.Api.Processing.Target.Messages { - public class ChangeKeyMessage + public class ChangeKeyMessage { /// /// Gets or sets the relative URL for the resource whose key is to be changed. @@ -21,7 +21,7 @@ public class ChangeKeyMessage /// Gets or sets the target API's existing resource body with the new key values applied -- to be PUT against the target API. /// public string Body { get; set; } - + /// /// Gets or sets the source API's resource identifier for the resource whose key was changed (primarily for correlating activity in log messages). /// diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/DeleteItemMessage.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/DeleteItemMessage.cs index a76add4..d883fc0 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/DeleteItemMessage.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/DeleteItemMessage.cs @@ -5,7 +5,7 @@ namespace EdFi.Tools.ApiPublisher.Connections.Api.Processing.Target.Messages { - public class DeleteItemMessage + public class DeleteItemMessage { /// /// Gets or sets the relative URL for the resource to be deleted. diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/GetItemForDeletionMessage.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/GetItemForDeletionMessage.cs index 32a4fe2..8611ffd 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/GetItemForDeletionMessage.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/GetItemForDeletionMessage.cs @@ -7,18 +7,18 @@ namespace EdFi.Tools.ApiPublisher.Connections.Api.Processing.Target.Messages { - public class GetItemForDeletionMessage + public class GetItemForDeletionMessage { /// /// Gets or sets the relative URL for the resource to be deleted. /// public string ResourceUrl { get; set; } - + /// /// Gets or sets the natural key values for the resource to be deleted on the target. /// public JToken KeyValues { get; set; } - + /// /// Gets or sets the source API's resource identifier for the resource that was deleted. /// diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/GetItemForKeyChangeMessage.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/GetItemForKeyChangeMessage.cs index c1a30ee..0b9bd5c 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/GetItemForKeyChangeMessage.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/GetItemForKeyChangeMessage.cs @@ -7,23 +7,23 @@ namespace EdFi.Tools.ApiPublisher.Connections.Api.Processing.Target.Messages { - public class GetItemForKeyChangeMessage + public class GetItemForKeyChangeMessage { /// /// Gets or sets the relative URL for the resource whose key is to be changed. /// public string ResourceUrl { get; set; } - + /// /// Gets or sets the existing natural key values for the resource on the target whose key is to be changed. /// public JToken ExistingKeyValues { get; set; } - + /// /// Gets or sets the new natural key values for the resource on the target whose key is to be changed. /// public JToken NewKeyValues { get; set; } - + /// /// Gets or sets the source API's resource identifier for the resource whose key was changed. /// diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/PostItemMessage.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/PostItemMessage.cs index ff8c2e1..a951c05 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/PostItemMessage.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Messages/PostItemMessage.cs @@ -7,12 +7,12 @@ namespace EdFi.Tools.ApiPublisher.Connections.Api.Processing.Target.Messages { - public class PostItemMessage + public class PostItemMessage { public string ResourceUrl { get; set; } - + public JObject Item { get; set; } - + public Action PostAuthorizationFailureRetry { get; set; } } } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Configuration/SqliteConnectionDetails.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Configuration/SqliteConnectionDetails.cs index ab99790..3efe498 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Configuration/SqliteConnectionDetails.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Configuration/SqliteConnectionDetails.cs @@ -10,7 +10,7 @@ namespace EdFi.Tools.ApiPublisher.Connections.Sqlite.Configuration; public class SqliteConnectionDetails : SourceConnectionDetailsBase, ITargetConnectionDetails { public string File { get; set; } - + public override bool IsFullyDefined() => !string.IsNullOrEmpty(File); /// diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Finalization/SavePublishingOperationMetadataFinalizationActivity.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Finalization/SavePublishingOperationMetadataFinalizationActivity.cs index 76c53c9..8a802bc 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Finalization/SavePublishingOperationMetadataFinalizationActivity.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Finalization/SavePublishingOperationMetadataFinalizationActivity.cs @@ -21,11 +21,11 @@ public SavePublishingOperationMetadataFinalizationActivity( _publishingOperationMetadataCollector = publishingOperationMetadataCollector; _createConnection = createConnection; } - + public async Task Execute() { await using var connection = _createConnection(); - + await connection.OpenAsync(); await CreatePublishingMetadataTableAsync(); @@ -73,11 +73,11 @@ async Task SavePublishingOperationMetadataAsync() INSERT INTO PublishingMetadata(CurrentChangeVersion, SourceVersionMetadata, TargetVersionMetadata, MinChangeVersion, MaxChangeVersion) VALUES ($changeVersion, $sourceVersionMetadata, $targetVersionMetadata, $minChangeVersion, $maxChangeVersion);"; - cmd.Parameters.AddWithValue("$changeVersion", metadata.CurrentChangeVersion ?? (object) DBNull.Value); - cmd.Parameters.AddWithValue("$sourceVersionMetadata", metadata.SourceVersionMetadata?.ToString() ?? (object) DBNull.Value); - cmd.Parameters.AddWithValue("$targetVersionMetadata", metadata.TargetVersionMetadata?.ToString() ?? (object) DBNull.Value); - cmd.Parameters.AddWithValue("$minChangeVersion", metadata.ChangeWindow?.MinChangeVersion ?? (object) DBNull.Value); - cmd.Parameters.AddWithValue("$maxChangeVersion", metadata.ChangeWindow?.MaxChangeVersion ?? (object) DBNull.Value); + cmd.Parameters.AddWithValue("$changeVersion", metadata.CurrentChangeVersion ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$sourceVersionMetadata", metadata.SourceVersionMetadata?.ToString() ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$targetVersionMetadata", metadata.TargetVersionMetadata?.ToString() ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$minChangeVersion", metadata.ChangeWindow?.MinChangeVersion ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$maxChangeVersion", metadata.ChangeWindow?.MaxChangeVersion ?? (object)DBNull.Value); await cmd.ExecuteNonQueryAsync(); } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Metadata/Versioning/SourceSqliteVersionMetadataProvider.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Metadata/Versioning/SourceSqliteVersionMetadataProvider.cs index 6d99894..dc75a7e 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Metadata/Versioning/SourceSqliteVersionMetadataProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Metadata/Versioning/SourceSqliteVersionMetadataProvider.cs @@ -30,7 +30,7 @@ public async Task GetVersionMetadata() if (rawValue != null) { - return JObject.Parse((string) rawValue); + return JObject.Parse((string)rawValue); } return null; diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Modules/SqliteAsSourceModule.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Modules/SqliteAsSourceModule.cs index 9584dc5..a9d6ac4 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Modules/SqliteAsSourceModule.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Modules/SqliteAsSourceModule.cs @@ -29,7 +29,7 @@ public SqliteAsSourceModule(IConfigurationRoot finalConfiguration) { _finalConfiguration = finalConfiguration; } - + protected override void Load(ContainerBuilder builder) { var connectionsConfiguration = _finalConfiguration.GetSection("Connections"); @@ -44,15 +44,15 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType() .As() .SingleInstance(); - + // Version metadata for a Source API builder.RegisterType() .As() .SingleInstance(); - + // Determine data source capabilities for Sqlite as source builder.RegisterType().As(); - + // Register resource page message producer using a page-based strategy builder.RegisterType() .As() diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Modules/SqliteAsTargetModule.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Modules/SqliteAsTargetModule.cs index 39c8746..085bd43 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Modules/SqliteAsTargetModule.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Modules/SqliteAsTargetModule.cs @@ -47,12 +47,12 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType() .As>() .SingleInstance(); - + // Register the processing stage initiators builder.RegisterType().Keyed(PublishingStage.KeyChanges); builder.RegisterType().Keyed(PublishingStage.Upserts); builder.RegisterType().Keyed(PublishingStage.Deletes); - + // Register a finalization step to record publishing operation metadata builder.RegisterType().As(); } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Plugin.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Plugin.cs index a1096c5..9e1d81a 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Plugin.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Plugin.cs @@ -33,7 +33,7 @@ public void PerformFinalRegistrations(ContainerBuilder containerBuilder, IConfig { // TODO: When support is added for Sqlite database as source string sourceConnectionType = ConfigurationHelper.GetSourceConnectionType(finalConfigurationRoot); - + if (sourceConnectionType == SqliteConnectionType) { containerBuilder.RegisterModule(new SqliteAsSourceModule(finalConfigurationRoot)); diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/Counting/SqliteSourceTotalCountProvider.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/Counting/SqliteSourceTotalCountProvider.cs index 4110ed7..a32d3c2 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/Counting/SqliteSourceTotalCountProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/Counting/SqliteSourceTotalCountProvider.cs @@ -20,7 +20,7 @@ public SqliteSourceTotalCountProvider(Func createConnection) { _createConnection = createConnection; } - + public async Task<(bool, long)> TryGetTotalCountAsync( string resourceUrl, Options options, @@ -30,13 +30,13 @@ public SqliteSourceTotalCountProvider(Func createConnection) { await using var connection = _createConnection(); await connection.OpenAsync(cancellationToken).ConfigureAwait(false); - + var cmd = connection.CreateCommand(); cmd.CommandText = @"SELECT ItemCount FROM ResourceItemCount WHERE ResourcePath = $resourcePath"; cmd.Parameters.AddWithValue("$resourcePath", resourceUrl); - long count = (long) (cmd.ExecuteScalar() ?? -1); + long count = (long)(cmd.ExecuteScalar() ?? -1); return (count >= 0, count); } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/MessageHandlers/SqliteStreamResourcePageMessageHandler.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/MessageHandlers/SqliteStreamResourcePageMessageHandler.cs index da23ea5..0432fc3 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/MessageHandlers/SqliteStreamResourcePageMessageHandler.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/MessageHandlers/SqliteStreamResourcePageMessageHandler.cs @@ -18,14 +18,14 @@ namespace EdFi.Tools.ApiPublisher.Connections.Sqlite.Processing.Source.MessageHa public class SqliteStreamResourcePageMessageHandler : IStreamResourcePageMessageHandler { private readonly Func _createConnection; - + private readonly ILogger _logger = Log.ForContext(typeof(SqliteStreamResourcePageMessageHandler)); public SqliteStreamResourcePageMessageHandler(Func createConnection) { _createConnection = createConnection; } - + public async Task> HandleStreamResourcePageAsync( StreamResourcePageMessage message, Options options, diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/MessageProducers/SqliteStreamResourcePageMessageProducer.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/MessageProducers/SqliteStreamResourcePageMessageProducer.cs index 9ebad71..a2f9ec9 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/MessageProducers/SqliteStreamResourcePageMessageProducer.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/MessageProducers/SqliteStreamResourcePageMessageProducer.cs @@ -21,7 +21,7 @@ public class SqliteStreamResourcePageMessageProducer : IStreamResourcePageMessag private readonly Func _createConnection; private readonly ILogger _logger = Log.ForContext(typeof(SqliteStreamResourcePageMessageProducer)); - + public SqliteStreamResourcePageMessageProducer( ISourceTotalCountProvider sourceTotalCountProvider, Func createConnection) @@ -29,7 +29,7 @@ public SqliteStreamResourcePageMessageProducer( _sourceTotalCountProvider = sourceTotalCountProvider; _createConnection = createConnection; } - + public async Task>> ProduceMessagesAsync( StreamResourceMessage message, Options options, @@ -44,7 +44,7 @@ public async Task>> P message.ChangeWindow, errorHandlingBlock, cancellationToken); - + if (!totalCountSuccess) { // Allow processing to continue without performing additional work on this resource. @@ -57,16 +57,16 @@ public async Task>> P { return Enumerable.Empty>(); } - + await using var connection = _createConnection(); var (schema, table, tableSuffix) = SqliteTableNameHelper.ParseDetailsFromResourcePath(message.ResourceUrl); - + var cmd = connection.CreateCommand(); cmd.CommandText = $"SELECT id FROM {schema}__{table}{tableSuffix} ORDER BY id"; - + await connection.OpenAsync(); - + await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.CloseConnection, cancellationToken); var pageMessages = new List>(); diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/Versioning/SqliteSourceCurrentChangeVersionProvider.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/Versioning/SqliteSourceCurrentChangeVersionProvider.cs index ac088d7..d2ee221 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/Versioning/SqliteSourceCurrentChangeVersionProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Source/Versioning/SqliteSourceCurrentChangeVersionProvider.cs @@ -16,7 +16,7 @@ public SqliteSourceCurrentChangeVersionProvider(Func createCon { _createConnection = createConnection; } - + public async Task GetCurrentChangeVersionAsync() { await using var connection = _createConnection(); diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Target/Blocks/SqlLiteProcessingBlocksFactoryBase.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Target/Blocks/SqlLiteProcessingBlocksFactoryBase.cs index f4ceb5d..6c13c2e 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Target/Blocks/SqlLiteProcessingBlocksFactoryBase.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Target/Blocks/SqlLiteProcessingBlocksFactoryBase.cs @@ -33,7 +33,7 @@ protected SqlLiteProcessingBlocksFactoryBase(Func createConnec protected abstract string TableSuffix { get; } - private readonly ConcurrentDictionary _tableTupleByResourceUrl + private readonly ConcurrentDictionary _tableTupleByResourceUrl = new(StringComparer.OrdinalIgnoreCase); public (ITargetBlock, ISourceBlock) CreateProcessingBlocks( @@ -43,7 +43,7 @@ protected SqlLiteProcessingBlocksFactoryBase(Func createConnec async msg => { Options options = createBlocksRequest.Options; - + try { var (schema, table, tableSuffix) = _tableTupleByResourceUrl.GetOrAdd( @@ -80,11 +80,11 @@ protected SqlLiteProcessingBlocksFactoryBase(Func createConnec connection.Open(); int attempts = 0; - + var delay = Backoff.ExponentialBackoff( TimeSpan.FromMilliseconds(options.RetryStartingDelayMilliseconds), options.MaxRetryAttempts); - + var sqlInsertResult = await Policy .Handle() .WaitAndRetryAsync(delay, (result, ts, retryAttempt, ctx) => diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Target/Messages/ResourceJsonMessage.cs b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Target/Messages/ResourceJsonMessage.cs index 18e9c05..1303ef8 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Target/Messages/ResourceJsonMessage.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Sqlite/Processing/Target/Messages/ResourceJsonMessage.cs @@ -18,6 +18,6 @@ public abstract class ResourceJsonMessage public string Json { get; set; } } -public class KeyChangesJsonMessage : ResourceJsonMessage {} -public class UpsertsJsonMessage : ResourceJsonMessage {} -public class DeletesJsonMessage : ResourceJsonMessage {} +public class KeyChangesJsonMessage : ResourceJsonMessage { } +public class UpsertsJsonMessage : ResourceJsonMessage { } +public class DeletesJsonMessage : ResourceJsonMessage { } diff --git a/src/EdFi.Tools.ApiPublisher.Core/ApiClientManagement/HttpClients.cs b/src/EdFi.Tools.ApiPublisher.Core/ApiClientManagement/HttpClients.cs index 98761c4..9ce076e 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/ApiClientManagement/HttpClients.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/ApiClientManagement/HttpClients.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.Core.ApiClientManagement { - public class HttpClients + public class HttpClients { public HttpClient Source { get; set; } public HttpClient Target { get; set; } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/AppSettingsConfigurationProvider.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/AppSettingsConfigurationProvider.cs index 2c60479..0524470 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/AppSettingsConfigurationProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/AppSettingsConfigurationProvider.cs @@ -8,16 +8,16 @@ namespace EdFi.Tools.ApiPublisher.Core.Configuration { - public class AppSettingsConfigurationProvider : IAppSettingsConfigurationProvider + public class AppSettingsConfigurationProvider : IAppSettingsConfigurationProvider { private readonly Lazy _configuration; - + public AppSettingsConfigurationProvider() { _configuration = new Lazy(() => { string environmentName = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); - + var configuration = new ConfigurationBuilder() .AddJsonFile("appSettings.json", optional: true, reloadOnChange: true) .AddJsonFile($"appSettings.{environmentName}.json", optional: true) @@ -26,7 +26,7 @@ public AppSettingsConfigurationProvider() return configuration; }); } - + public IConfiguration GetConfiguration() { return _configuration.Value; diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ChangeProcessorConfiguration.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ChangeProcessorConfiguration.cs index c14d5bd..5e03ffc 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ChangeProcessorConfiguration.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ChangeProcessorConfiguration.cs @@ -10,7 +10,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Configuration { - public class ChangeProcessorConfiguration + public class ChangeProcessorConfiguration { private readonly ILogger _logger = Log.ForContext(typeof(ChangeProcessorConfiguration)); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationHelper.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationHelper.cs index 5b23cae..74224f6 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationHelper.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationHelper.cs @@ -12,7 +12,7 @@ public static class ConfigurationHelper public static string GetConfigurationStoreProviderName(IConfigurationRoot configurationRoot) { var configurationStoreSection = configurationRoot.GetSection("configurationStore"); - + string configurationProviderName = configurationStoreSection.GetValue("provider"); return configurationProviderName; @@ -25,7 +25,7 @@ public static string GetSourceConnectionType(IConfigurationRoot configurationRoo return sourceConnectionConfiguration.GetValue("Type") ?? "api"; } - + public static string GetTargetConnectionType(IConfigurationRoot configurationRoot) { var connectionsConfiguration = configurationRoot.GetSection("Connections"); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationStoreHelper.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationStoreHelper.cs index 645a407..07654e9 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationStoreHelper.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationStoreHelper.cs @@ -5,7 +5,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Configuration { - public static class ConfigurationStoreHelper + public static class ConfigurationStoreHelper { public static string Key(string apiConnectionName) { diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConnectionConfiguration.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConnectionConfiguration.cs index bbb36b6..77d5c00 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConnectionConfiguration.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConnectionConfiguration.cs @@ -5,12 +5,12 @@ namespace EdFi.Tools.ApiPublisher.Core.Configuration -{ - public class ConnectionConfiguration +{ + public class ConnectionConfiguration { public Connections Connections { get; set; } } - + public class Connections { public NamedConnectionDetailsConfiguration Source { get; set; } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/Enhancers/IConfigurationBuilderEnhancer.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/Enhancers/IConfigurationBuilderEnhancer.cs index 1b39c94..3982ace 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/Enhancers/IConfigurationBuilderEnhancer.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/Enhancers/IConfigurationBuilderEnhancer.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Configuration.Enhancers { - public interface IConfigurationBuilderEnhancer + public interface IConfigurationBuilderEnhancer { void Enhance(IConfigurationRoot initialConfiguration, IConfigurationBuilder configurationBuilder); } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/IAppSettingsConfigurationProvider.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/IAppSettingsConfigurationProvider.cs index b5e1b4d..503e199 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/IAppSettingsConfigurationProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/IAppSettingsConfigurationProvider.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Configuration { - public interface IAppSettingsConfigurationProvider + public interface IAppSettingsConfigurationProvider { IConfiguration GetConfiguration(); } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/IChangeProcessorRuntimeConfigurationProvider.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/IChangeProcessorRuntimeConfigurationProvider.cs index 3c69cac..ddedb58 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/IChangeProcessorRuntimeConfigurationProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/IChangeProcessorRuntimeConfigurationProvider.cs @@ -5,7 +5,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Configuration { - public interface IChangeProcessorRuntimeConfigurationProvider + public interface IChangeProcessorRuntimeConfigurationProvider { ChangeProcessorConfiguration GetRuntimeConfiguration(string[] commandLineArgs); } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ISourceConnectionDetails.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ISourceConnectionDetails.cs index d385699..353b56f 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ISourceConnectionDetails.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ISourceConnectionDetails.cs @@ -12,15 +12,15 @@ public interface ISourceConnectionDetails : INamedConnectionDetails public bool? IgnoreIsolation { get; set; } IDictionary LastChangeVersionProcessedByTargetName { get; } - + public string Include { get; set; } - + public string IncludeOnly { get; set; } - + public string Exclude { get; set; } - + public string ExcludeOnly { get; set; } - + /// /// Gets or sets the explicitly provided value to use for the last change version processed. /// diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/PollyRateLimiter.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/PollyRateLimiter.cs index ce202d2..4a1b63c 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/PollyRateLimiter.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/PollyRateLimiter.cs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: Apache-2.0 // Licensed to the Ed-Fi Alliance under one or more agreements. // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. @@ -21,7 +21,7 @@ public class PollyRateLimiter : IRateLimiting public PollyRateLimiter(Options options) { _rateLimiter = Policy.RateLimitAsync( - options.RateLimitNumberExecutions, + options.RateLimitNumberExecutions, TimeSpan.FromSeconds(options.RateLimitTimeSeconds), options.RateLimitNumberExecutions); _retryPolicyForRateLimit = Policy @@ -35,17 +35,23 @@ public PollyRateLimiter(Options options) } ); } - + public async Task ExecuteAsync(Func> action) { try { return await _rateLimiter.ExecuteAsync(action); } - catch (RateLimitRejectedException) { + catch (RateLimitRejectedException) + { _logger.Fatal("Rate limit exceeded. Please try again later."); throw; } + catch (Exception ex) + { + _logger.Fatal(ex, ex.Message); + throw; + } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/Serilog/TextFormatter.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/Serilog/TextFormatter.cs index b5b3ae5..14a06f9 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/Serilog/TextFormatter.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/Serilog/TextFormatter.cs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: Apache-2.0 // Licensed to the Ed-Fi Alliance under one or more agreements. // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. @@ -43,8 +43,8 @@ public void Format(LogEvent logEvent, TextWriter output) public class LogEventFormatValues { private const string ThreadIdSerilogPropertyName = "ThreadId"; - private const string SourceContextSerilogPropertyName = "SourceContext"; - + private const string SourceContextSerilogPropertyName = "SourceContext"; + public DateTime Timestamp => _logEvent.Timestamp.DateTime; public string Level => GetShortFormatLevel(_logEvent.Level); public string SourceContext => GetStringValueFromProperty(_logEvent.Properties.GetValueOrDefault(SourceContextSerilogPropertyName)); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Dependencies/FallbackGraphMLDependencyMetadataProvider.cs b/src/EdFi.Tools.ApiPublisher.Core/Dependencies/FallbackGraphMLDependencyMetadataProvider.cs index 0f1bb1c..fc36d90 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Dependencies/FallbackGraphMLDependencyMetadataProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Dependencies/FallbackGraphMLDependencyMetadataProvider.cs @@ -18,7 +18,7 @@ public FallbackGraphMLDependencyMetadataProvider(Options options) { _options = options; } - + public Task<(XElement, XNamespace)> GetDependencyMetadataAsync() { if (!_options.UseSourceDependencyMetadata) diff --git a/src/EdFi.Tools.ApiPublisher.Core/Dependencies/IResourceDependencyProvider.cs b/src/EdFi.Tools.ApiPublisher.Core/Dependencies/IResourceDependencyProvider.cs index 30db70e..5ff5d15 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Dependencies/IResourceDependencyProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Dependencies/IResourceDependencyProvider.cs @@ -8,7 +8,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Dependencies { - public interface IResourceDependencyProvider + public interface IResourceDependencyProvider { Task> GetDependenciesByResourcePathAsync(bool includeDescriptors); } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Dependencies/ResourceDependencyProvider.cs b/src/EdFi.Tools.ApiPublisher.Core/Dependencies/ResourceDependencyProvider.cs index 08f8939..889d87b 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Dependencies/ResourceDependencyProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Dependencies/ResourceDependencyProvider.cs @@ -13,7 +13,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Dependencies { - public class ResourceDependencyProvider : IResourceDependencyProvider + public class ResourceDependencyProvider : IResourceDependencyProvider { private readonly IGraphMLDependencyMetadataProvider _graphMLDependencyMetadataProvider; diff --git a/src/EdFi.Tools.ApiPublisher.Core/Extensions/HttpStatusCodeExtensions.cs b/src/EdFi.Tools.ApiPublisher.Core/Extensions/HttpStatusCodeExtensions.cs index 8997fb6..40b39db 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Extensions/HttpStatusCodeExtensions.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Extensions/HttpStatusCodeExtensions.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Extensions { - public static class HttpStatusCodeExtensions + public static class HttpStatusCodeExtensions { public static bool IsPotentiallyTransientFailure(this HttpStatusCode httpStatusCode) { diff --git a/src/EdFi.Tools.ApiPublisher.Core/Extensions/IDictionaryExtensions.cs b/src/EdFi.Tools.ApiPublisher.Core/Extensions/IDictionaryExtensions.cs index 6561988..a2c965a 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Extensions/IDictionaryExtensions.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Extensions/IDictionaryExtensions.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Extensions { - public static class IDictionaryExtensions + public static class IDictionaryExtensions { public static TValue GetValueOrDefault(this IDictionary dictionary, TKey key) { diff --git a/src/EdFi.Tools.ApiPublisher.Core/Extensions/NullableBooleanExtensions.cs b/src/EdFi.Tools.ApiPublisher.Core/Extensions/NullableBooleanExtensions.cs index 77301ab..4a6ae4b 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Extensions/NullableBooleanExtensions.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Extensions/NullableBooleanExtensions.cs @@ -16,7 +16,7 @@ public static bool IsNotTrue(this bool? value) { return value != true; } - + public static bool IsFalse(this bool? value) { return (value == false); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Extensions/StringExtensions.cs b/src/EdFi.Tools.ApiPublisher.Core/Extensions/StringExtensions.cs index 5d730f2..4151be2 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Extensions/StringExtensions.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Extensions/StringExtensions.cs @@ -8,7 +8,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Extensions { - public static class StringExtensions + public static class StringExtensions { public static string EnsureSuffixApplied(this string text, string suffix) { @@ -16,7 +16,7 @@ public static string EnsureSuffixApplied(this string text, string suffix) { return suffix; } - + if (text.EndsWith(suffix)) { return text; @@ -24,7 +24,7 @@ public static string EnsureSuffixApplied(this string text, string suffix) return text + suffix; } - + public static bool TryTrimSuffix(this string text, string suffix, out string trimmedText) { trimmedText = null; @@ -61,7 +61,7 @@ public static string TrimSuffix(this string text, string suffix) return text; } - + /// /// Returns a string that is converted to camel casing, detecting and handling acronyms as prefixes and suffixes. /// @@ -103,7 +103,7 @@ public static string ToCamelCase(this string text) // Apply simple camel casing return char.ToLower(text[0]) + text.Substring(1); } - + public static bool EqualsIgnoreCase(this string text, string compareText) => text == null ? compareText == null : text.Equals(compareText, StringComparison.InvariantCultureIgnoreCase); public static bool StartsWithIgnoreCase(this string text, string compareText) => text == null ? compareText == null : text.StartsWith(compareText, StringComparison.InvariantCultureIgnoreCase); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Helpers/ApiRequestHelper.cs b/src/EdFi.Tools.ApiPublisher.Core/Helpers/ApiRequestHelper.cs index d9538b0..58b5dc7 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Helpers/ApiRequestHelper.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Helpers/ApiRequestHelper.cs @@ -8,14 +8,14 @@ namespace EdFi.Tools.ApiPublisher.Core.Helpers { - public static class ApiRequestHelper + public static class ApiRequestHelper { public static string GetChangeWindowQueryStringParameters(ChangeWindow changeWindow) { string changeWindowParms = changeWindow == null ? String.Empty : $"&minChangeVersion={changeWindow.MinChangeVersion}&maxChangeVersion={changeWindow.MaxChangeVersion}"; - + return changeWindowParms; } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Helpers/ExponentialBackOffHelper.cs b/src/EdFi.Tools.ApiPublisher.Core/Helpers/ExponentialBackOffHelper.cs index 523d97d..ec5f8af 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Helpers/ExponentialBackOffHelper.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Helpers/ExponentialBackOffHelper.cs @@ -9,17 +9,17 @@ namespace EdFi.Tools.ApiPublisher.Core.Helpers { - public static class ExponentialBackOffHelper + public static class ExponentialBackOffHelper { private static readonly ILogger _logger = Log.ForContext(typeof(ExponentialBackOffHelper)); - + public static void PerformDelay(ref int delay) { if (_logger.IsEnabled(LogEventLevel.Debug)) _logger.Debug($"Performing exponential \"back off\" of thread for {delay} milliseconds."); - + Thread.Sleep(delay); - + delay = delay * 2; } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Helpers/ResourcePathHelper.cs b/src/EdFi.Tools.ApiPublisher.Core/Helpers/ResourcePathHelper.cs index 38409ff..9252771 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Helpers/ResourcePathHelper.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Helpers/ResourcePathHelper.cs @@ -8,7 +8,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Helpers { - public static class ResourcePathHelper + public static class ResourcePathHelper { /// /// Parses the supplied text as CSV (comma separated values) into an array, removing diff --git a/src/EdFi.Tools.ApiPublisher.Core/Helpers/Version.cs b/src/EdFi.Tools.ApiPublisher.Core/Helpers/Version.cs index 497846e..cbb009f 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Helpers/Version.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Helpers/Version.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Helpers { - public class Version : IEquatable + public class Version : IEquatable { public Version(string versionText) { @@ -68,14 +68,14 @@ public override bool Equals(object obj) return false; } - return Equals((Version) obj); + return Equals((Version)obj); } public override int GetHashCode() { return HashCode.Combine(Major, Minor, Revision); } - + #endregion } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Helpers/VersionHelper.cs b/src/EdFi.Tools.ApiPublisher.Core/Helpers/VersionHelper.cs index 3e6d659..7c816a7 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Helpers/VersionHelper.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Helpers/VersionHelper.cs @@ -5,11 +5,11 @@ namespace EdFi.Tools.ApiPublisher.Core.Helpers { - public static class VersionExtensions + public static class VersionExtensions { public static bool IsAtLeast(this Version apiVersion, int majorVersion, int minorVersion) { - return apiVersion.Major > majorVersion + return apiVersion.Major > majorVersion || (apiVersion.Major == majorVersion && apiVersion.Minor >= minorVersion); } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Inflection/CompositeTermInflector.cs b/src/EdFi.Tools.ApiPublisher.Core/Inflection/CompositeTermInflector.cs index a94ac76..143a342 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Inflection/CompositeTermInflector.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Inflection/CompositeTermInflector.cs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: Apache-2.0 // Licensed to the Ed-Fi Alliance under one or more agreements. // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. @@ -9,14 +9,14 @@ namespace EdFi.Common.Inflection { - public static class CompositeTermInflector + public static class CompositeTermInflector { - private static readonly HashSet IgnoredSuffixes = new HashSet(); + private static readonly HashSet _ignoredSuffixes = new HashSet(); - private static readonly ConcurrentDictionary PluralizedByTerm + private static readonly ConcurrentDictionary _pluralizedByTerm = new ConcurrentDictionary(); - private static readonly ConcurrentDictionary SingularizedByTerm + private static readonly ConcurrentDictionary _singularizedByTerm = new ConcurrentDictionary(); static CompositeTermInflector() @@ -26,12 +26,12 @@ static CompositeTermInflector() public static void AddIgnoredSuffix(string suffix) { - IgnoredSuffixes.Add(suffix.ToLower()); + _ignoredSuffixes.Add(suffix.ToLower()); } public static string MakePlural(string compositeTerm) { - return PluralizedByTerm.GetOrAdd( + return _pluralizedByTerm.GetOrAdd( compositeTerm, t => { @@ -47,7 +47,7 @@ public static string MakePlural(string compositeTerm) string term = matches[i] .Value; - if (isCompositeTermPluralized || IgnoredSuffixes.Contains(term.ToLower())) + if (isCompositeTermPluralized || _ignoredSuffixes.Contains(term.ToLower())) { result.Insert(0, term); continue; @@ -65,7 +65,7 @@ public static string MakePlural(string compositeTerm) public static string MakeSingular(string compositeTerm) { - return SingularizedByTerm.GetOrAdd( + return _singularizedByTerm.GetOrAdd( compositeTerm, t => { @@ -81,7 +81,7 @@ public static string MakeSingular(string compositeTerm) string term = matches[i] .Value; - if (isCompositeTermSingularized || IgnoredSuffixes.Contains(term.ToLower())) + if (isCompositeTermSingularized || _ignoredSuffixes.Contains(term.ToLower())) { result.Insert(0, term); continue; diff --git a/src/EdFi.Tools.ApiPublisher.Core/Inflection/Inflector.cs b/src/EdFi.Tools.ApiPublisher.Core/Inflection/Inflector.cs index c4be3c7..07ed29d 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Inflection/Inflector.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Inflection/Inflector.cs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: Apache-2.0 // Licensed to the Ed-Fi Alliance under one or more agreements. // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. @@ -12,10 +12,10 @@ namespace EdFi.Common.Inflection { - // Done: GKM - Unify this with the instance in EdFi.Ods.CodeGen. - // In the meantime, duplicate any changes made to this file in the corresponding class. + // Done: GKM - Unify this with the instance in EdFi.Ods.CodeGen. + // In the meantime, duplicate any changes made to this file in the corresponding class. - /* + /* * SubSonic - http://subsonicproject.com * * The contents of this file are subject to the New BSD @@ -29,10 +29,10 @@ namespace EdFi.Common.Inflection * rights and limitations under the License. */ - /// - /// Summary for the Inflector class - /// - public static class Inflector + /// + /// Summary for the Inflector class + /// + public static class Inflector { private static readonly List _plurals = new List(); private static readonly List _singulars = new List(); @@ -189,7 +189,7 @@ public static string Inflect(string word, int basisCount, string singularWordOve { throw new ArgumentOutOfRangeException(nameof(basisCount), "Value must be greater than or equal to 0."); } - + if (basisCount == 1) { return singularWordOverride ?? MakeSingular(word); @@ -197,7 +197,7 @@ public static string Inflect(string word, int basisCount, string singularWordOve return pluralWordOverride ?? MakePlural(word); } - + /// /// Applies the rules. /// @@ -236,7 +236,7 @@ public static string ToTitleCase(string word) return Regex.Replace( ToHumanCase(AddUnderscores(word)), @"\b([a-z])", - delegate(Match match) + delegate (Match match) { return match.Captures[0] .Value.ToUpper(); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Metadata/CurrentChangeVersionCollector.cs b/src/EdFi.Tools.ApiPublisher.Core/Metadata/CurrentChangeVersionCollector.cs index f710fe7..963a78f 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Metadata/CurrentChangeVersionCollector.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Metadata/CurrentChangeVersionCollector.cs @@ -24,7 +24,7 @@ public CurrentChangeVersionCollector( public async Task GetCurrentChangeVersionAsync() { var currentChangeVersion = await _currentChangeVersionProvider.GetCurrentChangeVersionAsync(); - + _metadataCollector.SetCurrentChangeVersion(currentChangeVersion); return currentChangeVersion; diff --git a/src/EdFi.Tools.ApiPublisher.Core/Metadata/IPublishingOperationMetadataCollector.cs b/src/EdFi.Tools.ApiPublisher.Core/Metadata/IPublishingOperationMetadataCollector.cs index 3675e6d..26acc56 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Metadata/IPublishingOperationMetadataCollector.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Metadata/IPublishingOperationMetadataCollector.cs @@ -15,6 +15,6 @@ public interface IPublishingOperationMetadataCollector void SetTargetVersionMetadata(JObject versionMetadata); void SetChangeWindow(ChangeWindow changeWindow); void SetResourceItemCount(string resourcePath, long count); - + PublishingOperationMetadata GetMetadata(); } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Metadata/PublishingOperationMetadataCollector.cs b/src/EdFi.Tools.ApiPublisher.Core/Metadata/PublishingOperationMetadataCollector.cs index 916692b..f317d98 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Metadata/PublishingOperationMetadataCollector.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Metadata/PublishingOperationMetadataCollector.cs @@ -40,8 +40,8 @@ public void SetChangeWindow(ChangeWindow changeWindow) public void SetResourceItemCount(string resourcePath, long count) { - _resourceItemCountByPath.AddOrUpdate(resourcePath, - _ => count, + _resourceItemCountByPath.AddOrUpdate(resourcePath, + _ => count, (_, _) => count); } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Metadata/ResourceItemCountCollector.cs b/src/EdFi.Tools.ApiPublisher.Core/Metadata/ResourceItemCountCollector.cs index d9629d9..9aaba4c 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Metadata/ResourceItemCountCollector.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Metadata/ResourceItemCountCollector.cs @@ -23,7 +23,7 @@ public ResourceItemCountCollector(ISourceTotalCountProvider totalCountProvider, _totalCountProvider = totalCountProvider; _metadataCollector = metadataCollector; } - + public async Task<(bool, long)> TryGetTotalCountAsync( string resourceUrl, Options options, @@ -41,7 +41,7 @@ public ResourceItemCountCollector(ISourceTotalCountProvider totalCountProvider, _metadataCollector.SetResourceItemCount(resourceUrl, success ? count : -1); _metadataCollector.SetChangeWindow(changeWindow); - + return (success, count); } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Metadata/SourceEdFiVersionMetadataCollector.cs b/src/EdFi.Tools.ApiPublisher.Core/Metadata/SourceEdFiVersionMetadataCollector.cs index 9c3792c..2f1a639 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Metadata/SourceEdFiVersionMetadataCollector.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Metadata/SourceEdFiVersionMetadataCollector.cs @@ -25,7 +25,7 @@ public SourceEdFiVersionMetadataCollector( public async Task GetVersionMetadata() { var versionMetadata = await _sourceEdFiApiVersionMetadataProvider.GetVersionMetadata(); - + _metadataCollector.SetSourceVersionMetadata(versionMetadata); return versionMetadata; diff --git a/src/EdFi.Tools.ApiPublisher.Core/Metadata/TargetEdFiVersionMetadataCollector.cs b/src/EdFi.Tools.ApiPublisher.Core/Metadata/TargetEdFiVersionMetadataCollector.cs index ef7b6d1..d3327cd 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Metadata/TargetEdFiVersionMetadataCollector.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Metadata/TargetEdFiVersionMetadataCollector.cs @@ -25,7 +25,7 @@ public TargetEdFiVersionMetadataCollector( public async Task GetVersionMetadata() { var versionMetadata = await _targetEdFiApiVersionMetadataProvider.GetVersionMetadata(); - + _metadataCollector.SetTargetVersionMetadata(versionMetadata); return versionMetadata; diff --git a/src/EdFi.Tools.ApiPublisher.Core/Modules/CoreModule.cs b/src/EdFi.Tools.ApiPublisher.Core/Modules/CoreModule.cs index 692383e..65a034a 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Modules/CoreModule.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Modules/CoreModule.cs @@ -14,7 +14,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Modules { - public class CoreModule : Module + public class CoreModule : Module { protected override void Load(ContainerBuilder builder) { @@ -25,23 +25,23 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType() .As() .SingleInstance(); - + // General purpose version checker builder.RegisterType() .As() .SingleInstance(); - + // Register decorators for collecting publishing operation metadata builder.RegisterType() .As() .SingleInstance(); - + builder.RegisterDecorator(); builder.RegisterDecorator(); - + builder.RegisterDecorator(); builder.RegisterDecorator(); - + // Block factories builder.RegisterType(); //.SingleInstance(); builder.RegisterType(); //.SingleInstance(); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Modules/NodeJsRemediationsModule.cs b/src/EdFi.Tools.ApiPublisher.Core/Modules/NodeJsRemediationsModule.cs index 27fa679..c191951 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Modules/NodeJsRemediationsModule.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Modules/NodeJsRemediationsModule.cs @@ -20,7 +20,7 @@ public NodeJsRemediationsModule(IConfigurationRoot initialConfiguration) { this._initialConfiguration = initialConfiguration; } - + protected override void Load(ContainerBuilder builder) { string remediationsScriptFile = _initialConfiguration.GetValue("Options:RemediationsScriptFile"); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Plugin/IPlugin.cs b/src/EdFi.Tools.ApiPublisher.Core/Plugin/IPlugin.cs index 8d973f0..d5d2307 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Plugin/IPlugin.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Plugin/IPlugin.cs @@ -11,8 +11,8 @@ namespace EdFi.Tools.ApiPublisher.Core.Plugin; public interface IPlugin { void ApplyConfiguration(string[] args, IConfigurationBuilder configBuilder); - + void PerformConfigurationRegistrations(ContainerBuilder containerBuilder, IConfigurationRoot initialConfigurationRoot); - + void PerformFinalRegistrations(ContainerBuilder containerBuilder, IConfigurationRoot finalConfigurationRoot); } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/CreateBlocksRequest.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/CreateBlocksRequest.cs index e9bb228..53dcfda 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/CreateBlocksRequest.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/CreateBlocksRequest.cs @@ -10,26 +10,20 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing.Blocks { - public class CreateBlocksRequest + public class CreateBlocksRequest { public CreateBlocksRequest( - // EdFiApiClient sourceApiClient, - // EdFiApiClient targetApiClient, Options options, AuthorizationFailureHandling[] authorizationFailureHandling, ITargetBlock errorHandlingBlock, Func javaScriptModuleFactory) { - // SourceApiClient = sourceApiClient; - // TargetApiClient = targetApiClient; Options = options; AuthorizationFailureHandling = authorizationFailureHandling; ErrorHandlingBlock = errorHandlingBlock; JavaScriptModuleFactory = javaScriptModuleFactory; } - // public EdFiApiClient SourceApiClient { get; set; } - // public EdFiApiClient TargetApiClient { get; set; } public Options Options { get; set; } public AuthorizationFailureHandling[] AuthorizationFailureHandling { get; set; } public ITargetBlock ErrorHandlingBlock { get; set; } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/PublishErrorsBlocksFactory.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/PublishErrorsBlocksFactory.cs index 0bd7e48..1e115a6 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/PublishErrorsBlocksFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/PublishErrorsBlocksFactory.cs @@ -11,7 +11,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing.Blocks { - public class PublishErrorsBlocksFactory + public class PublishErrorsBlocksFactory { private static readonly ILogger _logger = Log.Logger.ForContext(typeof(PublishErrorsBlocksFactory)); private IErrorPublisher _errorPublisher; @@ -20,17 +20,17 @@ public PublishErrorsBlocksFactory(IErrorPublisher errorPublisher) { _errorPublisher = errorPublisher; } - + public ValueTuple, ActionBlock> CreateBlocks(Options options) { var publishErrorsIngestionBlock = new BatchBlock(options.ErrorPublishingBatchSize); var publishErrorsCompletionBlock = CreatePublishErrorsBlock(_errorPublisher); - - publishErrorsIngestionBlock.LinkTo(publishErrorsCompletionBlock, new DataflowLinkOptions {PropagateCompletion = true}); + + publishErrorsIngestionBlock.LinkTo(publishErrorsCompletionBlock, new DataflowLinkOptions { PropagateCompletion = true }); return (publishErrorsIngestionBlock, publishErrorsCompletionBlock); } - + private ActionBlock CreatePublishErrorsBlock(IErrorPublisher errorPublisher) { return new ActionBlock(async errors => diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/StreamResourceBlockFactory.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/StreamResourceBlockFactory.cs index dcc8a11..a9b69f8 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/StreamResourceBlockFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/StreamResourceBlockFactory.cs @@ -6,6 +6,7 @@ using EdFi.Tools.ApiPublisher.Core.Configuration; using EdFi.Tools.ApiPublisher.Core.Processing.Handlers; using EdFi.Tools.ApiPublisher.Core.Processing.Messages; +using Polly.RateLimit; using Serilog; using Serilog.Events; using System; @@ -17,7 +18,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing.Blocks { - public class StreamResourceBlockFactory + public class StreamResourceBlockFactory { private readonly ILogger _logger = Log.ForContext(typeof(StreamResourceBlockFactory)); private readonly IStreamResourcePageMessageProducer _streamResourcePageMessageProducer; @@ -63,6 +64,11 @@ public TransformManyBlock>> createProcessDataMessages, cancellationToken); } + catch (RateLimitRejectedException ex) + { + _logger.Fatal(ex, ex.Message); + throw; + } catch (Exception ex) { _logger.Error($"{message.ResourceUrl}: An unhandled exception occurred while producing streaming page messages:{Environment.NewLine}{ex}"); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/StreamResourcePagesBlockFactory.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/StreamResourcePagesBlockFactory.cs index 1c257a3..b46eaef 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/StreamResourcePagesBlockFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/Blocks/StreamResourcePagesBlockFactory.cs @@ -6,25 +6,26 @@ using EdFi.Tools.ApiPublisher.Core.Configuration; using EdFi.Tools.ApiPublisher.Core.Processing.Handlers; using EdFi.Tools.ApiPublisher.Core.Processing.Messages; +using Polly.RateLimit; using Serilog; using System; using System.Threading.Tasks.Dataflow; namespace EdFi.Tools.ApiPublisher.Core.Processing.Blocks { - public class StreamResourcePagesBlockFactory + public class StreamResourcePagesBlockFactory { private readonly IStreamResourcePageMessageHandler _streamResourcePageMessageHandler; - + private readonly ILogger _logger = Log.ForContext(typeof(StreamResourcePagesBlockFactory)); public StreamResourcePagesBlockFactory(IStreamResourcePageMessageHandler streamResourcePageMessageHandler) { _streamResourcePageMessageHandler = streamResourcePageMessageHandler; } - + public TransformManyBlock, TProcessDataMessage> CreateBlock( - Options options, + Options options, ITargetBlock errorHandlingBlock) { var streamResourcePagesBlock = @@ -35,6 +36,11 @@ public TransformManyBlock, TProce { return await _streamResourcePageMessageHandler.HandleStreamResourcePageAsync(msg, options, errorHandlingBlock).ConfigureAwait(false); } + catch (RateLimitRejectedException ex) + { + _logger.Fatal(ex, "{ResourceUrl}: Rate limit exceeded. Please try again later.", msg.ResourceUrl); + throw; + } catch (Exception ex) { _logger.Error($"{msg.ResourceUrl}: An unhandled exception occurred in the StreamResourcePages block: {ex}"); @@ -45,8 +51,8 @@ public TransformManyBlock, TProce { MaxDegreeOfParallelism = options.MaxDegreeOfParallelismForStreamResourcePages }); - + return streamResourcePagesBlock; } - } + } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeProcessor.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeProcessor.cs index b3efc7d..23f1a83 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeProcessor.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeProcessor.cs @@ -29,13 +29,13 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing { - public enum PublishingStage + public enum PublishingStage { KeyChanges, Upserts, Deletes } - + public class ChangeProcessor { private ILogger _logger = Log.ForContext(typeof(ChangeProcessor)); @@ -80,13 +80,13 @@ public ChangeProcessor( _publishingStageInitiatorByStage = publishingStageInitiatorByStage; _finalizationActivities = finalizationActivities; } - + public async Task ProcessChangesAsync(ChangeProcessorConfiguration configuration, CancellationToken cancellationToken) { var processStopwatch = new Stopwatch(); processStopwatch.Start(); - - var authorizationFailureHandling= configuration.AuthorizationFailureHandling; + + var authorizationFailureHandling = configuration.AuthorizationFailureHandling; var options = configuration.Options; var javascriptModuleFactory = configuration.JavascriptModuleFactory; @@ -109,7 +109,7 @@ await _sourceIsolationApplicator.ApplySourceSnapshotIdentifierAsync(configuratio ChangeWindow changeWindow = null; // Only named (managed) connections can use a Change Window for processing. - if ((!string.IsNullOrWhiteSpace(_sourceConnectionDetails.Name) + if ((!string.IsNullOrWhiteSpace(_sourceConnectionDetails.Name) && !string.IsNullOrWhiteSpace(_targetConnectionDetails.Name)) || options.UseChangeVersionPaging) { @@ -133,7 +133,7 @@ await _sourceIsolationApplicator.ApplySourceSnapshotIdentifierAsync(configuratio { return; } - + // Create the shared error processing block var (publishErrorsIngestionBlock, publishErrorsCompletionBlock) = _publishErrorsBlocksFactory.CreateBlocks(options); @@ -147,21 +147,21 @@ await _sourceIsolationApplicator.ApplySourceSnapshotIdentifierAsync(configuratio publishErrorsIngestionBlock, cancellationToken) .ConfigureAwait(false); - + // Process all the "Upserts" var postTaskStatuses = ProcessUpsertsToCompletion( - dependencyKeysByResourceKey, + dependencyKeysByResourceKey, options, authorizationFailureHandling, - changeWindow, + changeWindow, publishErrorsIngestionBlock, javascriptModuleFactory, cancellationToken); // Process all the deletions var deleteTaskStatuses = await ProcessDeletesToCompletionAsync( - changeWindow, - dependencyKeysByResourceKey, + changeWindow, + dependencyKeysByResourceKey, options, authorizationFailureHandling, publishErrorsIngestionBlock, @@ -170,7 +170,7 @@ await _sourceIsolationApplicator.ApplySourceSnapshotIdentifierAsync(configuratio // Indicate to the error handling that we're done feeding it errors. publishErrorsIngestionBlock.Complete(); - + // Wait for all errors to be published. _logger.Debug($"Waiting for all errors to be published."); publishErrorsCompletionBlock.Completion.Wait(); @@ -210,14 +210,14 @@ await UpdateChangeVersionAsync(configuration, changeWindow) } private async Task UpdateChangeVersionAsync( - ChangeProcessorConfiguration configuration, + ChangeProcessorConfiguration configuration, ChangeWindow changeWindow) { var sourceDetails = _sourceConnectionDetails; var sinkDetails = _targetConnectionDetails; - + var configurationStoreSection = configuration.ConfigurationStoreSection; - + // If we have a name for source and target connections, write the change version if (!string.IsNullOrEmpty(sourceDetails.Name) && !string.IsNullOrEmpty(sinkDetails.Name)) @@ -253,7 +253,7 @@ await _changeVersionProcessedWriter.SetProcessedChangeVersionAsync( { _logger.Information($"Unable to update the last change version processed because no name was provided for the target."); } - + _logger.Information($"Last Change Version Processed for source to target: {changeWindow.MaxChangeVersion}"); } } @@ -264,14 +264,14 @@ private async Task> PrepareResourceDependenciesAsy AuthorizationFailureHandling[] authorizationFailureHandling) { // Get the dependencies - var postDependencyKeysByResourceKey = + var postDependencyKeysByResourceKey = await _resourceDependencyProvider.GetDependenciesByResourcePathAsync(options.IncludeDescriptors) .ConfigureAwait(false); // Ensure the Publishing extension is not present in dependencies -- we don't want to publish snapshots as a resource // NOTE: This logic is unnecessary starting with Ed-Fi ODS API v5.2 postDependencyKeysByResourceKey.Remove("/publishing/snapshots"); - + AdjustDependenciesForConfiguredAuthorizationConcerns(); // Filter resources down to just those requested, if an explicit inclusion list provided @@ -282,18 +282,18 @@ await _resourceDependencyProvider.GetDependenciesByResourcePathAsync(options.Inc var includeResourcePaths = ResourcePathHelper.ParseResourcesCsvToResourcePathArray(_sourceConnectionDetails.Include); var includeOnlyResourcePaths = ResourcePathHelper.ParseResourcesCsvToResourcePathArray(_sourceConnectionDetails.IncludeOnly); - + // Evaluate whether any of the included resources have a "retry" dependency var retryDependenciesForIncludeResourcePaths = includeResourcePaths .Where(p => postDependencyKeysByResourceKey.ContainsKey($"{p}{Conventions.RetryKeySuffix}")) .Select(p => $"{p}{Conventions.RetryKeySuffix}") .ToArray(); - + var retryDependenciesForIncludeOnlyResourcePaths = includeOnlyResourcePaths .Where(p => postDependencyKeysByResourceKey.ContainsKey($"{p}{Conventions.RetryKeySuffix}")) .Select(p => $"{p}{Conventions.RetryKeySuffix}") .ToArray(); - + postDependencyKeysByResourceKey = ApplyResourceInclusionsToDependencies( postDependencyKeysByResourceKey, includeResourcePaths.Concat(retryDependenciesForIncludeResourcePaths).ToArray(), @@ -314,7 +314,7 @@ await _resourceDependencyProvider.GetDependenciesByResourcePathAsync(options.Inc // _logger.Debug(resourceListMessage); // } } - + if (!string.IsNullOrWhiteSpace(_sourceConnectionDetails.Exclude) || !string.IsNullOrWhiteSpace(_sourceConnectionDetails.ExcludeOnly)) { _logger.Information("Applying resource exclusions..."); @@ -322,7 +322,7 @@ await _resourceDependencyProvider.GetDependenciesByResourcePathAsync(options.Inc var excludeResourcePaths = ResourcePathHelper.ParseResourcesCsvToResourcePathArray(_sourceConnectionDetails.Exclude); var excludeOnlyResourcePaths = ResourcePathHelper.ParseResourcesCsvToResourcePathArray(_sourceConnectionDetails.ExcludeOnly); - + // Evaluate whether any of the included resources have a "retry" dependency var retryDependenciesForExcludeResourcePaths = excludeResourcePaths .Where(p => postDependencyKeysByResourceKey.ContainsKey($"{p}{Conventions.RetryKeySuffix}")) @@ -333,7 +333,7 @@ await _resourceDependencyProvider.GetDependenciesByResourcePathAsync(options.Inc .Where(p => postDependencyKeysByResourceKey.ContainsKey($"{p}{Conventions.RetryKeySuffix}")) .Select(p => $"{p}{Conventions.RetryKeySuffix}") .ToArray(); - + postDependencyKeysByResourceKey = ApplyResourceExclusionsToDependencies( postDependencyKeysByResourceKey, excludeResourcePaths.Concat(retryDependenciesForExcludeResourcePaths).ToArray(), @@ -380,18 +380,18 @@ await _resourceDependencyProvider.GetDependenciesByResourcePathAsync(options.Inc } _logger.Information($"{postDependencyKeysByResourceKey.Count} resources to be processed after applying configuration for source API resource inclusions and/or exclusions."); - + var reportableResources = GetReportableResources(); - + var resourceListMessage = $"The following resources are to be published:{Environment.NewLine}{string.Join(Environment.NewLine, reportableResources.Select(kvp => kvp.Key + string.Join(string.Empty, kvp.Value.Select(x => Environment.NewLine + "\t" + x))))}"; - + // if (options.WhatIf) // { - _logger.Information(resourceListMessage); + _logger.Information(resourceListMessage); // } // else // { - // _logger.Debug(resourceListMessage); + // _logger.Debug(resourceListMessage); // } return postDependencyKeysByResourceKey; @@ -420,7 +420,7 @@ void AdjustDependenciesForConfiguredAuthorizationConcerns() { postDependencyKeysByResourceKey[dependencyEntryKey] = postDependencyKeysByResourceKey[dependencyEntryKey] - .Concat(new[] {dependencyAdjustment.RetryResourceKey}) + .Concat(new[] { dependencyAdjustment.RetryResourceKey }) .ToArray(); } @@ -432,7 +432,7 @@ void AdjustDependenciesForConfiguredAuthorizationConcerns() IDictionary ApplyResourceExclusionsToDependencies( IDictionary dependenciesByResourcePath, - string[] excludeResourcePaths, + string[] excludeResourcePaths, string[] excludeOnlyResourcePaths) { var resourcesToInclude = new HashSet(dependenciesByResourcePath.Keys, StringComparer.OrdinalIgnoreCase); @@ -442,9 +442,9 @@ IDictionary ApplyResourceExclusionsToDependencies( { allExclusionTraceEntries.Add( $"Excluding resource '{excludedResourcePath}' and its dependents..."); - + var exclusionTraceEntries = new List(); - + RemoveDependentResources(excludedResourcePath, exclusionTraceEntries); allExclusionTraceEntries.AddRange(exclusionTraceEntries.Distinct()); @@ -454,10 +454,10 @@ IDictionary ApplyResourceExclusionsToDependencies( { allExclusionTraceEntries.Add( $"Excluding resource '{excludeOnlyResourcePath}' leaving dependents intact..."); - + resourcesToInclude.Remove(excludeOnlyResourcePath); } - + var filteredResources = new Dictionary( dependenciesByResourcePath.Where(kvp => resourcesToInclude.Contains(kvp.Key)), StringComparer.OrdinalIgnoreCase); @@ -491,7 +491,7 @@ void RemoveDependentResources(string resourcePath, List exclusionTraceEn var dependentResourcePaths = dependenciesByResourcePath .Where(kvp => kvp.Value.Contains(resourcePath)) .Select(kvp => kvp.Key); - + foreach (string dependentResourcePath in dependentResourcePaths) { if (!dependentResourcePath.EndsWith("Descriptors")) @@ -504,7 +504,7 @@ void RemoveDependentResources(string resourcePath, List exclusionTraceEn } } } - + IDictionary ApplyResourceInclusionsToDependencies( IDictionary dependenciesByResourcePath, string[] includeResourcePaths, string[] includeOnlyResourcePaths) @@ -528,7 +528,7 @@ IDictionary ApplyResourceInclusionsToDependencies( foreach (string includeOnlyResourcePath in includeOnlyResourcePaths) { allInclusionTraceEntries.Add($"Including resource '{includeOnlyResourcePath}' without its dependencies..."); - + resourcesToInclude.Add(includeOnlyResourcePath); } @@ -542,7 +542,7 @@ IDictionary ApplyResourceInclusionsToDependencies( filteredResources[resourceItem.Key] = resourceItem.Value.Where(dp => filteredResources.ContainsKey(dp)).ToArray(); } - + if (_logger.IsEnabled(LogEventLevel.Debug)) { if (allInclusionTraceEntries.Count > 0) @@ -612,7 +612,7 @@ private long GetLastChangeVersionProcessed() { return _sourceConnectionDetails.LastChangeVersionProcessed.Value; } - + // Fall back to using the pre-configured change version return _sourceConnectionDetails .LastChangeVersionProcessedByTargetName @@ -644,7 +644,7 @@ private TaskStatus[] ProcessUpsertsToCompletion( Array.Empty(), javascriptModuleFactory, null); - + // Start processing resources in dependency order var streamingPagesOfPostsByResourcePath = initiator.Start(processingContext, cancellationToken); @@ -654,7 +654,7 @@ private TaskStatus[] ProcessUpsertsToCompletion( streamingPagesOfPostsByResourcePath, processingSemaphore, options); - + return postTaskStatuses; } @@ -678,11 +678,11 @@ private async Task ProcessDeletesToCompletionAsync( _logger.Information($"Change window starting value indicates all values are being published, and so there is no need to perform delete processing."); return Array.Empty(); } - + TaskStatus[] deleteTaskStatuses = Array.Empty(); - + // Invert the dependencies for use in deletion, excluding descriptors (if present) and the special #Retry nodes - var deleteDependenciesByResourcePath = InvertDependencies(postDependenciesByResourcePath, + var deleteDependenciesByResourcePath = InvertDependencies(postDependenciesByResourcePath, path => path.EndsWith("Descriptors") || path.EndsWith(Conventions.RetryKeySuffix)); if (deleteDependenciesByResourcePath.Any()) @@ -769,7 +769,7 @@ private async Task ProcessKeyChangesToCompletionAsync( } var supportsKeyChanges = await _sourceCapabilities.SupportsKeyChangesAsync(probeResourceKey); - + if (supportsKeyChanges) { _logger.Debug($"Source supports key changes. Initiating key changes processing."); @@ -821,10 +821,10 @@ private IDictionary GetKeyChangeDependencies(IDictionary(postDependenciesByResourcePath); - + int infiniteLoopProtectionThreshold = keyChangeDependenciesByResourcePath.Count(); int i = 0; - + while (i < infiniteLoopProtectionThreshold) { // Identify all resources that have no more dependencies (except retained dependencies) @@ -834,32 +834,32 @@ private IDictionary GetKeyChangeDependencies(IDictionary kvp.Key) .Except(resourcesWithUpdatableKeys) .ToArray(); - + // Exit processing if there are no more resources that can be removed from the dependency graph if (!resourcesWithoutRetainedDependencies.Any()) { break; } - + var retainedDependenciesByResourcePath = new Dictionary(); - + // Iterate through all the resources that have no more dependencies (except retain dependencies) foreach (var resourcePathToBeRemoved in resourcesWithoutRetainedDependencies) { var dependenciesWithUpdatableKeys = keyChangeDependenciesByResourcePath[resourcePathToBeRemoved] .Intersect(resourcesWithUpdatableKeys) .ToArray(); - + if (dependenciesWithUpdatableKeys.Any()) { // Capture the dependencies of this resource that must be retained (used in place of dependencies on the resource being removed) retainedDependenciesByResourcePath.Add(resourcePathToBeRemoved, dependenciesWithUpdatableKeys); } - + // Remove the resource from the graph keyChangeDependenciesByResourcePath.Remove(resourcePathToBeRemoved); } - + // Iterate through the remaining graph resources foreach (var kvp in keyChangeDependenciesByResourcePath.ToArray()) { @@ -868,14 +868,14 @@ private IDictionary GetKeyChangeDependencies(IDictionary GetKeyChangeDependencies(IDictionary 0) @@ -962,24 +962,24 @@ private IDictionary InvertDependencies( postDependenciesByResourcePath .SelectMany(kvp => kvp.Value.Select(x => (x, kvp.Key))) .Where(tuple => !excludeResourcePath(tuple.Key) && !excludeResourcePath(tuple.x)); - + var allResourceTuples = postDependenciesByResourcePath .Select(kvp => (kvp.Key, null as string)) .Where(tuple => !excludeResourcePath(tuple.Key)); - + var deleteDependenciesByResourcePath = reverseDependencyTuples .Concat(allResourceTuples) .GroupBy(x => x.Item1) .ToDictionary( - g => g.Key, + g => g.Key, g => g .Where(x => !string.IsNullOrEmpty(x.Item2)) .Select(x => x.Item2) - .ToArray(), + .ToArray(), StringComparer.OrdinalIgnoreCase); - + return deleteDependenciesByResourcePath; } @@ -995,7 +995,7 @@ private TaskStatus[] WaitForResourceStreamingToComplete( _logger.Information($"Waiting for {streamingPagesByResourcePath.Count} {activityDescription} streaming sources to complete..."); var lastProgressUpdate = DateTime.Now; - + while (streamingPagesByResourcePath.Any()) { string[] resourcePaths = streamingPagesByResourcePath.Keys.OrderBy(x => x).ToArray(); @@ -1005,10 +1005,10 @@ private TaskStatus[] WaitForResourceStreamingToComplete( int paddedDisplayLength = resourcePaths.Max(x => x.Length) + 2; var remainingResources = resourcePaths.Select(rp => new - { - ResourcePath = rp, - DependentItems = streamingPagesByResourcePath[rp].DependencyPaths.Where(streamingPagesByResourcePath.ContainsKey).ToArray() - }) + { + ResourcePath = rp, + DependentItems = streamingPagesByResourcePath[rp].DependencyPaths.Where(streamingPagesByResourcePath.ContainsKey).ToArray() + }) .Select(x => new { ResourcePath = x.ResourcePath, @@ -1020,17 +1020,17 @@ private TaskStatus[] WaitForResourceStreamingToComplete( .Where(x => x.Count == 0) .OrderBy(x => x.ResourcePath) .ToArray(); - + var itemsMessage = new StringBuilder(); itemsMessage.AppendLine($"The following {resourcesBeingProcessed.Length} resources are processing (or ready for processing):"); - + foreach (var resource in resourcesBeingProcessed) { itemsMessage.Append(" "); itemsMessage.AppendLine($"{GetResourcePathDisplayText(resource.ResourcePath)}"); } - + var resourcesWaiting = remainingResources .Where(x => x.Count > 0) .OrderBy(x => x.ResourcePath) @@ -1047,7 +1047,7 @@ private TaskStatus[] WaitForResourceStreamingToComplete( itemsMessage.AppendLine($"{GetResourcePathDisplayText(resource.ResourcePath, paddedDisplayLength)} ({resource.DependentItems.Length} dependencies remaining --> {Truncate(string.Join(", ", resource.DependentItems.Select(x => GetResourcePathDisplayText(x))), 50)})"); } } - + string Truncate(string text, int length) { if (text.Length <= length) @@ -1063,7 +1063,7 @@ string GetResourcePathDisplayText(string resourcePath, int padToLength = 0) string[] parts = resourcePath.Split('/', StringSplitOptions.RemoveEmptyEntries); string displayName; - + if (parts[0] == "ed-fi") { displayName = parts[1]; @@ -1077,24 +1077,24 @@ string GetResourcePathDisplayText(string resourcePath, int padToLength = 0) { return displayName; } - + return $"{displayName}{new string(' ', (padToLength + 2) - displayName.Length)}"; } - + string logMessage = $"Waiting for the {activityDescription} streaming of {resourcePaths.Length} resources to complete...{Environment.NewLine}{itemsMessage}"; - + if (_logger.IsEnabled(LogEventLevel.Debug)) { _logger.Debug(logMessage); } - else + else { _logger.Information(logMessage); } - + lastProgressUpdate = DateTime.Now; } - + int completedIndex = Task.WaitAny( resourcePaths.Select(k => streamingPagesByResourcePath[k].CompletionBlock.Completion).ToArray(), TimeSpan.FromSeconds(options.StreamingPagesWaitDurationSeconds)); @@ -1104,20 +1104,20 @@ string GetResourcePathDisplayText(string resourcePath, int padToLength = 0) // Check for unhandled task failure var resourcePath = resourcePaths.ElementAt(completedIndex); var streamingPagesItem = streamingPagesByResourcePath[resourcePath]; - + var blockCompletion = streamingPagesItem.CompletionBlock.Completion; - + if (blockCompletion.IsFaulted) { _logger.Fatal($"Streaming task failure for {resourcePath}: {blockCompletion.Exception}"); } - + completedStreamingPagesByResourcePath.Add( resourcePaths[completedIndex], streamingPagesByResourcePath[resourcePaths[completedIndex]].CompletionBlock.Completion.Status); streamingPagesItem.CompletionBlock = null; - + streamingPagesByResourcePath.Remove(resourcePaths[completedIndex]); if (_logger.IsEnabled(LogEventLevel.Debug)) diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeWindow.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeWindow.cs index 1d559db..106e63a 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeWindow.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeWindow.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing { - public class ChangeWindow + public class ChangeWindow { private long _minChangeVersion; private long _maxChangeVersion; @@ -21,7 +21,7 @@ public long MinChangeVersion { throw new ArgumentOutOfRangeException(nameof(value), "Change versions must be greater than 0."); } - + _minChangeVersion = value; } } @@ -35,7 +35,7 @@ public long MaxChangeVersion { throw new ArgumentOutOfRangeException(nameof(value), "Change versions must be greater than 0."); } - + _maxChangeVersion = value; } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/EdFiApiConstants.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/EdFiApiConstants.cs index ff0f845..b607a37 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/EdFiApiConstants.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/EdFiApiConstants.cs @@ -5,28 +5,28 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing { - public static class EdFiApiConstants + public static class EdFiApiConstants { /// /// Gets the path suffix to the "deletes" child resource under each of the data management API's resources. /// public const string DeletesPathSuffix = "/deletes"; - + /// /// Gets the path suffix to the "deletes" child resource under each of the data management API's resources. /// public const string KeyChangesPathSuffix = "/keyChanges"; - + /// /// Gets the path segment to the data management API, including the version. /// public const string DataManagementApiSegment = "data/v3"; - + /// /// Gets the path segment to the change queries feature of the API, including the version. /// public const string ChangeQueriesApiSegment = "changeQueries/v1"; - + /// /// Gets the name of the property which holds the natural key values for a deleted item (if supported by the source API). /// diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/IChangeVersionProcessedWriter.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/IChangeVersionProcessedWriter.cs index 757307a..78ef443 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/IChangeVersionProcessedWriter.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/IChangeVersionProcessedWriter.cs @@ -8,7 +8,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing { - public interface IChangeVersionProcessedWriter + public interface IChangeVersionProcessedWriter { Task SetProcessedChangeVersionAsync( string sourceConnectionName, diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/IErrorPublisher.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/IErrorPublisher.cs index f5698d1..79dac25 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/IErrorPublisher.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/IErrorPublisher.cs @@ -8,7 +8,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing { - public interface IErrorPublisher + public interface IErrorPublisher { Task PublishErrorsAsync(ErrorItemMessage[] messages); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/ErrorItemMessage.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/ErrorItemMessage.cs index 8c1b552..83bad50 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/ErrorItemMessage.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/ErrorItemMessage.cs @@ -9,7 +9,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing.Messages { - public class ErrorItemMessage + public class ErrorItemMessage { public ErrorItemMessage() { @@ -19,19 +19,19 @@ public ErrorItemMessage() public DateTime DateTime { get; } public string Method { get; set; } - + public string ResourceUrl { get; set; } #nullable enable - public string? Id { get; set; } + public string? Id { get; set; } //[JsonIgnore] public JObject? Body { get; set; } public HttpStatusCode? ResponseStatus { get; set; } - + public string ResponseContent { get; set; } - + public Exception Exception { get; set; } } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/StreamResourceMessage.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/StreamResourceMessage.cs index 2a13f65..dd9d553 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/StreamResourceMessage.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/StreamResourceMessage.cs @@ -9,7 +9,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing.Messages { - public class StreamResourceMessage + public class StreamResourceMessage { // ---------------------------- // Resource-specific context @@ -26,10 +26,10 @@ public class StreamResourceMessage // Source Ed-Fi ODS API processing context (shared) // ------------------------------------------------- // public EdFiApiClient EdFiApiClient { get; set; } - + // NOTE: This is potentially not Ed-Fi ODs API-specific, but likely so public int PageSize { get; set; } - + // ---------------------------- // Global processing context // ---------------------------- diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/StreamResourcePageMessage.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/StreamResourcePageMessage.cs index f211771..b1a01d6 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/StreamResourcePageMessage.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/Messages/StreamResourcePageMessage.cs @@ -9,10 +9,10 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing.Messages { - /// - /// Represents details needed for obtaining a page of JSON data from the source connection. - /// - public class StreamResourcePageMessage + /// + /// Represents details needed for obtaining a page of JSON data from the source connection. + /// + public class StreamResourcePageMessage { // ---------------------------- // Resource-specific context @@ -31,7 +31,7 @@ public class StreamResourcePageMessage public string PartitionFrom { get; set; } public string PartitionUntil { get; set; } public bool IsFinalPage { get; set; } - + // ------------------------------------------------- // Source Ed-Fi ODS API processing context (shared) // ------------------------------------------------- diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/SerilogErrorPublisher.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/SerilogErrorPublisher.cs index 4b7129b..1601059 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/SerilogErrorPublisher.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/SerilogErrorPublisher.cs @@ -11,16 +11,16 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing { - /// - /// Publishes errors without the original request content (due to security considerations) by logging - /// the JSON serialized representations of the . - /// - public class SerilogErrorPublisher : IErrorPublisher + /// + /// Publishes errors without the original request content (due to security considerations) by logging + /// the JSON serialized representations of the . + /// + public class SerilogErrorPublisher : IErrorPublisher { private readonly ILogger _logger = Log.Logger.ForContext(typeof(SerilogErrorPublisher)); private long _publishedErrorCount; - + public Task PublishErrorsAsync(ErrorItemMessage[] messages) { return Task.Run(() => diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/StreamingPagesItem.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/StreamingPagesItem.cs index ded85e9..93bba7a 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/StreamingPagesItem.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/StreamingPagesItem.cs @@ -8,7 +8,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Processing { - public class StreamingPagesItem + public class StreamingPagesItem { public string[] DependencyPaths { get; set; } public ISourceBlock CompletionBlock { get; set; } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/StreamingResourceProcessor.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/StreamingResourceProcessor.cs index 3f102f9..3d7f460 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/StreamingResourceProcessor.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/StreamingResourceProcessor.cs @@ -61,7 +61,7 @@ public IDictionary Start( _logger.Information($"Initiating resource streaming."); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; - + var streamingPagesByResourceKey = new Dictionary(StringComparer.OrdinalIgnoreCase); var streamingResourceBlockByResourceKey = @@ -88,7 +88,7 @@ public IDictionary Start( if (resourceKey.EndsWith(Conventions.RetryKeySuffix)) { // Save an action delegate for processing the item, keyed by the resource path - postAuthorizationRetryByResourceKey.Add(resourcePath, msg => processingInputBlock.Post((TProcessDataMessage) msg)); + postAuthorizationRetryByResourceKey.Add(resourcePath, msg => processingInputBlock.Post((TProcessDataMessage)msg)); } streamingPagesByResourceKey.Add(resourceKey, new StreamingPagesItem { CompletionBlock = processingOutputBlock }); diff --git a/src/EdFi.Tools.ApiPublisher.Core/Registration/AssemblyLoaderHelper.cs b/src/EdFi.Tools.ApiPublisher.Core/Registration/AssemblyLoaderHelper.cs index b90e9b2..385aeb0 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Registration/AssemblyLoaderHelper.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Registration/AssemblyLoaderHelper.cs @@ -15,7 +15,7 @@ namespace EdFi.Ods.Api.Helpers { - public static class AssemblyLoaderHelper + public static class AssemblyLoaderHelper { private static readonly ILogger _logger = Log.ForContext(typeof(AssemblyLoaderHelper)); private const string AssemblyMetadataSearchString = "assemblyMetadata.json"; @@ -102,141 +102,5 @@ private static bool IsNotNetFramework(string assemblyName) && assemblyName != "netstandard" && !assemblyName.StartsWithIgnoreCase("Autofac"); } - - // public static IEnumerable FindPluginAssemblies(string pluginFolder, bool includeFramework = false) - // { - // // Storage to ensure not loading the same assembly twice and optimize calls to GetAssemblies() - // IDictionary loaded = new ConcurrentDictionary(); - // - // CacheAlreadyLoadedAssemblies(loaded, includeFramework); - // - // if (!IsPluginFolderNameSupplied()) - // { - // yield break; - // } - // - // pluginFolder = Path.GetFullPath(pluginFolder); - // - // if (!Directory.Exists(pluginFolder)) - // { - // _logger.Debug($"Plugin folder '{pluginFolder}' does not exist. No plugins will be loaded."); - // yield break; - // } - // - // var assemblies = Directory.GetFiles(pluginFolder, "*.dll", SearchOption.AllDirectories); - // - // var pluginAssemblyLoadContext = new PluginAssemblyLoadContext(); - // - // try - // { - // foreach (var assemblyPath in assemblies) - // { - // var assembly = pluginAssemblyLoadContext.LoadFromAssemblyPath(assemblyPath); - // - // if (!HasPlugin(assembly)) - // { - // continue; - // } - // - // string assemblyDirectory = Path.GetDirectoryName(assemblyPath); - // var assemblyMetadata = ReadAssemblyMetadata(assembly); - // - // if (IsExtensionAssembly(assemblyMetadata)) - // { - // var validator = GetExtensionValidator(); - // var validationResult = validator.ValidateObject(assemblyDirectory); - // - // if (!validationResult.Any()) - // { - // yield return assembly.Location; - // } - // else - // { - // _logger.Warn($"Assembly: {assembly.GetName()} - {string.Join(",", validationResult)}"); - // } - // } - // else if (IsProfileAssembly(assemblyMetadata)) - // { - // yield return assembly.Location; - // } - // } - // } - // finally - // { - // pluginAssemblyLoadContext.Unload(); - // } - // - // static FluentValidationObjectValidator GetExtensionValidator() - // { - // return new FluentValidationObjectValidator( - // new IValidator[] - // { - // new ApiModelExistsValidator(), - // new IsExtensionPluginValidator(), - // new IsApiVersionValidValidator(ApiVersionConstants.InformationalVersion) - // }); - // } - // - // bool IsPluginFolderNameSupplied() - // { - // if (!string.IsNullOrWhiteSpace(pluginFolder)) - // { - // return true; - // } - // - // _logger.Debug($"Plugin folder was not specified so no plugins will be loaded."); - // return false; - // } - // - // static bool HasPlugin(Assembly assembly) - // { - // return assembly.GetTypes().Any( - // t => t.GetInterfaces() - // .Any(i => i.AssemblyQualifiedName == typeof(IPluginMarker).AssemblyQualifiedName)); - // } - // } - - // private static bool IsExtensionAssembly(AssemblyMetadata assemblyMetadata) - // { - // return assemblyMetadata.AssemblyModelType.EqualsIgnoreCase(PluginConventions.Extension); - // } - // - // private static bool IsProfileAssembly(AssemblyMetadata assemblyMetadata) - // { - // return assemblyMetadata.AssemblyModelType.EqualsIgnoreCase(PluginConventions.Profile); - // } - - // private static AssemblyMetadata ReadAssemblyMetadata(Assembly assembly) - // { - // var resourceName = assembly.GetManifestResourceNames() - // .FirstOrDefault(p => p.EndsWith(AssemblyMetadataSearchString)); - // - // if (resourceName == null) - // { - // throw new Exception($"Assembly metadata embedded resource '{AssemblyMetadataSearchString}' not found in assembly '{Path.GetFileName(assembly.Location)}'."); - // } - // - // var stream = assembly.GetManifestResourceStream(resourceName); - // - // using var reader = new StreamReader(stream); - // - // string jsonFile = reader.ReadToEnd(); - // - // _logger.Debug($"Deserializing object type '{typeof(AssemblyMetadata)}' from embedded resource '{resourceName}'."); - // - // return JsonConvert.DeserializeObject(jsonFile); - // } - - // private class PluginAssemblyLoadContext : AssemblyLoadContext - // { - // public PluginAssemblyLoadContext() - // : base(isCollectible: true) { } - // } - - // private class AssemblyMetadata - // { - // public string AssemblyModelType { get; set; } - // public string AssemblyMetadataFormatVersion { get; set; } - // } } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Registration/AutofacRegistrationExtensions.cs b/src/EdFi.Tools.ApiPublisher.Core/Registration/AutofacRegistrationExtensions.cs index 1fd53fb..c46b804 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Registration/AutofacRegistrationExtensions.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Registration/AutofacRegistrationExtensions.cs @@ -10,7 +10,7 @@ namespace EdFi.Tools.ApiPublisher.Core.Registration { - public static class AutofacRegistrationExtensions + public static class AutofacRegistrationExtensions { public static IRegistrationBuilder UsingDefaultImplementationConvention(this IRegistrationBuilder registrationBuilder) { @@ -23,13 +23,13 @@ public static IRegistrationBuilder("version"); string targetApiVersionText = targetVersionObject.Value("version"); @@ -63,7 +63,7 @@ public async Task CheckApiVersionsAsync(ChangeProcessorConfiguration configurati // TODO: Consider splitting this into a separate context object configuration.SourceApiVersion = sourceApiVersion; configuration.TargetApiVersion = targetApiVersion; - + // Warn if API versions don't match if (!sourceApiVersion.Equals(targetApiVersion)) { @@ -91,16 +91,16 @@ string GetEdFiStandardVersion(JObject jObject) { string edFiVersion; - var dataModels = (JArray) jObject["dataModels"]; + var dataModels = (JArray)jObject["dataModels"]; edFiVersion = dataModels.Where(o => Newtonsoft.Json.Linq.Extensions.Value(o["name"]) == "Ed-Fi") .Select(o => o["version"].Value()) .SingleOrDefault(); - + return edFiVersion; } -#region Sample Version Metadata + #region Sample Version Metadata /* Sample version metadata: @@ -166,6 +166,6 @@ string GetEdFiStandardVersion(JObject jObject) ] } */ -#endregion + #endregion } } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Configuration/Serilog/TextFormatterTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Configuration/Serilog/TextFormatterTests.cs index 0aa9ce0..2caf1cb 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Configuration/Serilog/TextFormatterTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Configuration/Serilog/TextFormatterTests.cs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: Apache-2.0 // Licensed to the Ed-Fi Alliance under one or more agreements. // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. @@ -24,7 +24,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Configuration.Serilog; public class TextFormatterTests { [TestFixture] - public class When_use_the_TextFormatter_in_Serilog + public class When_use_the_TextFormatter_in_Serilog { private const string Message = "My text format message"; private const string LevelInfoPlain = "INFO"; @@ -71,8 +71,8 @@ public void Should_render_the_messages_with_default_format() textWriter.ToString().ShouldContain(LevelInfoFormatted); textWriter.ToString().ShouldContain(logEvent.Timestamp.ToString("yyyy-MM-dd HH:mm:ss,fff")); } - } - + } + [TestCase("[{Level}] - {Message}")] public void Should_display_the_messages_with_custom_format(string format) { @@ -154,7 +154,5 @@ private LogEvent CreateSerilogEntry(LogEventLevel logEventLevel = LogEventLevel. return logEvent; } - } - - + } } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Extensions/HttpRequestMessageExtensions.cs b/src/EdFi.Tools.ApiPublisher.Tests/Extensions/HttpRequestMessageExtensions.cs index 0676af2..f0f4265 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Extensions/HttpRequestMessageExtensions.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Extensions/HttpRequestMessageExtensions.cs @@ -9,7 +9,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Extensions { - public static class HttpRequestMessageExtensions + public static class HttpRequestMessageExtensions { public static bool HasParameter(this HttpRequestMessage request, string parameterName) { @@ -28,8 +28,8 @@ public static T QueryString(this HttpRequestMessage request, string parameter { return default; } - - return (T) Convert.ChangeType(value, typeof(T)); + + return (T)Convert.ChangeType(value, typeof(T)); } } } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Extensions/ObjectExtensions.cs b/src/EdFi.Tools.ApiPublisher.Tests/Extensions/ObjectExtensions.cs index 2725764..b195f5f 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Extensions/ObjectExtensions.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Extensions/ObjectExtensions.cs @@ -15,7 +15,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Extensions { - public static class ObjectExtensions + public static class ObjectExtensions { private static readonly ILogger _logger = Log.ForContext(typeof(ObjectExtensions)); @@ -99,7 +99,7 @@ public static IDictionary ToQueryStringParams(this object instan MockRequests.SerializerSettings); var obj = JObject.Parse(json); - + var queryStringParms = new Dictionary(); foreach (var property in obj.Properties()) @@ -109,7 +109,7 @@ public static IDictionary ToQueryStringParams(this object instan return queryStringParms; } - + public static NameValueCollection ParseQueryString(this Uri uri) { return HttpUtility.ParseQueryString(uri.Query); diff --git a/src/EdFi.Tools.ApiPublisher.Tests/FakeResponse.cs b/src/EdFi.Tools.ApiPublisher.Tests/FakeResponse.cs index 3821258..70cc987 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/FakeResponse.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/FakeResponse.cs @@ -10,7 +10,7 @@ namespace EdFi.Tools.ApiPublisher.Tests { - public static class FakeResponse + public static class FakeResponse { public static HttpResponseMessage OK(string content) => new HttpResponseMessage(HttpStatusCode.OK) @@ -25,11 +25,6 @@ public static HttpResponseMessage OK(object data) => }; public static HttpResponseMessage NotFound() => new HttpResponseMessage(HttpStatusCode.NotFound); - - // public static HttpResponseMessage StatusCodeResult(HttpStatusCode responseCode) - // { - // return new HttpResponseMessage(responseCode); - // } } public static class HttpResponseMessageExtensions diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Helpers/TestHelpers.cs b/src/EdFi.Tools.ApiPublisher.Tests/Helpers/TestHelpers.cs index 010b5cc..bca323a 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Helpers/TestHelpers.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Helpers/TestHelpers.cs @@ -34,7 +34,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Helpers { - public class TestHelpers + public class TestHelpers { public const string AnyResourcePattern = "/(ed-fi|tpdm)/\\w+"; // public const string AnyResourcePattern = "/ed-fi/\\w+"; @@ -56,7 +56,7 @@ public static Faker> GetGenericResourceFaker() public static Faker GetKeyValueFaker() { var linkValueFaker = GetLinkValueFaker(); - + // Initialize a generator for the fake natural key class var keyValueFaker = new Faker().StrictMode(true) .RuleFor(o => o.Name, f => f.Name.FirstName()) @@ -113,14 +113,14 @@ public static ApiConnectionDetails GetSourceApiConnectionDetails( Secret = "secret", Scope = null, SchoolYear = schoolYear, - + Include = include == null ? null : string.Join(",", include), IncludeOnly = includeOnly == null ? null : string.Join(",", includeOnly), Exclude = exclude == null ? null : string.Join(",", exclude), ExcludeOnly = excludeOnly == null ? null : string.Join(",", excludeOnly), IgnoreIsolation = ignoreIsolation, - + // LastChangeVersionProcessed = null, // LastChangeVersionsProcessed = "{ 'TestTarget': 1234 }", TreatForbiddenPostAsWarning = true, @@ -146,13 +146,13 @@ public static ApiConnectionDetails GetTargetApiConnectionDetails(int? schoolYear Include = null, // "abc,def,ghi", Exclude = null, ExcludeOnly = null, - + IgnoreIsolation = true, - + LastChangeVersionProcessed = null, LastChangeVersionsProcessed = null, TreatForbiddenPostAsWarning = true, - LastChangeVersionProcessedByTargetName = {}, + LastChangeVersionProcessedByTargetName = { }, ProfileName = profileName, }; @@ -177,7 +177,7 @@ public static string[] GetResourcesWithUpdatableKeys() public static AuthorizationFailureHandling[] GetAuthorizationFailureHandling() { - return new [] + return new[] { new AuthorizationFailureHandling { @@ -221,7 +221,7 @@ public static IFakeHttpRequestHandler GetFakeBaselineSourceApiRequestHandler( .SetChangeQueriesUrlSegment(changeQueriesUrlSegment) .OAuthToken() .ApiVersionMetadata() - .Snapshots(new []{ new Snapshot { Id = Guid.NewGuid(), SnapshotIdentifier = "ABC123", SnapshotDateTime = DateTime.Now } }) + .Snapshots(new[] { new Snapshot { Id = Guid.NewGuid(), SnapshotIdentifier = "ABC123", SnapshotDateTime = DateTime.Now } }) .LegacySnapshotsNotFound(); } @@ -301,7 +301,7 @@ EdFiApiClient TargetApiClientFactory() => var streamingResourceProcessor = new StreamingResourceProcessor( new StreamResourceBlockFactory( - (withReversePaging) ? + (withReversePaging) ? new EdFiApiChangeVersionReversePagingStreamResourcePageMessageProducer( new EdFiApiSourceTotalCountProvider(sourceEdFiApiClientProvider)) : new EdFiApiLimitOffsetPagingStreamResourcePageMessageProducer( diff --git a/src/EdFi.Tools.ApiPublisher.Tests/HttpClientHandlerFakeBridge.cs b/src/EdFi.Tools.ApiPublisher.Tests/HttpClientHandlerFakeBridge.cs index 495fc7a..4c07cbe 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/HttpClientHandlerFakeBridge.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/HttpClientHandlerFakeBridge.cs @@ -10,7 +10,7 @@ namespace EdFi.Tools.ApiPublisher.Tests { - public class HttpClientHandlerFakeBridge : HttpClientHandler + public class HttpClientHandlerFakeBridge : HttpClientHandler { private readonly IFakeHttpRequestHandler _handler; @@ -24,7 +24,7 @@ protected override Task SendAsync(HttpRequestMessage reques Console.WriteLine($"Requested URL: {request.Method} {request.RequestUri}"); string requestPath = $"{request.RequestUri.Scheme}://{request.RequestUri.Host}{request.RequestUri.LocalPath}"; - + switch (request.Method.ToString().ToUpper()) { case "GET": diff --git a/src/EdFi.Tools.ApiPublisher.Tests/IFakeHttpRequestHandler.cs b/src/EdFi.Tools.ApiPublisher.Tests/IFakeHttpRequestHandler.cs index 865219d..326263e 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/IFakeHttpRequestHandler.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/IFakeHttpRequestHandler.cs @@ -7,7 +7,7 @@ namespace EdFi.Tools.ApiPublisher.Tests { - public interface IFakeHttpRequestHandler + public interface IFakeHttpRequestHandler { string BaseUrl { get; } string DataManagementUrlSegment { get; } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/JavascriptHosting/ScriptExecutionTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/JavascriptHosting/ScriptExecutionTests.cs index f39580f..997f56b 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/JavascriptHosting/ScriptExecutionTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/JavascriptHosting/ScriptExecutionTests.cs @@ -11,24 +11,24 @@ namespace EdFi.Tools.ApiPublisher.Tests.JavascriptHosting { - [TestFixture] + [TestFixture] public class ScriptExecutionTests { - class Result + private class Result { public string? greeting { get; set; } } - class Person + private class Person { public string? name { get; set; } public int age { get; set; } } - + [Test] public async Task Should_execute_JavaScript() { - + var result = await StaticNodeJSService.InvokeFromStringAsync( @" module.exports = (callback, name) => { @@ -42,7 +42,7 @@ public async Task Should_execute_JavaScript() result!.greeting.ShouldBe("Hello Bob!"); } - const string HelloNameSource = @" + private const string HelloNameSource = @" module.exports = { sayHello: async (name) => { const result = { greeting: `Hello ${name}!` }; @@ -60,19 +60,19 @@ public async Task Should_execute_JavaScript() public async Task Should_execute_JavaScript_object_with_functions() { var result = await StaticNodeJSService.InvokeFromStringAsync( - HelloNameSource, + HelloNameSource, cacheIdentifier: "helloNameModule", - exportName: "sayHello", + exportName: "sayHello", args: new object?[] { "Bob" }); - + result!.greeting.ShouldBe("Hello Bob!"); var result2 = await StaticNodeJSService.InvokeFromStringAsync( - HelloNameSource, + HelloNameSource, cacheIdentifier: "helloNameModule", - exportName: "sayGoodbye", + exportName: "sayGoodbye", args: new object?[] { "Bob" }); - + result2!.greeting.ShouldBe("Goodbye Bob!"); } @@ -89,11 +89,11 @@ await StaticNodeJSService.InvokeFromStringAsync( args: new object?[] { "Bob" }); }); } - + [Test] public async Task Should_execute_JavaScript_object_with_object_argument() { - const string helloPersonSource = @" + const string HelloPersonSource = @" module.exports = { sayHello: async (person) => { const result = { greeting: `Hello ${person.name}! You are ${person.age} years old already!` }; @@ -107,26 +107,26 @@ public async Task Should_execute_JavaScript_object_with_object_argument() "; var results = await StaticNodeJSService.InvokeFromStringAsync( - helloPersonSource, + HelloPersonSource, cacheIdentifier: "helloPersonModule", exportName: "sayHello", - args: new object?[] { new Person { name = "Bob", age = 42 }}); + args: new object?[] { new Person { name = "Bob", age = 42 } }); results?.greeting.ShouldBe("Hello Bob! You are 42 years old already!"); - + var results2 = await StaticNodeJSService.InvokeFromStringAsync( - helloPersonSource, + HelloPersonSource, cacheIdentifier: "helloPersonModule", exportName: "sayGoodbye", - args: new object?[] { new Person { name = "Bob", age = 42 }}); + args: new object?[] { new Person { name = "Bob", age = 42 } }); results2?.greeting.ShouldBe("Goodbye Bob! You are 42 years old already!"); } - + [Test] public async Task Should_execute_JavaScript_object_with_status_codes_with_object_argument() { - const string helloPersonSource2 = @" + const string HelloPersonSource2 = @" module.exports = { 200: async (person) => { const result = { greeting: `Hello ${person.name}! You are ${person.age} years old already!` }; @@ -140,18 +140,18 @@ public async Task Should_execute_JavaScript_object_with_status_codes_with_object "; var results = await StaticNodeJSService.InvokeFromStringAsync( - helloPersonSource2, + HelloPersonSource2, cacheIdentifier: "helloPersonModule2", exportName: "200", - args: new object?[] { new Person { name = "Bob", age = 42 }}); + args: new object?[] { new Person { name = "Bob", age = 42 } }); results?.greeting.ShouldBe("Hello Bob! You are 42 years old already!"); - + var results2 = await StaticNodeJSService.InvokeFromStringAsync( - helloPersonSource2, + HelloPersonSource2, cacheIdentifier: "helloPersonModule2", exportName: "500", - args: new object?[] { new Person { name = "Bob", age = 42 }}); + args: new object?[] { new Person { name = "Bob", age = 42 } }); results2?.greeting.ShouldBe("Goodbye Bob! You are 42 years old already!"); } @@ -159,7 +159,7 @@ public async Task Should_execute_JavaScript_object_with_status_codes_with_object [Test] public async Task Should_execute_JavaScript_object_with_resource_paths_with_object_argument() { - const string helloPersonSource3 = @" + const string HelloPersonSource3 = @" module.exports = { '/ed-fi/students/200': async (person) => { const result = { greeting: `Hello ${person.name}! You are ${person.age} years old already!` }; @@ -173,18 +173,18 @@ public async Task Should_execute_JavaScript_object_with_resource_paths_with_obje "; var results = await StaticNodeJSService.InvokeFromStringAsync( - helloPersonSource3, + HelloPersonSource3, cacheIdentifier: "helloPersonModule3", exportName: "/ed-fi/students/200", - args: new object?[] { new Person { name = "Bob", age = 42 }}); + args: new object?[] { new Person { name = "Bob", age = 42 } }); results?.greeting.ShouldBe("Hello Bob! You are 42 years old already!"); - + var results2 = await StaticNodeJSService.InvokeFromStringAsync( - helloPersonSource3, + HelloPersonSource3, cacheIdentifier: "helloPersonModule3", exportName: "/ed-fi/students/500", - args: new object?[] { new Person { name = "Bob", age = 42 }}); + args: new object?[] { new Person { name = "Bob", age = 42 } }); results2?.greeting.ShouldBe("Goodbye Bob! You are 42 years old already!"); } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/MockRateLimitingMethod.cs b/src/EdFi.Tools.ApiPublisher.Tests/MockRateLimitingMethod.cs index 3b6059a..81937b4 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/MockRateLimitingMethod.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/MockRateLimitingMethod.cs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: Apache-2.0 // Licensed to the Ed-Fi Alliance under one or more agreements. // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. @@ -20,11 +20,11 @@ public MockRateLimitingMethod(IRateLimiting rateLimiter) _rateLimiter = rateLimiter; } - public async Task ExecuteAsync(int id=0) + public async Task ExecuteAsync(int id = 0) { return await _rateLimiter.ExecuteAsync(async () => { - await Task.Delay(100); + await Task.Delay(100); return new HttpResponseMessage(System.Net.HttpStatusCode.OK) { Content = new StringContent("Execution completed successfully!") diff --git a/src/EdFi.Tools.ApiPublisher.Tests/MockRequests.cs b/src/EdFi.Tools.ApiPublisher.Tests/MockRequests.cs index e195acb..ea11636 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/MockRequests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/MockRequests.cs @@ -21,13 +21,13 @@ namespace EdFi.Tools.ApiPublisher.Tests { - public static class MockRequests + public static class MockRequests { - public const string SourceApiBaseUrl = "https://test.source"; + public const string SourceApiBaseUrl = "https://test.source"; public const string TargetApiBaseUrl = "https://test.target"; public const string DataManagementPath = "/data/v3"; - + public static readonly string SchoolYearSpecificDataManagementPath = $"/data/v3/{SchoolYear}"; public const int SchoolYear = 2099; @@ -41,7 +41,7 @@ public static class MockRequests }, Converters = new JsonConverter[] { new Iso8601UtcDateOnlyConverter() } }; - + // public static void NotFound(this IFakeHttpClientHandler fakeRequestHandler, string url) // { // A.CallTo(() => fakeRequestHandler.Get(url, A.Ignored)) @@ -66,9 +66,9 @@ public static IFakeHttpRequestHandler GetResourceData( () => fakeRequestHandler.Get( A.Ignored, A.That.Matches( - msg => - (msg.RequestUri.LocalPath == url || Regex.IsMatch(msg.RequestUri.LocalPath, url)) - && !HasTotalCountParameter(msg) + msg => + (msg.RequestUri.LocalPath == url || Regex.IsMatch(msg.RequestUri.LocalPath, url)) + && !HasTotalCountParameter(msg) && RequestMatchesParameters(msg, parameters)))) .Returns(FakeResponse.OK(data)); @@ -87,13 +87,13 @@ public static IFakeHttpRequestHandler GetResourceDataItem( return fakeRequestHandler; } - private static bool RequestMatchesParameters(HttpRequestMessage request, IDictionary parameters) + private static bool RequestMatchesParameters(HttpRequestMessage request, IDictionary parameters) { if (parameters == null) { return true; } - + var queryString = request.RequestUri.ParseQueryString(); foreach (var parameter in parameters) @@ -108,8 +108,8 @@ private static bool RequestMatchesParameters(HttpRequestMessage request, IDictio } public static IFakeHttpRequestHandler PostResource( - this IFakeHttpRequestHandler fakeRequestHandler, - string url, + this IFakeHttpRequestHandler fakeRequestHandler, + string url, params HttpStatusCode[] responseCodes) { var mocker = A.CallTo( @@ -170,7 +170,7 @@ HttpResponseMessage CreateMessageWithAppropriateBody((HttpStatusCode, JObject) r { return new HttpResponseMessage(statusCode); } - + // Return a stock body in the response for a 400 or 500 series status code return new HttpResponseMessage(statusCode) { @@ -201,7 +201,7 @@ public static IFakeHttpRequestHandler ResourceCount( A.Ignored, A.That.Matches(msg => HasTotalCountParameter(msg)))); } - + fakeCall.Returns(FakeResponse.OK("[]").AppendHeaders(("Total-Count", responseTotalCountHeader.ToString()))); return fakeRequestHandler; @@ -224,7 +224,7 @@ public static IFakeHttpRequestHandler Dependencies(this IFakeHttpRequestHandler return fakeRequestHandler; } - + public static IFakeHttpRequestHandler Dependencies(this IFakeHttpRequestHandler fakeRequestHandler, string resourcePath) { A.CallTo( @@ -276,7 +276,7 @@ public static IFakeHttpRequestHandler OAuthToken(this IFakeHttpRequestHandler fa access_token = "TheAccessToken", // scope = "255901" })); - + return fakeRequestHandler; } @@ -284,15 +284,15 @@ public static IFakeHttpRequestHandler Snapshots(this IFakeHttpRequestHandler fak { A.CallTo(() => fakeRequestHandler.Get($"{fakeRequestHandler.BaseUrl}/{fakeRequestHandler.ChangeQueriesUrlSegment}/snapshots", A.Ignored)) .Returns(FakeResponse.OK(JsonConvert.SerializeObject(data))); - + return fakeRequestHandler; } - + public static IFakeHttpRequestHandler SnapshotsEmpty(this IFakeHttpRequestHandler fakeRequestHandler) { A.CallTo(() => fakeRequestHandler.Get($"{fakeRequestHandler.BaseUrl}/{fakeRequestHandler.ChangeQueriesUrlSegment}/snapshots", A.Ignored)) .Returns(FakeResponse.OK("[]")); - + return fakeRequestHandler; } @@ -300,15 +300,15 @@ public static IFakeHttpRequestHandler SnapshotsNotFound(this IFakeHttpRequestHan { A.CallTo(() => fakeRequestHandler.Get($"{fakeRequestHandler.BaseUrl}/{fakeRequestHandler.ChangeQueriesUrlSegment}/snapshots", A.Ignored)) .Returns(FakeResponse.NotFound()); - + return fakeRequestHandler; } - + public static IFakeHttpRequestHandler LegacySnapshotsNotFound(this IFakeHttpRequestHandler fakeRequestHandler) { A.CallTo(() => fakeRequestHandler.Get($"{fakeRequestHandler.BaseUrl}/{fakeRequestHandler.DataManagementUrlSegment}/publishing/snapshots", A.Ignored)) .Returns(FakeResponse.NotFound()); - + return fakeRequestHandler; } @@ -326,21 +326,21 @@ public static IFakeHttpRequestHandler AvailableChangeVersions(this IFakeHttpRequ public static IFakeHttpRequestHandler SetBaseUrl(this IFakeHttpRequestHandler fakeRequestHandler, string apiBaseUrl) { A.CallTo(() => fakeRequestHandler.BaseUrl).Returns(apiBaseUrl); - + return fakeRequestHandler; } public static IFakeHttpRequestHandler SetDataManagementUrlSegment(this IFakeHttpRequestHandler fakeRequestHandler, string urlSegment) { A.CallTo(() => fakeRequestHandler.DataManagementUrlSegment).Returns(urlSegment); - + return fakeRequestHandler; } public static IFakeHttpRequestHandler SetChangeQueriesUrlSegment(this IFakeHttpRequestHandler fakeRequestHandler, string urlSegment) { A.CallTo(() => fakeRequestHandler.ChangeQueriesUrlSegment).Returns(urlSegment); - + return fakeRequestHandler; } } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Models/FakeKey.cs b/src/EdFi.Tools.ApiPublisher.Tests/Models/FakeKey.cs index e66c0eb..8205954 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Models/FakeKey.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Models/FakeKey.cs @@ -8,14 +8,14 @@ namespace EdFi.Tools.ApiPublisher.Tests.Models { - public class FakeKey + public class FakeKey { [JsonProperty("name")] public string Name { get; set; } - + [JsonProperty("birthDate")] public DateTime BirthDate { get; set; } - + [JsonProperty("retirementAge")] public int RetirementAge { get; set; } @@ -27,7 +27,7 @@ public class Link { [JsonProperty("rel")] public string Rel { get; set; } - + [JsonProperty("href")] public string Href { get; set; } } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Models/GenericResource.cs b/src/EdFi.Tools.ApiPublisher.Tests/Models/GenericResource.cs index 18e823b..56c12b1 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Models/GenericResource.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Models/GenericResource.cs @@ -7,17 +7,17 @@ namespace EdFi.Tools.ApiPublisher.Tests.Models { - public class GenericResource + public class GenericResource { [JsonProperty("id")] public string Id { get; set; } - + [JsonProperty("someReference")] public TKey SomeReference { get; set; } [JsonProperty("vehicleYear")] public int VehicleYear { get; set; } - + [JsonProperty("vehicleManufacturer")] public string VehicleManufacturer { get; set; } } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Models/KeyChange.cs b/src/EdFi.Tools.ApiPublisher.Tests/Models/KeyChange.cs index 6c381cd..48e0874 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Models/KeyChange.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Models/KeyChange.cs @@ -9,7 +9,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Models { - public class KeyChange + public class KeyChange { public string Id { get; set; } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Models/Snapshot.cs b/src/EdFi.Tools.ApiPublisher.Tests/Models/Snapshot.cs index 9caa9a1..0ad2862 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Models/Snapshot.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Models/Snapshot.cs @@ -9,10 +9,10 @@ namespace EdFi.Tools.ApiPublisher.Tests.Models { - /// - /// A class which represents the changes.Snapshot table of the Snapshot aggregate in the ODS Database. - /// - public class Snapshot + /// + /// A class which represents the changes.Snapshot table of the Snapshot aggregate in the ODS Database. + /// + public class Snapshot { /// /// The unique identifier for the Snapshot resource. diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeExtensionResourcesTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeExtensionResourcesTests.cs index be12895..02464c8 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeExtensionResourcesTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeExtensionResourcesTests.cs @@ -18,7 +18,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing { - [TestFixture] + [TestFixture] public class ExcludeExtensionResourcesTests { [TestFixture] @@ -52,7 +52,7 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- // Target Requests // ----------------------------------------------------------------- - + _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); // Every POST succeeds @@ -63,8 +63,8 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( - exclude: new []{ "assessments", "/ed-fi/sections", "/tpdm/candidates" }); - + exclude: new[] { "assessments", "/ed-fi/sections", "/tpdm/candidates" }); + var targetApiConnectionDetails = TestHelpers.GetTargetApiConnectionDetails(); // ----------------------------------------------------------------- diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeOnlyResourcesTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeOnlyResourcesTests.cs index 71d25d7..688282f 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeOnlyResourcesTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeOnlyResourcesTests.cs @@ -17,7 +17,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing { - [TestFixture] + [TestFixture] public class ExcludeOnlyResourcesTests { [TestFixture] @@ -50,19 +50,19 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- // Target Requests // ----------------------------------------------------------------- - + _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); // Every POST succeeds _fakeTargetRequestHandler.EveryDataManagementPostReturns200Ok(); - + // ----------------------------------------------------------------- // Source/Target Connection Details // ----------------------------------------------------------------- - + var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( - excludeOnly: new []{ "schools" }); - + excludeOnly: new[] { "schools" }); + var targetApiConnectionDetails = TestHelpers.GetTargetApiConnectionDetails(); // ----------------------------------------------------------------- @@ -87,8 +87,8 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeResourcesTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeResourcesTests.cs index b8199b3..3ed43c9 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeResourcesTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ExcludeResourcesTests.cs @@ -17,7 +17,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing { - [TestFixture] + [TestFixture] public class ExcludeResourcesTests { [TestFixture] @@ -51,7 +51,7 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- // Target Requests // ----------------------------------------------------------------- - + _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); // Every POST succeeds @@ -62,8 +62,8 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( - exclude: new []{ "schools" }); - + exclude: new[] { "schools" }); + var targetApiConnectionDetails = TestHelpers.GetTargetApiConnectionDetails(); // ----------------------------------------------------------------- @@ -88,8 +88,8 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -162,7 +162,7 @@ public void Should_NOT_attempt_to_publish_resources_that_are_dependent_on_the_ex A.Ignored)) .MustNotHaveHappened(); } - + [TestCase("/ed-fi/assessments")] // Dependent on EducationOrganization [TestCase("/ed-fi/programs")] // Dependent on EducationOrganization [TestCase("/ed-fi/educationOrganizationNetworkAssociations")] // Dependent on EducationOrganization diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/IgnoreIsolationTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/IgnoreIsolationTests.cs index 0aac068..ae5b3dc 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/IgnoreIsolationTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/IgnoreIsolationTests.cs @@ -15,7 +15,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing { - [TestFixture] + [TestFixture] public class IgnoreIsolationTests { [TestFixture] @@ -47,7 +47,7 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- // Target Requests // ----------------------------------------------------------------- - + _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); // Every POST succeeds @@ -82,8 +82,8 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/IncludeOnlyResources.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/IncludeOnlyResources.cs index 29584a5..743706e 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/IncludeOnlyResources.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/IncludeOnlyResources.cs @@ -17,7 +17,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing { - [TestFixture] + [TestFixture] public class IncludeOnlyResourcesTests { [TestFixture] @@ -51,7 +51,7 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- // Target Requests // ----------------------------------------------------------------- - + _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); // Every POST succeeds @@ -62,8 +62,8 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( - includeOnly: new []{ "schools" }); - + includeOnly: new[] { "schools" }); + var targetApiConnectionDetails = TestHelpers.GetTargetApiConnectionDetails(); // ----------------------------------------------------------------- @@ -88,8 +88,8 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -134,7 +134,7 @@ public void Should_attempt_to_publish_the_resource_that_is_included() A.Ignored)) .MustHaveHappened(); } - + [Test] public void Should_reflect_the_processing_as_an_inclusion_without_its_dependencies_in_the_log() { diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/IncludeResourcesTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/IncludeResourcesTests.cs index f159d7c..dc8598c 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/IncludeResourcesTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/IncludeResourcesTests.cs @@ -18,7 +18,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing { - [TestFixture] + [TestFixture] public class IncludeResourcesTests { [TestFixture] @@ -52,7 +52,7 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- // Target Requests // ----------------------------------------------------------------- - + _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); // Every POST succeeds @@ -63,14 +63,14 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( - include: new []{ "schools" }); - + include: new[] { "schools" }); + var targetApiConnectionDetails = TestHelpers.GetTargetApiConnectionDetails(); - + // ----------------------------------------------------------------- // Options and Configuration // ----------------------------------------------------------------- - + var options = TestHelpers.GetOptions(); options.IncludeDescriptors = false; // Shorten test execution time @@ -89,8 +89,8 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -135,7 +135,7 @@ public void Should_attempt_to_publish_the_resource_that_is_included() A.Ignored)) .MustHaveHappened(); } - + [Test] public void Should_reflect_the_processing_as_an_inclusion_with_its_dependencies_in_the_log() { diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/KeyChangesTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/KeyChangesTests.cs index bc5b782..bb0bd41 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/KeyChangesTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/KeyChangesTests.cs @@ -26,13 +26,13 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing { - [TestFixture] + [TestFixture] public class KeyChangesTests { [TestFixture] public class When_publishing_natural_key_changes : TestFixtureAsyncBase { - const int TestItemQuantity = 2; + private const int TestItemQuantity = 2; private ChangeProcessor _changeProcessor; private ChangeProcessorConfiguration _changeProcessorConfiguration; @@ -51,7 +51,7 @@ protected override async Task ArrangeAsync() // Initialize a generator for the fake natural key class var keyValueFaker = TestHelpers.GetKeyValueFaker(); - + // Initialize a generator for the /keyChanges API response var keyChangeFaker = new Faker>().StrictMode(true) .RuleFor(o => o.Id, f => Guid.NewGuid().ToString("n")) @@ -74,7 +74,7 @@ protected override async Task ArrangeAsync() // Target Requests // ----------------------------------------------------------------- int i = 0; - + // Initialize a generator for a generic resource with a reference containing the key values var targetResourceFaker = new Faker>().StrictMode(true) .RuleFor(o => o.Id, f => Guid.NewGuid().ToString("n")) @@ -83,7 +83,7 @@ protected override async Task ArrangeAsync() .RuleFor(o => o.VehicleYear, f => f.Date.Between(DateTime.Today.AddYears(-50), DateTime.Today).Year); _suppliedTargetResources = targetResourceFaker.Generate(TestItemQuantity); - + _fakeTargetRequestHandler = A.Fake() .SetBaseUrl(MockRequests.TargetApiBaseUrl) .SetDataManagementUrlSegment(EdFiApiConstants.DataManagementApiSegment) @@ -99,7 +99,7 @@ protected override async Task ArrangeAsync() _suppliedKeyChanges[j].OldKeyValuesObject.ToQueryStringParams(), new[] { _suppliedTargetResources[j] }); } - + // ----------------------------------------------------------------- // Source/Target Connection Details // ----------------------------------------------------------------- @@ -132,8 +132,8 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -144,20 +144,20 @@ protected override async Task ActAsync() public void Should_probe_the_source_API_for_keyChanges_support_by_calling_the_first_resource_with_a_limit_of_1() { string keyChangeSupportProbingResourceName = _resourcesWithUpdatableKeys.OrderBy(x => x).FirstOrDefault(); - + A.CallTo( () => _fakeSourceRequestHandler.Get( $"{MockRequests.SourceApiBaseUrl}{MockRequests.DataManagementPath}{keyChangeSupportProbingResourceName}{EdFiApiConstants.KeyChangesPathSuffix}", A.That.Matches(msg => !msg.HasParameter("totalCount") && msg.QueryString("limit") == 1 ))) - .MustHaveHappenedOnceExactly(); + .MustHaveHappenedOnceExactly(); } [Test] public void Should_GET_keyChanges_from_source_API_for_each_resource_whose_keys_are_updatable() { // Console.WriteLine(_loggerRepository.LoggedContent()); - + foreach (var resourceWithUpdatableKey in _resourcesWithUpdatableKeys) { // One request for the count @@ -185,11 +185,11 @@ public void Should_not_attempt_to_GET_keyChanges_from_source_API_for_any_resourc var ns = new XmlNamespaceManager(new NameTable()); ns.AddNamespace("g", "http://graphml.graphdrawing.org/xmlns"); - + var resourcePaths = dependencies .XPathSelectElements("//g:node", ns) .Select(x => x.Attribute("id")?.Value).ToArray(); - + var resourcesWithoutUpdatableKeys = resourcePaths.Except(_resourcesWithUpdatableKeys).ToArray(); foreach (var resourceWithoutUpdatableKeys in resourcesWithoutUpdatableKeys) @@ -234,19 +234,19 @@ private bool IsOriginalTargetResourceWithNewKeyValuesApplied(HttpRequestMessage string content = request.Content.ReadAsStringAsync().Result; var requestItem = JsonConvert.DeserializeObject>(content); - + requestItem.ShouldSatisfyAllConditions( // The main values of the object should match the target - () => request.RequestUri.LocalPath.Split('/').Last().ShouldBe(suppliedTargetResource.Id), - () => requestItem.VehicleManufacturer.ShouldBe(suppliedTargetResource.VehicleManufacturer), + () => request.RequestUri.LocalPath.Split('/').Last().ShouldBe(suppliedTargetResource.Id), + () => requestItem.VehicleManufacturer.ShouldBe(suppliedTargetResource.VehicleManufacturer), () => requestItem.VehicleYear.ShouldBe(suppliedTargetResource.VehicleYear), // The key should match the source () => requestItem.SomeReference.Name.ShouldBe(suppliedKeyChange.NewKeyValuesObject.Name), - () => requestItem.SomeReference.BirthDate.ShouldBe(suppliedKeyChange.NewKeyValuesObject.BirthDate), + () => requestItem.SomeReference.BirthDate.ShouldBe(suppliedKeyChange.NewKeyValuesObject.BirthDate), () => requestItem.SomeReference.RetirementAge.ShouldBe(suppliedKeyChange.NewKeyValuesObject.RetirementAge)); return true; } } - } + } } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/PostRetryTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/PostRetryTests.cs index 172c4f1..2aefcab 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/PostRetryTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/PostRetryTests.cs @@ -14,12 +14,12 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing { - [TestFixture] + [TestFixture] public class PostRetryTests { private const string StateEducationAgencies = "/ed-fi/stateEducationAgencies"; private const string AddressTypeDescriptors = "/ed-fi/addressTypeDescriptors"; - + #region Test Cases [TestCase(HttpStatusCode.OK, StateEducationAgencies)] [TestCase(HttpStatusCode.OK, AddressTypeDescriptors)] @@ -106,7 +106,7 @@ public async Task When_a_POST_fails_with_certain_errors_should_retry_on_non_perm // Source Requests // ----------------------------------------------------------------- var sourceResourceFaker = TestHelpers.GetGenericResourceFaker(); - + var suppliedSourceResources = sourceResourceFaker.Generate(1); // Prepare the fake source API endpoint @@ -120,16 +120,16 @@ public async Task When_a_POST_fails_with_certain_errors_should_retry_on_non_perm // ----------------------------------------------------------------- // Target Requests // ----------------------------------------------------------------- - + var fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); if (initialResponseCodeOnPost == HttpStatusCode.OK) { - fakeTargetRequestHandler.PostResource( $"{EdFiApiConstants.DataManagementApiSegment}{resourcePath}", HttpStatusCode.OK); + fakeTargetRequestHandler.PostResource($"{EdFiApiConstants.DataManagementApiSegment}{resourcePath}", HttpStatusCode.OK); } else { - fakeTargetRequestHandler.PostResource( $"{EdFiApiConstants.DataManagementApiSegment}{resourcePath}", initialResponseCodeOnPost, HttpStatusCode.OK); + fakeTargetRequestHandler.PostResource($"{EdFiApiConstants.DataManagementApiSegment}{resourcePath}", initialResponseCodeOnPost, HttpStatusCode.OK); } // ----------------------------------------------------------------- @@ -137,8 +137,8 @@ public async Task When_a_POST_fails_with_certain_errors_should_retry_on_non_perm // ----------------------------------------------------------------- var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( - include: new []{ resourcePath }); - + include: new[] { resourcePath }); + var targetApiConnectionDetails = TestHelpers.GetTargetApiConnectionDetails(); // ----------------------------------------------------------------- diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ProfileApplicationTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ProfileApplicationTests.cs index 7ee07d8..0340dec 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ProfileApplicationTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ProfileApplicationTests.cs @@ -29,7 +29,7 @@ public class When_applying_profiles_to_source_and_target_connections : TestFixtu private IFakeHttpRequestHandler _fakeTargetRequestHandler; private IFakeHttpRequestHandler _fakeSourceRequestHandler; private ChangeProcessorConfiguration _changeProcessorConfiguration; - + private const string TestWritableProfileName = "Unit-Test-Target-Profile"; private const string TestReadableProfileName = "Unit-Test-Source-Profile"; @@ -54,7 +54,7 @@ protected override async Task ArrangeAsync() // Target Requests // ----------------------------------------------------------------- _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); - + // Every POST succeeds _fakeTargetRequestHandler.EveryDataManagementPostReturns200Ok(); @@ -80,10 +80,10 @@ protected override async Task ArrangeAsync() // Initialize logging TestHelpers.InitializeLogging(); - + // Configuration _changeProcessorConfiguration = TestHelpers.CreateChangeProcessorConfiguration(options); - + // Create change processor with dependencies _changeProcessor = TestHelpers.CreateChangeProcessorWithDefaultDependencies( options, @@ -91,8 +91,8 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -107,7 +107,7 @@ public void Should_NOT_apply_profile_content_types_to_descriptors_GET_requests() A.That.Matches(msg => DoesNotUseProfileContentType(msg)))) .MustHaveHappenedTwiceExactly(); // Once for count, once for data } - + [Test] public void Should_NOT_apply_profile_content_types_to_descriptors_POST_requests() { @@ -116,7 +116,7 @@ public void Should_NOT_apply_profile_content_types_to_descriptors_POST_requests( A.That.Matches(msg => DoesNotUseProfileContentType(msg)))) .MustHaveHappened(3, Times.Exactly); } - + [Test] public void Should_apply_readable_profile_content_type_to_count_requests() { @@ -162,7 +162,7 @@ private bool QueryStringHasTotalCount(Uri msgRequestUri) { return msgRequestUri?.ParseQueryString().AllKeys.Contains("totalCount", StringComparer.OrdinalIgnoreCase) ?? false; } - + private bool UsesReadableContentType(HttpRequestMessage requestMessage) { var match = Regex.Match( @@ -173,7 +173,7 @@ private bool UsesReadableContentType(HttpRequestMessage requestMessage) { return false; } - + return match.Groups["ProfileName"].Value == TestReadableProfileName.ToLower(); } @@ -187,11 +187,11 @@ private bool UsesWritableContentType(HttpRequestMessage requestMessage) { return false; } - + return match.Groups["ProfileName"].Value == TestWritableProfileName.ToLower(); } } - + [TestFixture] public class When_applying_a_profile_to_the_source_and_not_to_the_target_connections { @@ -199,7 +199,7 @@ public class When_applying_a_profile_to_the_source_and_not_to_the_target_connect private IFakeHttpRequestHandler _fakeTargetRequestHandler; private IFakeHttpRequestHandler _fakeSourceRequestHandler; private ChangeProcessorConfiguration _changeProcessorConfiguration; - + private const string TestReadableProfileName = "Unit-Test-Source-Profile"; [Test] @@ -224,7 +224,7 @@ public async Task Should_throw_an_exception_to_prevent_data_loss() // Target Requests // ----------------------------------------------------------------- _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); - + // Every POST succeeds _fakeTargetRequestHandler.EveryDataManagementPostReturns200Ok(); @@ -251,12 +251,12 @@ public async Task Should_throw_an_exception_to_prevent_data_loss() // Initialize logging TestHelpers.InitializeLogging(); - + // Configuration _changeProcessorConfiguration = TestHelpers.CreateChangeProcessorConfiguration(options); - + Should.Throw( - () => + () => // Create change processor with dependencies _changeProcessor = TestHelpers.CreateChangeProcessorWithDefaultDependencies( options, @@ -265,7 +265,7 @@ public async Task Should_throw_an_exception_to_prevent_data_loss() targetApiConnectionDetails, _fakeTargetRequestHandler)) .Message.ShouldBe("The source API connection has a ProfileName specified, but the target API connection does not. POST requests against a target API without the Profile-based context of the source data can lead to accidental data loss."); - await Task.Yield(); - } + await Task.Yield(); + } } } diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/RateLimitingTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/RateLimitingTests.cs index dc04718..ea3c68a 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/RateLimitingTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/RateLimitingTests.cs @@ -56,7 +56,7 @@ public void RateLimitedMethod_Should_Throw_RateLimiterRejectedException_On_Overl options.RateLimitTimeSeconds = 1; options.RateLimitMaxRetries = 1; var rateLimiter = new PollyRateLimiter(options); - + var methodToTest = new MockRateLimitingMethod(rateLimiter); var tasks = new List>(); diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/RemediationIntegrationTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/RemediationIntegrationTests.cs index 54c3183..82e7e38 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/RemediationIntegrationTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/RemediationIntegrationTests.cs @@ -36,8 +36,8 @@ public class When_a_POST_initially_fails_with_a_permanent_failure_but_has_JavaSc private ChangeProcessorConfiguration _changeProcessorConfiguration; private IFakeHttpRequestHandler _fakeTargetRequestHandler; - const HttpStatusCode InitialResponseCodeOnPost = HttpStatusCode.BadRequest; - const string ResourcePath = DisciplineActionsResourcePath; + private const HttpStatusCode InitialResponseCodeOnPost = HttpStatusCode.BadRequest; + private const string ResourcePath = DisciplineActionsResourcePath; private static string DisciplineActionsItemJson = $@" {{ @@ -116,7 +116,7 @@ public class When_a_POST_initially_fails_with_a_permanent_failure_but_has_JavaSc private const string DisciplineIncidentActionsPostResponseErrorJson = $@" {{ ""message"": ""Validation of 'DisciplineAction' failed.\\r\\n\\tValidation of 'DisciplineActionStaffs' failed.\\n\\t\\tDisciplineActionStaff[0]: Staff reference could not be resolved.\\n\\t\\tDisciplineActionStaff[2]: Staff reference could not be resolved.\\n""}}"; - readonly Func _remediationJavaScriptModuleFactory = () => @$" + private readonly Func _remediationJavaScriptModuleFactory = () => @$" module.exports = {{ '/ed-fi/disciplineActions/400': async (failureContext) => {{ // Parse the request/response data @@ -165,8 +165,8 @@ protected override async Task ArrangeAsync() // Test-specific mocks .AvailableChangeVersions(1100) .ResourceCount(responseTotalCountHeader: 1) - .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{ResourcePath}", - new [] { JObject.Parse(DisciplineActionsItemJson) }) + .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{ResourcePath}", + new[] { JObject.Parse(DisciplineActionsItemJson) }) .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{AssessmentsResourcePath}", suppliedSourceResources); @@ -227,8 +227,8 @@ protected override async Task ArrangeAsync() targetApiConnectionDetails, _fakeTargetRequestHandler, nodeJsService); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -281,7 +281,7 @@ public void Should_perform_remediation_for_items_with_error_messages() .MustHaveHappenedOnceExactly(); }); } - + [Test] public void Should_NOT_perform_remediation_for_items_WITHOUT_error_messages() { @@ -301,7 +301,7 @@ private bool WithMatchingStaffUniqueId(HttpRequestMessage msg, string studentUni { return false; } - + var data = JObject.Parse(content); return data["staffUniqueId"].Value() == studentUniqueId; @@ -314,8 +314,8 @@ public class When_a_POST_initially_fails_with_a_permanent_failure_but_has_JavaSc private ChangeProcessorConfiguration _changeProcessorConfiguration; private IFakeHttpRequestHandler _fakeTargetRequestHandler; - const HttpStatusCode InitialResponseCodeOnPost = HttpStatusCode.BadRequest; - const string ResourcePath = DisciplineActionsResourcePath; + private const HttpStatusCode InitialResponseCodeOnPost = HttpStatusCode.BadRequest; + private const string ResourcePath = DisciplineActionsResourcePath; private static string DisciplineActionsItemJson = $@" {{ @@ -396,7 +396,7 @@ public class When_a_POST_initially_fails_with_a_permanent_failure_but_has_JavaSc private const string AssessmentsResourcePath = "/ed-fi/assessments"; - readonly Func _remediationJavaScriptModuleFactory = () => @$" + private readonly Func _remediationJavaScriptModuleFactory = () => @$" module.exports = {{ '/ed-fi/disciplineActions/400': async (failureContext) => {{ // Parse the request/response data @@ -437,8 +437,8 @@ protected override async Task ArrangeAsync() // Test-specific mocks .AvailableChangeVersions(1100) .ResourceCount(responseTotalCountHeader: 1) - .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{ResourcePath}", - new [] { JObject.Parse(DisciplineActionsItemJson) }) + .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{ResourcePath}", + new[] { JObject.Parse(DisciplineActionsItemJson) }) .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{AssessmentsResourcePath}", suppliedSourceResources); @@ -495,8 +495,8 @@ protected override async Task ArrangeAsync() targetApiConnectionDetails, _fakeTargetRequestHandler, nodeJsService); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -556,9 +556,9 @@ private bool WithExactEntriesForStaffUniqueIds(HttpRequestMessage msg, params st { return false; } - - return ((IEnumerable) data.staffs) - .All(s => requiredStaffUniqueIds.Contains((string) s.staffReference.staffUniqueId)); + + return ((IEnumerable)data.staffs) + .All(s => requiredStaffUniqueIds.Contains((string)s.staffReference.staffUniqueId)); } } @@ -572,7 +572,7 @@ public TestNodeJsService(string testCacheIdentifier) { _testCacheIdentifier = testCacheIdentifier; } - + public Task InvokeFromStringAsync( Func moduleFactory, string cacheIdentifier, diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/RemediationTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/RemediationTests.cs index 9186b3f..0dc5a2f 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/RemediationTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/RemediationTests.cs @@ -33,9 +33,9 @@ public class When_a_POST_fails_but_has_JavaScript_extension_module_for_remediati private IFakeHttpRequestHandler _fakeTargetRequestHandler; // [TestCase(HttpStatusCode.Forbidden, StaffDisciplineIncidentAssociations, true)] - HttpStatusCode initialResponseCodeOnPost = HttpStatusCode.Forbidden; - const string resourcePath = StaffDisciplineIncidentAssociations; - const bool shouldRetry = true; + private HttpStatusCode initialResponseCodeOnPost = HttpStatusCode.Forbidden; + private const string ResourcePath = StaffDisciplineIncidentAssociations; + private const bool ShouldRetry = true; protected override async Task ArrangeAsync() { @@ -52,7 +52,7 @@ protected override async Task ArrangeAsync() // Test-specific mocks .AvailableChangeVersions(1100) .ResourceCount(responseTotalCountHeader: 1) - .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{resourcePath}", suppliedSourceResources); + .GetResourceData($"{EdFiApiConstants.DataManagementApiSegment}{ResourcePath}", suppliedSourceResources); // ----------------------------------------------------------------- @@ -65,13 +65,13 @@ protected override async Task ArrangeAsync() if (initialResponseCodeOnPost == HttpStatusCode.OK) { _fakeTargetRequestHandler.PostResource( - $"{EdFiApiConstants.DataManagementApiSegment}{resourcePath}", + $"{EdFiApiConstants.DataManagementApiSegment}{ResourcePath}", HttpStatusCode.OK); } else { _fakeTargetRequestHandler.PostResource( - $"{EdFiApiConstants.DataManagementApiSegment}{resourcePath}", + $"{EdFiApiConstants.DataManagementApiSegment}{ResourcePath}", initialResponseCodeOnPost, HttpStatusCode.OK); } @@ -80,7 +80,7 @@ protected override async Task ArrangeAsync() // Source/Target Connection Details // ----------------------------------------------------------------- - var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails(includeOnly: new[] { resourcePath }); + var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails(includeOnly: new[] { ResourcePath }); var targetApiConnectionDetails = TestHelpers.GetTargetApiConnectionDetails(); // ----------------------------------------------------------------- @@ -89,7 +89,7 @@ protected override async Task ArrangeAsync() var options = TestHelpers.GetOptions(); // Only include descriptors if our test subject resource is a descriptor (trying to avoid any dependencies to keep things simpler) - options.IncludeDescriptors = resourcePath.EndsWith("Descriptors"); + options.IncludeDescriptors = ResourcePath.EndsWith("Descriptors"); // ----------------------------------------------------------------- // Initialize logging @@ -137,8 +137,8 @@ protected override async Task ArrangeAsync() targetApiConnectionDetails, _fakeTargetRequestHandler, nodeJsService); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -151,18 +151,18 @@ public void Should_attempt_original_unmodified_request_once() // Should try POST once with original (unmodified) request body A.CallTo( () => _fakeTargetRequestHandler.Post( - $"{MockRequests.TargetApiBaseUrl}{MockRequests.DataManagementPath}{resourcePath}", + $"{MockRequests.TargetApiBaseUrl}{MockRequests.DataManagementPath}{ResourcePath}", A.That.Matches(req => !HasModifiedRequestBody(req)))) .MustHaveHappened(1, Times.Exactly); } - + [Test] public void Should_retry_request_even_after_an_otherwise_permanent_failure_with_the_modified_request_provided_by_the_remediation_extension() { // Should try POST once with modified request body, as directed by the JavaScript extension A.CallTo( () => _fakeTargetRequestHandler.Post( - $"{MockRequests.TargetApiBaseUrl}{MockRequests.DataManagementPath}{resourcePath}", + $"{MockRequests.TargetApiBaseUrl}{MockRequests.DataManagementPath}{ResourcePath}", A.That.Matches(req => HasModifiedRequestBody(req)))) .MustHaveHappened(1, Times.Exactly); } @@ -175,7 +175,7 @@ private bool HasModifiedRequestBody(HttpRequestMessage requestMessage) return (modifiedRequest.type == "modified"); } - + [Test] public void Should_process_remediation_requests_returned_from_nodejs_service_invocation_with_the_supplied_message_bodies() { @@ -196,7 +196,7 @@ public void Should_process_remediation_requests_returned_from_nodejs_service_inv msg => WithMatchingBody(msg, "Staff EdOrg Assignment Request Body")))) .MustHaveHappenedOnceExactly(); }); - } + } private bool WithMatchingBody(HttpRequestMessage msg, string expectedMessage) { diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs index eb44a7d..6bb83fd 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/ReversePagingTests.cs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: Apache-2.0 // Licensed to the Ed-Fi Alliance under one or more agreements. // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/SchoolYearSpecificClientTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/SchoolYearSpecificClientTests.cs index 1d6d44b..c77440a 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/SchoolYearSpecificClientTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/SchoolYearSpecificClientTests.cs @@ -16,7 +16,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing { - [TestFixture] + [TestFixture] public class SchoolYearSpecificClientTests { [TestFixture] @@ -26,7 +26,7 @@ public class When_publishing_resources_to_SchoolYear_specific_deployment : TestF private IFakeHttpRequestHandler _fakeTargetRequestHandler; private IFakeHttpRequestHandler _fakeSourceRequestHandler; private ChangeProcessorConfiguration _changeProcessorConfiguration; - + private const int SuppliedSchoolYear = MockRequests.SchoolYear; protected override async Task ArrangeAsync() @@ -68,7 +68,7 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- // Options and Configuration // ----------------------------------------------------------------- - + var options = TestHelpers.GetOptions(); options.IncludeDescriptors = false; // Shorten test execution time @@ -87,8 +87,8 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -104,7 +104,7 @@ public void Should_get_dependencies_from_the_target_API_using_the_schoolyear_spe A.Ignored)) .MustHaveHappened(); } - + [Test] public void Should_attempt_to_read_resources_WITHOUT_schoolyear_in_the_source_path() { @@ -164,14 +164,14 @@ protected override async Task ArrangeAsync() // Target Requests // ----------------------------------------------------------------- _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); - + // Every POST succeeds - _fakeTargetRequestHandler.PostResource( $"{TestHelpers.AnyResourcePattern}", HttpStatusCode.OK); + _fakeTargetRequestHandler.PostResource($"{TestHelpers.AnyResourcePattern}", HttpStatusCode.OK); // ----------------------------------------------------------------- var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails(schoolYear: SuppliedSchoolYear); var targetApiConnectionDetails = TestHelpers.GetTargetApiConnectionDetails(); - + // ----------------------------------------------------------------- // Options and Configuration // ----------------------------------------------------------------- @@ -194,8 +194,8 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -211,7 +211,7 @@ public void Should_get_dependencies_from_the_target_API_using_the_default_URL() A.Ignored)) .MustHaveHappened(); } - + [Test] public void Should_attempt_to_read_resources_WITH_schoolyear_in_the_source_path() { @@ -272,9 +272,9 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler( $"{EdFiApiConstants.DataManagementApiSegment}/{SuppliedSchoolYear}"); - + // Every POST succeeds - _fakeTargetRequestHandler.PostResource( $"{EdFiApiConstants.DataManagementApiSegment}/{SuppliedSchoolYear}{TestHelpers.AnyResourcePattern}", HttpStatusCode.OK); + _fakeTargetRequestHandler.PostResource($"{EdFiApiConstants.DataManagementApiSegment}/{SuppliedSchoolYear}{TestHelpers.AnyResourcePattern}", HttpStatusCode.OK); // ----------------------------------------------------------------- // Source/Target Connection Details @@ -305,9 +305,9 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); + await Task.Yield(); - } + } protected override async Task ActAsync() { @@ -323,7 +323,7 @@ public void Should_get_dependencies_from_the_target_API_using_the_schoolyear_spe A.Ignored)) .MustHaveHappened(); } - + [Test] public void Should_attempt_to_read_resources_WITH_schoolyear_in_the_source_path() { diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/UnresolvedDependencyOnPrimaryRelationshipTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/UnresolvedDependencyOnPrimaryRelationshipTests.cs index 7fc9086..ee885ef 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/UnresolvedDependencyOnPrimaryRelationshipTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/UnresolvedDependencyOnPrimaryRelationshipTests.cs @@ -20,7 +20,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Processing { - [TestFixture] + [TestFixture] public class UnresolvedDependencyOnPrimaryRelationshipTests { [TestFixture] @@ -36,18 +36,18 @@ public class MissingPerson { [JsonProperty("id")] public string Id { get; set; } - + [JsonProperty("firstName")] public string FirstName { get; set; } - + [JsonProperty("lastSurname")] public string LastSurname { get; set; } - + [JsonProperty("_etag")] public string ETag { get; set; } } - const string TestResourcePath = "/ed-fi/studentSchoolAssociations"; + private const string TestResourcePath = "/ed-fi/studentSchoolAssociations"; protected override async Task ArrangeAsync() { @@ -55,7 +55,7 @@ protected override async Task ArrangeAsync() // Source Requests // ----------------------------------------------------------------- var sourceResourceFaker = TestHelpers.GetGenericResourceFaker(); - + var suppliedSourceResources = sourceResourceFaker.Generate(1); // Prepare the fake source API endpoint @@ -76,7 +76,7 @@ protected override async Task ArrangeAsync() LastSurname = "Jones", ETag = "etagvalue" }; - + _fakeSourceRequestHandler.GetResourceDataItem( $"{EdFiApiConstants.DataManagementApiSegment}{_suppliedSourceLinkHref}", suppliedMissingPerson); @@ -85,12 +85,12 @@ protected override async Task ArrangeAsync() // Target Requests // ----------------------------------------------------------------- _fakeTargetRequestHandler = TestHelpers.GetFakeBaselineTargetApiRequestHandler(); - + // Override dependencies to a single resource to minimize extraneous noise _fakeTargetRequestHandler.Dependencies(TestResourcePath); - - _fakeTargetRequestHandler.PostResource( $"{EdFiApiConstants.DataManagementApiSegment}{TestResourcePath}", - (HttpStatusCode.BadRequest, JObject.Parse("{\r\n \"message\": \"Validation of 'StudentSchoolAssociation' failed.\\r\\n\\tSome reference could not be resolved.\\n\"\r\n}")), + + _fakeTargetRequestHandler.PostResource($"{EdFiApiConstants.DataManagementApiSegment}{TestResourcePath}", + (HttpStatusCode.BadRequest, JObject.Parse("{\r\n \"message\": \"Validation of 'StudentSchoolAssociation' failed.\\r\\n\\tSome reference could not be resolved.\\n\"\r\n}")), (HttpStatusCode.OK, null)); // ----------------------------------------------------------------- @@ -98,10 +98,10 @@ protected override async Task ArrangeAsync() // ----------------------------------------------------------------- var sourceApiConnectionDetails = TestHelpers.GetSourceApiConnectionDetails( - include: new []{ TestResourcePath }); - + include: new[] { TestResourcePath }); + var targetApiConnectionDetails = TestHelpers.GetTargetApiConnectionDetails(); - + // ----------------------------------------------------------------- // Options and Configuration // ----------------------------------------------------------------- @@ -124,8 +124,8 @@ protected override async Task ArrangeAsync() _fakeSourceRequestHandler, targetApiConnectionDetails, _fakeTargetRequestHandler); - await Task.Yield(); - } + await Task.Yield(); + } protected override async Task ActAsync() { @@ -151,7 +151,7 @@ public void Should_attempt_to_get_the_item_for_the_unresolved_reference_from_the A.Ignored)) .MustHaveHappened(); } - + [Test] public void Should_attempt_to_post_the_item_obtained_from_the_source_API_for_the_unresolved_reference_to_the_target_API() { @@ -161,7 +161,7 @@ public void Should_attempt_to_post_the_item_obtained_from_the_source_API_for_the A.That.Matches(HasSuppliedStudentInPostRequestBody, "has supplied source item in POST request body"))) .MustHaveHappened(); } - + private bool HasSuppliedStudentInPostRequestBody(HttpRequestMessage req) { string content = req.Content.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult(); @@ -172,7 +172,7 @@ private bool HasSuppliedStudentInPostRequestBody(HttpRequestMessage req) o => o.ShouldNotBeNull(), o => o.ShouldNotContainKey("id"), o => o.ShouldNotContainKey("_etag"), - + o => o.ShouldContainKey("firstName"), o => o.ShouldContainKey("lastSurname"), @@ -184,4 +184,4 @@ private bool HasSuppliedStudentInPostRequestBody(HttpRequestMessage req) } } } -} \ No newline at end of file +} diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Resources/TestData.cs b/src/EdFi.Tools.ApiPublisher.Tests/Resources/TestData.cs index d414177..39ebd74 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Resources/TestData.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Resources/TestData.cs @@ -8,7 +8,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Resources { - public static class TestData + public static class TestData { public static class Dependencies { diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Serialization/GuidConverter.cs b/src/EdFi.Tools.ApiPublisher.Tests/Serialization/GuidConverter.cs index ef98b19..25f6f88 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Serialization/GuidConverter.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Serialization/GuidConverter.cs @@ -8,7 +8,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Serialization { - public class GuidConverter : JsonConverter + public class GuidConverter : JsonConverter { public override bool CanRead { @@ -17,7 +17,7 @@ public override bool CanRead public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { - writer.WriteValue(((Guid) value).ToString("N")); + writer.WriteValue(((Guid)value).ToString("N")); } public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Serialization/Iso8601UtcDateOnlyConverter.cs b/src/EdFi.Tools.ApiPublisher.Tests/Serialization/Iso8601UtcDateOnlyConverter.cs index 99a6937..62e34b7 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Serialization/Iso8601UtcDateOnlyConverter.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Serialization/Iso8601UtcDateOnlyConverter.cs @@ -10,7 +10,7 @@ namespace EdFi.Tools.ApiPublisher.Tests.Serialization { - public class Iso8601UtcDateOnlyConverter : IsoDateTimeConverter + public class Iso8601UtcDateOnlyConverter : IsoDateTimeConverter { // All valid US English time formats will contain either a time separator ':' or an AM/PM designator private readonly Regex _timePortionRegex = new Regex(":|am|pm", RegexOptions.Compiled | RegexOptions.IgnoreCase); diff --git a/src/EdFi.Tools.ApiPublisher.Tests/TestFixtureAsyncBase.cs b/src/EdFi.Tools.ApiPublisher.Tests/TestFixtureAsyncBase.cs index 06ba538..d008c14 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/TestFixtureAsyncBase.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/TestFixtureAsyncBase.cs @@ -15,7 +15,7 @@ namespace EdFi.Tools.ApiPublisher.Tests { - [TestFixture] + [TestFixture] public abstract class TestFixtureAsyncBase {