From 6c1caa1121e16e2db6dd6a1c853f7ef50609a101 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 23 Nov 2022 11:30:53 +1000 Subject: [PATCH 01/47] Implement endpoint definitions using Attributes & interfaces api --- .editorconfig | 226 ++++++++++++++++++ .../ApiEndpointAttributeFactoryTests.cs | 69 ++++++ .../ApiEndpointRegisterTests.cs | 14 ++ .../MIFCore.Hangfire.APIETL.Tests.csproj | 23 ++ MIFCore.Hangfire.APIETL/ApiData.cs | 17 ++ MIFCore.Hangfire.APIETL/ApiEndpoint.cs | 32 +++ .../ApiEndpointAttributeFactory.cs | 79 ++++++ .../ApiEndpointNameAttribute.cs | 15 ++ .../ApiEndpointRegister.cs | 58 +++++ .../ApiEndpointSelectorAttribute.cs | 15 ++ MIFCore.Hangfire.APIETL/EndpointExtractJob.cs | 120 ++++++++++ .../EndpointServiceCollectionExtensions.cs | 58 +++++ .../IApiEndpointAttributeFactory.cs | 9 + MIFCore.Hangfire.APIETL/IDefineEndpoints.cs | 9 + .../IPrepareNextRequest.cs | 10 + MIFCore.Hangfire.APIETL/IPrepareRequest.cs | 9 + .../MIFCore.Hangfire.APIETL.csproj | 33 +++ .../PrepareNextRequestArgs.cs | 16 ++ MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs | 17 ++ MIFCore.sln | 16 ++ 20 files changed, 845 insertions(+) create mode 100644 .editorconfig create mode 100644 MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs create mode 100644 MIFCore.Hangfire.APIETL.Tests/ApiEndpointRegisterTests.cs create mode 100644 MIFCore.Hangfire.APIETL.Tests/MIFCore.Hangfire.APIETL.Tests.csproj create mode 100644 MIFCore.Hangfire.APIETL/ApiData.cs create mode 100644 MIFCore.Hangfire.APIETL/ApiEndpoint.cs create mode 100644 MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs create mode 100644 MIFCore.Hangfire.APIETL/ApiEndpointNameAttribute.cs create mode 100644 MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs create mode 100644 MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs create mode 100644 MIFCore.Hangfire.APIETL/EndpointExtractJob.cs create mode 100644 MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs create mode 100644 MIFCore.Hangfire.APIETL/IApiEndpointAttributeFactory.cs create mode 100644 MIFCore.Hangfire.APIETL/IDefineEndpoints.cs create mode 100644 MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs create mode 100644 MIFCore.Hangfire.APIETL/IPrepareRequest.cs create mode 100644 MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj create mode 100644 MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs create mode 100644 MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..10e5a38 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,226 @@ +# Remove the line below if you want to inherit .editorconfig settings from higher directories +root = true + +# C# files +[*.cs] + +#### Core EditorConfig Options #### + +# Indentation and spacing +indent_size = 4 +indent_style = space +tab_width = 4 + +# New line preferences +end_of_line = crlf +insert_final_newline = false + +#### .NET Coding Conventions #### + +# Organize usings +dotnet_separate_import_directive_groups = false +dotnet_sort_system_directives_first = false +file_header_template = unset + +# this. and Me. preferences +dotnet_style_qualification_for_event = true +dotnet_style_qualification_for_field = true +dotnet_style_qualification_for_method = true +dotnet_style_qualification_for_property = true + +# Language keywords vs BCL types preferences +dotnet_style_predefined_type_for_locals_parameters_members = true +dotnet_style_predefined_type_for_member_access = true + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity +dotnet_style_parentheses_in_other_operators = never_if_unnecessary +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity + +# Modifier preferences +dotnet_style_require_accessibility_modifiers = for_non_interface_members + +# Expression-level preferences +dotnet_style_coalesce_expression = true +dotnet_style_collection_initializer = true +dotnet_style_explicit_tuple_names = true +dotnet_style_namespace_match_folder = true +dotnet_style_null_propagation = true +dotnet_style_object_initializer = true +dotnet_style_operator_placement_when_wrapping = beginning_of_line +dotnet_style_prefer_auto_properties = false +dotnet_style_prefer_compound_assignment = true +dotnet_style_prefer_conditional_expression_over_assignment = true +dotnet_style_prefer_conditional_expression_over_return = true +dotnet_style_prefer_foreach_explicit_cast_in_source = when_strongly_typed +dotnet_style_prefer_inferred_anonymous_type_member_names = true +dotnet_style_prefer_inferred_tuple_names = true +dotnet_style_prefer_is_null_check_over_reference_equality_method = true +dotnet_style_prefer_simplified_boolean_expressions = true +dotnet_style_prefer_simplified_interpolation = true + +# Field preferences +dotnet_style_readonly_field = true + +# Parameter preferences +dotnet_code_quality_unused_parameters = all + +# Suppression preferences +dotnet_remove_unnecessary_suppression_exclusions = none + +# New line preferences +dotnet_style_allow_multiple_blank_lines_experimental = true +dotnet_style_allow_statement_immediately_after_block_experimental = true + +#### C# Coding Conventions #### + +# var preferences +csharp_style_var_elsewhere = false +csharp_style_var_for_built_in_types = false +csharp_style_var_when_type_is_apparent = false + +# Expression-bodied members +csharp_style_expression_bodied_accessors = true +csharp_style_expression_bodied_constructors = false +csharp_style_expression_bodied_indexers = true +csharp_style_expression_bodied_lambdas = when_on_single_line +csharp_style_expression_bodied_local_functions = false +csharp_style_expression_bodied_methods = false +csharp_style_expression_bodied_operators = false +csharp_style_expression_bodied_properties = true + +# Pattern matching preferences +csharp_style_pattern_matching_over_as_with_null_check = true +csharp_style_pattern_matching_over_is_with_cast_check = true +csharp_style_prefer_extended_property_pattern = true +csharp_style_prefer_not_pattern = true +csharp_style_prefer_pattern_matching = true +csharp_style_prefer_switch_expression = true + +# Null-checking preferences +csharp_style_conditional_delegate_call = true + +# Modifier preferences +csharp_prefer_static_local_function = true +csharp_preferred_modifier_order = public,private,protected,internal,file,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,required,volatile,async +csharp_style_prefer_readonly_struct = true + +# Code-block preferences +csharp_prefer_braces = true +csharp_prefer_simple_using_statement = true +csharp_style_namespace_declarations = block_scoped +csharp_style_prefer_method_group_conversion = true +csharp_style_prefer_top_level_statements = true + +# Expression-level preferences +csharp_prefer_simple_default_expression = true +csharp_style_deconstructed_variable_declaration = true +csharp_style_implicit_object_creation_when_type_is_apparent = true +csharp_style_inlined_variable_declaration = true +csharp_style_prefer_index_operator = true +csharp_style_prefer_local_over_anonymous_function = true +csharp_style_prefer_null_check_over_type_check = true +csharp_style_prefer_range_operator = true +csharp_style_prefer_tuple_swap = true +csharp_style_prefer_utf8_string_literals = true +csharp_style_throw_expression = true +csharp_style_unused_value_assignment_preference = discard_variable +csharp_style_unused_value_expression_statement_preference = discard_variable + +# 'using' directive preferences +csharp_using_directive_placement = outside_namespace + +# New line preferences +csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = true +csharp_style_allow_blank_lines_between_consecutive_braces_experimental = true +csharp_style_allow_embedded_statements_on_same_line_experimental = true + +#### C# Formatting Rules #### + +# New line preferences +csharp_new_line_before_catch = true +csharp_new_line_before_else = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_open_brace = all +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_labels = one_less_than_current +csharp_indent_switch_labels = true + +# Space preferences +csharp_space_after_cast = false +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_after_semicolon_in_for_statement = true +csharp_space_around_binary_operators = before_and_after +csharp_space_around_declaration_statements = false +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_between_square_brackets = false + +# Wrapping preferences +csharp_preserve_single_line_blocks = true +csharp_preserve_single_line_statements = true + +#### Naming styles #### + +# Naming rules + +dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion +dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface +dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i + +dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.types_should_be_pascal_case.symbols = types +dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case + +dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members +dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case + +# Symbol specifications + +dotnet_naming_symbols.interface.applicable_kinds = interface +dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.interface.required_modifiers = + +dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum +dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.types.required_modifiers = + +dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method +dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.non_field_members.required_modifiers = + +# Naming styles + +dotnet_naming_style.pascal_case.required_prefix = +dotnet_naming_style.pascal_case.required_suffix = +dotnet_naming_style.pascal_case.word_separator = +dotnet_naming_style.pascal_case.capitalization = pascal_case + +dotnet_naming_style.begins_with_i.required_prefix = I +dotnet_naming_style.begins_with_i.required_suffix = +dotnet_naming_style.begins_with_i.word_separator = +dotnet_naming_style.begins_with_i.capitalization = pascal_case diff --git a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs new file mode 100644 index 0000000..650f96e --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs @@ -0,0 +1,69 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MIFCore.Hangfire.APIETL.Tests +{ + [TestClass()] + public class ApiEndpointAttributeFactoryTests + { + private static string[] Tenants = new[] { "tenant1", "tenant2" }; + + [TestMethod()] + public async Task Create_WithEndpointDefiner_CreatesEndpointsForEachTenant() + { + var serviceProvider = this.GetServiceProvider(typeof(Endpoint1), typeof(TenantedEndpointRegister)); + var factory = new ApiEndpointAttributeFactory( + endpointNameAttributes: serviceProvider.GetRequiredService>(), + endpointDefiners: serviceProvider.GetRequiredService>() + ); + + // There should be four endpoints total, api/endpoint1, api/endpoint2 but duplicated for each "tenant" + var endpoints = await factory.Create().ToListAsync(); + + Assert.AreEqual(4, endpoints.Count); + Assert.AreEqual(2, endpoints.Count(x => x.Name == "api/endpoint1")); + Assert.AreEqual(2, endpoints.Count(x => x.Name == "api/endpoint2")); + + Assert.AreEqual(1, endpoints.Count(x => x.Name == "api/endpoint1" && x.AdditionalHeaders["tenantId"] == "tenant1")); + Assert.AreEqual(1, endpoints.Count(x => x.Name == "api/endpoint2" && x.AdditionalHeaders["tenantId"] == "tenant1")); + + Assert.AreEqual(1, endpoints.Count(x => x.Name == "api/endpoint1" && x.AdditionalHeaders["tenantId"] == "tenant2")); + Assert.AreEqual(1, endpoints.Count(x => x.Name == "api/endpoint2" && x.AdditionalHeaders["tenantId"] == "tenant2")); + } + + private IServiceProvider GetServiceProvider(params Type[] endpoints) + { + var serviceCollection = new ServiceCollection(); + serviceCollection.AddEndpoints(endpoints); + return serviceCollection.BuildServiceProvider(); + } + + [ApiEndpointName("api/endpoint1")] + class Endpoint1 : IPrepareRequest + { + public Task OnPrepareRequest(PrepareRequestArgs args) + { + throw new NotImplementedException(); + } + } + + [ApiEndpointName("api/endpoint2")] + [ApiEndpointSelector(".*")] + class TenantedEndpointRegister : IDefineEndpoints + { + public async IAsyncEnumerable DefineEndpoints(string endpointName) + { + foreach (var t in Tenants) + { + yield return new ApiEndpoint($"{endpointName} ({t})") + { + AdditionalHeaders = + { + { "tenantId", t } + } + }; + } + } + } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointRegisterTests.cs b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointRegisterTests.cs new file mode 100644 index 0000000..391f116 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointRegisterTests.cs @@ -0,0 +1,14 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MIFCore.Hangfire.APIETL.Tests +{ + [TestClass()] + public class ApiEndpointRegisterTests + { + [TestMethod()] + public void RegisterTest() + { + + } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL.Tests/MIFCore.Hangfire.APIETL.Tests.csproj b/MIFCore.Hangfire.APIETL.Tests/MIFCore.Hangfire.APIETL.Tests.csproj new file mode 100644 index 0000000..a710fbb --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/MIFCore.Hangfire.APIETL.Tests.csproj @@ -0,0 +1,23 @@ + + + + net6 + enable + enable + + false + + + + + + + + + + + + + + + diff --git a/MIFCore.Hangfire.APIETL/ApiData.cs b/MIFCore.Hangfire.APIETL/ApiData.cs new file mode 100644 index 0000000..450e5b8 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiData.cs @@ -0,0 +1,17 @@ +using System; + +namespace MIFCore.Hangfire.APIETL +{ + public class ApiData + { + public Guid Id { get; set; } = Guid.NewGuid(); + public Guid? ParentId { get; set; } + + public string Endpoint { get; set; } + public string Uri { get; set; } + + public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.Now; + + public string Data { get; set; } + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpoint.cs b/MIFCore.Hangfire.APIETL/ApiEndpoint.cs new file mode 100644 index 0000000..69f9738 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpoint.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; + +[assembly: InternalsVisibleTo("MIFCore.Hangfire.APIETL.Tests")] +namespace MIFCore.Hangfire.APIETL +{ + public class ApiEndpoint + { + public ApiEndpoint(string jobName, string httpClientName = null) + { + this.JobName = jobName; + this.HttpClientName = httpClientName; + } + + internal ApiEndpoint(string name) + { + this.Name = name; + this.JobName = name; + } + + public string Name { get; internal set; } + + public string JobName { get; } + public string HttpClientName { get; } + + public IDictionary AdditionalHeaders { get; } = new Dictionary(); + + public delegate Task PrepareRequestDelegate(PrepareRequestArgs args); + public delegate Task> PrepareNextRequestDelegate(PrepareNextRequestArgs args); + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs b/MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs new file mode 100644 index 0000000..427bbdb --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs @@ -0,0 +1,79 @@ +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; + +namespace MIFCore.Hangfire.APIETL +{ + internal class ApiEndpointAttributeFactory : IApiEndpointAttributeFactory + { + private readonly IEnumerable endpointNameAttributes; + private readonly IEnumerable endpointDefiners; + + public ApiEndpointAttributeFactory(IEnumerable endpointNameAttributes, IEnumerable endpointDefiners) + { + this.endpointNameAttributes = endpointNameAttributes; + this.endpointDefiners = endpointDefiners; + } + + public async IAsyncEnumerable Create() + { + // Get the endpoints defined by the ApiEndpointNameAttribute + var endpointsDefinedByAttributes = this.endpointNameAttributes + .Select(y => y.EndpointName) + .Distinct(); + + // Create ApiEndpoint records from the ApiEndpointNameAttribute + foreach (var endpointName in endpointsDefinedByAttributes) + { + var endpointDefiners = this.GetEndpointDefiners(endpointName); + + // If the endpoint corresponds to an IDefineEndpoint, then the endpoints will be created / transformed from it instead. + if (endpointDefiners.Any()) + { + // It is possible for the endpointDefiner to create multiple endpoints from a single name (such as when tenancy is used, same endpoint, different tenant) + foreach (var ed in endpointDefiners) + { + var apiEndpoints = ed.DefineEndpoints(endpointName); + + await foreach (var ep in apiEndpoints) + { + // Ensure the endpoint name is set as this is an internal field + ep.Name = endpointName; + + yield return ep; + } + } + } + + // Otherwise create a single endpoint from the name, using all defaults + else + { + yield return new ApiEndpoint(endpointName); + } + } + } + + private IEnumerable GetEndpointDefiners(string endpointName) + { + foreach (var ed in this.endpointDefiners) + { + var edType = ed.GetType(); + var endpointNameAttribute = edType.GetCustomAttribute(); + var endpointSelectorAttribute = edType.GetCustomAttribute(); + + if (endpointNameAttribute?.EndpointName == endpointName) + { + yield return ed; + } + else if (endpointSelectorAttribute != null) + { + var regex = new Regex(endpointSelectorAttribute.Regex); + + if (regex.IsMatch(endpointName)) + yield return ed; + } + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointNameAttribute.cs b/MIFCore.Hangfire.APIETL/ApiEndpointNameAttribute.cs new file mode 100644 index 0000000..aac32be --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpointNameAttribute.cs @@ -0,0 +1,15 @@ +using System; + +namespace MIFCore.Hangfire.APIETL +{ + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] + public class ApiEndpointNameAttribute : Attribute + { + public ApiEndpointNameAttribute(string endpointName) + { + this.EndpointName = endpointName; + } + + public string EndpointName { get; } + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs new file mode 100644 index 0000000..6a2ddbc --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs @@ -0,0 +1,58 @@ +using Hangfire; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL +{ + public class ApiEndpointRegister + { + private readonly IDictionary endpoints = new Dictionary(); + private readonly IRecurringJobManager recurringJobManager; + private readonly IApiEndpointAttributeFactory apiEndpointAttributeFactory; + + public ApiEndpointRegister(IRecurringJobManager recurringJobManager, IApiEndpointAttributeFactory apiEndpointAttributeFactory) + { + this.recurringJobManager = recurringJobManager; + this.apiEndpointAttributeFactory = apiEndpointAttributeFactory; + } + + public IEnumerable Endpoints { get => this.endpoints.Values.ToArray(); } + + public ApiEndpointRegister Register(ApiEndpoint endpoint) + { + this.endpoints.Add(endpoint.Name, endpoint); + + this.recurringJobManager.AddOrUpdate( + endpoint.JobName, + job => job.Extract(endpoint.Name, null), + Cron.Daily() + ); + + return this; + } + + /// + /// Automatically registers all endpoints defined using the Attributes API as recurring jobs. + /// + /// + public async Task Register() + { + // If there are already endpoints registered, then we don't need to do anything otherwise it will cause double ups. + if (this.Endpoints.Any()) + return; + + var endpoints = this.apiEndpointAttributeFactory.Create(); + + await foreach (var ep in endpoints) + { + this.Register(ep); + } + } + + public ApiEndpoint Get(string endpointName) + { + return this.endpoints[endpointName]; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs b/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs new file mode 100644 index 0000000..ecc3b98 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs @@ -0,0 +1,15 @@ +using System; + +namespace MIFCore.Hangfire.APIETL +{ + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] + public class ApiEndpointSelectorAttribute : Attribute + { + public ApiEndpointSelectorAttribute(string regex) + { + this.Regex = regex; + } + + public string Regex { get; } + } +} diff --git a/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs new file mode 100644 index 0000000..3bf0d26 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs @@ -0,0 +1,120 @@ +using Hangfire; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL +{ + internal class EndpointExtractJob + { + private readonly IHttpClientFactory httpClientFactory; + private readonly ApiEndpointRegister apiEndpointRegister; + private readonly IBackgroundJobClient backgroundJobClient; + + public EndpointExtractJob( + IHttpClientFactory httpClientFactory, + ApiEndpointRegister apiEndpointRegister, + IBackgroundJobClient backgroundJobClient) + { + this.httpClientFactory = httpClientFactory; + this.apiEndpointRegister = apiEndpointRegister; + this.backgroundJobClient = backgroundJobClient; + } + + [DisableIdenticalQueuedItems] + public async Task Extract(string endpointName, ExtractArgs extractArgs = null) + { + // If there aren't any endpoints registered, then we can't do anything. So let's just reschedule this job for later. + if (this.apiEndpointRegister.Endpoints.Any() == false) + { + throw new RescheduleJobException(DateTime.Now.AddMinutes(1)); + } + + var endpoint = this.apiEndpointRegister.Get(endpointName); + await this.ExtractEndpoint(endpoint, extractArgs); + } + + private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs = null) + { + if (endpoint is null) + { + throw new ArgumentNullException(nameof(endpoint)); + } + + var httpClient = this.httpClientFactory.CreateClient(endpoint.HttpClientName); + var request = await CreateRequest(endpoint, extractArgs); + var apiData = await this.ExecuteRequest(endpoint, httpClient, request, extractArgs); + + if (endpoint is IPrepareNextRequest prepareNextRequest) + { + // If OnPrepareNextRequest returns null or an empty dict, then we're done with this endpoint. + var nextRequestData = await prepareNextRequest.OnPrepareNextRequest(new PrepareNextRequestArgs(apiData: apiData, data: extractArgs?.RequestData)); + + if (nextRequestData == default(IDictionary) + || nextRequestData.Keys.Any() == false) + return; + + // Otherwise we need to schedule another job to continue extracting this endpoint. + this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.ParentId))); + } + } + + private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient httpClient, HttpRequestMessage request, ExtractArgs extractArgs = null) + { + // Get the response payload as a string + var response = await httpClient.SendAsync(request); + var data = await response.Content.ReadAsStringAsync(); + + // Turn the payload response into an ApiData instance + var apiData = new ApiData + { + Endpoint = endpoint.Name, + Uri = response.RequestMessage.RequestUri.ToString(), + Data = data, + ParentId = extractArgs?.ParentApiDataId + }; + + // TODO: Push apiData to a another service for any additional handling + + return apiData; + } + + private static async Task CreateRequest(ApiEndpoint endpoint, ExtractArgs extractArgs = null) + { + // Create a new request, using endpoint.Name as the relative uri + // i.e endpoint.Name = "getStuff" and httpClient.BaseAddress = "https://someapi/api/" + var request = new HttpRequestMessage + { + RequestUri = new Uri(endpoint.Name, UriKind.Relative) + }; + + // Stitch on any additional headers + foreach (var h in endpoint.AdditionalHeaders) + { + request.Headers.Add(h.Key, h.Value); + } + + if (endpoint is IPrepareRequest prepareRequest) + { + // OnPrepareRequest after this system has finished up with the request + await prepareRequest.OnPrepareRequest(new PrepareRequestArgs(request, extractArgs?.RequestData)); + } + + return request; + } + + public class ExtractArgs + { + public ExtractArgs(IDictionary requestData, Guid? parentApiDataId) + { + this.RequestData = requestData; + this.ParentApiDataId = parentApiDataId; + } + + public IDictionary RequestData { get; set; } + public Guid? ParentApiDataId { get; set; } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs new file mode 100644 index 0000000..3f95e17 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs @@ -0,0 +1,58 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace MIFCore.Hangfire.APIETL +{ + public static class EndpointServiceCollectionExtensions + { + public static IServiceCollection AddEndpoints(this IServiceCollection serviceDescriptors, Assembly assembly = null) + { + assembly ??= Assembly.GetCallingAssembly(); + + // Find all endpoint related types in the assembl + var endpoints = assembly + .GetTypes() + .Where(y => + y.GetCustomAttribute() != null + || y.GetCustomAttribute() != null); + + return serviceDescriptors.AddEndpoints(endpoints); + } + + public static IServiceCollection AddEndpoints(this IServiceCollection serviceDescriptors, IEnumerable endpoints) + { + // Register the services used to register jobs and create ApiEndpoint definitions + serviceDescriptors.TryAddSingleton(); + serviceDescriptors.TryAddTransient(); + + foreach (var t in endpoints) + { + var endpointNameAttribute = t.GetCustomAttribute(); + var endpointSelectorAttribute = t.GetCustomAttribute(); + + if (endpointNameAttribute == null + && endpointSelectorAttribute == null) + throw new ArgumentException($"The type {t.FullName} does not have an {nameof(ApiEndpointNameAttribute)} or {nameof(ApiEndpointSelectorAttribute)} attribute."); + + if (typeof(IDefineEndpoints).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(IDefineEndpoints), t); + + if (typeof(IPrepareRequest).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(IPrepareRequest), t); + + if (typeof(IPrepareNextRequest).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(IPrepareNextRequest), t); + + // Register the endpoint name attribute, so an ApiEndpoint is created from it + if (endpointNameAttribute != null) + serviceDescriptors.AddSingleton(endpointNameAttribute); + } + + return serviceDescriptors; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointAttributeFactory.cs b/MIFCore.Hangfire.APIETL/IApiEndpointAttributeFactory.cs new file mode 100644 index 0000000..64f8e78 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IApiEndpointAttributeFactory.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL +{ + public interface IApiEndpointAttributeFactory + { + IAsyncEnumerable Create(); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs b/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs new file mode 100644 index 0000000..c12ba90 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL +{ + public interface IDefineEndpoints + { + IAsyncEnumerable DefineEndpoints(string endpointName); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs b/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs new file mode 100644 index 0000000..33e03ac --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL +{ + internal interface IPrepareNextRequest + { + Task> OnPrepareNextRequest(PrepareNextRequestArgs args); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/IPrepareRequest.cs b/MIFCore.Hangfire.APIETL/IPrepareRequest.cs new file mode 100644 index 0000000..80e1755 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IPrepareRequest.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL +{ + public interface IPrepareRequest + { + Task OnPrepareRequest(PrepareRequestArgs args); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj new file mode 100644 index 0000000..6726135 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj @@ -0,0 +1,33 @@ + + + + netstandard2.1 + Maitland Marshall + MAIT DEV + https://github.com/maitlandmarshall/MIFCore + https://github.com/maitlandmarshall/MIFCore + git + true + MIT + Hangfire, Scheduling, Jobs, Simplified, API, ETL + An add-on to MIFCore which allows users to define API endpoints to extract, transform and load. + + 1 + 0 + 0 + 0 + + $(Major).$(Minor).$(Build).$(Revision) + $(Major).$(Minor).$(Build) + + + + + + + + + + + + diff --git a/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs b/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs new file mode 100644 index 0000000..b7ce08f --- /dev/null +++ b/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL +{ + public class PrepareNextRequestArgs + { + public PrepareNextRequestArgs(ApiData apiData, IDictionary data) + { + this.ApiData = apiData; + this.Data = data ?? new Dictionary(); + } + + public ApiData ApiData { get; } + public IDictionary Data { get; } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs b/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs new file mode 100644 index 0000000..ad92fc4 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; +using System.Net.Http; + +namespace MIFCore.Hangfire.APIETL +{ + public class PrepareRequestArgs + { + public PrepareRequestArgs(HttpRequestMessage request, IDictionary data) + { + this.Request = request; + this.Data = data ?? new Dictionary(); + } + + public HttpRequestMessage Request { get; set; } + public IDictionary Data { get; set; } + } +} \ No newline at end of file diff --git a/MIFCore.sln b/MIFCore.sln index 8a4795f..41da9b6 100644 --- a/MIFCore.sln +++ b/MIFCore.sln @@ -17,6 +17,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MIFCore.Hangfire", "MIFCore EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MIFCore.Common", "MIFCore.Common\MIFCore.Common.csproj", "{A583D632-B2AD-4914-97EF-426AAF4725CB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MIFCore.Hangfire.APIETL", "MIFCore.Hangfire.APIETL\MIFCore.Hangfire.APIETL.csproj", "{0679FE87-1E69-4CE8-899B-BDC15340FDCB}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "APIETL", "APIETL", "{86C67505-D831-4A34-829E-D12EB7E2BDA7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MIFCore.Hangfire.APIETL.Tests", "MIFCore.Hangfire.APIETL.Tests\MIFCore.Hangfire.APIETL.Tests.csproj", "{33568C51-42C1-41A6-8CCF-0F62FF905378}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -47,6 +53,14 @@ Global {A583D632-B2AD-4914-97EF-426AAF4725CB}.Debug|Any CPU.Build.0 = Debug|Any CPU {A583D632-B2AD-4914-97EF-426AAF4725CB}.Release|Any CPU.ActiveCfg = Release|Any CPU {A583D632-B2AD-4914-97EF-426AAF4725CB}.Release|Any CPU.Build.0 = Release|Any CPU + {0679FE87-1E69-4CE8-899B-BDC15340FDCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0679FE87-1E69-4CE8-899B-BDC15340FDCB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0679FE87-1E69-4CE8-899B-BDC15340FDCB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0679FE87-1E69-4CE8-899B-BDC15340FDCB}.Release|Any CPU.Build.0 = Release|Any CPU + {33568C51-42C1-41A6-8CCF-0F62FF905378}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {33568C51-42C1-41A6-8CCF-0F62FF905378}.Debug|Any CPU.Build.0 = Debug|Any CPU + {33568C51-42C1-41A6-8CCF-0F62FF905378}.Release|Any CPU.ActiveCfg = Release|Any CPU + {33568C51-42C1-41A6-8CCF-0F62FF905378}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -54,6 +68,8 @@ Global GlobalSection(NestedProjects) = preSolution {B4B03402-85EA-49BE-BC98-D3AF0903F005} = {E057276C-548A-4190-AF5A-9A92E08268A1} {7FA32F14-5D33-4F0D-9F38-DF19D2C7F6E2} = {E057276C-548A-4190-AF5A-9A92E08268A1} + {0679FE87-1E69-4CE8-899B-BDC15340FDCB} = {86C67505-D831-4A34-829E-D12EB7E2BDA7} + {33568C51-42C1-41A6-8CCF-0F62FF905378} = {86C67505-D831-4A34-829E-D12EB7E2BDA7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {70D42A47-49DB-4784-8B40-89988134D6C8} From 3639977c4ed06f1e27dc8f11faa7fcdf1dfe4d58 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 23 Nov 2022 11:52:02 +1000 Subject: [PATCH 02/47] Endpoints defined by attributes / interface add support for PrepareRequest and PrepareNextRequest --- .../ApiEndpointAttributeFactory.cs | 17 +----- .../ApiEndpointServiceExtensions.cs | 28 ++++++++++ MIFCore.Hangfire.APIETL/EndpointExtractJob.cs | 32 +++++------ .../EndpointExtractPipeline.cs | 53 +++++++++++++++++++ .../EndpointServiceCollectionExtensions.cs | 1 + .../IApiEndpointService.cs | 4 ++ MIFCore.Hangfire.APIETL/IDefineEndpoints.cs | 2 +- .../IEndpointExtractPipeline.cs | 6 +++ .../IPrepareNextRequest.cs | 2 +- MIFCore.Hangfire.APIETL/IPrepareRequest.cs | 2 +- .../PrepareNextRequestArgs.cs | 4 +- MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs | 4 +- 12 files changed, 116 insertions(+), 39 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs create mode 100644 MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs create mode 100644 MIFCore.Hangfire.APIETL/IApiEndpointService.cs create mode 100644 MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs b/MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs index 427bbdb..c455d79 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs @@ -1,7 +1,5 @@ using System.Collections.Generic; using System.Linq; -using System.Reflection; -using System.Text.RegularExpressions; namespace MIFCore.Hangfire.APIETL { @@ -58,21 +56,8 @@ private IEnumerable GetEndpointDefiners(string endpointName) { foreach (var ed in this.endpointDefiners) { - var edType = ed.GetType(); - var endpointNameAttribute = edType.GetCustomAttribute(); - var endpointSelectorAttribute = edType.GetCustomAttribute(); - - if (endpointNameAttribute?.EndpointName == endpointName) - { + if (ed.RespondsToEndpointName(endpointName)) yield return ed; - } - else if (endpointSelectorAttribute != null) - { - var regex = new Regex(endpointSelectorAttribute.Regex); - - if (regex.IsMatch(endpointName)) - yield return ed; - } } } } diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs b/MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs new file mode 100644 index 0000000..07213cb --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs @@ -0,0 +1,28 @@ +using System.Reflection; +using System.Text.RegularExpressions; + +namespace MIFCore.Hangfire.APIETL +{ + public static class ApiEndpointServiceExtensions + { + public static bool RespondsToEndpointName(this IApiEndpointService apiEndpointService, string endpointName) + { + var type = apiEndpointService.GetType(); + var endpointNameAttribute = type.GetCustomAttribute(); + var endpointSelectorAttribute = type.GetCustomAttribute(); + + if (endpointNameAttribute?.EndpointName == endpointName) + { + return true; + } + else if (endpointSelectorAttribute != null) + { + var regex = new Regex(endpointSelectorAttribute.Regex); + + return regex.IsMatch(endpointName); + } + + return false; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs index 3bf0d26..64248b9 100644 --- a/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs @@ -12,15 +12,18 @@ internal class EndpointExtractJob private readonly IHttpClientFactory httpClientFactory; private readonly ApiEndpointRegister apiEndpointRegister; private readonly IBackgroundJobClient backgroundJobClient; + private readonly IEndpointExtractPipeline endpointExtractPipeline; public EndpointExtractJob( IHttpClientFactory httpClientFactory, ApiEndpointRegister apiEndpointRegister, - IBackgroundJobClient backgroundJobClient) + IBackgroundJobClient backgroundJobClient, + IEndpointExtractPipeline endpointExtractPipeline) { this.httpClientFactory = httpClientFactory; this.apiEndpointRegister = apiEndpointRegister; this.backgroundJobClient = backgroundJobClient; + this.endpointExtractPipeline = endpointExtractPipeline; } [DisableIdenticalQueuedItems] @@ -44,21 +47,17 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs } var httpClient = this.httpClientFactory.CreateClient(endpoint.HttpClientName); - var request = await CreateRequest(endpoint, extractArgs); + var request = await this.CreateRequest(endpoint, extractArgs); var apiData = await this.ExecuteRequest(endpoint, httpClient, request, extractArgs); - if (endpoint is IPrepareNextRequest prepareNextRequest) - { - // If OnPrepareNextRequest returns null or an empty dict, then we're done with this endpoint. - var nextRequestData = await prepareNextRequest.OnPrepareNextRequest(new PrepareNextRequestArgs(apiData: apiData, data: extractArgs?.RequestData)); + var nextRequestData = await this.endpointExtractPipeline.OnPrepareNextRequest(new PrepareNextRequestArgs(endpoint: endpoint, apiData: apiData, data: extractArgs?.RequestData)); - if (nextRequestData == default(IDictionary) - || nextRequestData.Keys.Any() == false) - return; + // If OnPrepareNextRequest returns an empty dict, then we're done with this endpoint. + if (nextRequestData.Keys.Any() == false) + return; - // Otherwise we need to schedule another job to continue extracting this endpoint. - this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.ParentId))); - } + // Otherwise we need to schedule another job to continue extracting this endpoint. + this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.ParentId))); } private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient httpClient, HttpRequestMessage request, ExtractArgs extractArgs = null) @@ -81,7 +80,7 @@ private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient http return apiData; } - private static async Task CreateRequest(ApiEndpoint endpoint, ExtractArgs extractArgs = null) + private async Task CreateRequest(ApiEndpoint endpoint, ExtractArgs extractArgs = null) { // Create a new request, using endpoint.Name as the relative uri // i.e endpoint.Name = "getStuff" and httpClient.BaseAddress = "https://someapi/api/" @@ -96,11 +95,8 @@ private static async Task CreateRequest(ApiEndpoint endpoint request.Headers.Add(h.Key, h.Value); } - if (endpoint is IPrepareRequest prepareRequest) - { - // OnPrepareRequest after this system has finished up with the request - await prepareRequest.OnPrepareRequest(new PrepareRequestArgs(request, extractArgs?.RequestData)); - } + // OnPrepareRequest after this system has finished up with the request + await this.endpointExtractPipeline.OnPrepareRequest(new PrepareRequestArgs(endpoint, request, extractArgs?.RequestData)); return request; } diff --git a/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs new file mode 100644 index 0000000..11c3af3 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL +{ + internal class EndpointExtractPipeline : IEndpointExtractPipeline + { + private readonly IEnumerable prepareRequests; + private readonly IEnumerable prepareNextRequests; + + public EndpointExtractPipeline( + IEnumerable prepareRequests, + IEnumerable prepareNextRequests) + { + this.prepareRequests = prepareRequests; + this.prepareNextRequests = prepareNextRequests; + } + + public async Task OnPrepareRequest(PrepareRequestArgs args) + { + var relatedPrepareRequests = this.prepareRequests + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + + foreach (var prepareRequest in relatedPrepareRequests) + { + await prepareRequest.OnPrepareRequest(args); + } + } + + public async Task> OnPrepareNextRequest(PrepareNextRequestArgs args) + { + var relatedPreparedNextRequests = this.prepareNextRequests + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + + var result = new Dictionary(); + + foreach (var prepareNextRequest in relatedPreparedNextRequests) + { + // Get the data from the prepareNextRequest + var data = await prepareNextRequest.OnPrepareNextRequest(args); + + if (data == default(IDictionary)) + continue; + + // Merge it with the resulting dictionary + result = result.Intersect(data).ToDictionary(x => x.Key, x => x.Value); + } + + return result; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs index 3f95e17..a5e09a5 100644 --- a/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs @@ -28,6 +28,7 @@ public static IServiceCollection AddEndpoints(this IServiceCollection serviceDes // Register the services used to register jobs and create ApiEndpoint definitions serviceDescriptors.TryAddSingleton(); serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); foreach (var t in endpoints) { diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointService.cs b/MIFCore.Hangfire.APIETL/IApiEndpointService.cs new file mode 100644 index 0000000..410b388 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IApiEndpointService.cs @@ -0,0 +1,4 @@ +namespace MIFCore.Hangfire.APIETL +{ + public interface IApiEndpointService { } +} diff --git a/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs b/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs index c12ba90..0cdb7c4 100644 --- a/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs +++ b/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs @@ -2,7 +2,7 @@ namespace MIFCore.Hangfire.APIETL { - public interface IDefineEndpoints + public interface IDefineEndpoints : IApiEndpointService { IAsyncEnumerable DefineEndpoints(string endpointName); } diff --git a/MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs new file mode 100644 index 0000000..02186ce --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs @@ -0,0 +1,6 @@ +namespace MIFCore.Hangfire.APIETL +{ + internal interface IEndpointExtractPipeline : IPrepareRequest, IPrepareNextRequest + { + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs b/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs index 33e03ac..3a08d5e 100644 --- a/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs +++ b/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs @@ -3,7 +3,7 @@ namespace MIFCore.Hangfire.APIETL { - internal interface IPrepareNextRequest + internal interface IPrepareNextRequest : IApiEndpointService { Task> OnPrepareNextRequest(PrepareNextRequestArgs args); } diff --git a/MIFCore.Hangfire.APIETL/IPrepareRequest.cs b/MIFCore.Hangfire.APIETL/IPrepareRequest.cs index 80e1755..96c1ff3 100644 --- a/MIFCore.Hangfire.APIETL/IPrepareRequest.cs +++ b/MIFCore.Hangfire.APIETL/IPrepareRequest.cs @@ -2,7 +2,7 @@ namespace MIFCore.Hangfire.APIETL { - public interface IPrepareRequest + public interface IPrepareRequest : IApiEndpointService { Task OnPrepareRequest(PrepareRequestArgs args); } diff --git a/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs b/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs index b7ce08f..dad0baf 100644 --- a/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs +++ b/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs @@ -4,12 +4,14 @@ namespace MIFCore.Hangfire.APIETL { public class PrepareNextRequestArgs { - public PrepareNextRequestArgs(ApiData apiData, IDictionary data) + public PrepareNextRequestArgs(ApiEndpoint endpoint, ApiData apiData, IDictionary data) { + this.Endpoint = endpoint; this.ApiData = apiData; this.Data = data ?? new Dictionary(); } + public ApiEndpoint Endpoint { get; } public ApiData ApiData { get; } public IDictionary Data { get; } } diff --git a/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs b/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs index ad92fc4..cb3ed89 100644 --- a/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs +++ b/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs @@ -5,12 +5,14 @@ namespace MIFCore.Hangfire.APIETL { public class PrepareRequestArgs { - public PrepareRequestArgs(HttpRequestMessage request, IDictionary data) + public PrepareRequestArgs(ApiEndpoint endpoint, HttpRequestMessage request, IDictionary data) { + this.Endpoint = endpoint; this.Request = request; this.Data = data ?? new Dictionary(); } + public ApiEndpoint Endpoint { get; } public HttpRequestMessage Request { get; set; } public IDictionary Data { get; set; } } From f0a1efa4261a6a1b78dd11496618a27ea914ab08 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 23 Nov 2022 11:58:11 +1000 Subject: [PATCH 03/47] Add hook for handling responses from an endpoint --- MIFCore.Hangfire.APIETL/EndpointExtractJob.cs | 2 +- .../EndpointExtractPipeline.cs | 16 +++++++++++++++- MIFCore.Hangfire.APIETL/HandleResponseArgs.cs | 14 ++++++++++++++ .../IEndpointExtractPipeline.cs | 2 +- MIFCore.Hangfire.APIETL/IHandleResponse.cs | 9 +++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL/HandleResponseArgs.cs create mode 100644 MIFCore.Hangfire.APIETL/IHandleResponse.cs diff --git a/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs index 64248b9..8f3ae42 100644 --- a/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs @@ -75,7 +75,7 @@ private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient http ParentId = extractArgs?.ParentApiDataId }; - // TODO: Push apiData to a another service for any additional handling + await this.endpointExtractPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData)); return apiData; } diff --git a/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs index 11c3af3..fe70554 100644 --- a/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs +++ b/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs @@ -8,13 +8,16 @@ internal class EndpointExtractPipeline : IEndpointExtractPipeline { private readonly IEnumerable prepareRequests; private readonly IEnumerable prepareNextRequests; + private readonly IEnumerable handleResponses; public EndpointExtractPipeline( IEnumerable prepareRequests, - IEnumerable prepareNextRequests) + IEnumerable prepareNextRequests, + IEnumerable handleResponses) { this.prepareRequests = prepareRequests; this.prepareNextRequests = prepareNextRequests; + this.handleResponses = handleResponses; } public async Task OnPrepareRequest(PrepareRequestArgs args) @@ -49,5 +52,16 @@ public async Task> OnPrepareNextRequest(PrepareNextR return result; } + + public async Task OnHandleResponse(HandleResponseArgs args) + { + var relatedHandleResponses = this.handleResponses + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + + foreach (var handleResponse in relatedHandleResponses) + { + await handleResponse.OnHandleResponse(args); + } + } } } diff --git a/MIFCore.Hangfire.APIETL/HandleResponseArgs.cs b/MIFCore.Hangfire.APIETL/HandleResponseArgs.cs new file mode 100644 index 0000000..2bd7c59 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/HandleResponseArgs.cs @@ -0,0 +1,14 @@ +namespace MIFCore.Hangfire.APIETL +{ + public class HandleResponseArgs + { + public HandleResponseArgs(ApiEndpoint endpoint, ApiData apiData) + { + this.Endpoint = endpoint; + this.ApiData = apiData; + } + + public ApiEndpoint Endpoint { get; } + public ApiData ApiData { get; } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs index 02186ce..1e1f020 100644 --- a/MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs +++ b/MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs @@ -1,6 +1,6 @@ namespace MIFCore.Hangfire.APIETL { - internal interface IEndpointExtractPipeline : IPrepareRequest, IPrepareNextRequest + internal interface IEndpointExtractPipeline : IPrepareRequest, IPrepareNextRequest, IHandleResponse { } } \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/IHandleResponse.cs b/MIFCore.Hangfire.APIETL/IHandleResponse.cs new file mode 100644 index 0000000..05a9e7d --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IHandleResponse.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL +{ + internal interface IHandleResponse : IApiEndpointService + { + Task OnHandleResponse(HandleResponseArgs args); + } +} \ No newline at end of file From 207bbe9d6f2fbfa3f49cdb3eb4ce55f8b43f27ca Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 23 Nov 2022 12:58:58 +1000 Subject: [PATCH 04/47] Convert RescheduleDate to UTC otherwise job scheduler will schedule for the wrong time --- MIFCore.Hangfire/RescheduleJobByDateFilter.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/MIFCore.Hangfire/RescheduleJobByDateFilter.cs b/MIFCore.Hangfire/RescheduleJobByDateFilter.cs index 4e18f85..5ccfb05 100644 --- a/MIFCore.Hangfire/RescheduleJobByDateFilter.cs +++ b/MIFCore.Hangfire/RescheduleJobByDateFilter.cs @@ -1,7 +1,4 @@ using Hangfire.States; -using System; -using System.Collections.Generic; -using System.Text; namespace MIFCore.Hangfire { @@ -17,7 +14,7 @@ public void OnStateElection(ElectStateContext context) if (failedState.Exception is RescheduleJobException exception) { context.SetJobParameter("RetryCount", 0); - context.CandidateState = new ScheduledState(exception.RescheduleDate) + context.CandidateState = new ScheduledState(exception.RescheduleDate.ToUniversalTime()) { Reason = $"Job has been rescheduled for {exception.RescheduleDate.ToLocalTime():dd/MM/yyyy HH:mm:ss}" }; From 58027d0e1e7616ed5200d5b3406772bb3ee83df4 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 23 Nov 2022 12:59:40 +1000 Subject: [PATCH 05/47] Add support for multiple attributes on one class --- .../ApiEndpointServiceExtensions.cs | 15 ++++---- MIFCore.Hangfire.APIETL/EndpointExtractJob.cs | 37 +++++++++++-------- .../EndpointExtractPipeline.cs | 2 +- .../EndpointServiceCollectionExtensions.cs | 25 +++++++++---- MIFCore.Hangfire.APIETL/IHandleResponse.cs | 2 +- .../IPrepareNextRequest.cs | 2 +- .../PrepareNextRequestArgs.cs | 2 +- MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs | 2 +- 8 files changed, 51 insertions(+), 36 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs b/MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs index 07213cb..670430c 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs @@ -1,4 +1,5 @@ -using System.Reflection; +using System.Linq; +using System.Reflection; using System.Text.RegularExpressions; namespace MIFCore.Hangfire.APIETL @@ -8,18 +9,16 @@ public static class ApiEndpointServiceExtensions public static bool RespondsToEndpointName(this IApiEndpointService apiEndpointService, string endpointName) { var type = apiEndpointService.GetType(); - var endpointNameAttribute = type.GetCustomAttribute(); - var endpointSelectorAttribute = type.GetCustomAttribute(); + var endpointNameAttributes = type.GetCustomAttributes(); + var endpointSelectorAttributes = type.GetCustomAttributes(); - if (endpointNameAttribute?.EndpointName == endpointName) + if (endpointNameAttributes.Any(y => y.EndpointName == endpointName)) { return true; } - else if (endpointSelectorAttribute != null) + else if (endpointSelectorAttributes.Any()) { - var regex = new Regex(endpointSelectorAttribute.Regex); - - return regex.IsMatch(endpointName); + return endpointSelectorAttributes.Any(y => Regex.IsMatch(endpointName, y.Regex)); } return false; diff --git a/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs index 8f3ae42..f8f8a7b 100644 --- a/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs @@ -32,7 +32,7 @@ public async Task Extract(string endpointName, ExtractArgs extractArgs = null) // If there aren't any endpoints registered, then we can't do anything. So let's just reschedule this job for later. if (this.apiEndpointRegister.Endpoints.Any() == false) { - throw new RescheduleJobException(DateTime.Now.AddMinutes(1)); + throw new RescheduleJobException(DateTime.Now.AddSeconds(5)); } var endpoint = this.apiEndpointRegister.Get(endpointName); @@ -46,21 +46,30 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs throw new ArgumentNullException(nameof(endpoint)); } + extractArgs ??= new ExtractArgs(new Dictionary(), null); + var httpClient = this.httpClientFactory.CreateClient(endpoint.HttpClientName); - var request = await this.CreateRequest(endpoint, extractArgs); + var request = await this.CreateRequest(httpClient.BaseAddress, endpoint, extractArgs); var apiData = await this.ExecuteRequest(endpoint, httpClient, request, extractArgs); - var nextRequestData = await this.endpointExtractPipeline.OnPrepareNextRequest(new PrepareNextRequestArgs(endpoint: endpoint, apiData: apiData, data: extractArgs?.RequestData)); + var nextRequestData = await this.endpointExtractPipeline.OnPrepareNextRequest(new PrepareNextRequestArgs(endpoint: endpoint, apiData: apiData, data: extractArgs.RequestData)); - // If OnPrepareNextRequest returns an empty dict, then we're done with this endpoint. - if (nextRequestData.Keys.Any() == false) - return; + try + { + // If OnPrepareNextRequest returns an empty dict, then we're done with this endpoint. + if (nextRequestData.Keys.Any() == false) + return; - // Otherwise we need to schedule another job to continue extracting this endpoint. - this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.ParentId))); + // Otherwise we need to schedule another job to continue extracting this endpoint. + this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.Id))); + } + finally + { + await this.endpointExtractPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData)); + } } - private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient httpClient, HttpRequestMessage request, ExtractArgs extractArgs = null) + private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient httpClient, HttpRequestMessage request, ExtractArgs extractArgs) { // Get the response payload as a string var response = await httpClient.SendAsync(request); @@ -72,21 +81,19 @@ private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient http Endpoint = endpoint.Name, Uri = response.RequestMessage.RequestUri.ToString(), Data = data, - ParentId = extractArgs?.ParentApiDataId + ParentId = extractArgs.ParentApiDataId }; - await this.endpointExtractPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData)); - return apiData; } - private async Task CreateRequest(ApiEndpoint endpoint, ExtractArgs extractArgs = null) + private async Task CreateRequest(Uri baseAddress, ApiEndpoint endpoint, ExtractArgs extractArgs) { // Create a new request, using endpoint.Name as the relative uri // i.e endpoint.Name = "getStuff" and httpClient.BaseAddress = "https://someapi/api/" var request = new HttpRequestMessage { - RequestUri = new Uri(endpoint.Name, UriKind.Relative) + RequestUri = new Uri(baseAddress, endpoint.Name) }; // Stitch on any additional headers @@ -96,7 +103,7 @@ private async Task CreateRequest(ApiEndpoint endpoint, Extra } // OnPrepareRequest after this system has finished up with the request - await this.endpointExtractPipeline.OnPrepareRequest(new PrepareRequestArgs(endpoint, request, extractArgs?.RequestData)); + await this.endpointExtractPipeline.OnPrepareRequest(new PrepareRequestArgs(endpoint, request, extractArgs.RequestData)); return request; } diff --git a/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs index fe70554..f123819 100644 --- a/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs +++ b/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs @@ -47,7 +47,7 @@ public async Task> OnPrepareNextRequest(PrepareNextR continue; // Merge it with the resulting dictionary - result = result.Intersect(data).ToDictionary(x => x.Key, x => x.Value); + result = result.Union(data).ToDictionary(x => x.Key, x => x.Value); } return result; diff --git a/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs index a5e09a5..9de128a 100644 --- a/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs @@ -17,8 +17,8 @@ public static IServiceCollection AddEndpoints(this IServiceCollection serviceDes var endpoints = assembly .GetTypes() .Where(y => - y.GetCustomAttribute() != null - || y.GetCustomAttribute() != null); + y.GetCustomAttributes().Any() + || y.GetCustomAttributes().Any()); return serviceDescriptors.AddEndpoints(endpoints); } @@ -29,14 +29,15 @@ public static IServiceCollection AddEndpoints(this IServiceCollection serviceDes serviceDescriptors.TryAddSingleton(); serviceDescriptors.TryAddTransient(); serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddScoped(); foreach (var t in endpoints) { - var endpointNameAttribute = t.GetCustomAttribute(); - var endpointSelectorAttribute = t.GetCustomAttribute(); + var endpointNameAttributes = t.GetCustomAttributes(); + var endpointSelectorAttribute = t.GetCustomAttributes(); - if (endpointNameAttribute == null - && endpointSelectorAttribute == null) + if (endpointNameAttributes.Any() == false + && endpointSelectorAttribute.Any() == false) throw new ArgumentException($"The type {t.FullName} does not have an {nameof(ApiEndpointNameAttribute)} or {nameof(ApiEndpointSelectorAttribute)} attribute."); if (typeof(IDefineEndpoints).IsAssignableFrom(t)) @@ -48,9 +49,17 @@ public static IServiceCollection AddEndpoints(this IServiceCollection serviceDes if (typeof(IPrepareNextRequest).IsAssignableFrom(t)) serviceDescriptors.AddScoped(typeof(IPrepareNextRequest), t); + if (typeof(IHandleResponse).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(IHandleResponse), t); + // Register the endpoint name attribute, so an ApiEndpoint is created from it - if (endpointNameAttribute != null) - serviceDescriptors.AddSingleton(endpointNameAttribute); + if (endpointNameAttributes.Any()) + { + foreach (var en in endpointNameAttributes) + { + serviceDescriptors.AddSingleton(en); + } + } } return serviceDescriptors; diff --git a/MIFCore.Hangfire.APIETL/IHandleResponse.cs b/MIFCore.Hangfire.APIETL/IHandleResponse.cs index 05a9e7d..e38b6c7 100644 --- a/MIFCore.Hangfire.APIETL/IHandleResponse.cs +++ b/MIFCore.Hangfire.APIETL/IHandleResponse.cs @@ -2,7 +2,7 @@ namespace MIFCore.Hangfire.APIETL { - internal interface IHandleResponse : IApiEndpointService + public interface IHandleResponse : IApiEndpointService { Task OnHandleResponse(HandleResponseArgs args); } diff --git a/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs b/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs index 3a08d5e..aa924e1 100644 --- a/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs +++ b/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs @@ -3,7 +3,7 @@ namespace MIFCore.Hangfire.APIETL { - internal interface IPrepareNextRequest : IApiEndpointService + public interface IPrepareNextRequest : IApiEndpointService { Task> OnPrepareNextRequest(PrepareNextRequestArgs args); } diff --git a/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs b/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs index dad0baf..e25ab7d 100644 --- a/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs +++ b/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs @@ -8,7 +8,7 @@ public PrepareNextRequestArgs(ApiEndpoint endpoint, ApiData apiData, IDictionary { this.Endpoint = endpoint; this.ApiData = apiData; - this.Data = data ?? new Dictionary(); + this.Data = data ?? throw new System.ArgumentNullException(nameof(data)); } public ApiEndpoint Endpoint { get; } diff --git a/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs b/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs index cb3ed89..a07112a 100644 --- a/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs +++ b/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs @@ -9,7 +9,7 @@ public PrepareRequestArgs(ApiEndpoint endpoint, HttpRequestMessage request, IDic { this.Endpoint = endpoint; this.Request = request; - this.Data = data ?? new Dictionary(); + this.Data = data ?? throw new System.ArgumentNullException(nameof(data)); } public ApiEndpoint Endpoint { get; } From 9649e7ba8a67a7040a7f4cd40435d7c3c2fa400e Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 28 Nov 2022 07:38:09 +1000 Subject: [PATCH 06/47] Extract interface for ApiEndpointRegister --- MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs | 2 +- .../EndpointServiceCollectionExtensions.cs | 2 +- MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs | 14 ++++++++++++++ MIFCore/MIFCore.csproj | 4 ++-- 4 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs index 6a2ddbc..049ed21 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs @@ -5,7 +5,7 @@ namespace MIFCore.Hangfire.APIETL { - public class ApiEndpointRegister + public class ApiEndpointRegister : IApiEndpointRegister { private readonly IDictionary endpoints = new Dictionary(); private readonly IRecurringJobManager recurringJobManager; diff --git a/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs index 9de128a..410921d 100644 --- a/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs @@ -26,7 +26,7 @@ public static IServiceCollection AddEndpoints(this IServiceCollection serviceDes public static IServiceCollection AddEndpoints(this IServiceCollection serviceDescriptors, IEnumerable endpoints) { // Register the services used to register jobs and create ApiEndpoint definitions - serviceDescriptors.TryAddSingleton(); + serviceDescriptors.TryAddSingleton(); serviceDescriptors.TryAddTransient(); serviceDescriptors.TryAddTransient(); serviceDescriptors.TryAddScoped(); diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs new file mode 100644 index 0000000..72447e4 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL +{ + public interface IApiEndpointRegister + { + IEnumerable Endpoints { get; } + + ApiEndpoint Get(string endpointName); + Task Register(); + ApiEndpointRegister Register(ApiEndpoint endpoint); + } +} \ No newline at end of file diff --git a/MIFCore/MIFCore.csproj b/MIFCore/MIFCore.csproj index 5188e12..f0f2c11 100644 --- a/MIFCore/MIFCore.csproj +++ b/MIFCore/MIFCore.csproj @@ -32,8 +32,8 @@ - - + + From c9739c498d4416640675428b83327ba6f48d0e82 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 28 Nov 2022 07:43:33 +1000 Subject: [PATCH 07/47] Move extraction feature set to extract namespace --- .../ApiEndpointAttributeFactoryTests.cs | 1 + MIFCore.Hangfire.APIETL/AssemblyInfo.cs | 6 ++++++ MIFCore.Hangfire.APIETL/{ => Extract}/ApiData.cs | 2 +- MIFCore.Hangfire.APIETL/{ => Extract}/ApiEndpoint.cs | 4 +--- .../{ => Extract}/ApiEndpointAttributeFactory.cs | 2 +- .../{ => Extract}/ApiEndpointNameAttribute.cs | 2 +- .../{ => Extract}/ApiEndpointRegister.cs | 2 +- .../{ => Extract}/ApiEndpointSelectorAttribute.cs | 2 +- .../{ => Extract}/ApiEndpointServiceExtensions.cs | 2 +- MIFCore.Hangfire.APIETL/{ => Extract}/EndpointExtractJob.cs | 2 +- .../{ => Extract}/EndpointExtractPipeline.cs | 2 +- .../{ => Extract}/EndpointServiceCollectionExtensions.cs | 2 +- MIFCore.Hangfire.APIETL/{ => Extract}/HandleResponseArgs.cs | 2 +- .../{ => Extract}/IApiEndpointAttributeFactory.cs | 2 +- .../{ => Extract}/IApiEndpointRegister.cs | 2 +- .../{ => Extract}/IApiEndpointService.cs | 2 +- MIFCore.Hangfire.APIETL/{ => Extract}/IDefineEndpoints.cs | 2 +- .../{ => Extract}/IEndpointExtractPipeline.cs | 2 +- MIFCore.Hangfire.APIETL/{ => Extract}/IHandleResponse.cs | 2 +- .../{ => Extract}/IPrepareNextRequest.cs | 2 +- MIFCore.Hangfire.APIETL/{ => Extract}/IPrepareRequest.cs | 2 +- .../{ => Extract}/PrepareNextRequestArgs.cs | 2 +- MIFCore.Hangfire.APIETL/{ => Extract}/PrepareRequestArgs.cs | 2 +- 23 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL/AssemblyInfo.cs rename MIFCore.Hangfire.APIETL/{ => Extract}/ApiData.cs (89%) rename MIFCore.Hangfire.APIETL/{ => Extract}/ApiEndpoint.cs (86%) rename MIFCore.Hangfire.APIETL/{ => Extract}/ApiEndpointAttributeFactory.cs (98%) rename MIFCore.Hangfire.APIETL/{ => Extract}/ApiEndpointNameAttribute.cs (88%) rename MIFCore.Hangfire.APIETL/{ => Extract}/ApiEndpointRegister.cs (97%) rename MIFCore.Hangfire.APIETL/{ => Extract}/ApiEndpointSelectorAttribute.cs (87%) rename MIFCore.Hangfire.APIETL/{ => Extract}/ApiEndpointServiceExtensions.cs (95%) rename MIFCore.Hangfire.APIETL/{ => Extract}/EndpointExtractJob.cs (99%) rename MIFCore.Hangfire.APIETL/{ => Extract}/EndpointExtractPipeline.cs (98%) rename MIFCore.Hangfire.APIETL/{ => Extract}/EndpointServiceCollectionExtensions.cs (98%) rename MIFCore.Hangfire.APIETL/{ => Extract}/HandleResponseArgs.cs (87%) rename MIFCore.Hangfire.APIETL/{ => Extract}/IApiEndpointAttributeFactory.cs (78%) rename MIFCore.Hangfire.APIETL/{ => Extract}/IApiEndpointRegister.cs (88%) rename MIFCore.Hangfire.APIETL/{ => Extract}/IApiEndpointService.cs (52%) rename MIFCore.Hangfire.APIETL/{ => Extract}/IDefineEndpoints.cs (81%) rename MIFCore.Hangfire.APIETL/{ => Extract}/IEndpointExtractPipeline.cs (72%) rename MIFCore.Hangfire.APIETL/{ => Extract}/IHandleResponse.cs (79%) rename MIFCore.Hangfire.APIETL/{ => Extract}/IPrepareNextRequest.cs (85%) rename MIFCore.Hangfire.APIETL/{ => Extract}/IPrepareRequest.cs (79%) rename MIFCore.Hangfire.APIETL/{ => Extract}/PrepareNextRequestArgs.cs (92%) rename MIFCore.Hangfire.APIETL/{ => Extract}/PrepareRequestArgs.cs (93%) diff --git a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs index 650f96e..95ee4eb 100644 --- a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs +++ b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.VisualStudio.TestTools.UnitTesting; +using MIFCore.Hangfire.APIETL.Extract; namespace MIFCore.Hangfire.APIETL.Tests { diff --git a/MIFCore.Hangfire.APIETL/AssemblyInfo.cs b/MIFCore.Hangfire.APIETL/AssemblyInfo.cs new file mode 100644 index 0000000..86d3157 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/AssemblyInfo.cs @@ -0,0 +1,6 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("MIFCore.Hangfire.APIETL.Tests")] +namespace MIFCore.Hangfire.APIETL +{ +} diff --git a/MIFCore.Hangfire.APIETL/ApiData.cs b/MIFCore.Hangfire.APIETL/Extract/ApiData.cs similarity index 89% rename from MIFCore.Hangfire.APIETL/ApiData.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiData.cs index 450e5b8..fb4eee2 100644 --- a/MIFCore.Hangfire.APIETL/ApiData.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiData.cs @@ -1,6 +1,6 @@ using System; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public class ApiData { diff --git a/MIFCore.Hangfire.APIETL/ApiEndpoint.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpoint.cs similarity index 86% rename from MIFCore.Hangfire.APIETL/ApiEndpoint.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiEndpoint.cs index 69f9738..4d03121 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpoint.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpoint.cs @@ -1,9 +1,7 @@ using System.Collections.Generic; -using System.Runtime.CompilerServices; using System.Threading.Tasks; -[assembly: InternalsVisibleTo("MIFCore.Hangfire.APIETL.Tests")] -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public class ApiEndpoint { diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttributeFactory.cs similarity index 98% rename from MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttributeFactory.cs index c455d79..444a5cb 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointAttributeFactory.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttributeFactory.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Linq; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { internal class ApiEndpointAttributeFactory : IApiEndpointAttributeFactory { diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointNameAttribute.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointNameAttribute.cs similarity index 88% rename from MIFCore.Hangfire.APIETL/ApiEndpointNameAttribute.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiEndpointNameAttribute.cs index aac32be..4b63b98 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointNameAttribute.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointNameAttribute.cs @@ -1,6 +1,6 @@ using System; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] public class ApiEndpointNameAttribute : Attribute diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs similarity index 97% rename from MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs index 049ed21..c98c89b 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public class ApiEndpointRegister : IApiEndpointRegister { diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointSelectorAttribute.cs similarity index 87% rename from MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiEndpointSelectorAttribute.cs index ecc3b98..982d443 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointSelectorAttribute.cs @@ -1,6 +1,6 @@ using System; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] public class ApiEndpointSelectorAttribute : Attribute diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointServiceExtensions.cs similarity index 95% rename from MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiEndpointServiceExtensions.cs index 670430c..45cc70f 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointServiceExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointServiceExtensions.cs @@ -2,7 +2,7 @@ using System.Reflection; using System.Text.RegularExpressions; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public static class ApiEndpointServiceExtensions { diff --git a/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs similarity index 99% rename from MIFCore.Hangfire.APIETL/EndpointExtractJob.cs rename to MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs index f8f8a7b..74e6919 100644 --- a/MIFCore.Hangfire.APIETL/EndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs @@ -5,7 +5,7 @@ using System.Net.Http; using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { internal class EndpointExtractJob { diff --git a/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractPipeline.cs similarity index 98% rename from MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs rename to MIFCore.Hangfire.APIETL/Extract/EndpointExtractPipeline.cs index f123819..65ded23 100644 --- a/MIFCore.Hangfire.APIETL/EndpointExtractPipeline.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractPipeline.cs @@ -2,7 +2,7 @@ using System.Linq; using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { internal class EndpointExtractPipeline : IEndpointExtractPipeline { diff --git a/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs similarity index 98% rename from MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs rename to MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs index 410921d..c7bd31c 100644 --- a/MIFCore.Hangfire.APIETL/EndpointServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs @@ -5,7 +5,7 @@ using System.Linq; using System.Reflection; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public static class EndpointServiceCollectionExtensions { diff --git a/MIFCore.Hangfire.APIETL/HandleResponseArgs.cs b/MIFCore.Hangfire.APIETL/Extract/HandleResponseArgs.cs similarity index 87% rename from MIFCore.Hangfire.APIETL/HandleResponseArgs.cs rename to MIFCore.Hangfire.APIETL/Extract/HandleResponseArgs.cs index 2bd7c59..9c8b688 100644 --- a/MIFCore.Hangfire.APIETL/HandleResponseArgs.cs +++ b/MIFCore.Hangfire.APIETL/Extract/HandleResponseArgs.cs @@ -1,4 +1,4 @@ -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public class HandleResponseArgs { diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointAttributeFactory.cs b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointAttributeFactory.cs similarity index 78% rename from MIFCore.Hangfire.APIETL/IApiEndpointAttributeFactory.cs rename to MIFCore.Hangfire.APIETL/Extract/IApiEndpointAttributeFactory.cs index 64f8e78..d05d664 100644 --- a/MIFCore.Hangfire.APIETL/IApiEndpointAttributeFactory.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointAttributeFactory.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public interface IApiEndpointAttributeFactory { diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs similarity index 88% rename from MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs rename to MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs index 72447e4..bf7b15a 100644 --- a/MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public interface IApiEndpointRegister { diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointService.cs b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointService.cs similarity index 52% rename from MIFCore.Hangfire.APIETL/IApiEndpointService.cs rename to MIFCore.Hangfire.APIETL/Extract/IApiEndpointService.cs index 410b388..c896eeb 100644 --- a/MIFCore.Hangfire.APIETL/IApiEndpointService.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointService.cs @@ -1,4 +1,4 @@ -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public interface IApiEndpointService { } } diff --git a/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs b/MIFCore.Hangfire.APIETL/Extract/IDefineEndpoints.cs similarity index 81% rename from MIFCore.Hangfire.APIETL/IDefineEndpoints.cs rename to MIFCore.Hangfire.APIETL/Extract/IDefineEndpoints.cs index 0cdb7c4..a3c6eef 100644 --- a/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IDefineEndpoints.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public interface IDefineEndpoints : IApiEndpointService { diff --git a/MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs similarity index 72% rename from MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs rename to MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs index 1e1f020..4599043 100644 --- a/MIFCore.Hangfire.APIETL/IEndpointExtractPipeline.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs @@ -1,4 +1,4 @@ -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { internal interface IEndpointExtractPipeline : IPrepareRequest, IPrepareNextRequest, IHandleResponse { diff --git a/MIFCore.Hangfire.APIETL/IHandleResponse.cs b/MIFCore.Hangfire.APIETL/Extract/IHandleResponse.cs similarity index 79% rename from MIFCore.Hangfire.APIETL/IHandleResponse.cs rename to MIFCore.Hangfire.APIETL/Extract/IHandleResponse.cs index e38b6c7..9db28d0 100644 --- a/MIFCore.Hangfire.APIETL/IHandleResponse.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IHandleResponse.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public interface IHandleResponse : IApiEndpointService { diff --git a/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs b/MIFCore.Hangfire.APIETL/Extract/IPrepareNextRequest.cs similarity index 85% rename from MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs rename to MIFCore.Hangfire.APIETL/Extract/IPrepareNextRequest.cs index aa924e1..48aa930 100644 --- a/MIFCore.Hangfire.APIETL/IPrepareNextRequest.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IPrepareNextRequest.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public interface IPrepareNextRequest : IApiEndpointService { diff --git a/MIFCore.Hangfire.APIETL/IPrepareRequest.cs b/MIFCore.Hangfire.APIETL/Extract/IPrepareRequest.cs similarity index 79% rename from MIFCore.Hangfire.APIETL/IPrepareRequest.cs rename to MIFCore.Hangfire.APIETL/Extract/IPrepareRequest.cs index 96c1ff3..fef9ff2 100644 --- a/MIFCore.Hangfire.APIETL/IPrepareRequest.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IPrepareRequest.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public interface IPrepareRequest : IApiEndpointService { diff --git a/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs b/MIFCore.Hangfire.APIETL/Extract/PrepareNextRequestArgs.cs similarity index 92% rename from MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs rename to MIFCore.Hangfire.APIETL/Extract/PrepareNextRequestArgs.cs index e25ab7d..7abe186 100644 --- a/MIFCore.Hangfire.APIETL/PrepareNextRequestArgs.cs +++ b/MIFCore.Hangfire.APIETL/Extract/PrepareNextRequestArgs.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public class PrepareNextRequestArgs { diff --git a/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs b/MIFCore.Hangfire.APIETL/Extract/PrepareRequestArgs.cs similarity index 93% rename from MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs rename to MIFCore.Hangfire.APIETL/Extract/PrepareRequestArgs.cs index a07112a..862805c 100644 --- a/MIFCore.Hangfire.APIETL/PrepareRequestArgs.cs +++ b/MIFCore.Hangfire.APIETL/Extract/PrepareRequestArgs.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Net.Http; -namespace MIFCore.Hangfire.APIETL +namespace MIFCore.Hangfire.APIETL.Extract { public class PrepareRequestArgs { From cc17ed346802d0536e4e11da84b09025bb0a4d54 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 28 Nov 2022 14:29:25 +1000 Subject: [PATCH 08/47] Automatic versioning for debugging local changes --- MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj index 6726135..7179d20 100644 --- a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj +++ b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj @@ -15,10 +15,10 @@ 1 0 0 - 0 + - $(Major).$(Minor).$(Build).$(Revision) - $(Major).$(Minor).$(Build) + + rev.$([System.DateTime]::UtcNow.ToString("MddHHmm")) From 2cd97c7dff32e0d12e3a3b3e309c80af4a98ea7d Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 28 Nov 2022 14:36:33 +1000 Subject: [PATCH 09/47] Rename AddEndpoints to AddApiEndpointsToExtract to disambigue from aspnetcore --- .../ApiEndpointAttributeFactoryTests.cs | 2 +- .../Extract/EndpointServiceCollectionExtensions.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs index 95ee4eb..3a075fc 100644 --- a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs +++ b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs @@ -35,7 +35,7 @@ public async Task Create_WithEndpointDefiner_CreatesEndpointsForEachTenant() private IServiceProvider GetServiceProvider(params Type[] endpoints) { var serviceCollection = new ServiceCollection(); - serviceCollection.AddEndpoints(endpoints); + serviceCollection.AddApiEndpointsToExtract(endpoints); return serviceCollection.BuildServiceProvider(); } diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs index c7bd31c..d9cb718 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs @@ -9,7 +9,7 @@ namespace MIFCore.Hangfire.APIETL.Extract { public static class EndpointServiceCollectionExtensions { - public static IServiceCollection AddEndpoints(this IServiceCollection serviceDescriptors, Assembly assembly = null) + public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollection serviceDescriptors, Assembly assembly = null) { assembly ??= Assembly.GetCallingAssembly(); @@ -20,10 +20,10 @@ public static IServiceCollection AddEndpoints(this IServiceCollection serviceDes y.GetCustomAttributes().Any() || y.GetCustomAttributes().Any()); - return serviceDescriptors.AddEndpoints(endpoints); + return serviceDescriptors.AddApiEndpointsToExtract(endpoints); } - public static IServiceCollection AddEndpoints(this IServiceCollection serviceDescriptors, IEnumerable endpoints) + public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollection serviceDescriptors, IEnumerable endpoints) { // Register the services used to register jobs and create ApiEndpoint definitions serviceDescriptors.TryAddSingleton(); From bf2c6e92e610b09a7378357da60682acbcab2faf Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Tue, 29 Nov 2022 09:14:09 +1000 Subject: [PATCH 10/47] Use IApiEndpointRegister instead of implementation --- MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs | 4 ++-- MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs | 4 ++-- MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs index c98c89b..bde2e22 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs @@ -5,7 +5,7 @@ namespace MIFCore.Hangfire.APIETL.Extract { - public class ApiEndpointRegister : IApiEndpointRegister + internal class ApiEndpointRegister : IApiEndpointRegister { private readonly IDictionary endpoints = new Dictionary(); private readonly IRecurringJobManager recurringJobManager; @@ -19,7 +19,7 @@ public ApiEndpointRegister(IRecurringJobManager recurringJobManager, IApiEndpoin public IEnumerable Endpoints { get => this.endpoints.Values.ToArray(); } - public ApiEndpointRegister Register(ApiEndpoint endpoint) + public IApiEndpointRegister Register(ApiEndpoint endpoint) { this.endpoints.Add(endpoint.Name, endpoint); diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs index 74e6919..2d198cb 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs @@ -10,13 +10,13 @@ namespace MIFCore.Hangfire.APIETL.Extract internal class EndpointExtractJob { private readonly IHttpClientFactory httpClientFactory; - private readonly ApiEndpointRegister apiEndpointRegister; + private readonly IApiEndpointRegister apiEndpointRegister; private readonly IBackgroundJobClient backgroundJobClient; private readonly IEndpointExtractPipeline endpointExtractPipeline; public EndpointExtractJob( IHttpClientFactory httpClientFactory, - ApiEndpointRegister apiEndpointRegister, + IApiEndpointRegister apiEndpointRegister, IBackgroundJobClient backgroundJobClient, IEndpointExtractPipeline endpointExtractPipeline) { diff --git a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs index bf7b15a..e198ae3 100644 --- a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs @@ -9,6 +9,6 @@ public interface IApiEndpointRegister ApiEndpoint Get(string endpointName); Task Register(); - ApiEndpointRegister Register(ApiEndpoint endpoint); + IApiEndpointRegister Register(ApiEndpoint endpoint); } } \ No newline at end of file From da6d30c8c44dff499062c2a6a99d2c15bb5d693d Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Tue, 29 Nov 2022 12:04:28 +1000 Subject: [PATCH 11/47] Rename ApiEndpointAttributeFactory to ApiEndpointFactory --- .../ApiEndpointAttributeFactoryTests.cs | 8 ++++---- ...ndpointNameAttribute.cs => ApiEndpointAttribute.cs} | 6 ++++-- ...dpointAttributeFactory.cs => ApiEndpointFactory.cs} | 10 +++++----- MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs | 4 ++-- .../Extract/ApiEndpointServiceExtensions.cs | 2 +- .../Extract/EndpointServiceCollectionExtensions.cs | 8 ++++---- ...pointAttributeFactory.cs => IApiEndpointFactory.cs} | 2 +- MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj | 5 ++--- 8 files changed, 23 insertions(+), 22 deletions(-) rename MIFCore.Hangfire.APIETL/Extract/{ApiEndpointNameAttribute.cs => ApiEndpointAttribute.cs} (52%) rename MIFCore.Hangfire.APIETL/Extract/{ApiEndpointAttributeFactory.cs => ApiEndpointFactory.cs} (81%) rename MIFCore.Hangfire.APIETL/Extract/{IApiEndpointAttributeFactory.cs => IApiEndpointFactory.cs} (74%) diff --git a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs index 3a075fc..4197d73 100644 --- a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs +++ b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs @@ -13,8 +13,8 @@ public class ApiEndpointAttributeFactoryTests public async Task Create_WithEndpointDefiner_CreatesEndpointsForEachTenant() { var serviceProvider = this.GetServiceProvider(typeof(Endpoint1), typeof(TenantedEndpointRegister)); - var factory = new ApiEndpointAttributeFactory( - endpointNameAttributes: serviceProvider.GetRequiredService>(), + var factory = new ApiEndpointFactory( + endpointNameAttributes: serviceProvider.GetRequiredService>(), endpointDefiners: serviceProvider.GetRequiredService>() ); @@ -39,7 +39,7 @@ private IServiceProvider GetServiceProvider(params Type[] endpoints) return serviceCollection.BuildServiceProvider(); } - [ApiEndpointName("api/endpoint1")] + [ApiEndpoint("api/endpoint1")] class Endpoint1 : IPrepareRequest { public Task OnPrepareRequest(PrepareRequestArgs args) @@ -48,7 +48,7 @@ public Task OnPrepareRequest(PrepareRequestArgs args) } } - [ApiEndpointName("api/endpoint2")] + [ApiEndpoint("api/endpoint2")] [ApiEndpointSelector(".*")] class TenantedEndpointRegister : IDefineEndpoints { diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointNameAttribute.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs similarity index 52% rename from MIFCore.Hangfire.APIETL/Extract/ApiEndpointNameAttribute.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs index 4b63b98..b6dd396 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointNameAttribute.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs @@ -3,13 +3,15 @@ namespace MIFCore.Hangfire.APIETL.Extract { [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] - public class ApiEndpointNameAttribute : Attribute + public class ApiEndpointAttribute : Attribute { - public ApiEndpointNameAttribute(string endpointName) + public ApiEndpointAttribute(string endpointName, string httpClientName) { this.EndpointName = endpointName; + this.HttpClientName = httpClientName; } public string EndpointName { get; } + public string HttpClientName { get; set; } } } diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttributeFactory.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointFactory.cs similarity index 81% rename from MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttributeFactory.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiEndpointFactory.cs index 444a5cb..98f0508 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttributeFactory.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointFactory.cs @@ -3,12 +3,12 @@ namespace MIFCore.Hangfire.APIETL.Extract { - internal class ApiEndpointAttributeFactory : IApiEndpointAttributeFactory + internal class ApiEndpointFactory : IApiEndpointFactory { - private readonly IEnumerable endpointNameAttributes; + private readonly IEnumerable endpointNameAttributes; private readonly IEnumerable endpointDefiners; - public ApiEndpointAttributeFactory(IEnumerable endpointNameAttributes, IEnumerable endpointDefiners) + public ApiEndpointFactory(IEnumerable endpointNameAttributes, IEnumerable endpointDefiners) { this.endpointNameAttributes = endpointNameAttributes; this.endpointDefiners = endpointDefiners; @@ -16,12 +16,12 @@ public ApiEndpointAttributeFactory(IEnumerable endpoin public async IAsyncEnumerable Create() { - // Get the endpoints defined by the ApiEndpointNameAttribute + // Get the endpoints defined by the ApiEndpointAttribute var endpointsDefinedByAttributes = this.endpointNameAttributes .Select(y => y.EndpointName) .Distinct(); - // Create ApiEndpoint records from the ApiEndpointNameAttribute + // Create ApiEndpoint records from the ApiEndpointAttribute foreach (var endpointName in endpointsDefinedByAttributes) { var endpointDefiners = this.GetEndpointDefiners(endpointName); diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs index bde2e22..588dc4b 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs @@ -9,9 +9,9 @@ internal class ApiEndpointRegister : IApiEndpointRegister { private readonly IDictionary endpoints = new Dictionary(); private readonly IRecurringJobManager recurringJobManager; - private readonly IApiEndpointAttributeFactory apiEndpointAttributeFactory; + private readonly IApiEndpointFactory apiEndpointAttributeFactory; - public ApiEndpointRegister(IRecurringJobManager recurringJobManager, IApiEndpointAttributeFactory apiEndpointAttributeFactory) + public ApiEndpointRegister(IRecurringJobManager recurringJobManager, IApiEndpointFactory apiEndpointAttributeFactory) { this.recurringJobManager = recurringJobManager; this.apiEndpointAttributeFactory = apiEndpointAttributeFactory; diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointServiceExtensions.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointServiceExtensions.cs index 45cc70f..b4349cc 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointServiceExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointServiceExtensions.cs @@ -9,7 +9,7 @@ public static class ApiEndpointServiceExtensions public static bool RespondsToEndpointName(this IApiEndpointService apiEndpointService, string endpointName) { var type = apiEndpointService.GetType(); - var endpointNameAttributes = type.GetCustomAttributes(); + var endpointNameAttributes = type.GetCustomAttributes(); var endpointSelectorAttributes = type.GetCustomAttributes(); if (endpointNameAttributes.Any(y => y.EndpointName == endpointName)) diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs index d9cb718..661eeb0 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs @@ -17,7 +17,7 @@ public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollectio var endpoints = assembly .GetTypes() .Where(y => - y.GetCustomAttributes().Any() + y.GetCustomAttributes().Any() || y.GetCustomAttributes().Any()); return serviceDescriptors.AddApiEndpointsToExtract(endpoints); @@ -27,18 +27,18 @@ public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollectio { // Register the services used to register jobs and create ApiEndpoint definitions serviceDescriptors.TryAddSingleton(); - serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); serviceDescriptors.TryAddTransient(); serviceDescriptors.TryAddScoped(); foreach (var t in endpoints) { - var endpointNameAttributes = t.GetCustomAttributes(); + var endpointNameAttributes = t.GetCustomAttributes(); var endpointSelectorAttribute = t.GetCustomAttributes(); if (endpointNameAttributes.Any() == false && endpointSelectorAttribute.Any() == false) - throw new ArgumentException($"The type {t.FullName} does not have an {nameof(ApiEndpointNameAttribute)} or {nameof(ApiEndpointSelectorAttribute)} attribute."); + throw new ArgumentException($"The type {t.FullName} does not have an {nameof(ApiEndpointAttribute)} or {nameof(ApiEndpointSelectorAttribute)} attribute."); if (typeof(IDefineEndpoints).IsAssignableFrom(t)) serviceDescriptors.AddScoped(typeof(IDefineEndpoints), t); diff --git a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointAttributeFactory.cs b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointFactory.cs similarity index 74% rename from MIFCore.Hangfire.APIETL/Extract/IApiEndpointAttributeFactory.cs rename to MIFCore.Hangfire.APIETL/Extract/IApiEndpointFactory.cs index d05d664..dcaf8d0 100644 --- a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointAttributeFactory.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointFactory.cs @@ -2,7 +2,7 @@ namespace MIFCore.Hangfire.APIETL.Extract { - public interface IApiEndpointAttributeFactory + public interface IApiEndpointFactory { IAsyncEnumerable Create(); } diff --git a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj index 7179d20..25e4c8c 100644 --- a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj +++ b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj @@ -12,9 +12,8 @@ Hangfire, Scheduling, Jobs, Simplified, API, ETL An add-on to MIFCore which allows users to define API endpoints to extract, transform and load. - 1 - 0 - 0 + 1.0.0 + From 51414963b5d45df470348986a982df2ed72cc5f1 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Tue, 29 Nov 2022 12:07:34 +1000 Subject: [PATCH 12/47] Add support for APIEndpoints without a defined HttpClientName --- MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs | 3 +-- MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs | 4 +++- .../Extract/EndpointServiceCollectionExtensions.cs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs index b6dd396..b454184 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs @@ -5,10 +5,9 @@ namespace MIFCore.Hangfire.APIETL.Extract [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] public class ApiEndpointAttribute : Attribute { - public ApiEndpointAttribute(string endpointName, string httpClientName) + public ApiEndpointAttribute(string endpointName) { this.EndpointName = endpointName; - this.HttpClientName = httpClientName; } public string EndpointName { get; } diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs index 2d198cb..2e05471 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs @@ -48,7 +48,9 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs extractArgs ??= new ExtractArgs(new Dictionary(), null); - var httpClient = this.httpClientFactory.CreateClient(endpoint.HttpClientName); + // Default to string.Empty to avoid an error if the endpoint.HttpClientName is null. + var httpClient = this.httpClientFactory.CreateClient(endpoint.HttpClientName ?? string.Empty); + var request = await this.CreateRequest(httpClient.BaseAddress, endpoint, extractArgs); var apiData = await this.ExecuteRequest(endpoint, httpClient, request, extractArgs); diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs index 661eeb0..9b4462b 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs @@ -13,7 +13,7 @@ public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollectio { assembly ??= Assembly.GetCallingAssembly(); - // Find all endpoint related types in the assembl + // Find all endpoint related types in the assembly var endpoints = assembly .GetTypes() .Where(y => From 7cf865d4771bb6c877f2abea763c33d2b51eed50 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Tue, 29 Nov 2022 13:11:32 +1000 Subject: [PATCH 13/47] Move common services to root namespace, move transform services to Transform namespace --- .../{Extract => }/ApiEndpoint.cs | 6 +---- .../{Extract => }/ApiEndpointAttribute.cs | 2 +- .../Extract/EndpointExtractJob.cs | 8 ++++-- .../Extract/EndpointExtractPipeline.cs | 14 ++-------- .../EndpointServiceCollectionExtensions.cs | 1 + .../Extract/IEndpointExtractPipeline.cs | 2 +- .../{Extract => }/IApiEndpointService.cs | 2 +- ...ns.cs => IApiEndpointServiceExtensions.cs} | 7 ++--- .../MIFCore.Hangfire.APIETL.csproj | 4 +++ .../Transform/ApiEndpointTransformPipeline.cs | 27 +++++++++++++++++++ .../HandleResponseArgs.cs | 4 ++- .../IApiEndpointTransformPipeline.cs | 6 +++++ .../{Extract => Transform}/IHandleResponse.cs | 2 +- 13 files changed, 58 insertions(+), 27 deletions(-) rename MIFCore.Hangfire.APIETL/{Extract => }/ApiEndpoint.cs (70%) rename MIFCore.Hangfire.APIETL/{Extract => }/ApiEndpointAttribute.cs (89%) rename MIFCore.Hangfire.APIETL/{Extract => }/IApiEndpointService.cs (52%) rename MIFCore.Hangfire.APIETL/{Extract/ApiEndpointServiceExtensions.cs => IApiEndpointServiceExtensions.cs} (84%) create mode 100644 MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs rename MIFCore.Hangfire.APIETL/{Extract => Transform}/HandleResponseArgs.cs (77%) create mode 100644 MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs rename MIFCore.Hangfire.APIETL/{Extract => Transform}/IHandleResponse.cs (78%) diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpoint.cs b/MIFCore.Hangfire.APIETL/ApiEndpoint.cs similarity index 70% rename from MIFCore.Hangfire.APIETL/Extract/ApiEndpoint.cs rename to MIFCore.Hangfire.APIETL/ApiEndpoint.cs index 4d03121..27699e9 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpoint.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpoint.cs @@ -1,7 +1,6 @@ using System.Collections.Generic; -using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { public class ApiEndpoint { @@ -23,8 +22,5 @@ internal ApiEndpoint(string name) public string HttpClientName { get; } public IDictionary AdditionalHeaders { get; } = new Dictionary(); - - public delegate Task PrepareRequestDelegate(PrepareRequestArgs args); - public delegate Task> PrepareNextRequestDelegate(PrepareNextRequestArgs args); } } diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs b/MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs similarity index 89% rename from MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs rename to MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs index b454184..7b065e4 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointAttribute.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs @@ -1,6 +1,6 @@ using System; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] public class ApiEndpointAttribute : Attribute diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs index 2e05471..1d26a3a 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs @@ -1,4 +1,5 @@ using Hangfire; +using MIFCore.Hangfire.APIETL.Transform; using System; using System.Collections.Generic; using System.Linq; @@ -13,17 +14,20 @@ internal class EndpointExtractJob private readonly IApiEndpointRegister apiEndpointRegister; private readonly IBackgroundJobClient backgroundJobClient; private readonly IEndpointExtractPipeline endpointExtractPipeline; + private readonly IApiEndpointTransformPipeline transformPipeline; public EndpointExtractJob( IHttpClientFactory httpClientFactory, IApiEndpointRegister apiEndpointRegister, IBackgroundJobClient backgroundJobClient, - IEndpointExtractPipeline endpointExtractPipeline) + IEndpointExtractPipeline endpointExtractPipeline, + IApiEndpointTransformPipeline transformPipeline) { this.httpClientFactory = httpClientFactory; this.apiEndpointRegister = apiEndpointRegister; this.backgroundJobClient = backgroundJobClient; this.endpointExtractPipeline = endpointExtractPipeline; + this.transformPipeline = transformPipeline; } [DisableIdenticalQueuedItems] @@ -67,7 +71,7 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs } finally { - await this.endpointExtractPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData)); + await this.transformPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData)); } } diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractPipeline.cs index 65ded23..ab0b589 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractPipeline.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointExtractPipeline.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using MIFCore.Hangfire.APIETL.Transform; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -52,16 +53,5 @@ public async Task> OnPrepareNextRequest(PrepareNextR return result; } - - public async Task OnHandleResponse(HandleResponseArgs args) - { - var relatedHandleResponses = this.handleResponses - .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); - - foreach (var handleResponse in relatedHandleResponses) - { - await handleResponse.OnHandleResponse(args); - } - } } } diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs index 9b4462b..0cec6c2 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using MIFCore.Hangfire.APIETL.Transform; using System; using System.Collections.Generic; using System.Linq; diff --git a/MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs index 4599043..997813f 100644 --- a/MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs +++ b/MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs @@ -1,6 +1,6 @@ namespace MIFCore.Hangfire.APIETL.Extract { - internal interface IEndpointExtractPipeline : IPrepareRequest, IPrepareNextRequest, IHandleResponse + internal interface IEndpointExtractPipeline : IPrepareRequest, IPrepareNextRequest { } } \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointService.cs b/MIFCore.Hangfire.APIETL/IApiEndpointService.cs similarity index 52% rename from MIFCore.Hangfire.APIETL/Extract/IApiEndpointService.cs rename to MIFCore.Hangfire.APIETL/IApiEndpointService.cs index c896eeb..410b388 100644 --- a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointService.cs +++ b/MIFCore.Hangfire.APIETL/IApiEndpointService.cs @@ -1,4 +1,4 @@ -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { public interface IApiEndpointService { } } diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointServiceExtensions.cs b/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs similarity index 84% rename from MIFCore.Hangfire.APIETL/Extract/ApiEndpointServiceExtensions.cs rename to MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs index b4349cc..a06ac56 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointServiceExtensions.cs +++ b/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs @@ -1,10 +1,11 @@ -using System.Linq; +using MIFCore.Hangfire.APIETL.Extract; +using System.Linq; using System.Reflection; using System.Text.RegularExpressions; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { - public static class ApiEndpointServiceExtensions + public static class IApiEndpointServiceExtensions { public static bool RespondsToEndpointName(this IApiEndpointService apiEndpointService, string endpointName) { diff --git a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj index 25e4c8c..c9061e2 100644 --- a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj +++ b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj @@ -29,4 +29,8 @@ + + + + diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs new file mode 100644 index 0000000..49d2105 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs @@ -0,0 +1,27 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + internal class ApiEndpointTransformPipeline : IApiEndpointTransformPipeline + { + private readonly IEnumerable handleResponses; + + public ApiEndpointTransformPipeline(IEnumerable handleResponses) + { + this.handleResponses = handleResponses; + } + + public async Task OnHandleResponse(HandleResponseArgs args) + { + var relatedHandleResponses = this.handleResponses + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + + foreach (var handleResponse in relatedHandleResponses) + { + await handleResponse.OnHandleResponse(args); + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Extract/HandleResponseArgs.cs b/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs similarity index 77% rename from MIFCore.Hangfire.APIETL/Extract/HandleResponseArgs.cs rename to MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs index 9c8b688..9e827fa 100644 --- a/MIFCore.Hangfire.APIETL/Extract/HandleResponseArgs.cs +++ b/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs @@ -1,4 +1,6 @@ -namespace MIFCore.Hangfire.APIETL.Extract +using MIFCore.Hangfire.APIETL.Extract; + +namespace MIFCore.Hangfire.APIETL.Transform { public class HandleResponseArgs { diff --git a/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs new file mode 100644 index 0000000..01d45c0 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs @@ -0,0 +1,6 @@ +namespace MIFCore.Hangfire.APIETL.Transform +{ + internal interface IApiEndpointTransformPipeline : IHandleResponse + { + } +} diff --git a/MIFCore.Hangfire.APIETL/Extract/IHandleResponse.cs b/MIFCore.Hangfire.APIETL/Transform/IHandleResponse.cs similarity index 78% rename from MIFCore.Hangfire.APIETL/Extract/IHandleResponse.cs rename to MIFCore.Hangfire.APIETL/Transform/IHandleResponse.cs index 9db28d0..8b14d66 100644 --- a/MIFCore.Hangfire.APIETL/Extract/IHandleResponse.cs +++ b/MIFCore.Hangfire.APIETL/Transform/IHandleResponse.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL.Transform { public interface IHandleResponse : IApiEndpointService { From 0e2efc2ace88260f3e0f00b8a480f0b06cbbc085 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Tue, 29 Nov 2022 13:13:39 +1000 Subject: [PATCH 14/47] Rename services starting with Endpoint to ApiEndpoint --- ...{EndpointExtractJob.cs => ApiEndpointExtractJob.cs} | 10 +++++----- ...xtractPipeline.cs => ApiEndpointExtractPipeline.cs} | 4 ++-- MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs | 2 +- ...nsions.cs => ExtractServiceCollectionExtensions.cs} | 6 +++--- .../Extract/IApiEndpointExtractPipeline.cs | 6 ++++++ .../Extract/IEndpointExtractPipeline.cs | 6 ------ 6 files changed, 17 insertions(+), 17 deletions(-) rename MIFCore.Hangfire.APIETL/Extract/{EndpointExtractJob.cs => ApiEndpointExtractJob.cs} (93%) rename MIFCore.Hangfire.APIETL/Extract/{EndpointExtractPipeline.cs => ApiEndpointExtractPipeline.cs} (94%) rename MIFCore.Hangfire.APIETL/Extract/{EndpointServiceCollectionExtensions.cs => ExtractServiceCollectionExtensions.cs} (92%) create mode 100644 MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractPipeline.cs delete mode 100644 MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs similarity index 93% rename from MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs index 1d26a3a..1cb8957 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs @@ -8,19 +8,19 @@ namespace MIFCore.Hangfire.APIETL.Extract { - internal class EndpointExtractJob + internal class ApiEndpointExtractJob { private readonly IHttpClientFactory httpClientFactory; private readonly IApiEndpointRegister apiEndpointRegister; private readonly IBackgroundJobClient backgroundJobClient; - private readonly IEndpointExtractPipeline endpointExtractPipeline; + private readonly IApiEndpointExtractPipeline endpointExtractPipeline; private readonly IApiEndpointTransformPipeline transformPipeline; - public EndpointExtractJob( + public ApiEndpointExtractJob( IHttpClientFactory httpClientFactory, IApiEndpointRegister apiEndpointRegister, IBackgroundJobClient backgroundJobClient, - IEndpointExtractPipeline endpointExtractPipeline, + IApiEndpointExtractPipeline endpointExtractPipeline, IApiEndpointTransformPipeline transformPipeline) { this.httpClientFactory = httpClientFactory; @@ -67,7 +67,7 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs return; // Otherwise we need to schedule another job to continue extracting this endpoint. - this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.Id))); + this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.Id))); } finally { diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractPipeline.cs similarity index 94% rename from MIFCore.Hangfire.APIETL/Extract/EndpointExtractPipeline.cs rename to MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractPipeline.cs index ab0b589..d5dbca8 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointExtractPipeline.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractPipeline.cs @@ -5,13 +5,13 @@ namespace MIFCore.Hangfire.APIETL.Extract { - internal class EndpointExtractPipeline : IEndpointExtractPipeline + internal class ApiEndpointExtractPipeline : IApiEndpointExtractPipeline { private readonly IEnumerable prepareRequests; private readonly IEnumerable prepareNextRequests; private readonly IEnumerable handleResponses; - public EndpointExtractPipeline( + public ApiEndpointExtractPipeline( IEnumerable prepareRequests, IEnumerable prepareNextRequests, IEnumerable handleResponses) diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs index 588dc4b..ce9e5c1 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs @@ -23,7 +23,7 @@ public IApiEndpointRegister Register(ApiEndpoint endpoint) { this.endpoints.Add(endpoint.Name, endpoint); - this.recurringJobManager.AddOrUpdate( + this.recurringJobManager.AddOrUpdate( endpoint.JobName, job => job.Extract(endpoint.Name, null), Cron.Daily() diff --git a/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/Extract/ExtractServiceCollectionExtensions.cs similarity index 92% rename from MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs rename to MIFCore.Hangfire.APIETL/Extract/ExtractServiceCollectionExtensions.cs index 0cec6c2..00ca192 100644 --- a/MIFCore.Hangfire.APIETL/Extract/EndpointServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ExtractServiceCollectionExtensions.cs @@ -8,7 +8,7 @@ namespace MIFCore.Hangfire.APIETL.Extract { - public static class EndpointServiceCollectionExtensions + public static class ExtractServiceCollectionExtensions { public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollection serviceDescriptors, Assembly assembly = null) { @@ -29,8 +29,8 @@ public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollectio // Register the services used to register jobs and create ApiEndpoint definitions serviceDescriptors.TryAddSingleton(); serviceDescriptors.TryAddTransient(); - serviceDescriptors.TryAddTransient(); - serviceDescriptors.TryAddScoped(); + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddScoped(); foreach (var t in endpoints) { diff --git a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractPipeline.cs new file mode 100644 index 0000000..1380407 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractPipeline.cs @@ -0,0 +1,6 @@ +namespace MIFCore.Hangfire.APIETL.Extract +{ + internal interface IApiEndpointExtractPipeline : IPrepareRequest, IPrepareNextRequest + { + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs deleted file mode 100644 index 997813f..0000000 --- a/MIFCore.Hangfire.APIETL/Extract/IEndpointExtractPipeline.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace MIFCore.Hangfire.APIETL.Extract -{ - internal interface IEndpointExtractPipeline : IPrepareRequest, IPrepareNextRequest - { - } -} \ No newline at end of file From 4d14fcaf2aecacf304b4aca86a3a5eb6b032f134 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Tue, 29 Nov 2022 13:15:56 +1000 Subject: [PATCH 15/47] Move common services from Extract namespace to root namespace --- MIFCore.Hangfire.APIETL/{Extract => }/ApiEndpointFactory.cs | 2 +- MIFCore.Hangfire.APIETL/{Extract => }/IApiEndpointFactory.cs | 2 +- MIFCore.Hangfire.APIETL/{Extract => }/IDefineEndpoints.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename MIFCore.Hangfire.APIETL/{Extract => }/ApiEndpointFactory.cs (98%) rename MIFCore.Hangfire.APIETL/{Extract => }/IApiEndpointFactory.cs (77%) rename MIFCore.Hangfire.APIETL/{Extract => }/IDefineEndpoints.cs (81%) diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointFactory.cs b/MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs similarity index 98% rename from MIFCore.Hangfire.APIETL/Extract/ApiEndpointFactory.cs rename to MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs index 98f0508..bbacb8d 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointFactory.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Linq; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { internal class ApiEndpointFactory : IApiEndpointFactory { diff --git a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointFactory.cs b/MIFCore.Hangfire.APIETL/IApiEndpointFactory.cs similarity index 77% rename from MIFCore.Hangfire.APIETL/Extract/IApiEndpointFactory.cs rename to MIFCore.Hangfire.APIETL/IApiEndpointFactory.cs index dcaf8d0..a14f444 100644 --- a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointFactory.cs +++ b/MIFCore.Hangfire.APIETL/IApiEndpointFactory.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { public interface IApiEndpointFactory { diff --git a/MIFCore.Hangfire.APIETL/Extract/IDefineEndpoints.cs b/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs similarity index 81% rename from MIFCore.Hangfire.APIETL/Extract/IDefineEndpoints.cs rename to MIFCore.Hangfire.APIETL/IDefineEndpoints.cs index a3c6eef..0cdb7c4 100644 --- a/MIFCore.Hangfire.APIETL/Extract/IDefineEndpoints.cs +++ b/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { public interface IDefineEndpoints : IApiEndpointService { From 7d6d942bd10883d40f26ba0a78948d70938c80c3 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Tue, 29 Nov 2022 13:16:37 +1000 Subject: [PATCH 16/47] Remove dependency on IHandleResponse --- .../Extract/ApiEndpointExtractPipeline.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractPipeline.cs index d5dbca8..6f6a7bc 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractPipeline.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractPipeline.cs @@ -1,5 +1,4 @@ -using MIFCore.Hangfire.APIETL.Transform; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -9,16 +8,13 @@ internal class ApiEndpointExtractPipeline : IApiEndpointExtractPipeline { private readonly IEnumerable prepareRequests; private readonly IEnumerable prepareNextRequests; - private readonly IEnumerable handleResponses; public ApiEndpointExtractPipeline( IEnumerable prepareRequests, - IEnumerable prepareNextRequests, - IEnumerable handleResponses) + IEnumerable prepareNextRequests) { this.prepareRequests = prepareRequests; this.prepareNextRequests = prepareNextRequests; - this.handleResponses = handleResponses; } public async Task OnPrepareRequest(PrepareRequestArgs args) From 965f018a288c6df6ae297deb6b93d24ed1bbcce1 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Tue, 29 Nov 2022 13:18:57 +1000 Subject: [PATCH 17/47] Move common services to root namespace --- ...ionExtensions.cs => APIETLServiceCollectionExtensions.cs} | 5 +++-- MIFCore.Hangfire.APIETL/{Extract => }/ApiEndpointRegister.cs | 3 ++- .../{Extract => }/ApiEndpointSelectorAttribute.cs | 2 +- .../{Extract => }/IApiEndpointRegister.cs | 2 +- MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs | 3 +-- 5 files changed, 8 insertions(+), 7 deletions(-) rename MIFCore.Hangfire.APIETL/{Extract/ExtractServiceCollectionExtensions.cs => APIETLServiceCollectionExtensions.cs} (95%) rename MIFCore.Hangfire.APIETL/{Extract => }/ApiEndpointRegister.cs (96%) rename MIFCore.Hangfire.APIETL/{Extract => }/ApiEndpointSelectorAttribute.cs (87%) rename MIFCore.Hangfire.APIETL/{Extract => }/IApiEndpointRegister.cs (88%) diff --git a/MIFCore.Hangfire.APIETL/Extract/ExtractServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs similarity index 95% rename from MIFCore.Hangfire.APIETL/Extract/ExtractServiceCollectionExtensions.cs rename to MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs index 00ca192..9fbe19c 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ExtractServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs @@ -1,14 +1,15 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using MIFCore.Hangfire.APIETL.Extract; using MIFCore.Hangfire.APIETL.Transform; using System; using System.Collections.Generic; using System.Linq; using System.Reflection; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { - public static class ExtractServiceCollectionExtensions + public static class APIETLServiceCollectionExtensions { public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollection serviceDescriptors, Assembly assembly = null) { diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs similarity index 96% rename from MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs rename to MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs index ce9e5c1..8641eee 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs @@ -1,9 +1,10 @@ using Hangfire; +using MIFCore.Hangfire.APIETL.Extract; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { internal class ApiEndpointRegister : IApiEndpointRegister { diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointSelectorAttribute.cs b/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs similarity index 87% rename from MIFCore.Hangfire.APIETL/Extract/ApiEndpointSelectorAttribute.cs rename to MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs index 982d443..ecc3b98 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointSelectorAttribute.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs @@ -1,6 +1,6 @@ using System; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] public class ApiEndpointSelectorAttribute : Attribute diff --git a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs similarity index 88% rename from MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs rename to MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs index e198ae3..0c18db8 100644 --- a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Threading.Tasks; -namespace MIFCore.Hangfire.APIETL.Extract +namespace MIFCore.Hangfire.APIETL { public interface IApiEndpointRegister { diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs b/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs index a06ac56..a00ed52 100644 --- a/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs +++ b/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs @@ -1,5 +1,4 @@ -using MIFCore.Hangfire.APIETL.Extract; -using System.Linq; +using System.Linq; using System.Reflection; using System.Text.RegularExpressions; From 1436c27ea3cf273e03d078699646b0a3b08427f9 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Tue, 29 Nov 2022 14:00:42 +1000 Subject: [PATCH 18/47] Add IApiEndpointTransformPipeline to service collection --- MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs index 9fbe19c..5f7ec6d 100644 --- a/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs @@ -31,6 +31,7 @@ public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollectio serviceDescriptors.TryAddSingleton(); serviceDescriptors.TryAddTransient(); serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); serviceDescriptors.TryAddScoped(); foreach (var t in endpoints) From aa9a2623b0c9b500f9c46d0d6c1f10e63bd7a317 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 30 Nov 2022 10:58:04 +1000 Subject: [PATCH 19/47] Add FlattenGraphExtensions --- .../Transform/FlattenGraphExtensionsTests.cs | 38 +++++++++++++++++ .../Transform/FlattenGraphExtensions.cs | 41 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 MIFCore.Hangfire.APIETL.Tests/Transform/FlattenGraphExtensionsTests.cs create mode 100644 MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs diff --git a/MIFCore.Hangfire.APIETL.Tests/Transform/FlattenGraphExtensionsTests.cs b/MIFCore.Hangfire.APIETL.Tests/Transform/FlattenGraphExtensionsTests.cs new file mode 100644 index 0000000..2de77c3 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/Transform/FlattenGraphExtensionsTests.cs @@ -0,0 +1,38 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Dynamic; + +namespace MIFCore.Hangfire.APIETL.Transform.Tests +{ + [TestClass()] + public class FlattenGraphExtensionsTests + { + [TestMethod()] + public void FlattenGraphTest() + { + dynamic expando = new ExpandoObject(); + expando.Id = "abc123"; + expando.Description = "does stuff"; + + dynamic nestedExpando = new ExpandoObject(); + nestedExpando.Id = "def456"; + nestedExpando.Plums = true; + + dynamic moreNested = new ExpandoObject(); + moreNested.Id = Guid.Empty; + moreNested.Date = DateTime.MaxValue; + moreNested.Items = new[] { "one", "two", "three" }; + + nestedExpando.Action = moreNested; + expando.Nested = nestedExpando; + + (expando as ExpandoObject).FlattenGraph(); + + Assert.AreEqual("def456", expando.Nested_Id); + Assert.AreEqual(true, expando.Nested_Plums); + + Assert.AreEqual(Guid.Empty, expando.Nested_Action_Id); + Assert.AreEqual(DateTime.MaxValue, expando.Nested_Action_Date); + Assert.AreEqual(3, expando.Nested_Action_Items.Length); + } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs new file mode 100644 index 0000000..cd476cd --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using System.Dynamic; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public static class FlattenGraphExtensions + { + public static void FlattenGraph(this ExpandoObject expando) + { + (expando as IDictionary).FlattenGraph(); + } + + public static void FlattenGraph(this IDictionary rootDict) + { + // iterate and check for nested objects + foreach (var rootKey in rootDict.Keys) + { + var rootValue = rootDict[rootKey]; + + // Is the property an object with keys and values? + if (rootValue is IDictionary nestedDict) + { + // Flatten the nestedDict so the entire graph flattens recursively + nestedDict.FlattenGraph(); + + // Get all the keys in the nestedDict and add them to the rootDict + foreach (var nestKey in nestedDict.Keys) + { + var nestValue = nestedDict[nestKey]; + + // Ensure the key is unique + rootDict.TryAdd($"{rootKey}_{nestKey}", nestValue); + } + + // Remove the original rootKey from the rootDict as all the properties have been merged into it + rootDict.Remove(rootKey, out _); + } + } + } + } +} From 6619e39e5319a817c1b40bdb8097e07613ef4970 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 30 Nov 2022 11:34:46 +1000 Subject: [PATCH 20/47] Add ExtractDistinctGraphObjectSetExtensions --- ...tDistinctGraphObjectSetsExtensionsTests.cs | 29 +++++++ ...xtractDistinctGraphObjectSetsExtensions.cs | 85 +++++++++++++++++++ .../Transform/FlattenGraphExtensions.cs | 6 ++ .../Transform/GraphObjectSet.cs | 13 +++ 4 files changed, 133 insertions(+) create mode 100644 MIFCore.Hangfire.APIETL.Tests/Transform/ExtractDistinctGraphObjectSetsExtensionsTests.cs create mode 100644 MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs create mode 100644 MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs diff --git a/MIFCore.Hangfire.APIETL.Tests/Transform/ExtractDistinctGraphObjectSetsExtensionsTests.cs b/MIFCore.Hangfire.APIETL.Tests/Transform/ExtractDistinctGraphObjectSetsExtensionsTests.cs new file mode 100644 index 0000000..6bdcfd7 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/Transform/ExtractDistinctGraphObjectSetsExtensionsTests.cs @@ -0,0 +1,29 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using System.Dynamic; + +namespace MIFCore.Hangfire.APIETL.Transform.Tests +{ + [TestClass()] + public class ExtractDistinctGraphObjectSetsExtensionsTests + { + [TestMethod()] + public void ExtractDistinctGraphObjectSetsTest() + { + var rootGraph = new + { + Id = "abc123", + Items = new[] + { + new { Id = "def456", Plums = true, Children = new[] { new { Value = "abc123" } } }, + new { Id = "def456", Plums = true, Children = new[] { new { Value = "def132" } } }, + new { Id = "def456", Plums = true, Children = new[] { new { Value = "ggjg35" } } }, + } + }; + + var rootGraphDynamic = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(rootGraph), new ExpandoObjectConverter()); + var sets = rootGraphDynamic.ExtractDistinctGraphObjectSets().ToList(); + } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs new file mode 100644 index 0000000..71e97ce --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs @@ -0,0 +1,85 @@ +using System.Collections.Generic; +using System.Linq; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public static class ExtractDistinctGraphObjectSetsExtensions + { + public static IEnumerable ExtractDistinctGraphObjectSets(this IEnumerable> root) + { + var rootObjectSet = new GraphObjectSet + { + Objects = root + }; + + yield return rootObjectSet; + + foreach (var rootItem in root) + { + var nestedObjectSets = rootItem.ExtractDistinctGraphObjectSets(rootObjectSet); + + foreach (var nos in nestedObjectSets) + { + if (nos.Parent is null) + continue; + + yield return nos; + } + } + } + + public static IEnumerable ExtractDistinctGraphObjectSets(this IDictionary rootItem, GraphObjectSet rootObjectSet = null) + { + // If no root object set was passed in (passed in from the IEnumerable version of ExtractDistinctGraphObjectSets) + // create a new GraphObjectSet to represent this single entity (root entity of the graph) + if (rootObjectSet is null) + { + rootObjectSet = new GraphObjectSet + { + Objects = new[] { rootItem } + }; + + yield return rootObjectSet; + } + + // loop through each key to find enumerable values + foreach (var (rootKey, rootValue) in rootItem) + { + IEnumerable nestedObjectSets; + + if (rootValue is IEnumerable childObjects) + { + var childDicts = childObjects.Cast>(); + + // Get the children of the children + nestedObjectSets = childDicts.ExtractDistinctGraphObjectSets(); + } + else if (rootValue is IDictionary childDict) + { + nestedObjectSets = childDict.ExtractDistinctGraphObjectSets(); + } + else + { + continue; + } + + // Link the children of the children via keys + foreach (var n in nestedObjectSets) + { + if (n.Parent is null) + { + n.Parent = rootItem; + n.ParentSet = rootObjectSet; + n.ParentKey = rootKey; + } + else + { + n.ParentKey = $"{rootKey}.{n.ParentKey}"; + } + + yield return n; + } + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs index cd476cd..34c52f4 100644 --- a/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs @@ -10,6 +10,12 @@ public static void FlattenGraph(this ExpandoObject expando) (expando as IDictionary).FlattenGraph(); } + public static void FlattenGraph(this IEnumerable> rootArray) + { + foreach (var a in rootArray) + a.FlattenGraph(); + } + public static void FlattenGraph(this IDictionary rootDict) { // iterate and check for nested objects diff --git a/MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs b/MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs new file mode 100644 index 0000000..0d2e13a --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public class GraphObjectSet + { + public IDictionary Parent { get; internal set; } + public GraphObjectSet ParentSet { get; internal set; } + public string ParentKey { get; internal set; } + + public IEnumerable> Objects { get; internal set; } + } +} From 261e3e7a91f49ecbfd34c2faa7a735d79e44d69c Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 30 Nov 2022 13:49:23 +1000 Subject: [PATCH 21/47] Refactor & add Transform callback --- ...xtractDistinctGraphObjectSetsExtensions.cs | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs index 71e97ce..4c88b40 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs @@ -5,31 +5,34 @@ namespace MIFCore.Hangfire.APIETL.Transform { public static class ExtractDistinctGraphObjectSetsExtensions { - public static IEnumerable ExtractDistinctGraphObjectSets(this IEnumerable> root) + public delegate void TransformObjectDelegate(TransformObjectArgs args); + + public static IEnumerable ExtractDistinctGraphObjectSets(this IEnumerable> root, ExtractDistinctGraphObjectSetsArgs args = null) { - var rootObjectSet = new GraphObjectSet + args ??= new ExtractDistinctGraphObjectSetsArgs(); + args.RootObjectSet ??= new GraphObjectSet { Objects = root }; - yield return rootObjectSet; + yield return args.RootObjectSet; foreach (var rootItem in root) { - var nestedObjectSets = rootItem.ExtractDistinctGraphObjectSets(rootObjectSet); + var nestedObjectSets = rootItem.ExtractDistinctGraphObjectSets(args); foreach (var nos in nestedObjectSets) { - if (nos.Parent is null) - continue; - yield return nos; } } } - public static IEnumerable ExtractDistinctGraphObjectSets(this IDictionary rootItem, GraphObjectSet rootObjectSet = null) + public static IEnumerable ExtractDistinctGraphObjectSets(this IDictionary rootItem, ExtractDistinctGraphObjectSetsArgs args = null) { + args ??= new ExtractDistinctGraphObjectSetsArgs(); + var rootObjectSet = args.RootObjectSet; + // If no root object set was passed in (passed in from the IEnumerable version of ExtractDistinctGraphObjectSets) // create a new GraphObjectSet to represent this single entity (root entity of the graph) if (rootObjectSet is null) @@ -39,6 +42,8 @@ public static IEnumerable ExtractDistinctGraphObjectSets(this ID Objects = new[] { rootItem } }; + args.RootObjectSet = rootObjectSet; + yield return rootObjectSet; } @@ -52,11 +57,17 @@ public static IEnumerable ExtractDistinctGraphObjectSets(this ID var childDicts = childObjects.Cast>(); // Get the children of the children - nestedObjectSets = childDicts.ExtractDistinctGraphObjectSets(); + nestedObjectSets = childDicts.ExtractDistinctGraphObjectSets(new ExtractDistinctGraphObjectSetsArgs + { + Transform = args.Transform + }); } else if (rootValue is IDictionary childDict) { - nestedObjectSets = childDict.ExtractDistinctGraphObjectSets(); + nestedObjectSets = childDict.ExtractDistinctGraphObjectSets(new ExtractDistinctGraphObjectSetsArgs + { + Transform = args.Transform + }); } else { @@ -80,6 +91,24 @@ public static IEnumerable ExtractDistinctGraphObjectSets(this ID yield return n; } } + + args.Transform?.Invoke(new TransformObjectArgs + { + Object = rootItem, + GraphObjectSet = rootObjectSet + }); + } + + public class TransformObjectArgs + { + public IDictionary Object { get; set; } + public GraphObjectSet GraphObjectSet { get; set; } + } + + public class ExtractDistinctGraphObjectSetsArgs + { + internal GraphObjectSet RootObjectSet { get; set; } + public TransformObjectDelegate Transform { get; set; } } } } From 02c419cb3ba1b527e90783ec44eb65014f64fa32 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 30 Nov 2022 13:52:43 +1000 Subject: [PATCH 22/47] GraphObjectSet Objects should be IList for index lookup --- .../Transform/ExtractDistinctGraphObjectSetsExtensions.cs | 6 +++++- MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs index 4c88b40..5cf7563 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs @@ -9,16 +9,20 @@ public static class ExtractDistinctGraphObjectSetsExtensions public static IEnumerable ExtractDistinctGraphObjectSets(this IEnumerable> root, ExtractDistinctGraphObjectSetsArgs args = null) { + var objects = new List>(); + args ??= new ExtractDistinctGraphObjectSetsArgs(); args.RootObjectSet ??= new GraphObjectSet { - Objects = root + Objects = objects }; yield return args.RootObjectSet; foreach (var rootItem in root) { + objects.Add(rootItem); + var nestedObjectSets = rootItem.ExtractDistinctGraphObjectSets(args); foreach (var nos in nestedObjectSets) diff --git a/MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs b/MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs index 0d2e13a..84b23d9 100644 --- a/MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs +++ b/MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs @@ -8,6 +8,6 @@ public class GraphObjectSet public GraphObjectSet ParentSet { get; internal set; } public string ParentKey { get; internal set; } - public IEnumerable> Objects { get; internal set; } + public IList> Objects { get; internal set; } } } From 0bf8474a56e50628ac7a1ea6ee5fdf4e7e7f5b3a Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 30 Nov 2022 14:14:38 +1000 Subject: [PATCH 23/47] Add GetKeys --- .../GraphObjectSetGetKeysExtensions.cs | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 MIFCore.Hangfire.APIETL/Transform/GraphObjectSetGetKeysExtensions.cs diff --git a/MIFCore.Hangfire.APIETL/Transform/GraphObjectSetGetKeysExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/GraphObjectSetGetKeysExtensions.cs new file mode 100644 index 0000000..414b4b6 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/GraphObjectSetGetKeysExtensions.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public static class GraphObjectSetGetKeysExtensions + { + public static IDictionary> GetKeyTypes(this IEnumerable graphObjectSets) + { + var allKeys = new Dictionary>(); + + foreach (var o in graphObjectSets) + { + var keys = o.GetKeyTypes(); + + foreach (var (key, value) in keys) + { + if (allKeys.TryGetValue(key, out var existingObjectValueType) == false) + { + existingObjectValueType = new HashSet(); + allKeys[key] = existingObjectValueType; + } + + foreach (var t in value) + { + existingObjectValueType.Add(t); + } + } + } + + return allKeys; + } + + public static IDictionary> GetKeyTypes(this GraphObjectSet graphObjectSet) + { + var keys = new Dictionary>(); + + foreach (var obj in graphObjectSet.Objects) + { + foreach (var (objKey, objectValue) in obj) + { + var objectValueType = objectValue == null ? null : objectValue.GetType(); + + if (keys.TryGetValue(objKey, out var existingObjectValueType) == false) + { + existingObjectValueType = new HashSet(); + keys[objKey] = existingObjectValueType; + } + + existingObjectValueType.Add(objectValueType); + } + } + + return keys; + } + } +} From c693a54334559bccc85e349126df84a9ba62e984 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 30 Nov 2022 15:40:11 +1000 Subject: [PATCH 24/47] Add ApiEndpointModel and SourcePropertyName attributes --- .../Load/ApiEndpointModelAttribute.cs | 15 +++++++++++++++ .../Load/SourcePropertyNameAttribute.cs | 14 ++++++++++++++ .../MIFCore.Hangfire.APIETL.csproj | 4 ---- 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/SourcePropertyNameAttribute.cs diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs new file mode 100644 index 0000000..89b6e25 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs @@ -0,0 +1,15 @@ +using System; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public class ApiEndpointModelAttribute : Attribute + { + public ApiEndpointModelAttribute(string endpointName) + { + this.EndpointName = endpointName; + } + + public string EndpointName { get; } + public string ModelPath { get; set; } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/SourcePropertyNameAttribute.cs b/MIFCore.Hangfire.APIETL/Load/SourcePropertyNameAttribute.cs new file mode 100644 index 0000000..c97be2b --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/SourcePropertyNameAttribute.cs @@ -0,0 +1,14 @@ +using System; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public class SourcePropertyNameAttribute : Attribute + { + public SourcePropertyNameAttribute(string name) + { + this.Name = name; + } + + public string Name { get; } + } +} diff --git a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj index c9061e2..25e4c8c 100644 --- a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj +++ b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj @@ -29,8 +29,4 @@ - - - - From 32318a36219ea1f12d9ae0f9583788501b0fbc96 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 30 Nov 2022 16:47:48 +1000 Subject: [PATCH 25/47] Add ApiEndpointModel and properties, class decorator attributes --- .../Extract/ApiEndpointExtractJob.cs | 4 +- .../Load/ApiEndpointModel.cs | 14 +++++ .../Load/ApiEndpointModelAttribute.cs | 11 +++- .../Load/ApiEndpointModelProperty.cs | 16 +++++ .../Load/ApiEndpointModelPropertyAttribute.cs | 27 ++++++++ .../Load/SourcePropertyNameAttribute.cs | 14 ----- .../Load/TypeExtensions.cs | 63 +++++++++++++++++++ .../MIFCore.Hangfire.APIETL.csproj | 1 + 8 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL/Load/ApiEndpointModel.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/ApiEndpointModelProperty.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs delete mode 100644 MIFCore.Hangfire.APIETL/Load/SourcePropertyNameAttribute.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs index 1cb8957..0db9859 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs @@ -95,8 +95,8 @@ private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient http private async Task CreateRequest(Uri baseAddress, ApiEndpoint endpoint, ExtractArgs extractArgs) { - // Create a new request, using endpoint.Name as the relative uri - // i.e endpoint.Name = "getStuff" and httpClient.BaseAddress = "https://someapi/api/" + // Create a new request, using endpoint.SourceName as the relative uri + // i.e endpoint.SourceName = "getStuff" and httpClient.BaseAddress = "https://someapi/api/" var request = new HttpRequestMessage { RequestUri = new Uri(baseAddress, endpoint.Name) diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModel.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModel.cs new file mode 100644 index 0000000..4dec596 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModel.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public class ApiEndpointModel + { + public string EndpointName { get; set; } + public string DestinationName { get; set; } + + public string InputPath { get; set; } + + public IDictionary MappedProperties { get; } = new Dictionary(); + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs index 89b6e25..ff06226 100644 --- a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs @@ -2,6 +2,7 @@ namespace MIFCore.Hangfire.APIETL.Load { + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] public class ApiEndpointModelAttribute : Attribute { public ApiEndpointModelAttribute(string endpointName) @@ -9,7 +10,15 @@ public ApiEndpointModelAttribute(string endpointName) this.EndpointName = endpointName; } + public ApiEndpointModelAttribute(string endpointName, string destinationName) + { + this.EndpointName = endpointName; + this.DestinationName = destinationName; + } + public string EndpointName { get; } - public string ModelPath { get; set; } + public string DestinationName { get; } + + public string InputPath { get; set; } } } diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelProperty.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelProperty.cs new file mode 100644 index 0000000..35b9e2c --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelProperty.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public class ApiEndpointModelProperty + { + public string SourceName { get; set; } + public string DestinationName { get; set; } + + public bool IsKey { get; set; } + + public HashSet SourceType { get; set; } + public string DestinationType { get; set; } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs new file mode 100644 index 0000000..46af40a --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs @@ -0,0 +1,27 @@ +using System; + +namespace MIFCore.Hangfire.APIETL.Load +{ + [AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] + public class ApiEndpointModelPropertyAttribute : Attribute + { + public ApiEndpointModelPropertyAttribute(string sourceName, string destinationName) + { + this.SourceName = sourceName; + this.DestinationName = destinationName; + } + + public ApiEndpointModelPropertyAttribute(string sourceName) + { + this.SourceName = sourceName; + this.DestinationName = sourceName; + } + + public ApiEndpointModelPropertyAttribute() { } + + public string SourceName { get; } + public string DestinationName { get; } + + public string DestinationType { get; set; } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/SourcePropertyNameAttribute.cs b/MIFCore.Hangfire.APIETL/Load/SourcePropertyNameAttribute.cs deleted file mode 100644 index c97be2b..0000000 --- a/MIFCore.Hangfire.APIETL/Load/SourcePropertyNameAttribute.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace MIFCore.Hangfire.APIETL.Load -{ - public class SourcePropertyNameAttribute : Attribute - { - public SourcePropertyNameAttribute(string name) - { - this.Name = name; - } - - public string Name { get; } - } -} diff --git a/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs b/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs new file mode 100644 index 0000000..4574c29 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Reflection; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public static class TypeExtensions + { + public static ApiEndpointModel GetApiEndpointModel(this Type type) + { + var attribute = type.GetCustomAttribute(); + + if (attribute is null) + return null; + + var model = new ApiEndpointModel + { + DestinationName = attribute.DestinationName, + EndpointName = attribute.EndpointName, + InputPath = attribute.InputPath + }; + + if (string.IsNullOrWhiteSpace(model.DestinationName)) + { + model.DestinationName = type.Name; + } + + var modelProperties = type.GetProperties() + .Select(y => new KeyValuePair(y, y.GetCustomAttribute())) + .Where(y => y.Value != null) + .ToDictionary( + keySelector: y => y.Key, + elementSelector: y => y.Value); + + foreach (var (propertyInfo, propertyAttribute) in modelProperties) + { + var sourceName = propertyAttribute.SourceName; + var destinationName = propertyAttribute.DestinationName; + + if (string.IsNullOrWhiteSpace(sourceName)) + { + sourceName = propertyInfo.Name; + destinationName = sourceName; + } + + var property = new ApiEndpointModelProperty + { + SourceName = sourceName, + DestinationName = destinationName, + IsKey = propertyInfo.GetCustomAttribute() != null, + SourceType = new HashSet { propertyInfo.PropertyType }, + DestinationType = propertyAttribute.DestinationType + }; + + model.MappedProperties.Add(sourceName, property); + } + + return model; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj index 25e4c8c..5083fa7 100644 --- a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj +++ b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj @@ -23,6 +23,7 @@ + From 9ffb97ef7042781b391befe963923da64747aeb4 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 30 Nov 2022 18:11:03 +1000 Subject: [PATCH 26/47] DestinationName should not prepopulate with sourceName --- .../Load/ApiEndpointModelPropertyAttribute.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs index 46af40a..d33b818 100644 --- a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs @@ -14,14 +14,13 @@ public ApiEndpointModelPropertyAttribute(string sourceName, string destinationNa public ApiEndpointModelPropertyAttribute(string sourceName) { this.SourceName = sourceName; - this.DestinationName = sourceName; } public ApiEndpointModelPropertyAttribute() { } public string SourceName { get; } - public string DestinationName { get; } + public string DestinationName { get; } public string DestinationType { get; set; } } } From d038969c93edf5e30279a4df561d6a7c70547f0c Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Wed, 30 Nov 2022 18:19:38 +1000 Subject: [PATCH 27/47] Add support for KeyAttribute on its own, without need for ApiEndpointModelPropertyAttribute --- .../Load/TypeExtensions.cs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs b/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs index 4574c29..fe2c4be 100644 --- a/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs @@ -28,7 +28,7 @@ public static ApiEndpointModel GetApiEndpointModel(this Type type) } var modelProperties = type.GetProperties() - .Select(y => new KeyValuePair(y, y.GetCustomAttribute())) + .Select(y => new KeyValuePair(y, y.GetCustomAttribute() as Attribute ?? y.GetCustomAttribute())) .Where(y => y.Value != null) .ToDictionary( keySelector: y => y.Key, @@ -36,25 +36,25 @@ public static ApiEndpointModel GetApiEndpointModel(this Type type) foreach (var (propertyInfo, propertyAttribute) in modelProperties) { - var sourceName = propertyAttribute.SourceName; - var destinationName = propertyAttribute.DestinationName; - - if (string.IsNullOrWhiteSpace(sourceName)) - { - sourceName = propertyInfo.Name; - destinationName = sourceName; - } - var property = new ApiEndpointModelProperty { - SourceName = sourceName, - DestinationName = destinationName, IsKey = propertyInfo.GetCustomAttribute() != null, SourceType = new HashSet { propertyInfo.PropertyType }, - DestinationType = propertyAttribute.DestinationType }; - model.MappedProperties.Add(sourceName, property); + if (propertyAttribute is ApiEndpointModelPropertyAttribute modelPropertyAttribute) + { + property.SourceName = modelPropertyAttribute.SourceName; + property.DestinationName = modelPropertyAttribute.DestinationName; + } + + if (string.IsNullOrWhiteSpace(property.SourceName)) + { + property.SourceName = propertyInfo.Name; + property.DestinationName = propertyInfo.Name; + } + + model.MappedProperties.Add(property.SourceName, property); } return model; From 62f9f933c330603494d5b8f147f485c1252a0704 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Thu, 1 Dec 2022 11:28:56 +1000 Subject: [PATCH 28/47] Add ApiEndpointTransformJob which automatically maps response payloads to a destination model --- .../MIFCore.Hangfire.APIETL.SqlServer.csproj | 30 +++++ .../SqlServerGetDestinationType.cs | 37 ++++++ .../Extract/ApiEndpointExtractJob.cs | 8 +- .../Load/IGetDestinationType.cs | 11 ++ .../MIFCore.Hangfire.APIETL.csproj | 9 +- .../Transform/ApiEndpointTransformJob.cs | 111 ++++++++++++++++++ .../Transform/ApiEndpointTransformPipeline.cs | 30 ++++- ...xtractDistinctGraphObjectSetsExtensions.cs | 3 +- .../Transform/HandleResponseArgs.cs | 9 +- .../Transform/IApiEndpointTransformJob.cs | 10 ++ .../IApiEndpointTransformPipeline.cs | 2 +- .../Transform/IParseResponse.cs | 10 ++ .../Transform/ITransformModel.cs | 9 ++ .../Transform/ParseResponseArgs.cs | 11 ++ .../Transform/ResponseArgsBase.cs | 16 +++ .../Transform/TransformModelArgs.cs | 15 +++ MIFCore.sln | 14 ++- 17 files changed, 315 insertions(+), 20 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj create mode 100644 MIFCore.Hangfire.APIETL.SqlServer/SqlServerGetDestinationType.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/IGetDestinationType.cs create mode 100644 MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs create mode 100644 MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs create mode 100644 MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs create mode 100644 MIFCore.Hangfire.APIETL/Transform/ITransformModel.cs create mode 100644 MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs create mode 100644 MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs create mode 100644 MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs diff --git a/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj b/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj new file mode 100644 index 0000000..ad0c142 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj @@ -0,0 +1,30 @@ + + + + netstandard2.1 + Maitland Marshall + MAIT DEV + https://github.com/maitlandmarshall/MIFCore + https://github.com/maitlandmarshall/MIFCore + git + true + MIT + Hangfire, Scheduling, Jobs, Simplified, API, ETL, SqlServer + ETL for SqlServer. + 10 + + + + 1.0.0 + + + + rev.$([System.DateTime]::UtcNow.ToString("MddHHmm")) + + + + + + + + diff --git a/MIFCore.Hangfire.APIETL.SqlServer/SqlServerGetDestinationType.cs b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerGetDestinationType.cs new file mode 100644 index 0000000..42bfb0d --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerGetDestinationType.cs @@ -0,0 +1,37 @@ +using MIFCore.Hangfire.APIETL.Load; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + internal class SqlServerGetDestinationType : IGetDestinationType + { + public Task GetDestinationType(ApiEndpointModel apiEndpointModel, string sourceKey, IEnumerable sourceModelTypes) + { + var clrType = sourceModelTypes.FirstOrDefault(y => y != null); + var destinationType = Type.GetTypeCode(clrType) switch + { + TypeCode.Empty => throw new NotImplementedException(), + TypeCode.Object => throw new NotImplementedException(), + TypeCode.DBNull => throw new NotImplementedException(), + TypeCode.Boolean => "bit", + TypeCode.Char => "char(max)", + TypeCode.SByte or TypeCode.Byte => "binary", + TypeCode.Int16 => "smallint", + TypeCode.UInt16 => "smallint", + TypeCode.Int32 or TypeCode.UInt32 => "int", + TypeCode.Int64 or TypeCode.UInt64 => "bigint", + TypeCode.Single => "real", + TypeCode.Double => "float", + TypeCode.Decimal => "decimal(18,4)", + TypeCode.DateTime => "datetimeoffset", + TypeCode.String => "nvarchar(max)", + _ => throw new NotImplementedException(), + }; + + return Task.FromResult(destinationType); + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs index 0db9859..2ee8072 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs @@ -14,20 +14,20 @@ internal class ApiEndpointExtractJob private readonly IApiEndpointRegister apiEndpointRegister; private readonly IBackgroundJobClient backgroundJobClient; private readonly IApiEndpointExtractPipeline endpointExtractPipeline; - private readonly IApiEndpointTransformPipeline transformPipeline; + private readonly IApiEndpointTransformJob endpointTransformJob; public ApiEndpointExtractJob( IHttpClientFactory httpClientFactory, IApiEndpointRegister apiEndpointRegister, IBackgroundJobClient backgroundJobClient, IApiEndpointExtractPipeline endpointExtractPipeline, - IApiEndpointTransformPipeline transformPipeline) + IApiEndpointTransformJob endpointTransformJob) { this.httpClientFactory = httpClientFactory; this.apiEndpointRegister = apiEndpointRegister; this.backgroundJobClient = backgroundJobClient; this.endpointExtractPipeline = endpointExtractPipeline; - this.transformPipeline = transformPipeline; + this.endpointTransformJob = endpointTransformJob; } [DisableIdenticalQueuedItems] @@ -71,7 +71,7 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs } finally { - await this.transformPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData)); + await this.endpointTransformJob.ExecuteTransformationJob(endpoint, apiData); } } diff --git a/MIFCore.Hangfire.APIETL/Load/IGetDestinationType.cs b/MIFCore.Hangfire.APIETL/Load/IGetDestinationType.cs new file mode 100644 index 0000000..4ccce14 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/IGetDestinationType.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public interface IGetDestinationType + { + Task GetDestinationType(ApiEndpointModel apiEndpointModel, string sourceKey, IEnumerable sourceModelTypes); + } +} diff --git a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj index 5083fa7..05e5133 100644 --- a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj +++ b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj @@ -21,13 +21,14 @@ - - - + + + + - + diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs new file mode 100644 index 0000000..a81fad9 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs @@ -0,0 +1,111 @@ +using Humanizer; +using MIFCore.Hangfire.APIETL.Extract; +using MIFCore.Hangfire.APIETL.Load; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + internal class ApiEndpointTransformJob : IApiEndpointTransformJob + { + private readonly IApiEndpointTransformPipeline transformPipeline; + private readonly IEnumerable apiEndpointModels; + private readonly IGetDestinationType getDestinationType; + + public ApiEndpointTransformJob(IApiEndpointTransformPipeline transformPipeline, IEnumerable apiEndpointModels, IGetDestinationType getDestinationType) + { + this.transformPipeline = transformPipeline; + this.apiEndpointModels = apiEndpointModels; + this.getDestinationType = getDestinationType; + } + + public async Task ExecuteTransformationJob(ApiEndpoint endpoint, ApiData apiData) + { + await this.transformPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData)); + var data = await this.transformPipeline.OnParse(new ParseResponseArgs(endpoint, apiData)); + + if (data != null) + { + // Get all the different object sets and group them by the ParentKey + var graphObjectSets = data.ExtractDistinctGraphObjectSets( + new ExtractDistinctGraphObjectSetsExtensions.ExtractDistinctGraphObjectSetsArgs + { + Transform = async (args) => + { + await this.transformPipeline.OnTransformModel(new TransformModelArgs(endpoint, apiData, args)); + } + }) + .GroupBy(y => y.ParentKey) + .ToList(); + + foreach (var set in graphObjectSets) + { + var model = this.apiEndpointModels. + FirstOrDefault(y => y.EndpointName == endpoint.Name && y.InputPath == set.Key); + + if (model is null) + continue; + + await this.AutoMapModelPropertiesFromApi(set, model); + } + } + } + + private async Task AutoMapModelPropertiesFromApi(IGrouping set, ApiEndpointModel apiEndpointModel) + { + var allKeyTypes = set.GetKeyTypes(); + + // Get the validKeyTypes and types from all objects with the same ParentKey + var validKeyTypes = allKeyTypes + + // Filter out the validKeyTypes which have collections + .Where(y => y.Value.Any(y => typeof(IEnumerable).IsAssignableFrom(y)) == false) + + // Filter out the validKeyTypes which only have null values + .Where(y => y.Value.All(y => y is null) == false) + .ToList(); + + // Now we have validKeyTypes which represent the destination table schema + foreach (var (key, types) in validKeyTypes) + { + if (apiEndpointModel.MappedProperties.TryGetValue(key, out var apiEndpointModelProperty) == false) + { + apiEndpointModelProperty = new ApiEndpointModelProperty + { + SourceName = key, + DestinationType = await this.getDestinationType.GetDestinationType(apiEndpointModel, key, types), + SourceType = types, + IsKey = false + }; + + apiEndpointModel.MappedProperties.Add(key, apiEndpointModelProperty); + } + else + { + if (string.IsNullOrWhiteSpace(apiEndpointModelProperty.DestinationType)) + { + apiEndpointModelProperty.DestinationType = await this.getDestinationType.GetDestinationType(apiEndpointModel, key, types); + } + } + + if (string.IsNullOrWhiteSpace(apiEndpointModelProperty.DestinationName)) + { + var underscoreReplacement = "****"; + + apiEndpointModelProperty.DestinationName = apiEndpointModelProperty + .SourceName + + // Pascalize removes underscores, and spaces, so replace them with a placeholder for now + // Add spaces so pascalize can capitalize words separated by underscores properly + .Replace("_", $" {underscoreReplacement} ") + .Pascalize() + + // Since spaces are removed, only the actual replacement characters should be left. + // Conver them back to underscores + .Replace(underscoreReplacement, "_"); + } + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs index 49d2105..2d79fa0 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs @@ -7,10 +7,14 @@ namespace MIFCore.Hangfire.APIETL.Transform internal class ApiEndpointTransformPipeline : IApiEndpointTransformPipeline { private readonly IEnumerable handleResponses; + private readonly IEnumerable parseResponses; + private readonly IEnumerable transformModels; - public ApiEndpointTransformPipeline(IEnumerable handleResponses) + public ApiEndpointTransformPipeline(IEnumerable handleResponses, IEnumerable parseResponses, IEnumerable transformModels) { this.handleResponses = handleResponses; + this.parseResponses = parseResponses; + this.transformModels = transformModels; } public async Task OnHandleResponse(HandleResponseArgs args) @@ -23,5 +27,29 @@ public async Task OnHandleResponse(HandleResponseArgs args) await handleResponse.OnHandleResponse(args); } } + + public async Task>> OnParse(ParseResponseArgs args) + { + var relatedParseResponses = this.parseResponses + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + + foreach (var handleResponse in relatedParseResponses) + { + return await handleResponse.OnParse(args); + } + + return null; + } + + public async Task OnTransformModel(TransformModelArgs args) + { + var relatedTransformModels = this.transformModels + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + + foreach (var transformModel in relatedTransformModels) + { + await transformModel.OnTransformModel(args); + } + } } } diff --git a/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs index 5cf7563..7d4d3b0 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs @@ -1,11 +1,12 @@ using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; namespace MIFCore.Hangfire.APIETL.Transform { public static class ExtractDistinctGraphObjectSetsExtensions { - public delegate void TransformObjectDelegate(TransformObjectArgs args); + public delegate Task TransformObjectDelegate(TransformObjectArgs args); public static IEnumerable ExtractDistinctGraphObjectSets(this IEnumerable> root, ExtractDistinctGraphObjectSetsArgs args = null) { diff --git a/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs b/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs index 9e827fa..30c3e91 100644 --- a/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs +++ b/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs @@ -2,15 +2,10 @@ namespace MIFCore.Hangfire.APIETL.Transform { - public class HandleResponseArgs + public class HandleResponseArgs : ResponseArgsBase { - public HandleResponseArgs(ApiEndpoint endpoint, ApiData apiData) + public HandleResponseArgs(ApiEndpoint endpoint, ApiData apiData) : base(endpoint, apiData) { - this.Endpoint = endpoint; - this.ApiData = apiData; } - - public ApiEndpoint Endpoint { get; } - public ApiData ApiData { get; } } } \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs new file mode 100644 index 0000000..6cb889c --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs @@ -0,0 +1,10 @@ +using MIFCore.Hangfire.APIETL.Extract; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public interface IApiEndpointTransformJob + { + Task ExecuteTransformationJob(ApiEndpoint endpoint, ApiData apiData); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs index 01d45c0..4ede5fe 100644 --- a/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs +++ b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs @@ -1,6 +1,6 @@ namespace MIFCore.Hangfire.APIETL.Transform { - internal interface IApiEndpointTransformPipeline : IHandleResponse + internal interface IApiEndpointTransformPipeline : IHandleResponse, IParseResponse, ITransformModel { } } diff --git a/MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs b/MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs new file mode 100644 index 0000000..bd512c2 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + internal interface IParseResponse : IApiEndpointService + { + Task>> OnParse(ParseResponseArgs args); + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/ITransformModel.cs b/MIFCore.Hangfire.APIETL/Transform/ITransformModel.cs new file mode 100644 index 0000000..1f06373 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ITransformModel.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public interface ITransformModel : IApiEndpointService + { + Task OnTransformModel(TransformModelArgs args); + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs b/MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs new file mode 100644 index 0000000..d059f7d --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs @@ -0,0 +1,11 @@ +using MIFCore.Hangfire.APIETL.Extract; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public class ParseResponseArgs : ResponseArgsBase + { + public ParseResponseArgs(ApiEndpoint endpoint, ApiData apiData) : base(endpoint, apiData) + { + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs b/MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs new file mode 100644 index 0000000..1a4bc00 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs @@ -0,0 +1,16 @@ +using MIFCore.Hangfire.APIETL.Extract; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public abstract class ResponseArgsBase + { + public ResponseArgsBase(ApiEndpoint endpoint, ApiData apiData) + { + this.Endpoint = endpoint; + this.ApiData = apiData; + } + + public ApiData ApiData { get; } + public ApiEndpoint Endpoint { get; } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs b/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs new file mode 100644 index 0000000..d2dbbf5 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs @@ -0,0 +1,15 @@ +using MIFCore.Hangfire.APIETL.Extract; +using static MIFCore.Hangfire.APIETL.Transform.ExtractDistinctGraphObjectSetsExtensions; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public class TransformModelArgs : ResponseArgsBase + { + public TransformModelArgs(ApiEndpoint endpoint, ApiData apiData, TransformObjectArgs transformArgs) : base(endpoint, apiData) + { + this.TransformArgs = transformArgs; + } + + public TransformObjectArgs TransformArgs { get; } + } +} diff --git a/MIFCore.sln b/MIFCore.sln index 41da9b6..3c10443 100644 --- a/MIFCore.sln +++ b/MIFCore.sln @@ -17,11 +17,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MIFCore.Hangfire", "MIFCore EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MIFCore.Common", "MIFCore.Common\MIFCore.Common.csproj", "{A583D632-B2AD-4914-97EF-426AAF4725CB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MIFCore.Hangfire.APIETL", "MIFCore.Hangfire.APIETL\MIFCore.Hangfire.APIETL.csproj", "{0679FE87-1E69-4CE8-899B-BDC15340FDCB}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MIFCore.Hangfire.APIETL", "MIFCore.Hangfire.APIETL\MIFCore.Hangfire.APIETL.csproj", "{0679FE87-1E69-4CE8-899B-BDC15340FDCB}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "APIETL", "APIETL", "{86C67505-D831-4A34-829E-D12EB7E2BDA7}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MIFCore.Hangfire.APIETL.Tests", "MIFCore.Hangfire.APIETL.Tests\MIFCore.Hangfire.APIETL.Tests.csproj", "{33568C51-42C1-41A6-8CCF-0F62FF905378}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MIFCore.Hangfire.APIETL.Tests", "MIFCore.Hangfire.APIETL.Tests\MIFCore.Hangfire.APIETL.Tests.csproj", "{33568C51-42C1-41A6-8CCF-0F62FF905378}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MIFCore.Hangfire.APIETL.SqlServer", "MIFCore.Hangfire.APIETL.SqlServer\MIFCore.Hangfire.APIETL.SqlServer.csproj", "{0BF5DF42-6060-457B-87CB-2C2DFB557EE7}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sources", "Sources", "{9E8E78A2-0ED5-46F7-A89E-5212FC2B5250}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -61,6 +65,10 @@ Global {33568C51-42C1-41A6-8CCF-0F62FF905378}.Debug|Any CPU.Build.0 = Debug|Any CPU {33568C51-42C1-41A6-8CCF-0F62FF905378}.Release|Any CPU.ActiveCfg = Release|Any CPU {33568C51-42C1-41A6-8CCF-0F62FF905378}.Release|Any CPU.Build.0 = Release|Any CPU + {0BF5DF42-6060-457B-87CB-2C2DFB557EE7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0BF5DF42-6060-457B-87CB-2C2DFB557EE7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0BF5DF42-6060-457B-87CB-2C2DFB557EE7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0BF5DF42-6060-457B-87CB-2C2DFB557EE7}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -70,6 +78,8 @@ Global {7FA32F14-5D33-4F0D-9F38-DF19D2C7F6E2} = {E057276C-548A-4190-AF5A-9A92E08268A1} {0679FE87-1E69-4CE8-899B-BDC15340FDCB} = {86C67505-D831-4A34-829E-D12EB7E2BDA7} {33568C51-42C1-41A6-8CCF-0F62FF905378} = {86C67505-D831-4A34-829E-D12EB7E2BDA7} + {0BF5DF42-6060-457B-87CB-2C2DFB557EE7} = {9E8E78A2-0ED5-46F7-A89E-5212FC2B5250} + {9E8E78A2-0ED5-46F7-A89E-5212FC2B5250} = {86C67505-D831-4A34-829E-D12EB7E2BDA7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {70D42A47-49DB-4784-8B40-89988134D6C8} From 08a7002a516c20b0d4b4cc18c4a9bf0add401742 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Thu, 1 Dec 2022 12:10:08 +1000 Subject: [PATCH 29/47] Refactor interface method name --- MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs | 5 +++-- MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs | 2 +- .../Transform/IApiEndpointTransformJob.cs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs index 2ee8072..8a94e24 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs @@ -59,11 +59,12 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs var apiData = await this.ExecuteRequest(endpoint, httpClient, request, extractArgs); var nextRequestData = await this.endpointExtractPipeline.OnPrepareNextRequest(new PrepareNextRequestArgs(endpoint: endpoint, apiData: apiData, data: extractArgs.RequestData)); + var isLastRequest = nextRequestData.Keys.Any() == false; try { // If OnPrepareNextRequest returns an empty dict, then we're done with this endpoint. - if (nextRequestData.Keys.Any() == false) + if (isLastRequest) return; // Otherwise we need to schedule another job to continue extracting this endpoint. @@ -71,7 +72,7 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs } finally { - await this.endpointTransformJob.ExecuteTransformationJob(endpoint, apiData); + await this.endpointTransformJob.Transform(endpoint, apiData); } } diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs index a81fad9..e90c6d7 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs @@ -20,7 +20,7 @@ public ApiEndpointTransformJob(IApiEndpointTransformPipeline transformPipeline, this.getDestinationType = getDestinationType; } - public async Task ExecuteTransformationJob(ApiEndpoint endpoint, ApiData apiData) + public async Task Transform(ApiEndpoint endpoint, ApiData apiData) { await this.transformPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData)); var data = await this.transformPipeline.OnParse(new ParseResponseArgs(endpoint, apiData)); diff --git a/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs index 6cb889c..2646136 100644 --- a/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs +++ b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs @@ -5,6 +5,6 @@ namespace MIFCore.Hangfire.APIETL.Transform { public interface IApiEndpointTransformJob { - Task ExecuteTransformationJob(ApiEndpoint endpoint, ApiData apiData); + Task Transform(ApiEndpoint endpoint, ApiData apiData); } } \ No newline at end of file From 06e7cf05a3508aef4ec0df61f7ab52d73c1b784a Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Thu, 1 Dec 2022 12:10:48 +1000 Subject: [PATCH 30/47] Add ICreateDestination and implement default sql server create destination for creating tables automatically --- .../DefaultSqlServerCreateDestination.cs | 41 +++++++++++++++++++ .../MIFCore.Hangfire.APIETL.SqlServer.csproj | 5 +++ .../Load/ApiEndpointLoadJob.cs | 10 +++++ .../Load/CreateDestinationArgs.cs | 12 ++++++ .../Load/ICreateDestination.cs | 9 ++++ 5 files changed, 77 insertions(+) create mode 100644 MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/ICreateDestination.cs diff --git a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs new file mode 100644 index 0000000..cb2da30 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs @@ -0,0 +1,41 @@ +using ETLBox.Connection; +using ETLBox.ControlFlow; +using ETLBox.ControlFlow.Tasks; +using MIFCore.Hangfire.APIETL.Load; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + internal class DefaultSqlServerCreateDestination : ICreateDestination + { + private readonly HangfireConfig hangfireConfig; + + public DefaultSqlServerCreateDestination(HangfireConfig hangfireConfig) + { + this.hangfireConfig = hangfireConfig; + } + + public async Task OnCreateDestination(CreateDestinationArgs args) + { + var apiEndpointModel = args.ApiEndpointModel; + var connMan = new SqlConnectionManager(this.hangfireConfig.ConnectionString); + + if (IfTableOrViewExistsTask.IsExisting(connMan, apiEndpointModel.DestinationName) == false) + { + var tableDefinition = new TableDefinition( + name: apiEndpointModel.DestinationName, + columns: apiEndpointModel.MappedProperties.Select(kvp => + { + var (key, value) = kvp; + return new TableColumn(value.DestinationName, value.DestinationType, value.IsKey == false, value.IsKey, false); + }).ToList()); + + await Task.Run(() => + { + CreateTableTask.Create(connMan, tableDefinition); + }); + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj b/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj index ad0c142..bd622e9 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj +++ b/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj @@ -22,6 +22,11 @@ rev.$([System.DateTime]::UtcNow.ToString("MddHHmm")) + + + + + diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs new file mode 100644 index 0000000..785b8ef --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace MIFCore.Hangfire.APIETL.Load +{ + internal class ApiEndpointLoadJob + { + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs b/MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs new file mode 100644 index 0000000..5f34a80 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs @@ -0,0 +1,12 @@ +namespace MIFCore.Hangfire.APIETL.Load +{ + public class CreateDestinationArgs + { + public CreateDestinationArgs(ApiEndpointModel apiEndpointModel) + { + this.ApiEndpointModel = apiEndpointModel; + } + + public ApiEndpointModel ApiEndpointModel { get; } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Load/ICreateDestination.cs b/MIFCore.Hangfire.APIETL/Load/ICreateDestination.cs new file mode 100644 index 0000000..5ef2f2c --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ICreateDestination.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public interface ICreateDestination : IApiEndpointService + { + Task OnCreateDestination(CreateDestinationArgs args); + } +} From 4f95c64de9ce1ee43fc9be42dce13d036cf29bbf Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Thu, 1 Dec 2022 12:35:02 +1000 Subject: [PATCH 31/47] Create ApiEndpointLoadJob --- .../IApiEndpointServiceExtensions.cs | 9 +++++++ .../Load/ApiEndpointLoadJob.cs | 21 ++++++++++++--- .../Load/ApiEndpointLoadPipeline.cs | 27 +++++++++++++++++++ .../Load/CreateDestinationArgs.cs | 4 ++- .../Load/IApiEndpointLoadJob.cs | 10 +++++++ .../Load/IApiEndpointLoadPipeline.cs | 6 +++++ .../Transform/ApiEndpointTransformJob.cs | 16 ++++++++--- 7 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadPipeline.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadJob.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs b/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs index a00ed52..47b121d 100644 --- a/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs +++ b/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs @@ -6,6 +6,15 @@ namespace MIFCore.Hangfire.APIETL { public static class IApiEndpointServiceExtensions { + public static bool IsDefaultResponder(this IApiEndpointService apiEndpointService) + { + var type = apiEndpointService.GetType(); + var endpointNameAttributes = type.GetCustomAttributes(); + var endpointSelectorAttributes = type.GetCustomAttributes(); + + return endpointNameAttributes.Any() == false && endpointSelectorAttributes.Any() == false; + } + public static bool RespondsToEndpointName(this IApiEndpointService apiEndpointService, string endpointName) { var type = apiEndpointService.GetType(); diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs index 785b8ef..533eb58 100644 --- a/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs @@ -1,10 +1,23 @@ -using System; -using System.Collections.Generic; -using System.Text; +using System.Collections.Generic; +using System.Threading.Tasks; namespace MIFCore.Hangfire.APIETL.Load { - internal class ApiEndpointLoadJob + internal class ApiEndpointLoadJob : IApiEndpointLoadJob { + private readonly IApiEndpointLoadPipeline loadPipeline; + + public ApiEndpointLoadJob(IApiEndpointLoadPipeline loadPipeline) + { + this.loadPipeline = loadPipeline; + } + + public async Task Load(ApiEndpoint apiEndpoint, ApiEndpointModel model, List> dataToLoad) + { + // Ensure the destination is created + await this.loadPipeline.OnCreateDestination(new CreateDestinationArgs(apiEndpoint, model)); + + + } } } diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadPipeline.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadPipeline.cs new file mode 100644 index 0000000..6dc6d90 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadPipeline.cs @@ -0,0 +1,27 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + internal class ApiEndpointLoadPipeline : IApiEndpointLoadPipeline + { + private readonly IEnumerable createDestinations; + + public ApiEndpointLoadPipeline(IEnumerable createDestinations) + { + this.createDestinations = createDestinations; + } + + public async Task OnCreateDestination(CreateDestinationArgs args) + { + var relatedCreateDestination = this.createDestinations + .LastOrDefault(y => y.RespondsToEndpointName(args.ApiEndpoint.Name) || y.IsDefaultResponder()); + + if (relatedCreateDestination is null) + return; + + await relatedCreateDestination.OnCreateDestination(args); + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs b/MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs index 5f34a80..47d3681 100644 --- a/MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs +++ b/MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs @@ -2,11 +2,13 @@ { public class CreateDestinationArgs { - public CreateDestinationArgs(ApiEndpointModel apiEndpointModel) + public CreateDestinationArgs(ApiEndpoint apiEndpoint, ApiEndpointModel apiEndpointModel) { + this.ApiEndpoint = apiEndpoint; this.ApiEndpointModel = apiEndpointModel; } + public ApiEndpoint ApiEndpoint { get; } public ApiEndpointModel ApiEndpointModel { get; } } } \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadJob.cs b/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadJob.cs new file mode 100644 index 0000000..490165e --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadJob.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public interface IApiEndpointLoadJob + { + Task Load(ApiEndpoint apiEndpoint, ApiEndpointModel model, List> dataToLoad); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs b/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs new file mode 100644 index 0000000..3045d6a --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs @@ -0,0 +1,6 @@ +namespace MIFCore.Hangfire.APIETL.Load +{ + internal interface IApiEndpointLoadPipeline : ICreateDestination + { + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs index e90c6d7..a60e993 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs @@ -12,12 +12,18 @@ internal class ApiEndpointTransformJob : IApiEndpointTransformJob private readonly IApiEndpointTransformPipeline transformPipeline; private readonly IEnumerable apiEndpointModels; private readonly IGetDestinationType getDestinationType; + private readonly IApiEndpointLoadJob apiEndpointLoadJob; - public ApiEndpointTransformJob(IApiEndpointTransformPipeline transformPipeline, IEnumerable apiEndpointModels, IGetDestinationType getDestinationType) + public ApiEndpointTransformJob( + IApiEndpointTransformPipeline transformPipeline, + IEnumerable apiEndpointModels, + IGetDestinationType getDestinationType, + IApiEndpointLoadJob apiEndpointLoadJob) { this.transformPipeline = transformPipeline; this.apiEndpointModels = apiEndpointModels; this.getDestinationType = getDestinationType; + this.apiEndpointLoadJob = apiEndpointLoadJob; } public async Task Transform(ApiEndpoint endpoint, ApiData apiData) @@ -47,12 +53,16 @@ public async Task Transform(ApiEndpoint endpoint, ApiData apiData) if (model is null) continue; - await this.AutoMapModelPropertiesFromApi(set, model); + await this.AutoMapModelPropertiesGraphObjectSet(set, model); + + var dataToLoad = set.SelectMany(y => y.Objects).ToList(); + + await this.apiEndpointLoadJob.Load(endpoint, model, dataToLoad); } } } - private async Task AutoMapModelPropertiesFromApi(IGrouping set, ApiEndpointModel apiEndpointModel) + private async Task AutoMapModelPropertiesGraphObjectSet(IGrouping set, ApiEndpointModel apiEndpointModel) { var allKeyTypes = set.GetKeyTypes(); From f3748cd769cb03c14e905189780d356c25183eb4 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Thu, 1 Dec 2022 12:54:08 +1000 Subject: [PATCH 32/47] Add ApiEndpointLoadJob and default SqlServer implementation for loading into database with ETLBox --- .../DefaultSqlServerCreateDestination.cs | 23 +++---- .../DefaultSqlServerLoadData.cs | 63 +++++++++++++++++++ .../SqlConnectionManagerFactory.cs | 19 ++++++ .../SqlServerConfig.cs | 7 +++ .../TableDefinitionFactory.cs | 36 +++++++++++ .../Load/ApiEndpointLoadJob.cs | 3 +- .../Load/ApiEndpointLoadPipeline.cs | 16 ++++- .../Load/IApiEndpointLoadPipeline.cs | 2 +- MIFCore.Hangfire.APIETL/Load/ILoadData.cs | 9 +++ MIFCore.Hangfire.APIETL/Load/LoadDataArgs.cs | 18 ++++++ .../Transform/TransformModelArgs.cs | 4 +- 11 files changed, 180 insertions(+), 20 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs create mode 100644 MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs create mode 100644 MIFCore.Hangfire.APIETL.SqlServer/SqlServerConfig.cs create mode 100644 MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/ILoadData.cs create mode 100644 MIFCore.Hangfire.APIETL/Load/LoadDataArgs.cs diff --git a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs index cb2da30..94587af 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs +++ b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs @@ -1,35 +1,28 @@ -using ETLBox.Connection; -using ETLBox.ControlFlow; -using ETLBox.ControlFlow.Tasks; +using ETLBox.ControlFlow.Tasks; using MIFCore.Hangfire.APIETL.Load; -using System.Linq; using System.Threading.Tasks; namespace MIFCore.Hangfire.APIETL.SqlServer { internal class DefaultSqlServerCreateDestination : ICreateDestination { - private readonly HangfireConfig hangfireConfig; + private readonly SqlConnectionManagerFactory sqlConnectionManagerFactory; + private readonly TableDefinitionFactory tableDefinitionFactory; - public DefaultSqlServerCreateDestination(HangfireConfig hangfireConfig) + public DefaultSqlServerCreateDestination(SqlConnectionManagerFactory sqlConnectionManagerFactory, TableDefinitionFactory tableDefinitionFactory) { - this.hangfireConfig = hangfireConfig; + this.sqlConnectionManagerFactory = sqlConnectionManagerFactory; + this.tableDefinitionFactory = tableDefinitionFactory; } public async Task OnCreateDestination(CreateDestinationArgs args) { var apiEndpointModel = args.ApiEndpointModel; - var connMan = new SqlConnectionManager(this.hangfireConfig.ConnectionString); + var connMan = this.sqlConnectionManagerFactory.Create(); if (IfTableOrViewExistsTask.IsExisting(connMan, apiEndpointModel.DestinationName) == false) { - var tableDefinition = new TableDefinition( - name: apiEndpointModel.DestinationName, - columns: apiEndpointModel.MappedProperties.Select(kvp => - { - var (key, value) = kvp; - return new TableColumn(value.DestinationName, value.DestinationType, value.IsKey == false, value.IsKey, false); - }).ToList()); + var tableDefinition = await this.tableDefinitionFactory.Create(apiEndpointModel); await Task.Run(() => { diff --git a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs new file mode 100644 index 0000000..dc102b8 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs @@ -0,0 +1,63 @@ +using ETLBox.DataFlow; +using ETLBox.DataFlow.Connectors; +using MIFCore.Hangfire.APIETL.Load; +using System.Collections.Generic; +using System.Dynamic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + internal class DefaultSqlServerLoadData : ILoadData + { + private readonly SqlConnectionManagerFactory sqlConnectionManagerFactory; + private readonly TableDefinitionFactory tableDefinitionFactory; + + public DefaultSqlServerLoadData(SqlConnectionManagerFactory sqlConnectionManagerFactory, TableDefinitionFactory tableDefinitionFactory) + { + this.sqlConnectionManagerFactory = sqlConnectionManagerFactory; + this.tableDefinitionFactory = tableDefinitionFactory; + } + + public async Task OnLoadData(LoadDataArgs args) + { + var apiEndpointModel = args.ApiEndpointModel; + var tableDefinition = await this.tableDefinitionFactory.Create(args.ApiEndpointModel.DestinationName); + var connMan = this.sqlConnectionManagerFactory.Create(); + + var source = new MemorySource(this.GetDataToLoad(args)); + var merge = new DbMerge(connMan, apiEndpointModel.DestinationName) + { + DestinationTableDefinition = tableDefinition, + MergeMode = MergeMode.InsertsAndUpdates, + ColumnMapping = apiEndpointModel + .MappedProperties + .Values + .Select(y => new ColumnMap + { + PropertyName = y.SourceName, + DbColumnName = y.DestinationName + }) + .ToList(), + MergeProperties = + { + IdColumns = apiEndpointModel + .MappedProperties + .Values + .Where(y => y.IsKey) + .Select(y => new IdColumn { IdPropertyName = y.SourceName }) + .ToList() + } + }; + + source.LinkTo(merge); + await source.ExecuteAsync(); + } + + private List GetDataToLoad(LoadDataArgs args) + { + var data = args.DataToLoad; + return data.Cast().ToList(); + } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs new file mode 100644 index 0000000..9fe9806 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs @@ -0,0 +1,19 @@ +using ETLBox.Connection; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + internal class SqlConnectionManagerFactory + { + private readonly SqlServerConfig sqlServerConfig; + + public SqlConnectionManagerFactory(SqlServerConfig sqlServerConfig) + { + this.sqlServerConfig = sqlServerConfig; + } + + public SqlConnectionManager Create() + { + return new SqlConnectionManager(this.sqlServerConfig.DestinationConnectionString); + } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/SqlServerConfig.cs b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerConfig.cs new file mode 100644 index 0000000..2f7066e --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerConfig.cs @@ -0,0 +1,7 @@ +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + public class SqlServerConfig + { + public string DestinationConnectionString { get; set; } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs new file mode 100644 index 0000000..4ead781 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs @@ -0,0 +1,36 @@ +using ETLBox.ControlFlow; +using MIFCore.Hangfire.APIETL.Load; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + internal class TableDefinitionFactory + { + private readonly SqlConnectionManagerFactory sqlConnectionManagerFactory; + + public TableDefinitionFactory(SqlConnectionManagerFactory sqlConnectionManagerFactory) + { + this.sqlConnectionManagerFactory = sqlConnectionManagerFactory; + } + + public async Task Create(string tableName) + { + var connMan = this.sqlConnectionManagerFactory.Create(); + return await Task.Run(() => TableDefinition.FromTableName(connMan, tableName)); + } + + public Task Create(ApiEndpointModel apiEndpointModel) + { + var tableDefinition = new TableDefinition( + name: apiEndpointModel.DestinationName, + columns: apiEndpointModel.MappedProperties.Select(kvp => + { + var (key, value) = kvp; + return new TableColumn(value.DestinationName, value.DestinationType, value.IsKey == false, value.IsKey, false); + }).ToList()); + + return Task.FromResult(tableDefinition); + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs index 533eb58..bd369c5 100644 --- a/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs @@ -17,7 +17,8 @@ public async Task Load(ApiEndpoint apiEndpoint, ApiEndpointModel model, List createDestinations; + private readonly IEnumerable loadDatas; - public ApiEndpointLoadPipeline(IEnumerable createDestinations) + public ApiEndpointLoadPipeline(IEnumerable createDestinations, IEnumerable loadDatas) { this.createDestinations = createDestinations; + this.loadDatas = loadDatas; } public async Task OnCreateDestination(CreateDestinationArgs args) @@ -23,5 +25,17 @@ public async Task OnCreateDestination(CreateDestinationArgs args) await relatedCreateDestination.OnCreateDestination(args); } + + public async Task OnLoadData(LoadDataArgs args) + { + var relatedLoadDatas = this.loadDatas + .Where(y => y.RespondsToEndpointName(args.ApiEndpoint.Name) || y.IsDefaultResponder()) + .ToList(); + + foreach (var r in relatedLoadDatas) + { + await r.OnLoadData(args); + } + } } } diff --git a/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs b/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs index 3045d6a..8745a97 100644 --- a/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs +++ b/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs @@ -1,6 +1,6 @@ namespace MIFCore.Hangfire.APIETL.Load { - internal interface IApiEndpointLoadPipeline : ICreateDestination + internal interface IApiEndpointLoadPipeline : ICreateDestination, ILoadData { } } \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Load/ILoadData.cs b/MIFCore.Hangfire.APIETL/Load/ILoadData.cs new file mode 100644 index 0000000..c023eea --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ILoadData.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public interface ILoadData : IApiEndpointService + { + public Task OnLoadData(LoadDataArgs args); + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/LoadDataArgs.cs b/MIFCore.Hangfire.APIETL/Load/LoadDataArgs.cs new file mode 100644 index 0000000..a855d93 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/LoadDataArgs.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public class LoadDataArgs + { + public LoadDataArgs(ApiEndpoint apiEndpoint, ApiEndpointModel apiEndpointModel, List> dataToLoad) + { + this.ApiEndpoint = apiEndpoint; + this.ApiEndpointModel = apiEndpointModel; + this.DataToLoad = dataToLoad; + } + + public ApiEndpoint ApiEndpoint { get; } + public ApiEndpointModel ApiEndpointModel { get; } + public List> DataToLoad { get; } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs b/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs index d2dbbf5..ac1b7f7 100644 --- a/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs +++ b/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs @@ -7,9 +7,9 @@ public class TransformModelArgs : ResponseArgsBase { public TransformModelArgs(ApiEndpoint endpoint, ApiData apiData, TransformObjectArgs transformArgs) : base(endpoint, apiData) { - this.TransformArgs = transformArgs; + this.TransformObjectArgs = transformArgs; } - public TransformObjectArgs TransformArgs { get; } + public TransformObjectArgs TransformObjectArgs { get; } } } From c917b9a71d3b7ec30aa3d428e9c700a079675cf6 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Thu, 1 Dec 2022 13:26:58 +1000 Subject: [PATCH 33/47] Add Service Collection helper for SqlServer --- .../DefaultSqlServerCreateDestination.cs | 6 +-- .../ISqlConnectionManagerFactory.cs | 9 +++++ .../ITableDefinitionFactory.cs | 12 ++++++ .../SqlConnectionManagerFactory.cs | 2 +- .../SqlServerServiceCollectionExtensions.cs | 39 +++++++++++++++++++ .../TableDefinitionFactory.cs | 2 +- .../Transform/TransformModelArgs.cs | 4 +- 7 files changed, 67 insertions(+), 7 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL.SqlServer/ISqlConnectionManagerFactory.cs create mode 100644 MIFCore.Hangfire.APIETL.SqlServer/ITableDefinitionFactory.cs create mode 100644 MIFCore.Hangfire.APIETL.SqlServer/SqlServerServiceCollectionExtensions.cs diff --git a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs index 94587af..91f927d 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs +++ b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs @@ -6,10 +6,10 @@ namespace MIFCore.Hangfire.APIETL.SqlServer { internal class DefaultSqlServerCreateDestination : ICreateDestination { - private readonly SqlConnectionManagerFactory sqlConnectionManagerFactory; - private readonly TableDefinitionFactory tableDefinitionFactory; + private readonly ISqlConnectionManagerFactory sqlConnectionManagerFactory; + private readonly ITableDefinitionFactory tableDefinitionFactory; - public DefaultSqlServerCreateDestination(SqlConnectionManagerFactory sqlConnectionManagerFactory, TableDefinitionFactory tableDefinitionFactory) + public DefaultSqlServerCreateDestination(ISqlConnectionManagerFactory sqlConnectionManagerFactory, ITableDefinitionFactory tableDefinitionFactory) { this.sqlConnectionManagerFactory = sqlConnectionManagerFactory; this.tableDefinitionFactory = tableDefinitionFactory; diff --git a/MIFCore.Hangfire.APIETL.SqlServer/ISqlConnectionManagerFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/ISqlConnectionManagerFactory.cs new file mode 100644 index 0000000..0864a4f --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/ISqlConnectionManagerFactory.cs @@ -0,0 +1,9 @@ +using ETLBox.Connection; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + public interface ISqlConnectionManagerFactory + { + SqlConnectionManager Create(); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL.SqlServer/ITableDefinitionFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/ITableDefinitionFactory.cs new file mode 100644 index 0000000..3e93c61 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/ITableDefinitionFactory.cs @@ -0,0 +1,12 @@ +using ETLBox.ControlFlow; +using MIFCore.Hangfire.APIETL.Load; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + public interface ITableDefinitionFactory + { + Task Create(ApiEndpointModel apiEndpointModel); + Task Create(string tableName); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs index 9fe9806..02d8967 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs +++ b/MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs @@ -2,7 +2,7 @@ namespace MIFCore.Hangfire.APIETL.SqlServer { - internal class SqlConnectionManagerFactory + internal class SqlConnectionManagerFactory : ISqlConnectionManagerFactory { private readonly SqlServerConfig sqlServerConfig; diff --git a/MIFCore.Hangfire.APIETL.SqlServer/SqlServerServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerServiceCollectionExtensions.cs new file mode 100644 index 0000000..1b61955 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerServiceCollectionExtensions.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using MIFCore.Hangfire.APIETL.Load; +using System; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + public static class SqlServerServiceCollectionExtensions + { + public static IServiceCollection AddApiEndpointsSqlServerDestination(this IServiceCollection serviceDescriptors, Action configAction = null) + { + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); + + serviceDescriptors.AddSingleton((svc) => + { + var sqlServerConfig = new SqlServerConfig(); + + if (configAction != null) + { + configAction(sqlServerConfig); + } + else + { + // If no configAction is supplied, default to the Hangfire config. + var hangfireConfig = svc.GetRequiredService(); + sqlServerConfig.DestinationConnectionString = hangfireConfig.ConnectionString; + } + + return sqlServerConfig; + }); + + return serviceDescriptors; + } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs index 4ead781..9187a46 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs +++ b/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs @@ -5,7 +5,7 @@ namespace MIFCore.Hangfire.APIETL.SqlServer { - internal class TableDefinitionFactory + internal class TableDefinitionFactory : ITableDefinitionFactory { private readonly SqlConnectionManagerFactory sqlConnectionManagerFactory; diff --git a/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs b/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs index ac1b7f7..b18a44c 100644 --- a/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs +++ b/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs @@ -7,9 +7,9 @@ public class TransformModelArgs : ResponseArgsBase { public TransformModelArgs(ApiEndpoint endpoint, ApiData apiData, TransformObjectArgs transformArgs) : base(endpoint, apiData) { - this.TransformObjectArgs = transformArgs; + this.Transform = transformArgs; } - public TransformObjectArgs TransformObjectArgs { get; } + public TransformObjectArgs Transform { get; } } } From 242f52365979d3906713b9b5a8152989318b4487 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 5 Dec 2022 10:21:55 +1000 Subject: [PATCH 34/47] Add services to service collection --- .../DefaultSqlServerLoadData.cs | 6 +-- .../TableDefinitionFactory.cs | 4 +- .../APIETLServiceCollectionExtensions.cs | 38 ++++++++++++++++--- .../ApiEndpointRegister.cs | 2 +- .../Extract/ApiEndpointExtractJob.cs | 16 +------- .../Extract/IApiEndpointExtractJob.cs | 23 +++++++++++ .../Transform/IParseResponse.cs | 2 +- 7 files changed, 64 insertions(+), 27 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractJob.cs diff --git a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs index dc102b8..27e66a4 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs +++ b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs @@ -10,10 +10,10 @@ namespace MIFCore.Hangfire.APIETL.SqlServer { internal class DefaultSqlServerLoadData : ILoadData { - private readonly SqlConnectionManagerFactory sqlConnectionManagerFactory; - private readonly TableDefinitionFactory tableDefinitionFactory; + private readonly ISqlConnectionManagerFactory sqlConnectionManagerFactory; + private readonly ITableDefinitionFactory tableDefinitionFactory; - public DefaultSqlServerLoadData(SqlConnectionManagerFactory sqlConnectionManagerFactory, TableDefinitionFactory tableDefinitionFactory) + public DefaultSqlServerLoadData(ISqlConnectionManagerFactory sqlConnectionManagerFactory, ITableDefinitionFactory tableDefinitionFactory) { this.sqlConnectionManagerFactory = sqlConnectionManagerFactory; this.tableDefinitionFactory = tableDefinitionFactory; diff --git a/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs index 9187a46..a7832ee 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs +++ b/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs @@ -7,9 +7,9 @@ namespace MIFCore.Hangfire.APIETL.SqlServer { internal class TableDefinitionFactory : ITableDefinitionFactory { - private readonly SqlConnectionManagerFactory sqlConnectionManagerFactory; + private readonly ISqlConnectionManagerFactory sqlConnectionManagerFactory; - public TableDefinitionFactory(SqlConnectionManagerFactory sqlConnectionManagerFactory) + public TableDefinitionFactory(ISqlConnectionManagerFactory sqlConnectionManagerFactory) { this.sqlConnectionManagerFactory = sqlConnectionManagerFactory; } diff --git a/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs index 5f7ec6d..051ae7c 100644 --- a/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using MIFCore.Hangfire.APIETL.Extract; +using MIFCore.Hangfire.APIETL.Load; using MIFCore.Hangfire.APIETL.Transform; using System; using System.Collections.Generic; @@ -20,29 +21,43 @@ public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollectio .GetTypes() .Where(y => y.GetCustomAttributes().Any() - || y.GetCustomAttributes().Any()); + || y.GetCustomAttributes().Any() + || y.GetCustomAttributes().Any()); return serviceDescriptors.AddApiEndpointsToExtract(endpoints); } - public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollection serviceDescriptors, IEnumerable endpoints) + public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollection serviceDescriptors, IEnumerable types) { // Register the services used to register jobs and create ApiEndpoint definitions serviceDescriptors.TryAddSingleton(); serviceDescriptors.TryAddTransient(); + + // Register the extract jobs & pipelines serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddScoped(); + + // Register the transform jobs & pipelines serviceDescriptors.TryAddTransient(); - serviceDescriptors.TryAddScoped(); + serviceDescriptors.TryAddScoped(); + + // Register the load jobs & pipelines + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddScoped(); + - foreach (var t in endpoints) + foreach (var t in types) { var endpointNameAttributes = t.GetCustomAttributes(); var endpointSelectorAttribute = t.GetCustomAttributes(); + var endpointModelAttributes = t.GetCustomAttributes(); if (endpointNameAttributes.Any() == false - && endpointSelectorAttribute.Any() == false) - throw new ArgumentException($"The type {t.FullName} does not have an {nameof(ApiEndpointAttribute)} or {nameof(ApiEndpointSelectorAttribute)} attribute."); + && endpointSelectorAttribute.Any() == false + && endpointModelAttributes.Any() == false) + throw new ArgumentException($"The type {t.FullName} does not have an {nameof(ApiEndpointAttribute)}, {nameof(ApiEndpointSelectorAttribute)} or {nameof(ApiEndpointModel)} attributes."); + // Register the extract services if (typeof(IDefineEndpoints).IsAssignableFrom(t)) serviceDescriptors.AddScoped(typeof(IDefineEndpoints), t); @@ -52,9 +67,20 @@ public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollectio if (typeof(IPrepareNextRequest).IsAssignableFrom(t)) serviceDescriptors.AddScoped(typeof(IPrepareNextRequest), t); + // Register the transform services if (typeof(IHandleResponse).IsAssignableFrom(t)) serviceDescriptors.AddScoped(typeof(IHandleResponse), t); + if (typeof(IParseResponse).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(IParseResponse), t); + + if (typeof(ITransformModel).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(ITransformModel), t); + + // Register the load services + if (typeof(ILoadData).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(ILoadData), t); + // Register the endpoint name attribute, so an ApiEndpoint is created from it if (endpointNameAttributes.Any()) { diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs index 8641eee..f31c5cb 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs @@ -24,7 +24,7 @@ public IApiEndpointRegister Register(ApiEndpoint endpoint) { this.endpoints.Add(endpoint.Name, endpoint); - this.recurringJobManager.AddOrUpdate( + this.recurringJobManager.AddOrUpdate( endpoint.JobName, job => job.Extract(endpoint.Name, null), Cron.Daily() diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs index 8a94e24..3106e9d 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs @@ -8,7 +8,7 @@ namespace MIFCore.Hangfire.APIETL.Extract { - internal class ApiEndpointExtractJob + internal class ApiEndpointExtractJob : IApiEndpointExtractJob { private readonly IHttpClientFactory httpClientFactory; private readonly IApiEndpointRegister apiEndpointRegister; @@ -68,7 +68,7 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs return; // Otherwise we need to schedule another job to continue extracting this endpoint. - this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.Id))); + this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.Id))); } finally { @@ -114,17 +114,5 @@ private async Task CreateRequest(Uri baseAddress, ApiEndpoin return request; } - - public class ExtractArgs - { - public ExtractArgs(IDictionary requestData, Guid? parentApiDataId) - { - this.RequestData = requestData; - this.ParentApiDataId = parentApiDataId; - } - - public IDictionary RequestData { get; set; } - public Guid? ParentApiDataId { get; set; } - } } } diff --git a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractJob.cs new file mode 100644 index 0000000..8b39eb6 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractJob.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Extract +{ + public interface IApiEndpointExtractJob + { + Task Extract(string endpointName, ExtractArgs extractArgs = null); + } + + public class ExtractArgs + { + public ExtractArgs(IDictionary requestData, Guid? parentApiDataId) + { + this.RequestData = requestData; + this.ParentApiDataId = parentApiDataId; + } + + public IDictionary RequestData { get; set; } + public Guid? ParentApiDataId { get; set; } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs b/MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs index bd512c2..1aec00b 100644 --- a/MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs +++ b/MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs @@ -3,7 +3,7 @@ namespace MIFCore.Hangfire.APIETL.Transform { - internal interface IParseResponse : IApiEndpointService + public interface IParseResponse : IApiEndpointService { Task>> OnParse(ParseResponseArgs args); } From 0e728d234e8feb695bf1f8d9c56b5031b41f377f Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 5 Dec 2022 10:56:51 +1000 Subject: [PATCH 35/47] Use ApiEndpointAttribute or ApiEndpointSelectorAttribute to define transformations and load serivces --- .../APIETLServiceCollectionExtensions.cs | 6 ++++++ MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs | 7 +++++++ .../ApiEndpointSelectorAttribute.cs | 7 +++++++ .../IApiEndpointServiceExtensions.cs | 12 +++++++----- .../Transform/ApiEndpointTransformPipeline.cs | 2 +- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs index 051ae7c..b8f7e4e 100644 --- a/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs @@ -81,6 +81,12 @@ public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollectio if (typeof(ILoadData).IsAssignableFrom(t)) serviceDescriptors.AddScoped(typeof(ILoadData), t); + if (endpointModelAttributes.Any()) + { + var endpointModel = t.GetApiEndpointModel(); + serviceDescriptors.AddSingleton(endpointModel); + } + // Register the endpoint name attribute, so an ApiEndpoint is created from it if (endpointNameAttributes.Any()) { diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs b/MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs index 7b065e4..534e446 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs @@ -10,7 +10,14 @@ public ApiEndpointAttribute(string endpointName) this.EndpointName = endpointName; } + public ApiEndpointAttribute(string endpointName, string inputPath) + { + this.EndpointName = endpointName; + this.InputPath = inputPath; + } + public string EndpointName { get; } public string HttpClientName { get; set; } + public string InputPath { get; } } } diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs b/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs index ecc3b98..f5b6226 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs @@ -10,6 +10,13 @@ public ApiEndpointSelectorAttribute(string regex) this.Regex = regex; } + public ApiEndpointSelectorAttribute(string regex, string inputPath) + { + this.Regex = regex; + this.InputPath = inputPath; + } + public string Regex { get; } + public string InputPath { get; } } } diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs b/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs index 47b121d..14c7fcf 100644 --- a/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs +++ b/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs @@ -15,22 +15,24 @@ public static bool IsDefaultResponder(this IApiEndpointService apiEndpointServic return endpointNameAttributes.Any() == false && endpointSelectorAttributes.Any() == false; } - public static bool RespondsToEndpointName(this IApiEndpointService apiEndpointService, string endpointName) + public static bool RespondsToEndpointName(this IApiEndpointService apiEndpointService, string endpointName, string inputPath = null) { var type = apiEndpointService.GetType(); var endpointNameAttributes = type.GetCustomAttributes(); var endpointSelectorAttributes = type.GetCustomAttributes(); - if (endpointNameAttributes.Any(y => y.EndpointName == endpointName)) + if (endpointNameAttributes.Any(y => y.EndpointName == endpointName && y.InputPath == inputPath)) { return true; } else if (endpointSelectorAttributes.Any()) { - return endpointSelectorAttributes.Any(y => Regex.IsMatch(endpointName, y.Regex)); + return endpointSelectorAttributes.Any(y => Regex.IsMatch(endpointName, y.Regex) && y.InputPath == inputPath); + } + else + { + return false; } - - return false; } } } diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs index 2d79fa0..33db7c8 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs @@ -44,7 +44,7 @@ public async Task>> OnParse(ParseRespons public async Task OnTransformModel(TransformModelArgs args) { var relatedTransformModels = this.transformModels - .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name, args.Transform.GraphObjectSet.ParentKey)); foreach (var transformModel in relatedTransformModels) { From fdc216c3cd2896594be922ef906a593dcb408836 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 5 Dec 2022 12:47:36 +1000 Subject: [PATCH 36/47] Scheduling the next page should only occur when all pipelines are successful --- .../Extract/ApiEndpointExtractJob.cs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs index 3106e9d..96b2664 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs @@ -61,19 +61,14 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs var nextRequestData = await this.endpointExtractPipeline.OnPrepareNextRequest(new PrepareNextRequestArgs(endpoint: endpoint, apiData: apiData, data: extractArgs.RequestData)); var isLastRequest = nextRequestData.Keys.Any() == false; - try - { - // If OnPrepareNextRequest returns an empty dict, then we're done with this endpoint. - if (isLastRequest) - return; + await this.endpointTransformJob.Transform(endpoint, apiData); - // Otherwise we need to schedule another job to continue extracting this endpoint. - this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.Id))); - } - finally - { - await this.endpointTransformJob.Transform(endpoint, apiData); - } + // If OnPrepareNextRequest returns an empty dict, then we're done with this endpoint. + if (isLastRequest) + return; + + // Otherwise we need to schedule another job to continue extracting this endpoint. + this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.Id))); } private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient httpClient, HttpRequestMessage request, ExtractArgs extractArgs) From 2ed19fffff75e613733e42117beb4a338f733a75 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 5 Dec 2022 12:47:58 +1000 Subject: [PATCH 37/47] If the consumer of library provides a destination type, use it --- MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs b/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs index fe2c4be..53c8f3f 100644 --- a/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs @@ -39,13 +39,14 @@ public static ApiEndpointModel GetApiEndpointModel(this Type type) var property = new ApiEndpointModelProperty { IsKey = propertyInfo.GetCustomAttribute() != null, - SourceType = new HashSet { propertyInfo.PropertyType }, + SourceType = new HashSet { propertyInfo.PropertyType } }; if (propertyAttribute is ApiEndpointModelPropertyAttribute modelPropertyAttribute) { property.SourceName = modelPropertyAttribute.SourceName; property.DestinationName = modelPropertyAttribute.DestinationName; + property.DestinationType = modelPropertyAttribute.DestinationType; } if (string.IsNullOrWhiteSpace(property.SourceName)) From 66696a12c10a9f0e69cca851716e63e2addcea25 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 5 Dec 2022 12:48:15 +1000 Subject: [PATCH 38/47] Await completion of the merge task to get any errors --- MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs index 27e66a4..0cf9866 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs +++ b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs @@ -52,6 +52,7 @@ public async Task OnLoadData(LoadDataArgs args) source.LinkTo(merge); await source.ExecuteAsync(); + await merge.Completion; } private List GetDataToLoad(LoadDataArgs args) From 4d59714c27c7a92f0563729e3b2344eae0134aae Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 5 Dec 2022 12:48:35 +1000 Subject: [PATCH 39/47] Automatically convert source values to destination CLR types as defined in the ApiEndpointModel --- .../Transform/ApiEndpointTransformJob.cs | 48 +++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs index a60e993..4a900ec 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs @@ -2,6 +2,7 @@ using MIFCore.Hangfire.APIETL.Extract; using MIFCore.Hangfire.APIETL.Load; using System.Collections.Generic; +using System.ComponentModel; using System.Linq; using System.Threading.Tasks; @@ -29,12 +30,12 @@ public ApiEndpointTransformJob( public async Task Transform(ApiEndpoint endpoint, ApiData apiData) { await this.transformPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData)); - var data = await this.transformPipeline.OnParse(new ParseResponseArgs(endpoint, apiData)); + var parsedData = await this.transformPipeline.OnParse(new ParseResponseArgs(endpoint, apiData)); - if (data != null) + if (parsedData != null) { // Get all the different object sets and group them by the ParentKey - var graphObjectSets = data.ExtractDistinctGraphObjectSets( + var graphObjectSets = parsedData.ExtractDistinctGraphObjectSets( new ExtractDistinctGraphObjectSetsExtensions.ExtractDistinctGraphObjectSetsArgs { Transform = async (args) => @@ -47,16 +48,54 @@ public async Task Transform(ApiEndpoint endpoint, ApiData apiData) foreach (var set in graphObjectSets) { + // Has the user of the library defined any endpoint models for this object set? var model = this.apiEndpointModels. FirstOrDefault(y => y.EndpointName == endpoint.Name && y.InputPath == set.Key); + // If they haven't, do nothing if (model is null) continue; + // There may be properties in the API response that we haven't mapped. + // Automatically generate properties from the response so they can load into the destination await this.AutoMapModelPropertiesGraphObjectSet(set, model); + // Generate a flat list of items to load var dataToLoad = set.SelectMany(y => y.Objects).ToList(); + // The parsedData from the API may need to be parsed into .net CLR types as defined by the ApiEndpointModelProperty SourceTypes + var propertiesWithClrType = model.MappedProperties + .Values + .Where(y => y.SourceType.Where(y => y != null).Any()) + .ToList(); + + foreach (var prop in propertiesWithClrType) + { + // Get the clr type the property should have + var clrType = prop.SourceType.First(y => y != null); + var typeConverter = TypeDescriptor.GetConverter(clrType); + + foreach (var data in dataToLoad) + { + // Try to get the property from the data object + if (data.TryGetValue(prop.SourceName, out var propValue)) + { + // Do nothing if its null + if (propValue is null) + continue; + + var propValueType = propValue.GetType(); + + // Make sure the propValueType is different from the clrType, otherwise a conversion is not necessary + if (propValueType != clrType + && typeConverter.CanConvertFrom(propValueType)) + { + data[prop.SourceName] = typeConverter.ConvertFrom(propValue); + } + } + } + } + await this.apiEndpointLoadJob.Load(endpoint, model, dataToLoad); } } @@ -79,6 +118,7 @@ private async Task AutoMapModelPropertiesGraphObjectSet(IGrouping Date: Mon, 5 Dec 2022 15:35:52 +1000 Subject: [PATCH 40/47] APIEndpointModels should be sufficient enough on its own without requiring an ApiEndpoint definition --- .../APIETLServiceCollectionExtensions.cs | 19 +++++++++---------- MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs | 15 +++++++++++---- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs index b8f7e4e..23d50be 100644 --- a/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs +++ b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs @@ -45,16 +45,15 @@ public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollectio serviceDescriptors.TryAddTransient(); serviceDescriptors.TryAddScoped(); - foreach (var t in types) { - var endpointNameAttributes = t.GetCustomAttributes(); - var endpointSelectorAttribute = t.GetCustomAttributes(); - var endpointModelAttributes = t.GetCustomAttributes(); + var apiEndpointAttributes = t.GetCustomAttributes(); + var apiEndpointSelectorAttributes = t.GetCustomAttributes(); + var apiEndpointModelAttributes = t.GetCustomAttributes(); - if (endpointNameAttributes.Any() == false - && endpointSelectorAttribute.Any() == false - && endpointModelAttributes.Any() == false) + if (apiEndpointAttributes.Any() == false + && apiEndpointSelectorAttributes.Any() == false + && apiEndpointModelAttributes.Any() == false) throw new ArgumentException($"The type {t.FullName} does not have an {nameof(ApiEndpointAttribute)}, {nameof(ApiEndpointSelectorAttribute)} or {nameof(ApiEndpointModel)} attributes."); // Register the extract services @@ -81,16 +80,16 @@ public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollectio if (typeof(ILoadData).IsAssignableFrom(t)) serviceDescriptors.AddScoped(typeof(ILoadData), t); - if (endpointModelAttributes.Any()) + if (apiEndpointModelAttributes.Any()) { var endpointModel = t.GetApiEndpointModel(); serviceDescriptors.AddSingleton(endpointModel); } // Register the endpoint name attribute, so an ApiEndpoint is created from it - if (endpointNameAttributes.Any()) + if (apiEndpointAttributes.Any()) { - foreach (var en in endpointNameAttributes) + foreach (var en in apiEndpointAttributes) { serviceDescriptors.AddSingleton(en); } diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs b/MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs index bbacb8d..5f45e77 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using MIFCore.Hangfire.APIETL.Load; +using System.Collections.Generic; using System.Linq; namespace MIFCore.Hangfire.APIETL @@ -6,19 +7,25 @@ namespace MIFCore.Hangfire.APIETL internal class ApiEndpointFactory : IApiEndpointFactory { private readonly IEnumerable endpointNameAttributes; + private readonly IEnumerable apiEndpointModels; private readonly IEnumerable endpointDefiners; - public ApiEndpointFactory(IEnumerable endpointNameAttributes, IEnumerable endpointDefiners) + public ApiEndpointFactory( + IEnumerable apiEndpointAttributes, + IEnumerable apiEndpointModels, + IEnumerable endpointDefiners) { - this.endpointNameAttributes = endpointNameAttributes; + this.endpointNameAttributes = apiEndpointAttributes; + this.apiEndpointModels = apiEndpointModels; this.endpointDefiners = endpointDefiners; } public async IAsyncEnumerable Create() { - // Get the endpoints defined by the ApiEndpointAttribute + // Get the endpoints defined by the ApiEndpoint or by the ApiEndpointModels apis var endpointsDefinedByAttributes = this.endpointNameAttributes .Select(y => y.EndpointName) + .Union(this.apiEndpointModels.Select(x => x.EndpointName)) .Distinct(); // Create ApiEndpoint records from the ApiEndpointAttribute From a8d80fe0dc670386764baf672b050f3695fe6b0c Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 5 Dec 2022 16:53:00 +1000 Subject: [PATCH 41/47] Create a new keys dict so keys can be modified during iteration --- MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs index 34c52f4..eb7b089 100644 --- a/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Dynamic; +using System.Linq; namespace MIFCore.Hangfire.APIETL.Transform { @@ -18,8 +19,10 @@ public static void FlattenGraph(this IEnumerable> ro public static void FlattenGraph(this IDictionary rootDict) { + var keys = rootDict.Keys.ToList(); + // iterate and check for nested objects - foreach (var rootKey in rootDict.Keys) + foreach (var rootKey in keys) { var rootValue = rootDict[rootKey]; From 6ea6f3ec1f2414a70685ecaf8468162a79a65a8c Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 5 Dec 2022 16:53:17 +1000 Subject: [PATCH 42/47] Ensure table names have a schema specified, default to dbo --- .../DefaultSqlServerLoadData.cs | 2 +- MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs | 5 +++++ .../ApiEndpointAttributeFactoryTests.cs | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs index 0cf9866..77ee821 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs +++ b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs @@ -26,7 +26,7 @@ public async Task OnLoadData(LoadDataArgs args) var connMan = this.sqlConnectionManagerFactory.Create(); var source = new MemorySource(this.GetDataToLoad(args)); - var merge = new DbMerge(connMan, apiEndpointModel.DestinationName) + var merge = new DbMerge(connMan, tableDefinition.Name) { DestinationTableDefinition = tableDefinition, MergeMode = MergeMode.InsertsAndUpdates, diff --git a/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs index a7832ee..f8ba28c 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs +++ b/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs @@ -16,6 +16,11 @@ public TableDefinitionFactory(ISqlConnectionManagerFactory sqlConnectionManagerF public async Task Create(string tableName) { + if (tableName.Contains(".") == false) + { + tableName = $"dbo.{tableName}"; + } + var connMan = this.sqlConnectionManagerFactory.Create(); return await Task.Run(() => TableDefinition.FromTableName(connMan, tableName)); } diff --git a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs index 4197d73..0ce0e9a 100644 --- a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs +++ b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.VisualStudio.TestTools.UnitTesting; using MIFCore.Hangfire.APIETL.Extract; +using MIFCore.Hangfire.APIETL.Load; namespace MIFCore.Hangfire.APIETL.Tests { @@ -14,8 +15,9 @@ public async Task Create_WithEndpointDefiner_CreatesEndpointsForEachTenant() { var serviceProvider = this.GetServiceProvider(typeof(Endpoint1), typeof(TenantedEndpointRegister)); var factory = new ApiEndpointFactory( - endpointNameAttributes: serviceProvider.GetRequiredService>(), - endpointDefiners: serviceProvider.GetRequiredService>() + apiEndpointAttributes: serviceProvider.GetRequiredService>(), + endpointDefiners: serviceProvider.GetRequiredService>(), + apiEndpointModels: new List() ); // There should be four endpoints total, api/endpoint1, api/endpoint2 but duplicated for each "tenant" From ad3aa8ef0282a02b15b863036ed0c866c35a7b7c Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Thu, 12 Jan 2023 14:09:31 +1000 Subject: [PATCH 43/47] FlattenGraph should loop through array types as well --- .../Transform/FlattenGraphExtensions.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs index eb7b089..d8b13c7 100644 --- a/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs +++ b/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs @@ -44,6 +44,14 @@ public static void FlattenGraph(this IDictionary rootDict) // Remove the original rootKey from the rootDict as all the properties have been merged into it rootDict.Remove(rootKey, out _); } + else if (rootValue is IEnumerable array) + { + var items = array + .Cast>() + .ToList(); + + items.FlattenGraph(); + } } } } From 68e5e7ba74b0a0dde5b5995de0627c85d7a95dfc Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Thu, 12 Jan 2023 14:09:52 +1000 Subject: [PATCH 44/47] Update VersionSuffix so that the year is prepended --- .../MIFCore.Hangfire.APIETL.SqlServer.csproj | 2 +- MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj b/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj index bd622e9..3301371 100644 --- a/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj +++ b/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj @@ -19,7 +19,7 @@ - rev.$([System.DateTime]::UtcNow.ToString("MddHHmm")) + rev.$([System.DateTime]::UtcNow.ToString("yyyyMddHHmm")) diff --git a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj index 05e5133..3f40b1a 100644 --- a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj +++ b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj @@ -17,7 +17,7 @@ - rev.$([System.DateTime]::UtcNow.ToString("MddHHmm")) + rev.$([System.DateTime]::UtcNow.ToString("yyyyMddHHmm")) From fbed8ed0f8a3e28efe23ed260b5e61b4e5406ea4 Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Thu, 12 Jan 2023 18:01:52 +1000 Subject: [PATCH 45/47] Ensure there are keys for every mapped property in the data payloads, otherwise an error will be thrown during the Load phase --- MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs index 4a900ec..b9d05bb 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs @@ -93,6 +93,11 @@ public async Task Transform(ApiEndpoint endpoint, ApiData apiData) data[prop.SourceName] = typeConverter.ConvertFrom(propValue); } } + else + { + // Ensure all the dataToLoad items have all the properties defined in the model, even if just null. + data[prop.SourceName] = null; + } } } From 1f3b3a46caf7aa082a0853bf5201f746e73049fd Mon Sep 17 00:00:00 2001 From: maitlandmarshall Date: Mon, 20 Feb 2023 11:55:11 +1000 Subject: [PATCH 46/47] Add support for APIs which require another APIs result first i.e /api/{someId}/groups --- .../ApiEndpointTests.cs | 18 +++++++ MIFCore.Hangfire.APIETL/ApiEndpoint.cs | 34 ++++++++++--- .../ApiEndpointRegister.cs | 4 ++ .../Extract/ApiEndpointExtractJob.cs | 51 ++++++++++++------- .../Transform/ApiEndpointTransformJob.cs | 8 +-- .../Transform/HandleResponseArgs.cs | 2 +- .../Transform/IApiEndpointTransformJob.cs | 2 +- .../Transform/ParseResponseArgs.cs | 2 +- .../Transform/ResponseArgsBase.cs | 5 +- .../Transform/TransformModelArgs.cs | 2 +- 10 files changed, 94 insertions(+), 34 deletions(-) create mode 100644 MIFCore.Hangfire.APIETL.Tests/ApiEndpointTests.cs diff --git a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointTests.cs b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointTests.cs new file mode 100644 index 0000000..eeabcb1 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointTests.cs @@ -0,0 +1,18 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MIFCore.Hangfire.APIETL.Tests +{ + [TestClass] + public class ApiEndpointTests + { + [TestMethod] + public void ApiEndpoint_ParseRouteParameters_ShouldExtractParameterNames() + { + var endpoint = new ApiEndpoint("some/api/{param1}/{param2}"); + + Assert.AreEqual(2, endpoint.RouteParameters.Count()); + Assert.IsTrue(endpoint.RouteParameters.Contains("param1")); + Assert.IsTrue(endpoint.RouteParameters.Contains("param2")); + } + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpoint.cs b/MIFCore.Hangfire.APIETL/ApiEndpoint.cs index 27699e9..a044d66 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpoint.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpoint.cs @@ -1,26 +1,46 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; namespace MIFCore.Hangfire.APIETL { public class ApiEndpoint { - public ApiEndpoint(string jobName, string httpClientName = null) + private readonly Lazy> routeParameters; + + public ApiEndpoint(string jobName, string httpClientName = null) : this() { - this.JobName = jobName; this.HttpClientName = httpClientName; + this.JobName = jobName; } - internal ApiEndpoint(string name) + internal ApiEndpoint(string name) : this() { this.Name = name; this.JobName = name; } - public string Name { get; internal set; } + private ApiEndpoint() + { + this.routeParameters = new Lazy>(() => + { + // Match on {something123} + var parameterRegex = new Regex(@"\{([A-Za-z0-9_]+)\}"); + var matches = parameterRegex.Matches(this.Name); - public string JobName { get; } - public string HttpClientName { get; } + // Return the names of the groups, i.e path/to/{id}/{otherId}, extract: id, otherId + // Select the second item in the group [1] as it is the "id" or "otherId" instead of "{id}", etc. + return matches.Select(y => y.Groups[1].Value).ToList(); + }); + } + + public string Name { get; internal set; } + public string JobName { get; internal set; } + public IEnumerable RouteParameters => this.routeParameters.Value; public IDictionary AdditionalHeaders { get; } = new Dictionary(); + + public string HttpClientName { get; } } } diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs index f31c5cb..12093f3 100644 --- a/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs +++ b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs @@ -24,6 +24,10 @@ public IApiEndpointRegister Register(ApiEndpoint endpoint) { this.endpoints.Add(endpoint.Name, endpoint); + // If the endpoint has route parameters, then we don't want to register it as a recurring job. + if (endpoint.RouteParameters.Any()) + return this; + this.recurringJobManager.AddOrUpdate( endpoint.JobName, job => job.Extract(endpoint.Name, null), diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs index 96b2664..910e8a4 100644 --- a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs @@ -61,7 +61,7 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs var nextRequestData = await this.endpointExtractPipeline.OnPrepareNextRequest(new PrepareNextRequestArgs(endpoint: endpoint, apiData: apiData, data: extractArgs.RequestData)); var isLastRequest = nextRequestData.Keys.Any() == false; - await this.endpointTransformJob.Transform(endpoint, apiData); + await this.endpointTransformJob.Transform(endpoint, apiData, extractArgs); // If OnPrepareNextRequest returns an empty dict, then we're done with this endpoint. if (isLastRequest) @@ -71,31 +71,26 @@ private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.Id))); } - private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient httpClient, HttpRequestMessage request, ExtractArgs extractArgs) + private async Task CreateRequest(Uri baseAddress, ApiEndpoint endpoint, ExtractArgs extractArgs) { - // Get the response payload as a string - var response = await httpClient.SendAsync(request); - var data = await response.Content.ReadAsStringAsync(); + var endpointName = endpoint.Name; - // Turn the payload response into an ApiData instance - var apiData = new ApiData + if (endpoint.RouteParameters.Any()) { - Endpoint = endpoint.Name, - Uri = response.RequestMessage.RequestUri.ToString(), - Data = data, - ParentId = extractArgs.ParentApiDataId - }; - - return apiData; - } + // If there are route parameters, then we need to replace the route parameters with the values from the extractArgs.RequestData + // e.g. if the endpoint.SourceName = "getStuff/{id}" and extractArgs.RequestData["id"] = 123, then endpoint.SourceName = "getStuff/123" + foreach (var routeParameter in endpoint.RouteParameters) + { + var routeParameterValue = extractArgs.RequestData[routeParameter]; + endpointName = endpointName.Replace($"{{{routeParameter}}}", routeParameterValue.ToString()); + } + } - private async Task CreateRequest(Uri baseAddress, ApiEndpoint endpoint, ExtractArgs extractArgs) - { // Create a new request, using endpoint.SourceName as the relative uri // i.e endpoint.SourceName = "getStuff" and httpClient.BaseAddress = "https://someapi/api/" var request = new HttpRequestMessage { - RequestUri = new Uri(baseAddress, endpoint.Name) + RequestUri = new Uri(baseAddress, endpointName) }; // Stitch on any additional headers @@ -109,5 +104,25 @@ private async Task CreateRequest(Uri baseAddress, ApiEndpoin return request; } + + private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient httpClient, HttpRequestMessage request, ExtractArgs extractArgs) + { + // Get the response payload as a string + var response = await httpClient.SendAsync(request); + response.EnsureSuccessStatusCode(); + + var data = await response.Content.ReadAsStringAsync(); + + // Turn the payload response into an ApiData instance + var apiData = new ApiData + { + Endpoint = endpoint.Name, + Uri = response.RequestMessage.RequestUri.ToString(), + Data = data, + ParentId = extractArgs.ParentApiDataId + }; + + return apiData; + } } } diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs index b9d05bb..d62e7de 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs @@ -27,10 +27,10 @@ public ApiEndpointTransformJob( this.apiEndpointLoadJob = apiEndpointLoadJob; } - public async Task Transform(ApiEndpoint endpoint, ApiData apiData) + public async Task Transform(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs) { - await this.transformPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData)); - var parsedData = await this.transformPipeline.OnParse(new ParseResponseArgs(endpoint, apiData)); + await this.transformPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData, extractArgs)); + var parsedData = await this.transformPipeline.OnParse(new ParseResponseArgs(endpoint, apiData, extractArgs)); if (parsedData != null) { @@ -40,7 +40,7 @@ public async Task Transform(ApiEndpoint endpoint, ApiData apiData) { Transform = async (args) => { - await this.transformPipeline.OnTransformModel(new TransformModelArgs(endpoint, apiData, args)); + await this.transformPipeline.OnTransformModel(new TransformModelArgs(endpoint, apiData, extractArgs, args)); } }) .GroupBy(y => y.ParentKey) diff --git a/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs b/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs index 30c3e91..a1b97fa 100644 --- a/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs +++ b/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs @@ -4,7 +4,7 @@ namespace MIFCore.Hangfire.APIETL.Transform { public class HandleResponseArgs : ResponseArgsBase { - public HandleResponseArgs(ApiEndpoint endpoint, ApiData apiData) : base(endpoint, apiData) + public HandleResponseArgs(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs) : base(endpoint, apiData, extractArgs) { } } diff --git a/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs index 2646136..b56e4b5 100644 --- a/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs +++ b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs @@ -5,6 +5,6 @@ namespace MIFCore.Hangfire.APIETL.Transform { public interface IApiEndpointTransformJob { - Task Transform(ApiEndpoint endpoint, ApiData apiData); + Task Transform(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs); } } \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs b/MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs index d059f7d..1e4082e 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs @@ -4,7 +4,7 @@ namespace MIFCore.Hangfire.APIETL.Transform { public class ParseResponseArgs : ResponseArgsBase { - public ParseResponseArgs(ApiEndpoint endpoint, ApiData apiData) : base(endpoint, apiData) + public ParseResponseArgs(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs) : base(endpoint, apiData, extractArgs) { } } diff --git a/MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs b/MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs index 1a4bc00..ab8cc8d 100644 --- a/MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs +++ b/MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs @@ -4,13 +4,16 @@ namespace MIFCore.Hangfire.APIETL.Transform { public abstract class ResponseArgsBase { - public ResponseArgsBase(ApiEndpoint endpoint, ApiData apiData) + public ResponseArgsBase(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs) { this.Endpoint = endpoint; this.ApiData = apiData; + this.ExtractArgs = extractArgs; } public ApiData ApiData { get; } public ApiEndpoint Endpoint { get; } + + public ExtractArgs ExtractArgs { get; } } } \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs b/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs index b18a44c..e226dc7 100644 --- a/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs +++ b/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs @@ -5,7 +5,7 @@ namespace MIFCore.Hangfire.APIETL.Transform { public class TransformModelArgs : ResponseArgsBase { - public TransformModelArgs(ApiEndpoint endpoint, ApiData apiData, TransformObjectArgs transformArgs) : base(endpoint, apiData) + public TransformModelArgs(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs, TransformObjectArgs transformArgs) : base(endpoint, apiData, extractArgs) { this.Transform = transformArgs; } From 3ea7a36446ab0d5f1a55e892b97437cc78fa1ee4 Mon Sep 17 00:00:00 2001 From: Maitland Marshall Date: Tue, 12 Sep 2023 14:31:38 +1000 Subject: [PATCH 47/47] Add APIETL actions --- .../MIFCore.Hangfire.APIETL-Nuget.yml | 38 +++++++++++++++++++ ...IFCore.Hangfire.APIETL.SqlServer-Nuget.yml | 38 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 .github/workflows/MIFCore.Hangfire.APIETL-Nuget.yml create mode 100644 .github/workflows/MIFCore.Hangfire.APIETL.SqlServer-Nuget.yml diff --git a/.github/workflows/MIFCore.Hangfire.APIETL-Nuget.yml b/.github/workflows/MIFCore.Hangfire.APIETL-Nuget.yml new file mode 100644 index 0000000..c776f43 --- /dev/null +++ b/.github/workflows/MIFCore.Hangfire.APIETL-Nuget.yml @@ -0,0 +1,38 @@ +name: Nuget MIFCore Hangfire APIETL + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [feature/api-etl] + paths: + - 'MIFCore.Hangfire.APIETL/**' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: windows-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: Setup .NET + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '6.0.x' # SDK Version to use. + - name: Restore dependencies + run: dotnet restore MIFCore.Hangfire.APIETL + - name: Build + run: dotnet build MIFCore.Hangfire.APIETL --no-restore + - name: Test + run: dotnet test MIFCore.Hangfire.APIETL --no-build --verbosity normal + - name: Pack + run: dotnet pack --configuration Release MIFCore.Hangfire.APIETL + - name: Publish the package to nuget.org + run: dotnet nuget push */bin/Release/*.nupkg -k ${{ secrets.NUGET_AUTH_TOKEN }} -s https://api.nuget.org/v3/index.json diff --git a/.github/workflows/MIFCore.Hangfire.APIETL.SqlServer-Nuget.yml b/.github/workflows/MIFCore.Hangfire.APIETL.SqlServer-Nuget.yml new file mode 100644 index 0000000..387351c --- /dev/null +++ b/.github/workflows/MIFCore.Hangfire.APIETL.SqlServer-Nuget.yml @@ -0,0 +1,38 @@ +name: Nuget MIFCore Hangfire APIETL SqlServer + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [feature/api-etl] + paths: + - 'MIFCore.Hangfire.APIETL.SqlServer/**' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: windows-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: Setup .NET + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '6.0.x' # SDK Version to use. + - name: Restore dependencies + run: dotnet restore MIFCore.Hangfire.APIETL.SqlServer + - name: Build + run: dotnet build MIFCore.Hangfire.APIETL.SqlServer --no-restore + - name: Test + run: dotnet test MIFCore.Hangfire.APIETL.SqlServer --no-build --verbosity normal + - name: Pack + run: dotnet pack --configuration Release MIFCore.Hangfire.APIETL.SqlServer + - name: Publish the package to nuget.org + run: dotnet nuget push */bin/Release/*.nupkg -k ${{ secrets.NUGET_AUTH_TOKEN }} -s https://api.nuget.org/v3/index.json