diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..10e5a38 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,226 @@ +# Remove the line below if you want to inherit .editorconfig settings from higher directories +root = true + +# C# files +[*.cs] + +#### Core EditorConfig Options #### + +# Indentation and spacing +indent_size = 4 +indent_style = space +tab_width = 4 + +# New line preferences +end_of_line = crlf +insert_final_newline = false + +#### .NET Coding Conventions #### + +# Organize usings +dotnet_separate_import_directive_groups = false +dotnet_sort_system_directives_first = false +file_header_template = unset + +# this. and Me. preferences +dotnet_style_qualification_for_event = true +dotnet_style_qualification_for_field = true +dotnet_style_qualification_for_method = true +dotnet_style_qualification_for_property = true + +# Language keywords vs BCL types preferences +dotnet_style_predefined_type_for_locals_parameters_members = true +dotnet_style_predefined_type_for_member_access = true + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity +dotnet_style_parentheses_in_other_operators = never_if_unnecessary +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity + +# Modifier preferences +dotnet_style_require_accessibility_modifiers = for_non_interface_members + +# Expression-level preferences +dotnet_style_coalesce_expression = true +dotnet_style_collection_initializer = true +dotnet_style_explicit_tuple_names = true +dotnet_style_namespace_match_folder = true +dotnet_style_null_propagation = true +dotnet_style_object_initializer = true +dotnet_style_operator_placement_when_wrapping = beginning_of_line +dotnet_style_prefer_auto_properties = false +dotnet_style_prefer_compound_assignment = true +dotnet_style_prefer_conditional_expression_over_assignment = true +dotnet_style_prefer_conditional_expression_over_return = true +dotnet_style_prefer_foreach_explicit_cast_in_source = when_strongly_typed +dotnet_style_prefer_inferred_anonymous_type_member_names = true +dotnet_style_prefer_inferred_tuple_names = true +dotnet_style_prefer_is_null_check_over_reference_equality_method = true +dotnet_style_prefer_simplified_boolean_expressions = true +dotnet_style_prefer_simplified_interpolation = true + +# Field preferences +dotnet_style_readonly_field = true + +# Parameter preferences +dotnet_code_quality_unused_parameters = all + +# Suppression preferences +dotnet_remove_unnecessary_suppression_exclusions = none + +# New line preferences +dotnet_style_allow_multiple_blank_lines_experimental = true +dotnet_style_allow_statement_immediately_after_block_experimental = true + +#### C# Coding Conventions #### + +# var preferences +csharp_style_var_elsewhere = false +csharp_style_var_for_built_in_types = false +csharp_style_var_when_type_is_apparent = false + +# Expression-bodied members +csharp_style_expression_bodied_accessors = true +csharp_style_expression_bodied_constructors = false +csharp_style_expression_bodied_indexers = true +csharp_style_expression_bodied_lambdas = when_on_single_line +csharp_style_expression_bodied_local_functions = false +csharp_style_expression_bodied_methods = false +csharp_style_expression_bodied_operators = false +csharp_style_expression_bodied_properties = true + +# Pattern matching preferences +csharp_style_pattern_matching_over_as_with_null_check = true +csharp_style_pattern_matching_over_is_with_cast_check = true +csharp_style_prefer_extended_property_pattern = true +csharp_style_prefer_not_pattern = true +csharp_style_prefer_pattern_matching = true +csharp_style_prefer_switch_expression = true + +# Null-checking preferences +csharp_style_conditional_delegate_call = true + +# Modifier preferences +csharp_prefer_static_local_function = true +csharp_preferred_modifier_order = public,private,protected,internal,file,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,required,volatile,async +csharp_style_prefer_readonly_struct = true + +# Code-block preferences +csharp_prefer_braces = true +csharp_prefer_simple_using_statement = true +csharp_style_namespace_declarations = block_scoped +csharp_style_prefer_method_group_conversion = true +csharp_style_prefer_top_level_statements = true + +# Expression-level preferences +csharp_prefer_simple_default_expression = true +csharp_style_deconstructed_variable_declaration = true +csharp_style_implicit_object_creation_when_type_is_apparent = true +csharp_style_inlined_variable_declaration = true +csharp_style_prefer_index_operator = true +csharp_style_prefer_local_over_anonymous_function = true +csharp_style_prefer_null_check_over_type_check = true +csharp_style_prefer_range_operator = true +csharp_style_prefer_tuple_swap = true +csharp_style_prefer_utf8_string_literals = true +csharp_style_throw_expression = true +csharp_style_unused_value_assignment_preference = discard_variable +csharp_style_unused_value_expression_statement_preference = discard_variable + +# 'using' directive preferences +csharp_using_directive_placement = outside_namespace + +# New line preferences +csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = true +csharp_style_allow_blank_lines_between_consecutive_braces_experimental = true +csharp_style_allow_embedded_statements_on_same_line_experimental = true + +#### C# Formatting Rules #### + +# New line preferences +csharp_new_line_before_catch = true +csharp_new_line_before_else = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_open_brace = all +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_labels = one_less_than_current +csharp_indent_switch_labels = true + +# Space preferences +csharp_space_after_cast = false +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_after_semicolon_in_for_statement = true +csharp_space_around_binary_operators = before_and_after +csharp_space_around_declaration_statements = false +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_between_square_brackets = false + +# Wrapping preferences +csharp_preserve_single_line_blocks = true +csharp_preserve_single_line_statements = true + +#### Naming styles #### + +# Naming rules + +dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion +dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface +dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i + +dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.types_should_be_pascal_case.symbols = types +dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case + +dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members +dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case + +# Symbol specifications + +dotnet_naming_symbols.interface.applicable_kinds = interface +dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.interface.required_modifiers = + +dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum +dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.types.required_modifiers = + +dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method +dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.non_field_members.required_modifiers = + +# Naming styles + +dotnet_naming_style.pascal_case.required_prefix = +dotnet_naming_style.pascal_case.required_suffix = +dotnet_naming_style.pascal_case.word_separator = +dotnet_naming_style.pascal_case.capitalization = pascal_case + +dotnet_naming_style.begins_with_i.required_prefix = I +dotnet_naming_style.begins_with_i.required_suffix = +dotnet_naming_style.begins_with_i.word_separator = +dotnet_naming_style.begins_with_i.capitalization = pascal_case diff --git a/.github/workflows/MIFCore.Hangfire.APIETL-Nuget.yml b/.github/workflows/MIFCore.Hangfire.APIETL-Nuget.yml new file mode 100644 index 0000000..c776f43 --- /dev/null +++ b/.github/workflows/MIFCore.Hangfire.APIETL-Nuget.yml @@ -0,0 +1,38 @@ +name: Nuget MIFCore Hangfire APIETL + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [feature/api-etl] + paths: + - 'MIFCore.Hangfire.APIETL/**' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: windows-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: Setup .NET + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '6.0.x' # SDK Version to use. + - name: Restore dependencies + run: dotnet restore MIFCore.Hangfire.APIETL + - name: Build + run: dotnet build MIFCore.Hangfire.APIETL --no-restore + - name: Test + run: dotnet test MIFCore.Hangfire.APIETL --no-build --verbosity normal + - name: Pack + run: dotnet pack --configuration Release MIFCore.Hangfire.APIETL + - name: Publish the package to nuget.org + run: dotnet nuget push */bin/Release/*.nupkg -k ${{ secrets.NUGET_AUTH_TOKEN }} -s https://api.nuget.org/v3/index.json diff --git a/.github/workflows/MIFCore.Hangfire.APIETL.SqlServer-Nuget.yml b/.github/workflows/MIFCore.Hangfire.APIETL.SqlServer-Nuget.yml new file mode 100644 index 0000000..387351c --- /dev/null +++ b/.github/workflows/MIFCore.Hangfire.APIETL.SqlServer-Nuget.yml @@ -0,0 +1,38 @@ +name: Nuget MIFCore Hangfire APIETL SqlServer + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [feature/api-etl] + paths: + - 'MIFCore.Hangfire.APIETL.SqlServer/**' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: windows-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: Setup .NET + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '6.0.x' # SDK Version to use. + - name: Restore dependencies + run: dotnet restore MIFCore.Hangfire.APIETL.SqlServer + - name: Build + run: dotnet build MIFCore.Hangfire.APIETL.SqlServer --no-restore + - name: Test + run: dotnet test MIFCore.Hangfire.APIETL.SqlServer --no-build --verbosity normal + - name: Pack + run: dotnet pack --configuration Release MIFCore.Hangfire.APIETL.SqlServer + - name: Publish the package to nuget.org + run: dotnet nuget push */bin/Release/*.nupkg -k ${{ secrets.NUGET_AUTH_TOKEN }} -s https://api.nuget.org/v3/index.json diff --git a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs new file mode 100644 index 0000000..91f927d --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerCreateDestination.cs @@ -0,0 +1,34 @@ +using ETLBox.ControlFlow.Tasks; +using MIFCore.Hangfire.APIETL.Load; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + internal class DefaultSqlServerCreateDestination : ICreateDestination + { + private readonly ISqlConnectionManagerFactory sqlConnectionManagerFactory; + private readonly ITableDefinitionFactory tableDefinitionFactory; + + public DefaultSqlServerCreateDestination(ISqlConnectionManagerFactory sqlConnectionManagerFactory, ITableDefinitionFactory tableDefinitionFactory) + { + this.sqlConnectionManagerFactory = sqlConnectionManagerFactory; + this.tableDefinitionFactory = tableDefinitionFactory; + } + + public async Task OnCreateDestination(CreateDestinationArgs args) + { + var apiEndpointModel = args.ApiEndpointModel; + var connMan = this.sqlConnectionManagerFactory.Create(); + + if (IfTableOrViewExistsTask.IsExisting(connMan, apiEndpointModel.DestinationName) == false) + { + var tableDefinition = await this.tableDefinitionFactory.Create(apiEndpointModel); + + await Task.Run(() => + { + CreateTableTask.Create(connMan, tableDefinition); + }); + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs new file mode 100644 index 0000000..77ee821 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/DefaultSqlServerLoadData.cs @@ -0,0 +1,64 @@ +using ETLBox.DataFlow; +using ETLBox.DataFlow.Connectors; +using MIFCore.Hangfire.APIETL.Load; +using System.Collections.Generic; +using System.Dynamic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + internal class DefaultSqlServerLoadData : ILoadData + { + private readonly ISqlConnectionManagerFactory sqlConnectionManagerFactory; + private readonly ITableDefinitionFactory tableDefinitionFactory; + + public DefaultSqlServerLoadData(ISqlConnectionManagerFactory sqlConnectionManagerFactory, ITableDefinitionFactory tableDefinitionFactory) + { + this.sqlConnectionManagerFactory = sqlConnectionManagerFactory; + this.tableDefinitionFactory = tableDefinitionFactory; + } + + public async Task OnLoadData(LoadDataArgs args) + { + var apiEndpointModel = args.ApiEndpointModel; + var tableDefinition = await this.tableDefinitionFactory.Create(args.ApiEndpointModel.DestinationName); + var connMan = this.sqlConnectionManagerFactory.Create(); + + var source = new MemorySource(this.GetDataToLoad(args)); + var merge = new DbMerge(connMan, tableDefinition.Name) + { + DestinationTableDefinition = tableDefinition, + MergeMode = MergeMode.InsertsAndUpdates, + ColumnMapping = apiEndpointModel + .MappedProperties + .Values + .Select(y => new ColumnMap + { + PropertyName = y.SourceName, + DbColumnName = y.DestinationName + }) + .ToList(), + MergeProperties = + { + IdColumns = apiEndpointModel + .MappedProperties + .Values + .Where(y => y.IsKey) + .Select(y => new IdColumn { IdPropertyName = y.SourceName }) + .ToList() + } + }; + + source.LinkTo(merge); + await source.ExecuteAsync(); + await merge.Completion; + } + + private List GetDataToLoad(LoadDataArgs args) + { + var data = args.DataToLoad; + return data.Cast().ToList(); + } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/ISqlConnectionManagerFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/ISqlConnectionManagerFactory.cs new file mode 100644 index 0000000..0864a4f --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/ISqlConnectionManagerFactory.cs @@ -0,0 +1,9 @@ +using ETLBox.Connection; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + public interface ISqlConnectionManagerFactory + { + SqlConnectionManager Create(); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL.SqlServer/ITableDefinitionFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/ITableDefinitionFactory.cs new file mode 100644 index 0000000..3e93c61 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/ITableDefinitionFactory.cs @@ -0,0 +1,12 @@ +using ETLBox.ControlFlow; +using MIFCore.Hangfire.APIETL.Load; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + public interface ITableDefinitionFactory + { + Task Create(ApiEndpointModel apiEndpointModel); + Task Create(string tableName); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj b/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj new file mode 100644 index 0000000..3301371 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/MIFCore.Hangfire.APIETL.SqlServer.csproj @@ -0,0 +1,35 @@ + + + + netstandard2.1 + Maitland Marshall + MAIT DEV + https://github.com/maitlandmarshall/MIFCore + https://github.com/maitlandmarshall/MIFCore + git + true + MIT + Hangfire, Scheduling, Jobs, Simplified, API, ETL, SqlServer + ETL for SqlServer. + 10 + + + + 1.0.0 + + + + rev.$([System.DateTime]::UtcNow.ToString("yyyyMddHHmm")) + + + + + + + + + + + + + diff --git a/MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs new file mode 100644 index 0000000..02d8967 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/SqlConnectionManagerFactory.cs @@ -0,0 +1,19 @@ +using ETLBox.Connection; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + internal class SqlConnectionManagerFactory : ISqlConnectionManagerFactory + { + private readonly SqlServerConfig sqlServerConfig; + + public SqlConnectionManagerFactory(SqlServerConfig sqlServerConfig) + { + this.sqlServerConfig = sqlServerConfig; + } + + public SqlConnectionManager Create() + { + return new SqlConnectionManager(this.sqlServerConfig.DestinationConnectionString); + } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/SqlServerConfig.cs b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerConfig.cs new file mode 100644 index 0000000..2f7066e --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerConfig.cs @@ -0,0 +1,7 @@ +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + public class SqlServerConfig + { + public string DestinationConnectionString { get; set; } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/SqlServerGetDestinationType.cs b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerGetDestinationType.cs new file mode 100644 index 0000000..42bfb0d --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerGetDestinationType.cs @@ -0,0 +1,37 @@ +using MIFCore.Hangfire.APIETL.Load; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + internal class SqlServerGetDestinationType : IGetDestinationType + { + public Task GetDestinationType(ApiEndpointModel apiEndpointModel, string sourceKey, IEnumerable sourceModelTypes) + { + var clrType = sourceModelTypes.FirstOrDefault(y => y != null); + var destinationType = Type.GetTypeCode(clrType) switch + { + TypeCode.Empty => throw new NotImplementedException(), + TypeCode.Object => throw new NotImplementedException(), + TypeCode.DBNull => throw new NotImplementedException(), + TypeCode.Boolean => "bit", + TypeCode.Char => "char(max)", + TypeCode.SByte or TypeCode.Byte => "binary", + TypeCode.Int16 => "smallint", + TypeCode.UInt16 => "smallint", + TypeCode.Int32 or TypeCode.UInt32 => "int", + TypeCode.Int64 or TypeCode.UInt64 => "bigint", + TypeCode.Single => "real", + TypeCode.Double => "float", + TypeCode.Decimal => "decimal(18,4)", + TypeCode.DateTime => "datetimeoffset", + TypeCode.String => "nvarchar(max)", + _ => throw new NotImplementedException(), + }; + + return Task.FromResult(destinationType); + } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/SqlServerServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerServiceCollectionExtensions.cs new file mode 100644 index 0000000..1b61955 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/SqlServerServiceCollectionExtensions.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using MIFCore.Hangfire.APIETL.Load; +using System; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + public static class SqlServerServiceCollectionExtensions + { + public static IServiceCollection AddApiEndpointsSqlServerDestination(this IServiceCollection serviceDescriptors, Action configAction = null) + { + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddTransient(); + + serviceDescriptors.AddSingleton((svc) => + { + var sqlServerConfig = new SqlServerConfig(); + + if (configAction != null) + { + configAction(sqlServerConfig); + } + else + { + // If no configAction is supplied, default to the Hangfire config. + var hangfireConfig = svc.GetRequiredService(); + sqlServerConfig.DestinationConnectionString = hangfireConfig.ConnectionString; + } + + return sqlServerConfig; + }); + + return serviceDescriptors; + } + } +} diff --git a/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs b/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs new file mode 100644 index 0000000..f8ba28c --- /dev/null +++ b/MIFCore.Hangfire.APIETL.SqlServer/TableDefinitionFactory.cs @@ -0,0 +1,41 @@ +using ETLBox.ControlFlow; +using MIFCore.Hangfire.APIETL.Load; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.SqlServer +{ + internal class TableDefinitionFactory : ITableDefinitionFactory + { + private readonly ISqlConnectionManagerFactory sqlConnectionManagerFactory; + + public TableDefinitionFactory(ISqlConnectionManagerFactory sqlConnectionManagerFactory) + { + this.sqlConnectionManagerFactory = sqlConnectionManagerFactory; + } + + public async Task Create(string tableName) + { + if (tableName.Contains(".") == false) + { + tableName = $"dbo.{tableName}"; + } + + var connMan = this.sqlConnectionManagerFactory.Create(); + return await Task.Run(() => TableDefinition.FromTableName(connMan, tableName)); + } + + public Task Create(ApiEndpointModel apiEndpointModel) + { + var tableDefinition = new TableDefinition( + name: apiEndpointModel.DestinationName, + columns: apiEndpointModel.MappedProperties.Select(kvp => + { + var (key, value) = kvp; + return new TableColumn(value.DestinationName, value.DestinationType, value.IsKey == false, value.IsKey, false); + }).ToList()); + + return Task.FromResult(tableDefinition); + } + } +} diff --git a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs new file mode 100644 index 0000000..0ce0e9a --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointAttributeFactoryTests.cs @@ -0,0 +1,72 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using MIFCore.Hangfire.APIETL.Extract; +using MIFCore.Hangfire.APIETL.Load; + +namespace MIFCore.Hangfire.APIETL.Tests +{ + [TestClass()] + public class ApiEndpointAttributeFactoryTests + { + private static string[] Tenants = new[] { "tenant1", "tenant2" }; + + [TestMethod()] + public async Task Create_WithEndpointDefiner_CreatesEndpointsForEachTenant() + { + var serviceProvider = this.GetServiceProvider(typeof(Endpoint1), typeof(TenantedEndpointRegister)); + var factory = new ApiEndpointFactory( + apiEndpointAttributes: serviceProvider.GetRequiredService>(), + endpointDefiners: serviceProvider.GetRequiredService>(), + apiEndpointModels: new List() + ); + + // There should be four endpoints total, api/endpoint1, api/endpoint2 but duplicated for each "tenant" + var endpoints = await factory.Create().ToListAsync(); + + Assert.AreEqual(4, endpoints.Count); + Assert.AreEqual(2, endpoints.Count(x => x.Name == "api/endpoint1")); + Assert.AreEqual(2, endpoints.Count(x => x.Name == "api/endpoint2")); + + Assert.AreEqual(1, endpoints.Count(x => x.Name == "api/endpoint1" && x.AdditionalHeaders["tenantId"] == "tenant1")); + Assert.AreEqual(1, endpoints.Count(x => x.Name == "api/endpoint2" && x.AdditionalHeaders["tenantId"] == "tenant1")); + + Assert.AreEqual(1, endpoints.Count(x => x.Name == "api/endpoint1" && x.AdditionalHeaders["tenantId"] == "tenant2")); + Assert.AreEqual(1, endpoints.Count(x => x.Name == "api/endpoint2" && x.AdditionalHeaders["tenantId"] == "tenant2")); + } + + private IServiceProvider GetServiceProvider(params Type[] endpoints) + { + var serviceCollection = new ServiceCollection(); + serviceCollection.AddApiEndpointsToExtract(endpoints); + return serviceCollection.BuildServiceProvider(); + } + + [ApiEndpoint("api/endpoint1")] + class Endpoint1 : IPrepareRequest + { + public Task OnPrepareRequest(PrepareRequestArgs args) + { + throw new NotImplementedException(); + } + } + + [ApiEndpoint("api/endpoint2")] + [ApiEndpointSelector(".*")] + class TenantedEndpointRegister : IDefineEndpoints + { + public async IAsyncEnumerable DefineEndpoints(string endpointName) + { + foreach (var t in Tenants) + { + yield return new ApiEndpoint($"{endpointName} ({t})") + { + AdditionalHeaders = + { + { "tenantId", t } + } + }; + } + } + } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointRegisterTests.cs b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointRegisterTests.cs new file mode 100644 index 0000000..391f116 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointRegisterTests.cs @@ -0,0 +1,14 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MIFCore.Hangfire.APIETL.Tests +{ + [TestClass()] + public class ApiEndpointRegisterTests + { + [TestMethod()] + public void RegisterTest() + { + + } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL.Tests/ApiEndpointTests.cs b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointTests.cs new file mode 100644 index 0000000..eeabcb1 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/ApiEndpointTests.cs @@ -0,0 +1,18 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MIFCore.Hangfire.APIETL.Tests +{ + [TestClass] + public class ApiEndpointTests + { + [TestMethod] + public void ApiEndpoint_ParseRouteParameters_ShouldExtractParameterNames() + { + var endpoint = new ApiEndpoint("some/api/{param1}/{param2}"); + + Assert.AreEqual(2, endpoint.RouteParameters.Count()); + Assert.IsTrue(endpoint.RouteParameters.Contains("param1")); + Assert.IsTrue(endpoint.RouteParameters.Contains("param2")); + } + } +} diff --git a/MIFCore.Hangfire.APIETL.Tests/MIFCore.Hangfire.APIETL.Tests.csproj b/MIFCore.Hangfire.APIETL.Tests/MIFCore.Hangfire.APIETL.Tests.csproj new file mode 100644 index 0000000..a710fbb --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/MIFCore.Hangfire.APIETL.Tests.csproj @@ -0,0 +1,23 @@ + + + + net6 + enable + enable + + false + + + + + + + + + + + + + + + diff --git a/MIFCore.Hangfire.APIETL.Tests/Transform/ExtractDistinctGraphObjectSetsExtensionsTests.cs b/MIFCore.Hangfire.APIETL.Tests/Transform/ExtractDistinctGraphObjectSetsExtensionsTests.cs new file mode 100644 index 0000000..6bdcfd7 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/Transform/ExtractDistinctGraphObjectSetsExtensionsTests.cs @@ -0,0 +1,29 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using System.Dynamic; + +namespace MIFCore.Hangfire.APIETL.Transform.Tests +{ + [TestClass()] + public class ExtractDistinctGraphObjectSetsExtensionsTests + { + [TestMethod()] + public void ExtractDistinctGraphObjectSetsTest() + { + var rootGraph = new + { + Id = "abc123", + Items = new[] + { + new { Id = "def456", Plums = true, Children = new[] { new { Value = "abc123" } } }, + new { Id = "def456", Plums = true, Children = new[] { new { Value = "def132" } } }, + new { Id = "def456", Plums = true, Children = new[] { new { Value = "ggjg35" } } }, + } + }; + + var rootGraphDynamic = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(rootGraph), new ExpandoObjectConverter()); + var sets = rootGraphDynamic.ExtractDistinctGraphObjectSets().ToList(); + } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL.Tests/Transform/FlattenGraphExtensionsTests.cs b/MIFCore.Hangfire.APIETL.Tests/Transform/FlattenGraphExtensionsTests.cs new file mode 100644 index 0000000..2de77c3 --- /dev/null +++ b/MIFCore.Hangfire.APIETL.Tests/Transform/FlattenGraphExtensionsTests.cs @@ -0,0 +1,38 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Dynamic; + +namespace MIFCore.Hangfire.APIETL.Transform.Tests +{ + [TestClass()] + public class FlattenGraphExtensionsTests + { + [TestMethod()] + public void FlattenGraphTest() + { + dynamic expando = new ExpandoObject(); + expando.Id = "abc123"; + expando.Description = "does stuff"; + + dynamic nestedExpando = new ExpandoObject(); + nestedExpando.Id = "def456"; + nestedExpando.Plums = true; + + dynamic moreNested = new ExpandoObject(); + moreNested.Id = Guid.Empty; + moreNested.Date = DateTime.MaxValue; + moreNested.Items = new[] { "one", "two", "three" }; + + nestedExpando.Action = moreNested; + expando.Nested = nestedExpando; + + (expando as ExpandoObject).FlattenGraph(); + + Assert.AreEqual("def456", expando.Nested_Id); + Assert.AreEqual(true, expando.Nested_Plums); + + Assert.AreEqual(Guid.Empty, expando.Nested_Action_Id); + Assert.AreEqual(DateTime.MaxValue, expando.Nested_Action_Date); + Assert.AreEqual(3, expando.Nested_Action_Items.Length); + } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs new file mode 100644 index 0000000..23d50be --- /dev/null +++ b/MIFCore.Hangfire.APIETL/APIETLServiceCollectionExtensions.cs @@ -0,0 +1,102 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using MIFCore.Hangfire.APIETL.Extract; +using MIFCore.Hangfire.APIETL.Load; +using MIFCore.Hangfire.APIETL.Transform; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace MIFCore.Hangfire.APIETL +{ + public static class APIETLServiceCollectionExtensions + { + public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollection serviceDescriptors, Assembly assembly = null) + { + assembly ??= Assembly.GetCallingAssembly(); + + // Find all endpoint related types in the assembly + var endpoints = assembly + .GetTypes() + .Where(y => + y.GetCustomAttributes().Any() + || y.GetCustomAttributes().Any() + || y.GetCustomAttributes().Any()); + + return serviceDescriptors.AddApiEndpointsToExtract(endpoints); + } + + public static IServiceCollection AddApiEndpointsToExtract(this IServiceCollection serviceDescriptors, IEnumerable types) + { + // Register the services used to register jobs and create ApiEndpoint definitions + serviceDescriptors.TryAddSingleton(); + serviceDescriptors.TryAddTransient(); + + // Register the extract jobs & pipelines + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddScoped(); + + // Register the transform jobs & pipelines + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddScoped(); + + // Register the load jobs & pipelines + serviceDescriptors.TryAddTransient(); + serviceDescriptors.TryAddScoped(); + + foreach (var t in types) + { + var apiEndpointAttributes = t.GetCustomAttributes(); + var apiEndpointSelectorAttributes = t.GetCustomAttributes(); + var apiEndpointModelAttributes = t.GetCustomAttributes(); + + if (apiEndpointAttributes.Any() == false + && apiEndpointSelectorAttributes.Any() == false + && apiEndpointModelAttributes.Any() == false) + throw new ArgumentException($"The type {t.FullName} does not have an {nameof(ApiEndpointAttribute)}, {nameof(ApiEndpointSelectorAttribute)} or {nameof(ApiEndpointModel)} attributes."); + + // Register the extract services + if (typeof(IDefineEndpoints).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(IDefineEndpoints), t); + + if (typeof(IPrepareRequest).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(IPrepareRequest), t); + + if (typeof(IPrepareNextRequest).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(IPrepareNextRequest), t); + + // Register the transform services + if (typeof(IHandleResponse).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(IHandleResponse), t); + + if (typeof(IParseResponse).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(IParseResponse), t); + + if (typeof(ITransformModel).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(ITransformModel), t); + + // Register the load services + if (typeof(ILoadData).IsAssignableFrom(t)) + serviceDescriptors.AddScoped(typeof(ILoadData), t); + + if (apiEndpointModelAttributes.Any()) + { + var endpointModel = t.GetApiEndpointModel(); + serviceDescriptors.AddSingleton(endpointModel); + } + + // Register the endpoint name attribute, so an ApiEndpoint is created from it + if (apiEndpointAttributes.Any()) + { + foreach (var en in apiEndpointAttributes) + { + serviceDescriptors.AddSingleton(en); + } + } + } + + return serviceDescriptors; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpoint.cs b/MIFCore.Hangfire.APIETL/ApiEndpoint.cs new file mode 100644 index 0000000..a044d66 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpoint.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace MIFCore.Hangfire.APIETL +{ + public class ApiEndpoint + { + private readonly Lazy> routeParameters; + + public ApiEndpoint(string jobName, string httpClientName = null) : this() + { + this.HttpClientName = httpClientName; + this.JobName = jobName; + } + + internal ApiEndpoint(string name) : this() + { + this.Name = name; + this.JobName = name; + } + + private ApiEndpoint() + { + this.routeParameters = new Lazy>(() => + { + // Match on {something123} + var parameterRegex = new Regex(@"\{([A-Za-z0-9_]+)\}"); + var matches = parameterRegex.Matches(this.Name); + + // Return the names of the groups, i.e path/to/{id}/{otherId}, extract: id, otherId + // Select the second item in the group [1] as it is the "id" or "otherId" instead of "{id}", etc. + return matches.Select(y => y.Groups[1].Value).ToList(); + }); + } + + public string Name { get; internal set; } + public string JobName { get; internal set; } + + public IEnumerable RouteParameters => this.routeParameters.Value; + public IDictionary AdditionalHeaders { get; } = new Dictionary(); + + public string HttpClientName { get; } + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs b/MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs new file mode 100644 index 0000000..534e446 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpointAttribute.cs @@ -0,0 +1,23 @@ +using System; + +namespace MIFCore.Hangfire.APIETL +{ + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] + public class ApiEndpointAttribute : Attribute + { + public ApiEndpointAttribute(string endpointName) + { + this.EndpointName = endpointName; + } + + public ApiEndpointAttribute(string endpointName, string inputPath) + { + this.EndpointName = endpointName; + this.InputPath = inputPath; + } + + public string EndpointName { get; } + public string HttpClientName { get; set; } + public string InputPath { get; } + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs b/MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs new file mode 100644 index 0000000..5f45e77 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpointFactory.cs @@ -0,0 +1,71 @@ +using MIFCore.Hangfire.APIETL.Load; +using System.Collections.Generic; +using System.Linq; + +namespace MIFCore.Hangfire.APIETL +{ + internal class ApiEndpointFactory : IApiEndpointFactory + { + private readonly IEnumerable endpointNameAttributes; + private readonly IEnumerable apiEndpointModels; + private readonly IEnumerable endpointDefiners; + + public ApiEndpointFactory( + IEnumerable apiEndpointAttributes, + IEnumerable apiEndpointModels, + IEnumerable endpointDefiners) + { + this.endpointNameAttributes = apiEndpointAttributes; + this.apiEndpointModels = apiEndpointModels; + this.endpointDefiners = endpointDefiners; + } + + public async IAsyncEnumerable Create() + { + // Get the endpoints defined by the ApiEndpoint or by the ApiEndpointModels apis + var endpointsDefinedByAttributes = this.endpointNameAttributes + .Select(y => y.EndpointName) + .Union(this.apiEndpointModels.Select(x => x.EndpointName)) + .Distinct(); + + // Create ApiEndpoint records from the ApiEndpointAttribute + foreach (var endpointName in endpointsDefinedByAttributes) + { + var endpointDefiners = this.GetEndpointDefiners(endpointName); + + // If the endpoint corresponds to an IDefineEndpoint, then the endpoints will be created / transformed from it instead. + if (endpointDefiners.Any()) + { + // It is possible for the endpointDefiner to create multiple endpoints from a single name (such as when tenancy is used, same endpoint, different tenant) + foreach (var ed in endpointDefiners) + { + var apiEndpoints = ed.DefineEndpoints(endpointName); + + await foreach (var ep in apiEndpoints) + { + // Ensure the endpoint name is set as this is an internal field + ep.Name = endpointName; + + yield return ep; + } + } + } + + // Otherwise create a single endpoint from the name, using all defaults + else + { + yield return new ApiEndpoint(endpointName); + } + } + } + + private IEnumerable GetEndpointDefiners(string endpointName) + { + foreach (var ed in this.endpointDefiners) + { + if (ed.RespondsToEndpointName(endpointName)) + yield return ed; + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs new file mode 100644 index 0000000..12093f3 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpointRegister.cs @@ -0,0 +1,63 @@ +using Hangfire; +using MIFCore.Hangfire.APIETL.Extract; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL +{ + internal class ApiEndpointRegister : IApiEndpointRegister + { + private readonly IDictionary endpoints = new Dictionary(); + private readonly IRecurringJobManager recurringJobManager; + private readonly IApiEndpointFactory apiEndpointAttributeFactory; + + public ApiEndpointRegister(IRecurringJobManager recurringJobManager, IApiEndpointFactory apiEndpointAttributeFactory) + { + this.recurringJobManager = recurringJobManager; + this.apiEndpointAttributeFactory = apiEndpointAttributeFactory; + } + + public IEnumerable Endpoints { get => this.endpoints.Values.ToArray(); } + + public IApiEndpointRegister Register(ApiEndpoint endpoint) + { + this.endpoints.Add(endpoint.Name, endpoint); + + // If the endpoint has route parameters, then we don't want to register it as a recurring job. + if (endpoint.RouteParameters.Any()) + return this; + + this.recurringJobManager.AddOrUpdate( + endpoint.JobName, + job => job.Extract(endpoint.Name, null), + Cron.Daily() + ); + + return this; + } + + /// + /// Automatically registers all endpoints defined using the Attributes API as recurring jobs. + /// + /// + public async Task Register() + { + // If there are already endpoints registered, then we don't need to do anything otherwise it will cause double ups. + if (this.Endpoints.Any()) + return; + + var endpoints = this.apiEndpointAttributeFactory.Create(); + + await foreach (var ep in endpoints) + { + this.Register(ep); + } + } + + public ApiEndpoint Get(string endpointName) + { + return this.endpoints[endpointName]; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs b/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs new file mode 100644 index 0000000..f5b6226 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/ApiEndpointSelectorAttribute.cs @@ -0,0 +1,22 @@ +using System; + +namespace MIFCore.Hangfire.APIETL +{ + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] + public class ApiEndpointSelectorAttribute : Attribute + { + public ApiEndpointSelectorAttribute(string regex) + { + this.Regex = regex; + } + + public ApiEndpointSelectorAttribute(string regex, string inputPath) + { + this.Regex = regex; + this.InputPath = inputPath; + } + + public string Regex { get; } + public string InputPath { get; } + } +} diff --git a/MIFCore.Hangfire.APIETL/AssemblyInfo.cs b/MIFCore.Hangfire.APIETL/AssemblyInfo.cs new file mode 100644 index 0000000..86d3157 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/AssemblyInfo.cs @@ -0,0 +1,6 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("MIFCore.Hangfire.APIETL.Tests")] +namespace MIFCore.Hangfire.APIETL +{ +} diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiData.cs b/MIFCore.Hangfire.APIETL/Extract/ApiData.cs new file mode 100644 index 0000000..fb4eee2 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/ApiData.cs @@ -0,0 +1,17 @@ +using System; + +namespace MIFCore.Hangfire.APIETL.Extract +{ + public class ApiData + { + public Guid Id { get; set; } = Guid.NewGuid(); + public Guid? ParentId { get; set; } + + public string Endpoint { get; set; } + public string Uri { get; set; } + + public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.Now; + + public string Data { get; set; } + } +} diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs new file mode 100644 index 0000000..910e8a4 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractJob.cs @@ -0,0 +1,128 @@ +using Hangfire; +using MIFCore.Hangfire.APIETL.Transform; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Extract +{ + internal class ApiEndpointExtractJob : IApiEndpointExtractJob + { + private readonly IHttpClientFactory httpClientFactory; + private readonly IApiEndpointRegister apiEndpointRegister; + private readonly IBackgroundJobClient backgroundJobClient; + private readonly IApiEndpointExtractPipeline endpointExtractPipeline; + private readonly IApiEndpointTransformJob endpointTransformJob; + + public ApiEndpointExtractJob( + IHttpClientFactory httpClientFactory, + IApiEndpointRegister apiEndpointRegister, + IBackgroundJobClient backgroundJobClient, + IApiEndpointExtractPipeline endpointExtractPipeline, + IApiEndpointTransformJob endpointTransformJob) + { + this.httpClientFactory = httpClientFactory; + this.apiEndpointRegister = apiEndpointRegister; + this.backgroundJobClient = backgroundJobClient; + this.endpointExtractPipeline = endpointExtractPipeline; + this.endpointTransformJob = endpointTransformJob; + } + + [DisableIdenticalQueuedItems] + public async Task Extract(string endpointName, ExtractArgs extractArgs = null) + { + // If there aren't any endpoints registered, then we can't do anything. So let's just reschedule this job for later. + if (this.apiEndpointRegister.Endpoints.Any() == false) + { + throw new RescheduleJobException(DateTime.Now.AddSeconds(5)); + } + + var endpoint = this.apiEndpointRegister.Get(endpointName); + await this.ExtractEndpoint(endpoint, extractArgs); + } + + private async Task ExtractEndpoint(ApiEndpoint endpoint, ExtractArgs extractArgs = null) + { + if (endpoint is null) + { + throw new ArgumentNullException(nameof(endpoint)); + } + + extractArgs ??= new ExtractArgs(new Dictionary(), null); + + // Default to string.Empty to avoid an error if the endpoint.HttpClientName is null. + var httpClient = this.httpClientFactory.CreateClient(endpoint.HttpClientName ?? string.Empty); + + var request = await this.CreateRequest(httpClient.BaseAddress, endpoint, extractArgs); + var apiData = await this.ExecuteRequest(endpoint, httpClient, request, extractArgs); + + var nextRequestData = await this.endpointExtractPipeline.OnPrepareNextRequest(new PrepareNextRequestArgs(endpoint: endpoint, apiData: apiData, data: extractArgs.RequestData)); + var isLastRequest = nextRequestData.Keys.Any() == false; + + await this.endpointTransformJob.Transform(endpoint, apiData, extractArgs); + + // If OnPrepareNextRequest returns an empty dict, then we're done with this endpoint. + if (isLastRequest) + return; + + // Otherwise we need to schedule another job to continue extracting this endpoint. + this.backgroundJobClient.Enqueue(y => y.Extract(endpoint.Name, new ExtractArgs(nextRequestData, apiData.Id))); + } + + private async Task CreateRequest(Uri baseAddress, ApiEndpoint endpoint, ExtractArgs extractArgs) + { + var endpointName = endpoint.Name; + + if (endpoint.RouteParameters.Any()) + { + // If there are route parameters, then we need to replace the route parameters with the values from the extractArgs.RequestData + // e.g. if the endpoint.SourceName = "getStuff/{id}" and extractArgs.RequestData["id"] = 123, then endpoint.SourceName = "getStuff/123" + foreach (var routeParameter in endpoint.RouteParameters) + { + var routeParameterValue = extractArgs.RequestData[routeParameter]; + endpointName = endpointName.Replace($"{{{routeParameter}}}", routeParameterValue.ToString()); + } + } + + // Create a new request, using endpoint.SourceName as the relative uri + // i.e endpoint.SourceName = "getStuff" and httpClient.BaseAddress = "https://someapi/api/" + var request = new HttpRequestMessage + { + RequestUri = new Uri(baseAddress, endpointName) + }; + + // Stitch on any additional headers + foreach (var h in endpoint.AdditionalHeaders) + { + request.Headers.Add(h.Key, h.Value); + } + + // OnPrepareRequest after this system has finished up with the request + await this.endpointExtractPipeline.OnPrepareRequest(new PrepareRequestArgs(endpoint, request, extractArgs.RequestData)); + + return request; + } + + private async Task ExecuteRequest(ApiEndpoint endpoint, HttpClient httpClient, HttpRequestMessage request, ExtractArgs extractArgs) + { + // Get the response payload as a string + var response = await httpClient.SendAsync(request); + response.EnsureSuccessStatusCode(); + + var data = await response.Content.ReadAsStringAsync(); + + // Turn the payload response into an ApiData instance + var apiData = new ApiData + { + Endpoint = endpoint.Name, + Uri = response.RequestMessage.RequestUri.ToString(), + Data = data, + ParentId = extractArgs.ParentApiDataId + }; + + return apiData; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractPipeline.cs new file mode 100644 index 0000000..6f6a7bc --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/ApiEndpointExtractPipeline.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Extract +{ + internal class ApiEndpointExtractPipeline : IApiEndpointExtractPipeline + { + private readonly IEnumerable prepareRequests; + private readonly IEnumerable prepareNextRequests; + + public ApiEndpointExtractPipeline( + IEnumerable prepareRequests, + IEnumerable prepareNextRequests) + { + this.prepareRequests = prepareRequests; + this.prepareNextRequests = prepareNextRequests; + } + + public async Task OnPrepareRequest(PrepareRequestArgs args) + { + var relatedPrepareRequests = this.prepareRequests + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + + foreach (var prepareRequest in relatedPrepareRequests) + { + await prepareRequest.OnPrepareRequest(args); + } + } + + public async Task> OnPrepareNextRequest(PrepareNextRequestArgs args) + { + var relatedPreparedNextRequests = this.prepareNextRequests + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + + var result = new Dictionary(); + + foreach (var prepareNextRequest in relatedPreparedNextRequests) + { + // Get the data from the prepareNextRequest + var data = await prepareNextRequest.OnPrepareNextRequest(args); + + if (data == default(IDictionary)) + continue; + + // Merge it with the resulting dictionary + result = result.Union(data).ToDictionary(x => x.Key, x => x.Value); + } + + return result; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractJob.cs b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractJob.cs new file mode 100644 index 0000000..8b39eb6 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractJob.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Extract +{ + public interface IApiEndpointExtractJob + { + Task Extract(string endpointName, ExtractArgs extractArgs = null); + } + + public class ExtractArgs + { + public ExtractArgs(IDictionary requestData, Guid? parentApiDataId) + { + this.RequestData = requestData; + this.ParentApiDataId = parentApiDataId; + } + + public IDictionary RequestData { get; set; } + public Guid? ParentApiDataId { get; set; } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractPipeline.cs b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractPipeline.cs new file mode 100644 index 0000000..1380407 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/IApiEndpointExtractPipeline.cs @@ -0,0 +1,6 @@ +namespace MIFCore.Hangfire.APIETL.Extract +{ + internal interface IApiEndpointExtractPipeline : IPrepareRequest, IPrepareNextRequest + { + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Extract/IPrepareNextRequest.cs b/MIFCore.Hangfire.APIETL/Extract/IPrepareNextRequest.cs new file mode 100644 index 0000000..48aa930 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/IPrepareNextRequest.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Extract +{ + public interface IPrepareNextRequest : IApiEndpointService + { + Task> OnPrepareNextRequest(PrepareNextRequestArgs args); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Extract/IPrepareRequest.cs b/MIFCore.Hangfire.APIETL/Extract/IPrepareRequest.cs new file mode 100644 index 0000000..fef9ff2 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/IPrepareRequest.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Extract +{ + public interface IPrepareRequest : IApiEndpointService + { + Task OnPrepareRequest(PrepareRequestArgs args); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Extract/PrepareNextRequestArgs.cs b/MIFCore.Hangfire.APIETL/Extract/PrepareNextRequestArgs.cs new file mode 100644 index 0000000..7abe186 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/PrepareNextRequestArgs.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Extract +{ + public class PrepareNextRequestArgs + { + public PrepareNextRequestArgs(ApiEndpoint endpoint, ApiData apiData, IDictionary data) + { + this.Endpoint = endpoint; + this.ApiData = apiData; + this.Data = data ?? throw new System.ArgumentNullException(nameof(data)); + } + + public ApiEndpoint Endpoint { get; } + public ApiData ApiData { get; } + public IDictionary Data { get; } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Extract/PrepareRequestArgs.cs b/MIFCore.Hangfire.APIETL/Extract/PrepareRequestArgs.cs new file mode 100644 index 0000000..862805c --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Extract/PrepareRequestArgs.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; +using System.Net.Http; + +namespace MIFCore.Hangfire.APIETL.Extract +{ + public class PrepareRequestArgs + { + public PrepareRequestArgs(ApiEndpoint endpoint, HttpRequestMessage request, IDictionary data) + { + this.Endpoint = endpoint; + this.Request = request; + this.Data = data ?? throw new System.ArgumentNullException(nameof(data)); + } + + public ApiEndpoint Endpoint { get; } + public HttpRequestMessage Request { get; set; } + public IDictionary Data { get; set; } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointFactory.cs b/MIFCore.Hangfire.APIETL/IApiEndpointFactory.cs new file mode 100644 index 0000000..a14f444 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IApiEndpointFactory.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL +{ + public interface IApiEndpointFactory + { + IAsyncEnumerable Create(); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs b/MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs new file mode 100644 index 0000000..0c18db8 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IApiEndpointRegister.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL +{ + public interface IApiEndpointRegister + { + IEnumerable Endpoints { get; } + + ApiEndpoint Get(string endpointName); + Task Register(); + IApiEndpointRegister Register(ApiEndpoint endpoint); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointService.cs b/MIFCore.Hangfire.APIETL/IApiEndpointService.cs new file mode 100644 index 0000000..410b388 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IApiEndpointService.cs @@ -0,0 +1,4 @@ +namespace MIFCore.Hangfire.APIETL +{ + public interface IApiEndpointService { } +} diff --git a/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs b/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs new file mode 100644 index 0000000..14c7fcf --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IApiEndpointServiceExtensions.cs @@ -0,0 +1,38 @@ +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; + +namespace MIFCore.Hangfire.APIETL +{ + public static class IApiEndpointServiceExtensions + { + public static bool IsDefaultResponder(this IApiEndpointService apiEndpointService) + { + var type = apiEndpointService.GetType(); + var endpointNameAttributes = type.GetCustomAttributes(); + var endpointSelectorAttributes = type.GetCustomAttributes(); + + return endpointNameAttributes.Any() == false && endpointSelectorAttributes.Any() == false; + } + + public static bool RespondsToEndpointName(this IApiEndpointService apiEndpointService, string endpointName, string inputPath = null) + { + var type = apiEndpointService.GetType(); + var endpointNameAttributes = type.GetCustomAttributes(); + var endpointSelectorAttributes = type.GetCustomAttributes(); + + if (endpointNameAttributes.Any(y => y.EndpointName == endpointName && y.InputPath == inputPath)) + { + return true; + } + else if (endpointSelectorAttributes.Any()) + { + return endpointSelectorAttributes.Any(y => Regex.IsMatch(endpointName, y.Regex) && y.InputPath == inputPath); + } + else + { + return false; + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs b/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs new file mode 100644 index 0000000..0cdb7c4 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/IDefineEndpoints.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL +{ + public interface IDefineEndpoints : IApiEndpointService + { + IAsyncEnumerable DefineEndpoints(string endpointName); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs new file mode 100644 index 0000000..bd369c5 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadJob.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + internal class ApiEndpointLoadJob : IApiEndpointLoadJob + { + private readonly IApiEndpointLoadPipeline loadPipeline; + + public ApiEndpointLoadJob(IApiEndpointLoadPipeline loadPipeline) + { + this.loadPipeline = loadPipeline; + } + + public async Task Load(ApiEndpoint apiEndpoint, ApiEndpointModel model, List> dataToLoad) + { + // Ensure the destination is created + await this.loadPipeline.OnCreateDestination(new CreateDestinationArgs(apiEndpoint, model)); + + // Now load the data into the destination + await this.loadPipeline.OnLoadData(new LoadDataArgs(apiEndpoint, model, dataToLoad)); + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadPipeline.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadPipeline.cs new file mode 100644 index 0000000..d6eb1b8 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointLoadPipeline.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + internal class ApiEndpointLoadPipeline : IApiEndpointLoadPipeline + { + private readonly IEnumerable createDestinations; + private readonly IEnumerable loadDatas; + + public ApiEndpointLoadPipeline(IEnumerable createDestinations, IEnumerable loadDatas) + { + this.createDestinations = createDestinations; + this.loadDatas = loadDatas; + } + + public async Task OnCreateDestination(CreateDestinationArgs args) + { + var relatedCreateDestination = this.createDestinations + .LastOrDefault(y => y.RespondsToEndpointName(args.ApiEndpoint.Name) || y.IsDefaultResponder()); + + if (relatedCreateDestination is null) + return; + + await relatedCreateDestination.OnCreateDestination(args); + } + + public async Task OnLoadData(LoadDataArgs args) + { + var relatedLoadDatas = this.loadDatas + .Where(y => y.RespondsToEndpointName(args.ApiEndpoint.Name) || y.IsDefaultResponder()) + .ToList(); + + foreach (var r in relatedLoadDatas) + { + await r.OnLoadData(args); + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModel.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModel.cs new file mode 100644 index 0000000..4dec596 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModel.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public class ApiEndpointModel + { + public string EndpointName { get; set; } + public string DestinationName { get; set; } + + public string InputPath { get; set; } + + public IDictionary MappedProperties { get; } = new Dictionary(); + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs new file mode 100644 index 0000000..ff06226 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelAttribute.cs @@ -0,0 +1,24 @@ +using System; + +namespace MIFCore.Hangfire.APIETL.Load +{ + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] + public class ApiEndpointModelAttribute : Attribute + { + public ApiEndpointModelAttribute(string endpointName) + { + this.EndpointName = endpointName; + } + + public ApiEndpointModelAttribute(string endpointName, string destinationName) + { + this.EndpointName = endpointName; + this.DestinationName = destinationName; + } + + public string EndpointName { get; } + public string DestinationName { get; } + + public string InputPath { get; set; } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelProperty.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelProperty.cs new file mode 100644 index 0000000..35b9e2c --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelProperty.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public class ApiEndpointModelProperty + { + public string SourceName { get; set; } + public string DestinationName { get; set; } + + public bool IsKey { get; set; } + + public HashSet SourceType { get; set; } + public string DestinationType { get; set; } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs new file mode 100644 index 0000000..d33b818 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ApiEndpointModelPropertyAttribute.cs @@ -0,0 +1,26 @@ +using System; + +namespace MIFCore.Hangfire.APIETL.Load +{ + [AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] + public class ApiEndpointModelPropertyAttribute : Attribute + { + public ApiEndpointModelPropertyAttribute(string sourceName, string destinationName) + { + this.SourceName = sourceName; + this.DestinationName = destinationName; + } + + public ApiEndpointModelPropertyAttribute(string sourceName) + { + this.SourceName = sourceName; + } + + public ApiEndpointModelPropertyAttribute() { } + + public string SourceName { get; } + + public string DestinationName { get; } + public string DestinationType { get; set; } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs b/MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs new file mode 100644 index 0000000..47d3681 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/CreateDestinationArgs.cs @@ -0,0 +1,14 @@ +namespace MIFCore.Hangfire.APIETL.Load +{ + public class CreateDestinationArgs + { + public CreateDestinationArgs(ApiEndpoint apiEndpoint, ApiEndpointModel apiEndpointModel) + { + this.ApiEndpoint = apiEndpoint; + this.ApiEndpointModel = apiEndpointModel; + } + + public ApiEndpoint ApiEndpoint { get; } + public ApiEndpointModel ApiEndpointModel { get; } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadJob.cs b/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadJob.cs new file mode 100644 index 0000000..490165e --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadJob.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public interface IApiEndpointLoadJob + { + Task Load(ApiEndpoint apiEndpoint, ApiEndpointModel model, List> dataToLoad); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs b/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs new file mode 100644 index 0000000..8745a97 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/IApiEndpointLoadPipeline.cs @@ -0,0 +1,6 @@ +namespace MIFCore.Hangfire.APIETL.Load +{ + internal interface IApiEndpointLoadPipeline : ICreateDestination, ILoadData + { + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Load/ICreateDestination.cs b/MIFCore.Hangfire.APIETL/Load/ICreateDestination.cs new file mode 100644 index 0000000..5ef2f2c --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ICreateDestination.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public interface ICreateDestination : IApiEndpointService + { + Task OnCreateDestination(CreateDestinationArgs args); + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/IGetDestinationType.cs b/MIFCore.Hangfire.APIETL/Load/IGetDestinationType.cs new file mode 100644 index 0000000..4ccce14 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/IGetDestinationType.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public interface IGetDestinationType + { + Task GetDestinationType(ApiEndpointModel apiEndpointModel, string sourceKey, IEnumerable sourceModelTypes); + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/ILoadData.cs b/MIFCore.Hangfire.APIETL/Load/ILoadData.cs new file mode 100644 index 0000000..c023eea --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/ILoadData.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public interface ILoadData : IApiEndpointService + { + public Task OnLoadData(LoadDataArgs args); + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/LoadDataArgs.cs b/MIFCore.Hangfire.APIETL/Load/LoadDataArgs.cs new file mode 100644 index 0000000..a855d93 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/LoadDataArgs.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public class LoadDataArgs + { + public LoadDataArgs(ApiEndpoint apiEndpoint, ApiEndpointModel apiEndpointModel, List> dataToLoad) + { + this.ApiEndpoint = apiEndpoint; + this.ApiEndpointModel = apiEndpointModel; + this.DataToLoad = dataToLoad; + } + + public ApiEndpoint ApiEndpoint { get; } + public ApiEndpointModel ApiEndpointModel { get; } + public List> DataToLoad { get; } + } +} diff --git a/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs b/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs new file mode 100644 index 0000000..53c8f3f --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Load/TypeExtensions.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Reflection; + +namespace MIFCore.Hangfire.APIETL.Load +{ + public static class TypeExtensions + { + public static ApiEndpointModel GetApiEndpointModel(this Type type) + { + var attribute = type.GetCustomAttribute(); + + if (attribute is null) + return null; + + var model = new ApiEndpointModel + { + DestinationName = attribute.DestinationName, + EndpointName = attribute.EndpointName, + InputPath = attribute.InputPath + }; + + if (string.IsNullOrWhiteSpace(model.DestinationName)) + { + model.DestinationName = type.Name; + } + + var modelProperties = type.GetProperties() + .Select(y => new KeyValuePair(y, y.GetCustomAttribute() as Attribute ?? y.GetCustomAttribute())) + .Where(y => y.Value != null) + .ToDictionary( + keySelector: y => y.Key, + elementSelector: y => y.Value); + + foreach (var (propertyInfo, propertyAttribute) in modelProperties) + { + var property = new ApiEndpointModelProperty + { + IsKey = propertyInfo.GetCustomAttribute() != null, + SourceType = new HashSet { propertyInfo.PropertyType } + }; + + if (propertyAttribute is ApiEndpointModelPropertyAttribute modelPropertyAttribute) + { + property.SourceName = modelPropertyAttribute.SourceName; + property.DestinationName = modelPropertyAttribute.DestinationName; + property.DestinationType = modelPropertyAttribute.DestinationType; + } + + if (string.IsNullOrWhiteSpace(property.SourceName)) + { + property.SourceName = propertyInfo.Name; + property.DestinationName = propertyInfo.Name; + } + + model.MappedProperties.Add(property.SourceName, property); + } + + return model; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj new file mode 100644 index 0000000..3f40b1a --- /dev/null +++ b/MIFCore.Hangfire.APIETL/MIFCore.Hangfire.APIETL.csproj @@ -0,0 +1,34 @@ + + + + netstandard2.1 + Maitland Marshall + MAIT DEV + https://github.com/maitlandmarshall/MIFCore + https://github.com/maitlandmarshall/MIFCore + git + true + MIT + Hangfire, Scheduling, Jobs, Simplified, API, ETL + An add-on to MIFCore which allows users to define API endpoints to extract, transform and load. + + 1.0.0 + + + + + rev.$([System.DateTime]::UtcNow.ToString("yyyyMddHHmm")) + + + + + + + + + + + + + + diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs new file mode 100644 index 0000000..d62e7de --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformJob.cs @@ -0,0 +1,168 @@ +using Humanizer; +using MIFCore.Hangfire.APIETL.Extract; +using MIFCore.Hangfire.APIETL.Load; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + internal class ApiEndpointTransformJob : IApiEndpointTransformJob + { + private readonly IApiEndpointTransformPipeline transformPipeline; + private readonly IEnumerable apiEndpointModels; + private readonly IGetDestinationType getDestinationType; + private readonly IApiEndpointLoadJob apiEndpointLoadJob; + + public ApiEndpointTransformJob( + IApiEndpointTransformPipeline transformPipeline, + IEnumerable apiEndpointModels, + IGetDestinationType getDestinationType, + IApiEndpointLoadJob apiEndpointLoadJob) + { + this.transformPipeline = transformPipeline; + this.apiEndpointModels = apiEndpointModels; + this.getDestinationType = getDestinationType; + this.apiEndpointLoadJob = apiEndpointLoadJob; + } + + public async Task Transform(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs) + { + await this.transformPipeline.OnHandleResponse(new HandleResponseArgs(endpoint, apiData, extractArgs)); + var parsedData = await this.transformPipeline.OnParse(new ParseResponseArgs(endpoint, apiData, extractArgs)); + + if (parsedData != null) + { + // Get all the different object sets and group them by the ParentKey + var graphObjectSets = parsedData.ExtractDistinctGraphObjectSets( + new ExtractDistinctGraphObjectSetsExtensions.ExtractDistinctGraphObjectSetsArgs + { + Transform = async (args) => + { + await this.transformPipeline.OnTransformModel(new TransformModelArgs(endpoint, apiData, extractArgs, args)); + } + }) + .GroupBy(y => y.ParentKey) + .ToList(); + + foreach (var set in graphObjectSets) + { + // Has the user of the library defined any endpoint models for this object set? + var model = this.apiEndpointModels. + FirstOrDefault(y => y.EndpointName == endpoint.Name && y.InputPath == set.Key); + + // If they haven't, do nothing + if (model is null) + continue; + + // There may be properties in the API response that we haven't mapped. + // Automatically generate properties from the response so they can load into the destination + await this.AutoMapModelPropertiesGraphObjectSet(set, model); + + // Generate a flat list of items to load + var dataToLoad = set.SelectMany(y => y.Objects).ToList(); + + // The parsedData from the API may need to be parsed into .net CLR types as defined by the ApiEndpointModelProperty SourceTypes + var propertiesWithClrType = model.MappedProperties + .Values + .Where(y => y.SourceType.Where(y => y != null).Any()) + .ToList(); + + foreach (var prop in propertiesWithClrType) + { + // Get the clr type the property should have + var clrType = prop.SourceType.First(y => y != null); + var typeConverter = TypeDescriptor.GetConverter(clrType); + + foreach (var data in dataToLoad) + { + // Try to get the property from the data object + if (data.TryGetValue(prop.SourceName, out var propValue)) + { + // Do nothing if its null + if (propValue is null) + continue; + + var propValueType = propValue.GetType(); + + // Make sure the propValueType is different from the clrType, otherwise a conversion is not necessary + if (propValueType != clrType + && typeConverter.CanConvertFrom(propValueType)) + { + data[prop.SourceName] = typeConverter.ConvertFrom(propValue); + } + } + else + { + // Ensure all the dataToLoad items have all the properties defined in the model, even if just null. + data[prop.SourceName] = null; + } + } + } + + await this.apiEndpointLoadJob.Load(endpoint, model, dataToLoad); + } + } + } + + private async Task AutoMapModelPropertiesGraphObjectSet(IGrouping set, ApiEndpointModel apiEndpointModel) + { + var allKeyTypes = set.GetKeyTypes(); + + // Get the validKeyTypes and types from all objects with the same ParentKey + var validKeyTypes = allKeyTypes + + // Filter out the validKeyTypes which have collections + .Where(y => y.Value.Any(y => typeof(IEnumerable).IsAssignableFrom(y)) == false) + + // Filter out the validKeyTypes which only have null values + .Where(y => y.Value.All(y => y is null) == false) + .ToList(); + + // Now we have validKeyTypes which represent the destination table schema + foreach (var (key, types) in validKeyTypes) + { + // The key could have already been added to the model via ApiEndpointModel API + if (apiEndpointModel.MappedProperties.TryGetValue(key, out var apiEndpointModelProperty) == false) + { + apiEndpointModelProperty = new ApiEndpointModelProperty + { + SourceName = key, + DestinationType = await this.getDestinationType.GetDestinationType(apiEndpointModel, key, types), + SourceType = types, + IsKey = false + }; + + apiEndpointModel.MappedProperties.Add(key, apiEndpointModelProperty); + } + + // If it has already been added, the destination type may be unknown. If so we can try to determine it from the API response types. + else + { + if (string.IsNullOrWhiteSpace(apiEndpointModelProperty.DestinationType)) + { + apiEndpointModelProperty.DestinationType = await this.getDestinationType.GetDestinationType(apiEndpointModel, key, types); + } + } + + if (string.IsNullOrWhiteSpace(apiEndpointModelProperty.DestinationName)) + { + var underscoreReplacement = "****"; + + apiEndpointModelProperty.DestinationName = apiEndpointModelProperty + .SourceName + + // Pascalize removes underscores, and spaces, so replace them with a placeholder for now + // Add spaces so pascalize can capitalize words separated by underscores properly + .Replace("_", $" {underscoreReplacement} ") + .Pascalize() + + // Since spaces are removed, only the actual replacement characters should be left. + // Conver them back to underscores + .Replace(underscoreReplacement, "_"); + } + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs new file mode 100644 index 0000000..33db7c8 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ApiEndpointTransformPipeline.cs @@ -0,0 +1,55 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + internal class ApiEndpointTransformPipeline : IApiEndpointTransformPipeline + { + private readonly IEnumerable handleResponses; + private readonly IEnumerable parseResponses; + private readonly IEnumerable transformModels; + + public ApiEndpointTransformPipeline(IEnumerable handleResponses, IEnumerable parseResponses, IEnumerable transformModels) + { + this.handleResponses = handleResponses; + this.parseResponses = parseResponses; + this.transformModels = transformModels; + } + + public async Task OnHandleResponse(HandleResponseArgs args) + { + var relatedHandleResponses = this.handleResponses + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + + foreach (var handleResponse in relatedHandleResponses) + { + await handleResponse.OnHandleResponse(args); + } + } + + public async Task>> OnParse(ParseResponseArgs args) + { + var relatedParseResponses = this.parseResponses + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name)); + + foreach (var handleResponse in relatedParseResponses) + { + return await handleResponse.OnParse(args); + } + + return null; + } + + public async Task OnTransformModel(TransformModelArgs args) + { + var relatedTransformModels = this.transformModels + .Where(y => y.RespondsToEndpointName(args.Endpoint.Name, args.Transform.GraphObjectSet.ParentKey)); + + foreach (var transformModel in relatedTransformModels) + { + await transformModel.OnTransformModel(args); + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs new file mode 100644 index 0000000..7d4d3b0 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ExtractDistinctGraphObjectSetsExtensions.cs @@ -0,0 +1,119 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public static class ExtractDistinctGraphObjectSetsExtensions + { + public delegate Task TransformObjectDelegate(TransformObjectArgs args); + + public static IEnumerable ExtractDistinctGraphObjectSets(this IEnumerable> root, ExtractDistinctGraphObjectSetsArgs args = null) + { + var objects = new List>(); + + args ??= new ExtractDistinctGraphObjectSetsArgs(); + args.RootObjectSet ??= new GraphObjectSet + { + Objects = objects + }; + + yield return args.RootObjectSet; + + foreach (var rootItem in root) + { + objects.Add(rootItem); + + var nestedObjectSets = rootItem.ExtractDistinctGraphObjectSets(args); + + foreach (var nos in nestedObjectSets) + { + yield return nos; + } + } + } + + public static IEnumerable ExtractDistinctGraphObjectSets(this IDictionary rootItem, ExtractDistinctGraphObjectSetsArgs args = null) + { + args ??= new ExtractDistinctGraphObjectSetsArgs(); + var rootObjectSet = args.RootObjectSet; + + // If no root object set was passed in (passed in from the IEnumerable version of ExtractDistinctGraphObjectSets) + // create a new GraphObjectSet to represent this single entity (root entity of the graph) + if (rootObjectSet is null) + { + rootObjectSet = new GraphObjectSet + { + Objects = new[] { rootItem } + }; + + args.RootObjectSet = rootObjectSet; + + yield return rootObjectSet; + } + + // loop through each key to find enumerable values + foreach (var (rootKey, rootValue) in rootItem) + { + IEnumerable nestedObjectSets; + + if (rootValue is IEnumerable childObjects) + { + var childDicts = childObjects.Cast>(); + + // Get the children of the children + nestedObjectSets = childDicts.ExtractDistinctGraphObjectSets(new ExtractDistinctGraphObjectSetsArgs + { + Transform = args.Transform + }); + } + else if (rootValue is IDictionary childDict) + { + nestedObjectSets = childDict.ExtractDistinctGraphObjectSets(new ExtractDistinctGraphObjectSetsArgs + { + Transform = args.Transform + }); + } + else + { + continue; + } + + // Link the children of the children via keys + foreach (var n in nestedObjectSets) + { + if (n.Parent is null) + { + n.Parent = rootItem; + n.ParentSet = rootObjectSet; + n.ParentKey = rootKey; + } + else + { + n.ParentKey = $"{rootKey}.{n.ParentKey}"; + } + + yield return n; + } + } + + args.Transform?.Invoke(new TransformObjectArgs + { + Object = rootItem, + GraphObjectSet = rootObjectSet + }); + } + + public class TransformObjectArgs + { + public IDictionary Object { get; set; } + public GraphObjectSet GraphObjectSet { get; set; } + } + + public class ExtractDistinctGraphObjectSetsArgs + { + internal GraphObjectSet RootObjectSet { get; set; } + public TransformObjectDelegate Transform { get; set; } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs new file mode 100644 index 0000000..d8b13c7 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/FlattenGraphExtensions.cs @@ -0,0 +1,58 @@ +using System.Collections.Generic; +using System.Dynamic; +using System.Linq; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public static class FlattenGraphExtensions + { + public static void FlattenGraph(this ExpandoObject expando) + { + (expando as IDictionary).FlattenGraph(); + } + + public static void FlattenGraph(this IEnumerable> rootArray) + { + foreach (var a in rootArray) + a.FlattenGraph(); + } + + public static void FlattenGraph(this IDictionary rootDict) + { + var keys = rootDict.Keys.ToList(); + + // iterate and check for nested objects + foreach (var rootKey in keys) + { + var rootValue = rootDict[rootKey]; + + // Is the property an object with keys and values? + if (rootValue is IDictionary nestedDict) + { + // Flatten the nestedDict so the entire graph flattens recursively + nestedDict.FlattenGraph(); + + // Get all the keys in the nestedDict and add them to the rootDict + foreach (var nestKey in nestedDict.Keys) + { + var nestValue = nestedDict[nestKey]; + + // Ensure the key is unique + rootDict.TryAdd($"{rootKey}_{nestKey}", nestValue); + } + + // Remove the original rootKey from the rootDict as all the properties have been merged into it + rootDict.Remove(rootKey, out _); + } + else if (rootValue is IEnumerable array) + { + var items = array + .Cast>() + .ToList(); + + items.FlattenGraph(); + } + } + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs b/MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs new file mode 100644 index 0000000..84b23d9 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/GraphObjectSet.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public class GraphObjectSet + { + public IDictionary Parent { get; internal set; } + public GraphObjectSet ParentSet { get; internal set; } + public string ParentKey { get; internal set; } + + public IList> Objects { get; internal set; } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/GraphObjectSetGetKeysExtensions.cs b/MIFCore.Hangfire.APIETL/Transform/GraphObjectSetGetKeysExtensions.cs new file mode 100644 index 0000000..414b4b6 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/GraphObjectSetGetKeysExtensions.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public static class GraphObjectSetGetKeysExtensions + { + public static IDictionary> GetKeyTypes(this IEnumerable graphObjectSets) + { + var allKeys = new Dictionary>(); + + foreach (var o in graphObjectSets) + { + var keys = o.GetKeyTypes(); + + foreach (var (key, value) in keys) + { + if (allKeys.TryGetValue(key, out var existingObjectValueType) == false) + { + existingObjectValueType = new HashSet(); + allKeys[key] = existingObjectValueType; + } + + foreach (var t in value) + { + existingObjectValueType.Add(t); + } + } + } + + return allKeys; + } + + public static IDictionary> GetKeyTypes(this GraphObjectSet graphObjectSet) + { + var keys = new Dictionary>(); + + foreach (var obj in graphObjectSet.Objects) + { + foreach (var (objKey, objectValue) in obj) + { + var objectValueType = objectValue == null ? null : objectValue.GetType(); + + if (keys.TryGetValue(objKey, out var existingObjectValueType) == false) + { + existingObjectValueType = new HashSet(); + keys[objKey] = existingObjectValueType; + } + + existingObjectValueType.Add(objectValueType); + } + } + + return keys; + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs b/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs new file mode 100644 index 0000000..a1b97fa --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/HandleResponseArgs.cs @@ -0,0 +1,11 @@ +using MIFCore.Hangfire.APIETL.Extract; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public class HandleResponseArgs : ResponseArgsBase + { + public HandleResponseArgs(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs) : base(endpoint, apiData, extractArgs) + { + } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs new file mode 100644 index 0000000..b56e4b5 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformJob.cs @@ -0,0 +1,10 @@ +using MIFCore.Hangfire.APIETL.Extract; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public interface IApiEndpointTransformJob + { + Task Transform(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs new file mode 100644 index 0000000..4ede5fe --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/IApiEndpointTransformPipeline.cs @@ -0,0 +1,6 @@ +namespace MIFCore.Hangfire.APIETL.Transform +{ + internal interface IApiEndpointTransformPipeline : IHandleResponse, IParseResponse, ITransformModel + { + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/IHandleResponse.cs b/MIFCore.Hangfire.APIETL/Transform/IHandleResponse.cs new file mode 100644 index 0000000..8b14d66 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/IHandleResponse.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public interface IHandleResponse : IApiEndpointService + { + Task OnHandleResponse(HandleResponseArgs args); + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs b/MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs new file mode 100644 index 0000000..1aec00b --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/IParseResponse.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public interface IParseResponse : IApiEndpointService + { + Task>> OnParse(ParseResponseArgs args); + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/ITransformModel.cs b/MIFCore.Hangfire.APIETL/Transform/ITransformModel.cs new file mode 100644 index 0000000..1f06373 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ITransformModel.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public interface ITransformModel : IApiEndpointService + { + Task OnTransformModel(TransformModelArgs args); + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs b/MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs new file mode 100644 index 0000000..1e4082e --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ParseResponseArgs.cs @@ -0,0 +1,11 @@ +using MIFCore.Hangfire.APIETL.Extract; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public class ParseResponseArgs : ResponseArgsBase + { + public ParseResponseArgs(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs) : base(endpoint, apiData, extractArgs) + { + } + } +} diff --git a/MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs b/MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs new file mode 100644 index 0000000..ab8cc8d --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/ResponseArgsBase.cs @@ -0,0 +1,19 @@ +using MIFCore.Hangfire.APIETL.Extract; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public abstract class ResponseArgsBase + { + public ResponseArgsBase(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs) + { + this.Endpoint = endpoint; + this.ApiData = apiData; + this.ExtractArgs = extractArgs; + } + + public ApiData ApiData { get; } + public ApiEndpoint Endpoint { get; } + + public ExtractArgs ExtractArgs { get; } + } +} \ No newline at end of file diff --git a/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs b/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs new file mode 100644 index 0000000..e226dc7 --- /dev/null +++ b/MIFCore.Hangfire.APIETL/Transform/TransformModelArgs.cs @@ -0,0 +1,15 @@ +using MIFCore.Hangfire.APIETL.Extract; +using static MIFCore.Hangfire.APIETL.Transform.ExtractDistinctGraphObjectSetsExtensions; + +namespace MIFCore.Hangfire.APIETL.Transform +{ + public class TransformModelArgs : ResponseArgsBase + { + public TransformModelArgs(ApiEndpoint endpoint, ApiData apiData, ExtractArgs extractArgs, TransformObjectArgs transformArgs) : base(endpoint, apiData, extractArgs) + { + this.Transform = transformArgs; + } + + public TransformObjectArgs Transform { get; } + } +} diff --git a/MIFCore.Hangfire/RescheduleJobByDateFilter.cs b/MIFCore.Hangfire/RescheduleJobByDateFilter.cs index 4e18f85..5ccfb05 100644 --- a/MIFCore.Hangfire/RescheduleJobByDateFilter.cs +++ b/MIFCore.Hangfire/RescheduleJobByDateFilter.cs @@ -1,7 +1,4 @@ using Hangfire.States; -using System; -using System.Collections.Generic; -using System.Text; namespace MIFCore.Hangfire { @@ -17,7 +14,7 @@ public void OnStateElection(ElectStateContext context) if (failedState.Exception is RescheduleJobException exception) { context.SetJobParameter("RetryCount", 0); - context.CandidateState = new ScheduledState(exception.RescheduleDate) + context.CandidateState = new ScheduledState(exception.RescheduleDate.ToUniversalTime()) { Reason = $"Job has been rescheduled for {exception.RescheduleDate.ToLocalTime():dd/MM/yyyy HH:mm:ss}" }; diff --git a/MIFCore.sln b/MIFCore.sln index 8a4795f..3c10443 100644 --- a/MIFCore.sln +++ b/MIFCore.sln @@ -17,6 +17,16 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MIFCore.Hangfire", "MIFCore EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MIFCore.Common", "MIFCore.Common\MIFCore.Common.csproj", "{A583D632-B2AD-4914-97EF-426AAF4725CB}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MIFCore.Hangfire.APIETL", "MIFCore.Hangfire.APIETL\MIFCore.Hangfire.APIETL.csproj", "{0679FE87-1E69-4CE8-899B-BDC15340FDCB}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "APIETL", "APIETL", "{86C67505-D831-4A34-829E-D12EB7E2BDA7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MIFCore.Hangfire.APIETL.Tests", "MIFCore.Hangfire.APIETL.Tests\MIFCore.Hangfire.APIETL.Tests.csproj", "{33568C51-42C1-41A6-8CCF-0F62FF905378}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MIFCore.Hangfire.APIETL.SqlServer", "MIFCore.Hangfire.APIETL.SqlServer\MIFCore.Hangfire.APIETL.SqlServer.csproj", "{0BF5DF42-6060-457B-87CB-2C2DFB557EE7}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sources", "Sources", "{9E8E78A2-0ED5-46F7-A89E-5212FC2B5250}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -47,6 +57,18 @@ Global {A583D632-B2AD-4914-97EF-426AAF4725CB}.Debug|Any CPU.Build.0 = Debug|Any CPU {A583D632-B2AD-4914-97EF-426AAF4725CB}.Release|Any CPU.ActiveCfg = Release|Any CPU {A583D632-B2AD-4914-97EF-426AAF4725CB}.Release|Any CPU.Build.0 = Release|Any CPU + {0679FE87-1E69-4CE8-899B-BDC15340FDCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0679FE87-1E69-4CE8-899B-BDC15340FDCB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0679FE87-1E69-4CE8-899B-BDC15340FDCB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0679FE87-1E69-4CE8-899B-BDC15340FDCB}.Release|Any CPU.Build.0 = Release|Any CPU + {33568C51-42C1-41A6-8CCF-0F62FF905378}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {33568C51-42C1-41A6-8CCF-0F62FF905378}.Debug|Any CPU.Build.0 = Debug|Any CPU + {33568C51-42C1-41A6-8CCF-0F62FF905378}.Release|Any CPU.ActiveCfg = Release|Any CPU + {33568C51-42C1-41A6-8CCF-0F62FF905378}.Release|Any CPU.Build.0 = Release|Any CPU + {0BF5DF42-6060-457B-87CB-2C2DFB557EE7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0BF5DF42-6060-457B-87CB-2C2DFB557EE7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0BF5DF42-6060-457B-87CB-2C2DFB557EE7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0BF5DF42-6060-457B-87CB-2C2DFB557EE7}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -54,6 +76,10 @@ Global GlobalSection(NestedProjects) = preSolution {B4B03402-85EA-49BE-BC98-D3AF0903F005} = {E057276C-548A-4190-AF5A-9A92E08268A1} {7FA32F14-5D33-4F0D-9F38-DF19D2C7F6E2} = {E057276C-548A-4190-AF5A-9A92E08268A1} + {0679FE87-1E69-4CE8-899B-BDC15340FDCB} = {86C67505-D831-4A34-829E-D12EB7E2BDA7} + {33568C51-42C1-41A6-8CCF-0F62FF905378} = {86C67505-D831-4A34-829E-D12EB7E2BDA7} + {0BF5DF42-6060-457B-87CB-2C2DFB557EE7} = {9E8E78A2-0ED5-46F7-A89E-5212FC2B5250} + {9E8E78A2-0ED5-46F7-A89E-5212FC2B5250} = {86C67505-D831-4A34-829E-D12EB7E2BDA7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {70D42A47-49DB-4784-8B40-89988134D6C8} diff --git a/MIFCore/MIFCore.csproj b/MIFCore/MIFCore.csproj index 9d92aaf..dcdeba0 100644 --- a/MIFCore/MIFCore.csproj +++ b/MIFCore/MIFCore.csproj @@ -32,8 +32,8 @@ - - + +