diff --git a/Brightcove.Core/EmbedGenerator/Models/EmbedModel.cs b/Brightcove.Core/EmbedGenerator/Models/EmbedModel.cs index 06b5e71d..cb69746e 100644 --- a/Brightcove.Core/EmbedGenerator/Models/EmbedModel.cs +++ b/Brightcove.Core/EmbedGenerator/Models/EmbedModel.cs @@ -30,6 +30,8 @@ public class EmbedModel public string Language { get; set; } = ""; + public string Title { get; set; } = ""; + public EmbedModel() { diff --git a/Brightcove.Core/Models/Experience.cs b/Brightcove.Core/Models/Experience.cs index 51e867f0..b03c6d13 100644 --- a/Brightcove.Core/Models/Experience.cs +++ b/Brightcove.Core/Models/Experience.cs @@ -10,9 +10,12 @@ namespace Brightcove.Core.Models /// public class Experience : Asset { - [JsonProperty("created_at", NullValueHandling = NullValueHandling.Ignore)] + [JsonProperty("createdAt", NullValueHandling = NullValueHandling.Ignore)] public DateTime CreationDate { get; set; } + [JsonProperty("updatedAt", NullValueHandling = NullValueHandling.Ignore)] + public new DateTime? LastModifiedDate { get; set; } + [JsonProperty("publishedUrl", NullValueHandling = NullValueHandling.Ignore)] public string Url { get; set; } diff --git a/Brightcove.Core/Models/Video.cs b/Brightcove.Core/Models/Video.cs index d1cad2f3..1288aa99 100644 --- a/Brightcove.Core/Models/Video.cs +++ b/Brightcove.Core/Models/Video.cs @@ -62,6 +62,9 @@ public class Video : Asset [JsonProperty("schedule", NullValueHandling = NullValueHandling.Ignore)] public VideoSchedule Schedule { get; set; } + [JsonProperty("variants", NullValueHandling = NullValueHandling.Ignore)] + public List Variants { get; set; } + public Video ShallowCopy() { return (Video)this.MemberwiseClone(); diff --git a/Brightcove.Core/Services/BrightcoveHttpClient.cs b/Brightcove.Core/Services/BrightcoveHttpClient.cs index eaba8421..833da3e4 100644 --- a/Brightcove.Core/Services/BrightcoveHttpClient.cs +++ b/Brightcove.Core/Services/BrightcoveHttpClient.cs @@ -14,6 +14,7 @@ public static class BrightcoveHttpClient static BrightcoveHttpClient() { Instance = new HttpClient(); + Instance.Timeout = TimeSpan.FromSeconds(30); } } } diff --git a/Brightcove.Core/Services/BrightcoveService.cs b/Brightcove.Core/Services/BrightcoveService.cs index 14b7f4f0..53c9ed8f 100644 --- a/Brightcove.Core/Services/BrightcoveService.cs +++ b/Brightcove.Core/Services/BrightcoveService.cs @@ -16,6 +16,9 @@ public class BrightcoveService { readonly static HttpClient client = BrightcoveHttpClient.Instance; + int retryMax = 3; + int retryAttempt = 0; + readonly string cmsBaseUrl = "https://cms.api.brightcove.com/v1/accounts"; readonly string ingestBaseUrl = "https://ingest.api.brightcove.com/v1/accounts"; readonly string playersBaseUrl = "https://players.api.brightcove.com/v1/accounts"; @@ -306,6 +309,7 @@ public Video UpdateVideo(Video video) Video newVideo = video.ShallowCopy(); newVideo.Id = null; newVideo.Images = null; + newVideo.Variants = null; string content = JsonConvert.SerializeObject(newVideo); @@ -519,12 +523,12 @@ public PlayerList GetPlayers() return players; } - public ExperienceList GetExperiences() + public ExperienceList GetExperiences(string query = "", string sort = "") { HttpRequestMessage request = new HttpRequestMessage(); request.Method = HttpMethod.Get; - request.RequestUri = new Uri($"{experienceBaseUrl}/{accountId}/experiences"); + request.RequestUri = new Uri($"{experienceBaseUrl}/{accountId}/experiences?query={query}&sort={sort}"); HttpResponseMessage response = SendRequest(request); @@ -719,12 +723,12 @@ public bool TryGetPlaylist(string playlistId, out PlayList playlist) return true; } - public int VideosCount() + public int VideosCount(string query = "") { HttpRequestMessage request = new HttpRequestMessage(); request.Method = HttpMethod.Get; - request.RequestUri = new Uri($"{cmsBaseUrl}/{accountId}/counts/videos"); + request.RequestUri = new Uri($"{cmsBaseUrl}/{accountId}/counts/videos?query={query}"); HttpResponseMessage response = SendRequest(request); Count count = JsonConvert.DeserializeObject(response.Content.ReadAsString()); @@ -747,16 +751,30 @@ public int PlayListsCount() private HttpResponseMessage SendRequest(HttpRequestMessage request) { - request.Headers.Authorization = authenticationService.CreateAuthenticationHeader(); + try + { + request.Headers.Authorization = authenticationService.CreateAuthenticationHeader(); - HttpResponseMessage response = client.Send(request); + HttpResponseMessage response = client.Send(request); - if (!response.IsSuccessStatusCode) - { - throw new HttpStatusException(request, response); + if (!response.IsSuccessStatusCode) + { + throw new HttpStatusException(request, response); + } + + return response; } + catch(Exception ex) + { + if(retryAttempt < retryMax) + { + retryAttempt++; + return SendRequest(request); + } - return response; + retryAttempt = 0; + throw ex; + } } } } diff --git a/Brightcove.DataExchangeFramework/Brightcove.DataExchangeFramework.csproj b/Brightcove.DataExchangeFramework/Brightcove.DataExchangeFramework.csproj index ba1d3ea3..233e2a30 100644 --- a/Brightcove.DataExchangeFramework/Brightcove.DataExchangeFramework.csproj +++ b/Brightcove.DataExchangeFramework/Brightcove.DataExchangeFramework.csproj @@ -420,10 +420,13 @@ + + - + + @@ -435,6 +438,8 @@ + + @@ -449,18 +454,23 @@ - + + + + + + - - + + diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/ApplyMappingPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ApplyMappingPipelineStepConverter.cs new file mode 100644 index 00000000..d4f5bb4c --- /dev/null +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ApplyMappingPipelineStepConverter.cs @@ -0,0 +1,19 @@ +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data; +using Sitecore.Data.Items; +using Sitecore.DataExchange.ApplyMapping; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Providers.Sc.Converters.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Model; +using System; + +namespace Brightcove.DataExchangeFramework.Converters +{ + [SupportedIds("{AB4AF4DF-D282-4CD1-8268-FA12A9E457A3}")] + public class ApplyMappingPipelineStepConverter : ApplyMappingStepConverter + { + public ApplyMappingPipelineStepConverter(IItemModelRepository repository) : base(repository) { } + } +} \ No newline at end of file diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/ReadAssetItemsPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ReadAssetItemsPipelineStepConverter.cs index 7a25a228..b894ebce 100644 --- a/Brightcove.DataExchangeFramework/Converters/PipelineStep/ReadAssetItemsPipelineStepConverter.cs +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ReadAssetItemsPipelineStepConverter.cs @@ -31,12 +31,16 @@ protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep) if (endpointId != null) { - Item endpointItem = Sitecore.Context.ContentDatabase.GetItem(new ID(endpointId)); + ItemModel endpointItem = ItemModelRepository.Get(endpointId); if(endpointItem != null) { - resolveAssetItemSettings.AcccountItemId = endpointItem["Account"]; + resolveAssetItemSettings.AcccountItemId = this.GetStringValue(endpointItem, "Account") ?? ""; resolveAssetItemSettings.RelativePath = this.GetStringValue(source, "RelativePath") ?? ""; + + Database database = Sitecore.Configuration.Factory.GetDatabase(ItemModelRepository.DatabaseName); + resolveAssetItemSettings.AccountItem = database.GetItem(resolveAssetItemSettings.AcccountItemId); + resolveAssetItemSettings.ParentItem = database.GetItem(resolveAssetItemSettings.AccountItem?.Paths?.Path + "/" + resolveAssetItemSettings.RelativePath); } } diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/ResolveAssetItemPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ResolveAssetItemPipelineStepConverter.cs index ff11c4a6..fda30862 100644 --- a/Brightcove.DataExchangeFramework/Converters/PipelineStep/ResolveAssetItemPipelineStepConverter.cs +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ResolveAssetItemPipelineStepConverter.cs @@ -24,15 +24,26 @@ protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep) if (endpointId != null) { - Item endpointItem = Sitecore.Context.ContentDatabase.GetItem(new ID(endpointId)); + ItemModel endpointItem = ItemModelRepository.Get(endpointId); if (endpointItem != null) { - resolveAssetItemSettings.AcccountItemId = endpointItem["Account"]; + resolveAssetItemSettings.AcccountItemId = this.GetStringValue(endpointItem, "Account") ?? ""; resolveAssetItemSettings.RelativePath = this.GetStringValue(source, "RelativePath") ?? ""; + + Database database = Sitecore.Configuration.Factory.GetDatabase(ItemModelRepository.DatabaseName); + resolveAssetItemSettings.AccountItem = database.GetItem(resolveAssetItemSettings.AcccountItemId); + resolveAssetItemSettings.ParentItem = database.GetItem(resolveAssetItemSettings.AccountItem?.Paths?.Path + "/" + resolveAssetItemSettings.RelativePath); } } + //We need to store the resolve asset item plugin in the global Sitecore.DataExchangeContext so it + //can be used in the VideoIdsPropertyValueReader + if (Sitecore.DataExchange.Context.GetPlugin() == null) + { + Sitecore.DataExchange.Context.Plugins.Add(resolveAssetItemSettings); + } + pipelineStep.AddPlugin(resolveAssetItemSettings); } } diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/SyncPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/SyncPipelineStepConverter.cs new file mode 100644 index 00000000..a08f4ec5 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/SyncPipelineStepConverter.cs @@ -0,0 +1,44 @@ +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data; +using Sitecore.Data.Items; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Converters.PipelineSteps; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Providers.Sc.Converters.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Brightcove.DataExchangeFramework.Converters +{ + public class SyncPipelineStepConverter : BasePipelineStepConverter + { + public SyncPipelineStepConverter(IItemModelRepository repository) : base(repository) + { + + } + + protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep) + { + Guid endpointId = this.GetGuidValue(source, "EndpointFrom"); + BrightcoveSyncSettings settings = new BrightcoveSyncSettings(); + + if (endpointId != null) + { + ItemModel endpointItem = ItemModelRepository.Get(endpointId); + + if(endpointItem != null) + { + string accountId = this.GetStringValue(endpointItem, "Account") ?? ""; + settings.AccountItem = ItemModelRepository.Get(accountId); + } + } + + pipelineStep.AddPlugin(settings); + } + } +} diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateAssetItemPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateAssetItemPipelineStepConverter.cs new file mode 100644 index 00000000..bbc2b023 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateAssetItemPipelineStepConverter.cs @@ -0,0 +1,19 @@ +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data; +using Sitecore.Data.Items; +using Sitecore.DataExchange.ApplyMapping; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Providers.Sc.Converters.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Model; +using System; + +namespace Brightcove.DataExchangeFramework.Converters +{ + [SupportedIds("{F4AF37B2-F92D-4E49-A017-3D7489E23910}")] + public class UpdateAssetItemPipelineStepConverter : UpdateSitecoreItemStepConverter + { + public UpdateAssetItemPipelineStepConverter(IItemModelRepository repository) : base(repository) { } + } +} \ No newline at end of file diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateVideoPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateVideoPipelineStepConverter.cs new file mode 100644 index 00000000..a187cbe3 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateVideoPipelineStepConverter.cs @@ -0,0 +1,81 @@ +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data; +using Sitecore.Data.Items; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Converters.PipelineSteps; +using Sitecore.DataExchange.DataAccess; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Plugins; +using Sitecore.DataExchange.Providers.Sc.Converters.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Model; +using System; + +namespace Brightcove.DataExchangeFramework.Converters +{ + [SupportedIds("{D79A5F5C-9A5E-4B1C-B884-8E3B97CACA2D}", "{F598D123-2FE9-45D3-99E2-3E4B5063190A}")] + public class UpdateVideoPipelineStepConverter : BasePipelineStepConverter + { + public UpdateVideoPipelineStepConverter(IItemModelRepository repository) : base(repository) { } + + protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep) + { + MappingSettings mappingSettings = new MappingSettings() + { + ModelMappingSets = this.ConvertReferencesToModels(source, "ModelMappingSets"), + VariantMappingSets = this.ConvertReferencesToModels(source, "VariantMappingSets"), + SourceObjectLocation = this.GetGuidValue(source, "SourceObjectLocation"), + TargetObjectLocation = this.GetGuidValue(source, "TargetObjectLocation") + }; + + pipelineStep.AddPlugin(mappingSettings); + + BrightcoveEndpointSettings endpointSettings = new BrightcoveEndpointSettings() + { + BrightcoveEndpoint = this.ConvertReferenceToModel(source, "BrightcoveEndpoint"), + SitecoreEndpoint = this.ConvertReferenceToModel(source, "SitecoreEndpoint") + }; + + pipelineStep.AddPlugin(endpointSettings); + + Guid endpointId = this.GetGuidValue(source, "BrightcoveEndpoint"); + + if(endpointId == null) + { + return; + } + + ItemModel endpointModel = ItemModelRepository.Get(endpointId); + + if(endpointModel == null) + { + return; + } + + Guid accountItemId = this.GetGuidValue(endpointModel, "Account"); + + if(accountItemId == null) + { + return; + } + + ItemModel accountItem = ItemModelRepository.Get(accountItemId); + + if(accountItem == null) + { + return; + } + + WebApiSettings webApiSettings = new WebApiSettings(); + + if (accountItem != null) + { + webApiSettings.AccountId = this.GetStringValue(accountItem, "AccountId") ?? ""; + webApiSettings.ClientId = this.GetStringValue(accountItem, "ClientId") ?? ""; + webApiSettings.ClientSecret = this.GetStringValue(accountItem, "ClientSecret") ?? ""; + } + + endpointSettings.BrightcoveEndpoint.AddPlugin(webApiSettings); + } + } +} \ No newline at end of file diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdatePipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/UpdatePipelineStepConverter.cs similarity index 100% rename from Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdatePipelineStepConverter.cs rename to Brightcove.DataExchangeFramework/Converters/UpdatePipelineStepConverter.cs diff --git a/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs b/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs index 2ef5114d..f69368dc 100644 --- a/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs +++ b/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs @@ -26,16 +26,15 @@ public WebApiEndpointConverter(IItemModelRepository repository) : base(repositor protected override void AddPlugins(ItemModel source, Endpoint endpoint) { Guid accountItemId = this.GetGuidValue(source, TemplateAccount); - Item accountItem = Sitecore.Context.ContentDatabase.GetItem(new ID(accountItemId)); + ItemModel accountItem = ItemModelRepository.Get(accountItemId); WebApiSettings accountSettings = new WebApiSettings(); if(accountItem != null) { - accountSettings.AccountId = accountItem["AccountId"]; - accountSettings.ClientId = accountItem["ClientId"]; - accountSettings.ClientSecret = accountItem["ClientSecret"]; - accountSettings.AccountItem = accountItem; + accountSettings.AccountId = this.GetStringValue(accountItem, "AccountId") ?? ""; + accountSettings.ClientId = this.GetStringValue(accountItem, "ClientId") ?? ""; + accountSettings.ClientSecret = this.GetStringValue(accountItem, "ClientSecret") ?? ""; } endpoint.AddPlugin(accountSettings); diff --git a/Brightcove.DataExchangeFramework/Helpers/BrightcoveSyncSettingsHelper.cs b/Brightcove.DataExchangeFramework/Helpers/BrightcoveSyncSettingsHelper.cs new file mode 100644 index 00000000..02c6bce4 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Helpers/BrightcoveSyncSettingsHelper.cs @@ -0,0 +1,45 @@ +using Brightcove.Core.Exceptions; +using Brightcove.Core.Models; +using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data.Fields; +using Sitecore.Data.Items; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.DataAccess; +using Sitecore.DataExchange.Extensions; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Plugins; +using Sitecore.DataExchange.Processors.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Diagnostics; +using Sitecore.Services.Core.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using Brightcove.Core.Extensions; +using Sitecore.DataExchange.ApplyMapping; +using Sitecore.Data; + +namespace Brightcove.DataExchangeFramework.Helpers +{ + public static class BrightcoveSyncSettingsHelper + { + public static void SetErrorFlag(PipelineContext pipelineContext) + { + if (pipelineContext == null) + { + throw new ArgumentNullException(nameof(pipelineContext)); + } + + var settings = pipelineContext.GetCurrentPipelineBatch().GetPlugin(); + + if(settings == null || settings.ErrorFlag) + { + return; + } + + settings.ErrorFlag = true; + } + } +} diff --git a/Brightcove.DataExchangeFramework/Helpers/IItemModelRepositoryHelper.cs b/Brightcove.DataExchangeFramework/Helpers/IItemModelRepositoryHelper.cs new file mode 100644 index 00000000..5fc93245 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Helpers/IItemModelRepositoryHelper.cs @@ -0,0 +1,38 @@ +using Brightcove.Core.Exceptions; +using Brightcove.Core.Models; +using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data.Fields; +using Sitecore.Data.Items; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.DataAccess; +using Sitecore.DataExchange.Extensions; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Plugins; +using Sitecore.DataExchange.Processors.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Diagnostics; +using Sitecore.Services.Core.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using Brightcove.Core.Extensions; +using Sitecore.DataExchange.ApplyMapping; +using Sitecore.Data; + +namespace Brightcove.DataExchangeFramework.Helpers +{ + public static class IItemModelRepositoryHelper + { + public static string GetIndexName(IItemModelRepository itemModelRepository) + { + if (itemModelRepository == null) + { + throw new ArgumentNullException(nameof(itemModelRepository)); + } + + return $"sitecore_{itemModelRepository.DatabaseName}_index"; + } + } +} diff --git a/Brightcove.DataExchangeFramework/Helpers/ItemUpdater.cs b/Brightcove.DataExchangeFramework/Helpers/ItemUpdater.cs index 93432616..d17a7cf3 100644 --- a/Brightcove.DataExchangeFramework/Helpers/ItemUpdater.cs +++ b/Brightcove.DataExchangeFramework/Helpers/ItemUpdater.cs @@ -76,50 +76,5 @@ private static void FixItemModel(ItemModel itemModel) itemModel[key] = (object)obj.ToString(); } } - - public static SyncStatus GetSyncStatus(ItemModel itemModel) - { - DateTime lastSyncTime = DateTime.MinValue; - DateTime modelLastModifiedTime = DateTime.MinValue; - DateTime itemLastModifiedTime = DateTime.MinValue; - - if (string.IsNullOrWhiteSpace((string)itemModel["LastSyncTime"])) - { - return SyncStatus.NewItem; - } - lastSyncTime = DateTime.Parse((string)itemModel["LastSyncTime"]); - - Item item = Sitecore.Context.ContentDatabase.GetItem(new ID(itemModel.GetItemId())); - - if (item != null) - { - itemLastModifiedTime = ((DateField)item.Fields["__Updated"]).DateTime; - } - - if (itemModel.ContainsKey("LastModifiedTime") && !string.IsNullOrWhiteSpace((string)itemModel["LastModifiedTime"])) - { - modelLastModifiedTime = DateTime.Parse((string)itemModel["LastModifiedTime"]); - } - - if (modelLastModifiedTime > lastSyncTime) - { - return SyncStatus.ModelNewer; - } - - if (itemLastModifiedTime > lastSyncTime) - { - return SyncStatus.ItemNewer; - } - - return SyncStatus.Unmodified; - } - } - - public enum SyncStatus - { - NewItem, - ModelNewer, - ItemNewer, - Unmodified } } diff --git a/Brightcove.DataExchangeFramework/Processors/ApplyMappingPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/ApplyMappingPipelineStepProcessor.cs new file mode 100644 index 00000000..91962954 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Processors/ApplyMappingPipelineStepProcessor.cs @@ -0,0 +1,139 @@ +using Brightcove.Core.Models; +using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Extensions; +using Brightcove.DataExchangeFramework.Helpers; +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data.Fields; +using Sitecore.Data.Items; +using Sitecore.DataExchange.ApplyMapping; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.DataAccess; +using Sitecore.DataExchange.Extensions; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Plugins; +using Sitecore.DataExchange.Processors.PipelineSteps; +using Sitecore.DataExchange.Providers.Sc.Processors.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Globalization; +using Sitecore.Services.Core.Diagnostics; +using Sitecore.Services.Core.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using MappingSettings = Sitecore.DataExchange.ApplyMapping.MappingSettings; + +namespace Brightcove.DataExchangeFramework.Processors +{ + public class ApplyMappingPipelineStepProcessor : ApplyMappingStepProcessor + { + protected override void ProcessPipelineStep(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) + { + try + { + MappingSettings mappingSettings = pipelineStep.GetMappingSettings(); + if (mappingSettings == null) + { + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because the pipeline step is missing a plugin.", new string[1] + { + "plugin: " + typeof (MappingSettings).FullName + }); + } + else + { + IMappingSet mappingSet = mappingSettings.MappingSet; + if (mappingSet == null) + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because the pipeline step has no mapping set assigned.", new string[2] + { + "plugin: " + typeof (MappingSettings).FullName, + "property: MappingSet" + }); + else if (mappingSet.Mappings == null) + { + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because the pipeline step has no mappings assigned.", new string[2] + { + "plugin: " + typeof (MappingSettings).FullName, + "property: MappingSet" + }); + } + else + { + object sourceObject = this.GetSourceObject(mappingSettings, pipelineContext, logger); + if (sourceObject == null) + { + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because no source object could be resolved from the pipeline context.", new string[1] + { + string.Format("source location: {0}", (object) mappingSettings.SourceObjectLocation) + }); + } + else + { + object targetObject = this.GetTargetObject(mappingSettings, pipelineContext, logger); + if (targetObject == null) + { + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because no target object could be resolved from the pipeline context.", new string[1] + { + string.Format("target location: {0}", (object) mappingSettings.TargetObjectLocation) + }); + } + else + { + MappingContext mappingContext = new MappingContext() + { + Source = sourceObject, + Target = targetObject + }; + if (!mappingSet.Run(mappingContext)) + { + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because mapping set failed.", new string[3] + { + string.Format("mappings that succeeded: {0}", (object) mappingContext.RunSuccess.Count), + string.Format("mappings that were not attempted: {0}", (object) mappingContext.RunIgnore.Count), + string.Format("mappings that failed: {0}", (object) mappingContext.RunFail.Count) + }); + this.Log(new Action(logger.Error), pipelineContext, "At least one required value mapping failed.", new string[2] + { + string.Format("mappings that failed: {0}", (object) mappingContext.RunFail.Count), + "value mapping ids: " + string.Join(",", mappingContext.RunFail.Select((Func) (x => x.Identifier)).ToArray()) + }); + + HandleError(pipelineContext); + } + else + { + if (mappingContext.RunFail.Any()) + { + this.Log(new Action(logger.Error), pipelineContext, "At least one value mapping failed.", new string[2] + { + string.Format("mappings that failed: {0}", (object) mappingContext.RunFail.Count), + "value mapping ids: " + string.Join(",", mappingContext.RunFail.Select((Func) (x => x.Identifier)).ToArray()) + }); + + HandleError(pipelineContext); + } + + pipelineContext.GetSynchronizationSettings().IsTargetDirty = this.IsTargetDirty(mappingContext, mappingSettings, pipelineContext, logger); + if (!this.ShouldRunMappingsAppliedActions(mappingContext, mappingSettings, pipelineContext, logger)) + return; + this.RunMappingsAppliedActions(mappingContext, mappingSettings, pipelineContext, logger); + } + } + } + } + } + } + catch(Exception ex) + { + logger.Error("Failed to apply mapping(s) because an unexpected error occured", ex); + HandleError(pipelineContext); + } + } + + protected void HandleError(PipelineContext pipelineContext) + { + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; + pipelineContext.CriticalError = false; + } + } +} diff --git a/Brightcove.DataExchangeFramework/Processors/BasePipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/BasePipelineStepProcessor.cs index f08fffaa..3f611489 100644 --- a/Brightcove.DataExchangeFramework/Processors/BasePipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/BasePipelineStepProcessor.cs @@ -1,5 +1,6 @@ using Brightcove.Core.Models; using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Helpers; using Brightcove.DataExchangeFramework.Settings; using Sitecore.DataExchange; using Sitecore.DataExchange.Attributes; @@ -22,28 +23,34 @@ namespace Brightcove.DataExchangeFramework.Processors { public class BasePipelineStepProcessor : Sitecore.DataExchange.Processors.PipelineSteps.BasePipelineStepProcessor { + protected IItemModelRepository itemModelRepository { get; set; } + protected override void ProcessPipelineStep(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - if (pipelineStep == null) - { - throw new ArgumentNullException(nameof(pipelineStep)); - } - if (pipelineContext == null) - { - throw new ArgumentNullException(nameof(pipelineContext)); - } - if (logger == null) - { - throw new ArgumentNullException(nameof(logger)); - } - try { + if (pipelineStep == null) + { + throw new ArgumentNullException(nameof(pipelineStep)); + } + if (pipelineContext == null) + { + throw new ArgumentNullException(nameof(pipelineContext)); + } + if (logger == null) + { + throw new ArgumentNullException(nameof(logger)); + } + + itemModelRepository = Sitecore.DataExchange.Context.ItemModelRepository; + ProcessPipelineStepInternal(pipelineStep, pipelineContext, logger); } catch(Exception ex) { LogFatal("An unexpected error occured running the internal pipeline step", ex); + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; pipelineContext.CriticalError = false; } } diff --git a/Brightcove.DataExchangeFramework/Processors/BasePipelineStepWithWebApiEndpointProcessor.cs b/Brightcove.DataExchangeFramework/Processors/BasePipelineStepWithWebApiEndpointProcessor.cs index 224fdac4..f2f5e39e 100644 --- a/Brightcove.DataExchangeFramework/Processors/BasePipelineStepWithWebApiEndpointProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/BasePipelineStepWithWebApiEndpointProcessor.cs @@ -1,5 +1,6 @@ using Brightcove.Core.Models; using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Helpers; using Brightcove.DataExchangeFramework.Settings; using Sitecore.Data.Fields; using Sitecore.Data.Items; @@ -10,6 +11,7 @@ using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.DataExchange.Processors.PipelineSteps; +using Sitecore.DataExchange.Providers.Sc.Plugins; using Sitecore.DataExchange.Repositories; using Sitecore.SecurityModel; using Sitecore.Services.Core.Diagnostics; @@ -25,54 +27,71 @@ namespace Brightcove.DataExchangeFramework.Processors public class BasePipelineStepWithWebApiEndpointProcessor : BasePipelineStepProcessor { protected WebApiSettings WebApiSettings { get; set; } + protected Endpoint EndpointFrom { get; set; } + protected BrightcoveService service { get; set; } + protected override void ProcessPipelineStep(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - if (pipelineStep == null) - { - throw new ArgumentNullException(nameof(pipelineStep)); - } - if (pipelineContext == null) - { - throw new ArgumentNullException(nameof(pipelineContext)); - } - if (logger == null) + try { - throw new ArgumentNullException(nameof(logger)); - } + if (pipelineStep == null) + { + throw new ArgumentNullException(nameof(pipelineStep)); + } + if (pipelineContext == null) + { + throw new ArgumentNullException(nameof(pipelineContext)); + } + if (logger == null) + { + throw new ArgumentNullException(nameof(logger)); + } - EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings(); + EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings(); - if (endpointSettings == null) - { - LogFatal("Pipeline step processing will abort because the pipeline step is missing endpoint settings."); - return; - } + if (endpointSettings == null) + { + LogFatal("Pipeline step processing will abort because the pipeline step is missing endpoint settings."); + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; + pipelineContext.CriticalError = false; + return; + } - EndpointFrom = endpointSettings.EndpointFrom; + EndpointFrom = endpointSettings.EndpointFrom; - if (EndpointFrom == null) - { - LogFatal("Pipeline step processing will abort because the pipeline step is missing an endpoint to read from."); - return; - } + if (EndpointFrom == null) + { + LogFatal("Pipeline step processing will abort because the pipeline step is missing an endpoint to read from."); + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; + pipelineContext.CriticalError = false; + return; + } - WebApiSettings = GetPluginOrFail(EndpointFrom); + WebApiSettings = GetPluginOrFail(EndpointFrom); - if(!WebApiSettings.Validate()) - { - LogFatal("Pipeline step processing will abort because the Brightcove web API settings are invalid: "+WebApiSettings.ValidationMessage); - return; - } + if (!WebApiSettings.Validate()) + { + LogFatal("Pipeline step processing will abort because the Brightcove web API settings are invalid: " + WebApiSettings.ValidationMessage); + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; + pipelineContext.CriticalError = false; + return; + } + + itemModelRepository = Sitecore.DataExchange.Context.ItemModelRepository; + service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - try - { ProcessPipelineStepInternal(pipelineStep, pipelineContext, logger); } catch (Exception ex) { LogFatal("An unexpected error occured running the internal pipeline step", ex); + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; pipelineContext.CriticalError = false; } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetExperiencesPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetExperiencesPipelineStepProcessor.cs index 377b0fff..c903e3fe 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetExperiencesPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetExperiencesPipelineStepProcessor.cs @@ -1,8 +1,10 @@ using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Helpers; using Brightcove.DataExchangeFramework.Settings; using Sitecore.Data.Fields; using Sitecore.Data.Items; using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.SecurityModel; @@ -14,25 +16,15 @@ namespace Brightcove.DataExchangeFramework.Processors { class GetExperiencesPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; - protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try - { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - - var data = service.GetExperiences().Items; - var dataSettings = new IterableDataSettings(data); + DateTime lastSyncStartTime = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()).LastSyncStartTime; - LogDebug("Read " + data.Count() + " experience model(s) from web API"); + var data = service.GetExperiences().Items.Where(e => e.LastModifiedDate > lastSyncStartTime); + LogInfo("Identified " + data.Count() + " experience model(s) that have been modified since last sync "+ lastSyncStartTime); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + var dataSettings = new IterableDataSettings(data); + pipelineContext.AddPlugin(dataSettings); } } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetFoldersPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetFoldersPipelineStepProcessor.cs index 89a73471..2446d330 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetFoldersPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetFoldersPipelineStepProcessor.cs @@ -1,9 +1,11 @@ using Brightcove.Core.Models; using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Helpers; using Brightcove.DataExchangeFramework.Settings; using Sitecore.DataExchange.Attributes; using Sitecore.DataExchange.Contexts; using Sitecore.DataExchange.Converters.PipelineSteps; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.DataExchange.Processors.PipelineSteps; @@ -20,31 +22,15 @@ namespace Brightcove.DataExchangeFramework.Processors { public class GetFoldersPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; - protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try - { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - - var folders = service.GetFolders(); - - foreach (Folder folder in folders) - { - folder.LastSyncTime = DateTime.UtcNow; - } - - LogDebug("Read " + folders.Count() + " folder model(s) from web API"); + DateTime lastSyncStartTime = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()).LastSyncStartTime; - var dataSettings = new IterableDataSettings(folders); + var folders = service.GetFolders().Where(f => f.UpdatedDate > lastSyncStartTime); + LogInfo("Identified " + folders.Count() + " folder model(s) that have been modified since last sync " + lastSyncStartTime); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + var dataSettings = new IterableDataSettings(folders); + pipelineContext.AddPlugin(dataSettings); } } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetLabelsPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetLabelsPipelineStepProcessor.cs index 23ad8534..4c8c0627 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetLabelsPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetLabelsPipelineStepProcessor.cs @@ -1,9 +1,11 @@ using Brightcove.Core.Models; using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Helpers; using Brightcove.DataExchangeFramework.Settings; using Sitecore.DataExchange.Attributes; using Sitecore.DataExchange.Contexts; using Sitecore.DataExchange.Converters.PipelineSteps; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.DataExchange.Processors.PipelineSteps; @@ -20,31 +22,15 @@ namespace Brightcove.DataExchangeFramework.Processors { public class GetLabelsPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; - protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try - { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - - var labels = service.GetLabels(); - - foreach (Label label in labels) - { - label.LastSyncTime = DateTime.UtcNow; - } - - LogDebug("Read " + labels.Count() + " label model(s) from web API"); + //DateTime lastSyncStartTime = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()).LastSyncStartTime; - var dataSettings = new IterableDataSettings(labels); + var labels = service.GetLabels(); + LogDebug("Identified " + labels.Count() + "label model(s)"); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + var dataSettings = new IterableDataSettings(labels); + pipelineContext.AddPlugin(dataSettings); } } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetPlayListsPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetPlayListsPipelineStepProcessor.cs index 8eaf67b7..9c2166bd 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetPlayListsPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetPlayListsPipelineStepProcessor.cs @@ -5,6 +5,7 @@ using Sitecore.Data.Items; using Sitecore.DataExchange.Attributes; using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.DataExchange.Processors.PipelineSteps; @@ -20,38 +21,32 @@ namespace Brightcove.DataExchangeFramework.Processors { class GetPlayListsPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; + DateTime lastSyncStartTime; int totalCount = 0; protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try - { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - - totalCount = service.PlayListsCount(); - LogDebug("Read " + totalCount + " playlist model(s) from web API"); + lastSyncStartTime = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()).LastSyncStartTime; + totalCount = service.PlayListsCount(); - var data = this.GetIterableData(WebApiSettings, pipelineStep); - var dataSettings = new IterableDataSettings(data); + var data = this.GetIterableData(WebApiSettings, pipelineStep); + var dataSettings = new IterableDataSettings(data); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + pipelineContext.AddPlugin(dataSettings); } protected virtual IEnumerable GetIterableData(WebApiSettings settings, PipelineStep pipelineStep) { - int limit = 100; + IEnumerable playLists; + int limit = 1000; for (int offset = 0; offset < totalCount; offset += limit) { - foreach(PlayList playList in service.GetPlayLists(offset, limit)) + playLists = service.GetPlayLists(offset, limit).Where(p => p.LastModifiedDate > lastSyncStartTime); + LogInfo("Identified " + playLists.Count() + " playlist model(s) that have been modified since last sync " + lastSyncStartTime); + + foreach (PlayList playList in playLists) { - playList.LastSyncTime = DateTime.UtcNow; yield return playList; } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetPlayersPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetPlayersPipelineStepProcessor.cs index 58e69a4e..794385ae 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetPlayersPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetPlayersPipelineStepProcessor.cs @@ -3,6 +3,7 @@ using Sitecore.Data.Fields; using Sitecore.Data.Items; using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.SecurityModel; @@ -14,31 +15,15 @@ namespace Brightcove.DataExchangeFramework.Processors { class GetPlayersPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; - protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try - { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - - var data = service.GetPlayers().Items; - - foreach (var player in data) - { - player.LastSyncTime = DateTime.UtcNow; - } - - var dataSettings = new IterableDataSettings(data); + DateTime lastSyncStartTime = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()).LastSyncStartTime; - LogDebug("Read " + data.Count() + " player model(s) from web API"); + var data = service.GetPlayers().Items.Where(p => p.Branches.Master.UpdatedAt > lastSyncStartTime); + LogInfo("Identified " + data.Count() + " player model(s) that have been modified since last sync " + lastSyncStartTime); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + var dataSettings = new IterableDataSettings(data); + pipelineContext.AddPlugin(dataSettings); } } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetVideosPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetVideosPipelineStepProcessor.cs index 152bf2a9..d166b9e4 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetVideosPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetVideosPipelineStepProcessor.cs @@ -6,6 +6,7 @@ using Sitecore.DataExchange.Attributes; using Sitecore.DataExchange.Contexts; using Sitecore.DataExchange.Converters.PipelineSteps; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.DataExchange.Processors.PipelineSteps; @@ -22,38 +23,35 @@ namespace Brightcove.DataExchangeFramework.Processors { public class GetVideosPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; int totalCount = 0; + string query = ""; protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try + BrightcoveSyncSettings syncSettings = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()); + + if (syncSettings.LastSyncStartTime != DateTime.MinValue) { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); + query = $"+updated_at:[{syncSettings.LastSyncStartTime.ToString("s", System.Globalization.CultureInfo.InvariantCulture)}Z TO *]"; + } - totalCount = service.VideosCount(); - LogDebug("Read " + totalCount + " video model(s) from web API"); + totalCount = service.VideosCount(query); + LogInfo("Identified " + totalCount + " video model(s) that have been modified since last sync "+ syncSettings.LastSyncStartTime); - var data = GetIterableData(pipelineStep); - var dataSettings = new IterableDataSettings(data); + var data = GetIterableData(pipelineStep); + var dataSettings = new IterableDataSettings(data); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + pipelineContext.AddPlugin(dataSettings); } protected virtual IEnumerable