From 77ab30f7fbcc139374e063d988335eb1813f69a7 Mon Sep 17 00:00:00 2001 From: Cody Rodgers Date: Mon, 11 Nov 2024 13:01:03 -0500 Subject: [PATCH] V10.1.X-4.0 (#25) * Update the embed generator to include an optional alt-text field for iframe accessibility support * Update all Brightcove templates/items to be unpublishable to prevent unnecessary performance hits * Update the pull pipeline processors to use the ItemModelRepository and cleanup unnecessary DB requests * Update the pull pipelines to set and use a new global last sync timestamp * Update the push pipelines to use the new last sync time * Update the video push pipeline to suppor the new last sync time field * Update the value reader/writers to include consistent exception handling * Update playlist/video page size to 1000 * Update the resolve asset item pipeline to dedup content items * Add the new clean pipeline batch * Add retry logic to the brightcove service * Fix loadPlayer script * Content Updates * Fix various bugs --------- Co-authored-by: Cody Rodgers --- .../EmbedGenerator/Models/EmbedModel.cs | 2 + Brightcove.Core/Models/Experience.cs | 5 +- Brightcove.Core/Models/Video.cs | 3 + .../Services/BrightcoveHttpClient.cs | 1 + Brightcove.Core/Services/BrightcoveService.cs | 38 +++- .../Brightcove.DataExchangeFramework.csproj | 18 +- .../ApplyMappingPipelineStepConverter.cs | 19 ++ .../ReadAssetItemsPipelineStepConverter.cs | 8 +- .../ResolveAssetItemPipelineStepConverter.cs | 15 +- .../PipelineStep/SyncPipelineStepConverter.cs | 44 ++++ .../UpdateAssetItemPipelineStepConverter.cs | 19 ++ ...cs => UpdateVideoPipelineStepConverter.cs} | 12 +- .../Converters/WebApiEndpointConverter.cs | 9 +- .../Helpers/BrightcoveSyncSettingsHelper.cs | 45 ++++ .../Helpers/IItemModelRepositoryHelper.cs | 38 ++++ .../Helpers/ItemUpdater.cs | 45 ---- .../ApplyMappingPipelineStepProcessor.cs | 139 ++++++++++++ .../Processors/BasePipelineStepProcessor.cs | 33 +-- ...PipelineStepWithWebApiEndpointProcessor.cs | 81 ++++--- .../GetExperiencesPipelineStepProcessor.cs | 22 +- .../Get/GetFoldersPipelineStepProcessor.cs | 28 +-- .../Get/GetLabelsPipelineStepProcessor.cs | 28 +-- .../Get/GetPlayListsPipelineStepProcessor.cs | 31 ++- .../Get/GetPlayersPipelineStepProcessor.cs | 27 +-- .../Get/GetVideosPipelineStepProcessor.cs | 30 ++- .../ReadAssetItemsPipelineStepProcessor.cs | 55 ++++- .../ResolveAssetItemPipelineStepProcessor.cs | 80 ++++--- ...ResolveFolderModelPipelineStepProcessor.cs | 56 ++--- .../ResolveLabelModelPipelineStepProcessor.cs | 64 ++---- ...solvePlayListModelPipelineStepProcessor.cs | 57 ++--- ...ResolvePlayerModelPipelineStepProcessor.cs | 69 ------ .../ResolveVideoModelPipelineStepProcessor.cs | 55 ++--- .../Sync/FinishSyncPipelineStepProcessor.cs | 43 ++++ .../Sync/ResetSyncPipelineStepProcessor.cs | 37 ++++ .../Sync/StartSyncPipelineStepProcessor.cs | 44 ++++ .../Sync/VerifySyncPipelineStepProcessor.cs | 45 ++++ .../UpdateAssetItemPipelineStepProcessor.cs | 43 ++++ .../UpdateFolderModelPipelineStepProcessor.cs | 78 +++---- .../UpdateLabelModelPipelineStepProcessor.cs | 84 ++++---- .../UpdatePlayerModelPipelineStepProcessor.cs | 86 -------- ...pdatePlaylistModelPipelineStepProcessor.cs | 80 +++---- .../UpdateVideoItemPipelineStepProcessor.cs | 117 ++++++++++ .../Update/UpdateVideoItemProcessor.cs | 134 ------------ .../UpdateVideoModelPipelineStepProcessor.cs | 202 ++++++------------ .../Settings/BrightcoveSyncSettings.cs | 21 ++ .../Settings/ResolveAssetItemSettings.cs | 7 +- .../Settings/WebApiSettings.cs | 1 - .../ChainedPropertyValueReader.cs | 43 ++-- .../CsvStringPropertyValueReader.cs | 43 ++-- .../DateTimePropertyValueReader.cs | 35 +-- .../ValueReaders/FolderPropertyValueReader.cs | 6 +- .../ValueReaders/LabelsPropertyValueReader.cs | 6 +- .../ValueReaders/NullableEnumValueReader.cs | 20 +- .../StringDictionaryPropertyValueReader.cs | 45 ++-- .../VideoIdsPropertyValueReader.cs | 5 +- .../ChainedPropertyValueWriter.cs | 9 +- .../ValueWriters/LabelsPropertyValueWriter.cs | 2 +- .../VideoIdsPropertyValueWriter.cs | 26 ++- .../EmbedGenerator/SitecoreEmbedGenerator.cs | 60 +++++- .../Models/EmbedRenderingParameters.cs | 14 ++ Brightcove.Web/UI/Wizards/EmbedMediaWizard.cs | 32 ++- .../Web/Brightcove/js/loadPlayer.js | 2 +- .../Brightcove.MediaLibrary/Brightcove.yml | 4 + .../BrightcoveAccount/_name/Experiences.yml | 18 +- .../BrightcoveAccount/_name/Folders.yml | 11 +- .../BrightcoveAccount/_name/Labels.yml | 11 +- .../BrightcoveAccount/_name/Players.yml | 25 +-- .../BrightcoveAccount/_name/Playlists.yml | 14 +- .../BrightcoveAccount/_name/Videos.yml | 14 +- .../Apply Asset Model Source Mappi.yml | 2 +- .../Apply Playlist Model Source Ma.yml | 2 +- .../Apply Asset Model Source Mappi.yml | 5 +- .../Apply Experience Model Source_.yml | 2 +- .../LastSyncTime Property Mapping.yml | 26 --- .../Apply Asset Model Source Mappi.yml | 2 +- .../Apply Player Model Source Mapp.yml | 28 --- .../LastSyncTime Property Mapping.yml | 30 --- .../Apply Asset Item Source Mappin.yml | 2 +- .../Apply Playlist Item Source Map.yml | 2 +- .../Apply Folder Model Source Mapp.yml | 2 +- .../Apply Asset Item Source Mappin.yml | 28 --- .../Apply Player Item Source Mappi.yml | 28 --- .../LastSyncTime Property.yml | 22 -- .../LastSyncTime Property.yml | 22 -- .../LastSyncTime Property.yml | 22 -- .../Last Synchronization Time Fiel.yml | 22 -- .../Folder Item Fields/LastSyncTime Field.yml | 22 -- ...ine Batch.yml => Clean Pipeline Batch.yml} | 28 ++- ...tch.yml => Pull Latest Pipeline Batch.yml} | 10 +- ...tch.yml => Push Latest Pipeline Batch.yml} | 21 +- .../_name/Pipelines/Clean Pipelines.yml | 18 ++ .../Clean Folders Pipeline.yml | 22 ++ .../Read Folder Items.yml | 34 +++ .../Resolve Folder Models.yml | 33 +++ .../Select Languages Pipeline Step.yml | 26 +++ .../Clean Labels Pipeline.yml} | 6 +- .../Read Label Items.yml} | 8 +- .../Resolve Label Models.yml} | 8 +- .../Select Languages Pipeline Step.yml | 8 +- .../Clean Playlists Pipeline.yml | 18 ++ .../Read Playlist Items.yml | 31 +++ .../Resolve Playlist Models.yml | 26 +++ .../Select Languages Pipeline Step.yml | 26 +++ .../Clean Pipelines/Clean Videos Pipeline.yml | 18 ++ .../Read Video Items.yml | 37 ++++ .../Resolve Video Models.yml | 30 +++ .../Select Languages Pipeline Step.yml | 26 +++ .../Push Videos Pipeline/Read Video Items.yml | 4 +- .../Subpipelines/Resolve Model Pipelines.yml | 18 ++ .../Resolve Folder Models Pipeline.yml | 22 ++ .../Resolve Folder Model.yml | 30 +++ .../Resolve Label Models Pipeline.yml} | 6 +- .../Resolve Label Model.yml | 27 +++ .../Resolve Playlist Models Pipeli.yml | 18 ++ .../Resolve Playlist Model.yml | 24 +++ .../Resolve Video Models Pipeline.yml | 23 ++ .../Resolve Video Model.yml | 26 +++ .../Resolve Experience Item.yml | 4 + .../Update Experience Item.yml | 2 +- .../Resolve Folder Item.yml | 4 + .../Update Folder Item.yml | 2 +- .../Resolve Label Item.yml | 4 + .../Update Label Item.yml | 2 +- .../Resolve Player Item.yml | 4 + .../Update Player Item.yml | 2 +- .../Resolve Playlist Item.yml | 4 + .../Update Playlist Item.yml | 2 +- .../Resolve Video Item.yml | 4 + .../Update Video Item.yml | 2 +- .../Resolve Player Model.yml | 21 -- .../Update Player Model.yml | 27 --- .../_name/Pipelines/Sync Pipelines.yml | 18 ++ .../Sync Pipelines/Finish Sync Pipeline.yml | 18 ++ .../Finish Sync Pipeline/Finish Sync.yml | 17 ++ .../Sync Pipelines/Reset Sync Pipeline.yml | 18 ++ .../Reset Sync Pipeline/Reset Sync.yml | 17 ++ .../Sync Pipelines/Start Sync Pipeline.yml | 18 ++ .../Start Sync Pipeline/Start Sync.yml | 17 ++ .../Sync Pipelines/Verify Sync Pipeline.yml | 18 ++ .../Verify Sync Pipeline/Verify Sync.yml | 17 ++ .../Name Property Mapping.yml | 4 + .../LastSyncTime Property.yml | 26 --- .../Folder Property Mapping.yml | 30 --- .../Labels Property Mapping.yml | 26 --- .../Apply Label Item Source Mappin.yml | 2 +- .../LastModifiedDate Property Mapp.yml | 30 --- .../Apply Label Model Source Mappi.yml | 2 +- .../LastModifiedDate Property Mapp.yml | 33 --- .../Apply Mapping Pipeline Step.yml | 29 +++ .../__Standard Values.yml | 21 ++ .../Finish Sync Pipeline Step.yml | 25 +++ .../Finish Sync Pipeline Step/Settings.yml | 14 ++ .../Settings/EndpointFrom.yml | 24 +++ .../__Standard Values.yml | 25 +++ .../Reset Sync Pipeline Step.yml | 25 +++ .../Reset Sync Pipeline Step/Settings.yml | 14 ++ .../Settings/EndpointFrom.yml | 24 +++ .../__Standard Values.yml | 25 +++ .../Start Sync Pipeline Step.yml | 25 +++ .../Start Sync Pipeline Step/Settings.yml | 14 ++ .../Settings/EndpointFrom.yml | 24 +++ .../__Standard Values.yml | 25 +++ .../Update Asset Item Pipeline Ste.yml | 32 +++ .../__Standard Values.yml | 21 ++ .../__Standard Values.yml | 4 +- .../__Standard Values.yml | 2 +- .../Verify Sync Pipeline Step.yml | 25 +++ .../Verify Sync Pipeline Step/Settings.yml | 14 ++ .../Settings/EndpointFrom.yml | 24 +++ .../__Standard Values.yml | 25 +++ .../__Standard Values.yml | 4 + .../Brightcove Folder/__Standard Values.yml | 4 + .../Brightcove Media Library Folde.yml | 28 +++ .../__Standard Values.yml | 26 +++ .../Synchronization Metadata/LastSyncTime.yml | 29 --- .../__Standard Values.yml | 5 + .../Settings/IframeTitleCustomField.yml | 35 +++ .../Settings/IframeTitleFallbackField.yml | 35 +++ .../BrightcoveAccount/Synchronization.yml | 18 ++ .../Synchronization/LastSyncFinishTime.yml | 21 ++ .../Synchronization/LastSyncStartTime.yml | 21 ++ .../BrightcoveAccount/__Standard Values.yml | 4 + .../Synchronization/LastSyncTime.yml | 29 --- .../BrightcovePlaylist/__Standard Values.yml | 4 + .../Brightcove/BrightcoveVideo.yml | 2 +- .../BrightcoveFolder.yml | 2 +- .../Brightcove Video Data/Labels.yml | 2 +- .../BrightcoveVideo/Scheduling/EndsAt.yml | 4 + .../BrightcoveVideo/Scheduling/StartsAt.yml | 4 + .../BrightcoveVideo/Scheduling/State.yml | 4 + .../BrightcoveVideo/__Standard Values.yml | 33 +-- .../__Standard Values.yml | 6 +- .../__Standard Values.yml | 4 + 193 files changed, 2966 insertions(+), 1758 deletions(-) create mode 100644 Brightcove.DataExchangeFramework/Converters/PipelineStep/ApplyMappingPipelineStepConverter.cs create mode 100644 Brightcove.DataExchangeFramework/Converters/PipelineStep/SyncPipelineStepConverter.cs create mode 100644 Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateAssetItemPipelineStepConverter.cs rename Brightcove.DataExchangeFramework/Converters/PipelineStep/{UpdatePipelineStepConverter.cs => UpdateVideoPipelineStepConverter.cs} (80%) create mode 100644 Brightcove.DataExchangeFramework/Helpers/BrightcoveSyncSettingsHelper.cs create mode 100644 Brightcove.DataExchangeFramework/Helpers/IItemModelRepositoryHelper.cs create mode 100644 Brightcove.DataExchangeFramework/Processors/ApplyMappingPipelineStepProcessor.cs delete mode 100644 Brightcove.DataExchangeFramework/Processors/Resolve/ResolvePlayerModelPipelineStepProcessor.cs create mode 100644 Brightcove.DataExchangeFramework/Processors/Sync/FinishSyncPipelineStepProcessor.cs create mode 100644 Brightcove.DataExchangeFramework/Processors/Sync/ResetSyncPipelineStepProcessor.cs create mode 100644 Brightcove.DataExchangeFramework/Processors/Sync/StartSyncPipelineStepProcessor.cs create mode 100644 Brightcove.DataExchangeFramework/Processors/Sync/VerifySyncPipelineStepProcessor.cs create mode 100644 Brightcove.DataExchangeFramework/Processors/Update/UpdateAssetItemPipelineStepProcessor.cs delete mode 100644 Brightcove.DataExchangeFramework/Processors/Update/UpdatePlayerModelPipelineStepProcessor.cs create mode 100644 Brightcove.DataExchangeFramework/Processors/Update/UpdateVideoItemPipelineStepProcessor.cs delete mode 100644 Brightcove.DataExchangeFramework/Processors/Update/UpdateVideoItemProcessor.cs create mode 100644 Brightcove.DataExchangeFramework/Settings/BrightcoveSyncSettings.cs delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/38b6b14a-dc3b-489d-a599-50bbf205052f/LastSyncTime Property Mapping.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/66c52e36-fb67-4007-9f8a-94b8a36f94f4/Apply Player Model Source Mapp.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/6f7b8f33-05d9-4ef3-b5ee-83d9bdcdac16/LastSyncTime Property Mapping.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/915f8c16-6e0f-4963-9452-75e501e8822a/Apply Asset Item Source Mappin.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/915f8c16-6e0f-4963-9452-75e501e8822a/Apply Player Item Source Mappi.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Data Access/Value Accessor Sets/Providers/Brightcove/Asset Model Properties/LastSyncTime Property.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Data Access/Value Accessor Sets/Providers/Brightcove/Folder Model Properties/LastSyncTime Property.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Data Access/Value Accessor Sets/Providers/Brightcove/Label Model Properties/LastSyncTime Property.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Data Access/Value Accessor Sets/Providers/Sitecore/Asset Item Fields/Last Synchronization Time Fiel.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Data Access/Value Accessor Sets/Providers/Sitecore/Folder Item Fields/LastSyncTime Field.yml rename Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipeline Batches/{Push Pipeline Batch.yml => Clean Pipeline Batch.yml} (56%) rename Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipeline Batches/{Pull Pipeline Batch.yml => Pull Latest Pipeline Batch.yml} (82%) rename Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipeline Batches/{Sync Pipeline Batch.yml => Push Latest Pipeline Batch.yml} (76%) create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Folders Pipeline.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Folders Pipeline/Read Folder Items.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Folders Pipeline/Resolve Folder Models.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Folders Pipeline/Select Languages Pipeline Step.yml rename Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/{Push Pipelines/Push Players Pipeline.yml => Clean Pipelines/Clean Labels Pipeline.yml} (74%) rename Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/{Push Pipelines/Push Players Pipeline/Read Player Items.yml => Clean Pipelines/Clean Labels Pipeline/Read Label Items.yml} (79%) rename Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/{Push Pipelines/Push Players Pipeline/Update Player Models.yml => Clean Pipelines/Clean Labels Pipeline/Resolve Label Models.yml} (73%) rename Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/{Push Pipelines/Push Players Pipeline => Clean Pipelines/Clean Labels Pipeline}/Select Languages Pipeline Step.yml (70%) create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Playlists Pipeline.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Playlists Pipeline/Read Playlist Items.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Playlists Pipeline/Resolve Playlist Models.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Playlists Pipeline/Select Languages Pipeline Step.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Videos Pipeline.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Videos Pipeline/Read Video Items.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Videos Pipeline/Resolve Video Models.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Clean Pipelines/Clean Videos Pipeline/Select Languages Pipeline Step.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/Resolve Model Pipelines.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/Resolve Model Pipelines/Resolve Folder Models Pipeline.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/Resolve Model Pipelines/Resolve Folder Models Pipeline/Resolve Folder Model.yml rename Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/{Update Model Pipelines/Update Player Models Pipeline.yml => Resolve Model Pipelines/Resolve Label Models Pipeline.yml} (71%) create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/Resolve Model Pipelines/Resolve Label Models Pipeline/Resolve Label Model.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/Resolve Model Pipelines/Resolve Playlist Models Pipeli.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/Resolve Model Pipelines/Resolve Playlist Models Pipeli/Resolve Playlist Model.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/Resolve Model Pipelines/Resolve Video Models Pipeline.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/Resolve Model Pipelines/Resolve Video Models Pipeline/Resolve Video Model.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/Update Model Pipelines/Update Player Models Pipeline/Resolve Player Model.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Subpipelines/Update Model Pipelines/Update Player Models Pipeline/Update Player Model.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Sync Pipelines.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Sync Pipelines/Finish Sync Pipeline.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Sync Pipelines/Finish Sync Pipeline/Finish Sync.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Sync Pipelines/Reset Sync Pipeline.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Sync Pipelines/Reset Sync Pipeline/Reset Sync.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Sync Pipelines/Start Sync Pipeline.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Sync Pipelines/Start Sync Pipeline/Start Sync.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Sync Pipelines/Verify Sync Pipeline.yml create mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Pipelines/Sync Pipelines/Verify Sync Pipeline/Verify Sync.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Value Mapping Sets/Model Source Mapping Sets/Label Model Source Mapping Set/LastSyncTime Property.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Value Mapping Sets/Model Source Mapping Sets/Video Model Source Mapping Set/Folder Property Mapping.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/Brightcove Tenant/_name/Value Mapping Sets/Model Source Mapping Sets/Video Model Source Mapping Set/Labels Property Mapping.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/b515fa0d-a5b2-4792-85f7-b415b091356d/LastModifiedDate Property Mapp.yml delete mode 100644 Serialization/Brightcove.Template.Branches.DataExchangeFramework.Framework.Branches/ef1e96bf-0500-4da4-a1ba-9ce1a318eda6/LastModifiedDate Property Mapp.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Apply Mapping Pipeline Step.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Apply Mapping Pipeline Step/__Standard Values.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Finish Sync Pipeline Step.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Finish Sync Pipeline Step/Settings.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Finish Sync Pipeline Step/Settings/EndpointFrom.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Finish Sync Pipeline Step/__Standard Values.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Reset Sync Pipeline Step.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Reset Sync Pipeline Step/Settings.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Reset Sync Pipeline Step/Settings/EndpointFrom.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Reset Sync Pipeline Step/__Standard Values.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Start Sync Pipeline Step.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Start Sync Pipeline Step/Settings.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Start Sync Pipeline Step/Settings/EndpointFrom.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Start Sync Pipeline Step/__Standard Values.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Update Asset Item Pipeline Ste.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Update Asset Item Pipeline Ste/__Standard Values.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Verify Sync Pipeline Step.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Verify Sync Pipeline Step/Settings.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Verify Sync Pipeline Step/Settings/EndpointFrom.yml create mode 100644 Serialization/Brightcove.Template.DataExchangeFramework/Brightcove/Pipeline Steps/Verify Sync Pipeline Step/__Standard Values.yml create mode 100644 Serialization/Brightcove.Template/Brightcove/Brightcove Media Library Folde.yml create mode 100644 Serialization/Brightcove.Template/Brightcove/Brightcove Media Library Folde/__Standard Values.yml delete mode 100644 Serialization/Brightcove.Template/Brightcove/Brightcove Synchronization Met/Synchronization Metadata/LastSyncTime.yml create mode 100644 Serialization/Brightcove.Template/Brightcove/BrightcoveAccount/Settings/IframeTitleCustomField.yml create mode 100644 Serialization/Brightcove.Template/Brightcove/BrightcoveAccount/Settings/IframeTitleFallbackField.yml create mode 100644 Serialization/Brightcove.Template/Brightcove/BrightcoveAccount/Synchronization.yml create mode 100644 Serialization/Brightcove.Template/Brightcove/BrightcoveAccount/Synchronization/LastSyncFinishTime.yml create mode 100644 Serialization/Brightcove.Template/Brightcove/BrightcoveAccount/Synchronization/LastSyncStartTime.yml delete mode 100644 Serialization/Brightcove.Template/Brightcove/BrightcoveMediaElement/Synchronization/LastSyncTime.yml diff --git a/Brightcove.Core/EmbedGenerator/Models/EmbedModel.cs b/Brightcove.Core/EmbedGenerator/Models/EmbedModel.cs index 06b5e71d..cb69746e 100644 --- a/Brightcove.Core/EmbedGenerator/Models/EmbedModel.cs +++ b/Brightcove.Core/EmbedGenerator/Models/EmbedModel.cs @@ -30,6 +30,8 @@ public class EmbedModel public string Language { get; set; } = ""; + public string Title { get; set; } = ""; + public EmbedModel() { diff --git a/Brightcove.Core/Models/Experience.cs b/Brightcove.Core/Models/Experience.cs index 51e867f0..b03c6d13 100644 --- a/Brightcove.Core/Models/Experience.cs +++ b/Brightcove.Core/Models/Experience.cs @@ -10,9 +10,12 @@ namespace Brightcove.Core.Models /// public class Experience : Asset { - [JsonProperty("created_at", NullValueHandling = NullValueHandling.Ignore)] + [JsonProperty("createdAt", NullValueHandling = NullValueHandling.Ignore)] public DateTime CreationDate { get; set; } + [JsonProperty("updatedAt", NullValueHandling = NullValueHandling.Ignore)] + public new DateTime? LastModifiedDate { get; set; } + [JsonProperty("publishedUrl", NullValueHandling = NullValueHandling.Ignore)] public string Url { get; set; } diff --git a/Brightcove.Core/Models/Video.cs b/Brightcove.Core/Models/Video.cs index d1cad2f3..1288aa99 100644 --- a/Brightcove.Core/Models/Video.cs +++ b/Brightcove.Core/Models/Video.cs @@ -62,6 +62,9 @@ public class Video : Asset [JsonProperty("schedule", NullValueHandling = NullValueHandling.Ignore)] public VideoSchedule Schedule { get; set; } + [JsonProperty("variants", NullValueHandling = NullValueHandling.Ignore)] + public List Variants { get; set; } + public Video ShallowCopy() { return (Video)this.MemberwiseClone(); diff --git a/Brightcove.Core/Services/BrightcoveHttpClient.cs b/Brightcove.Core/Services/BrightcoveHttpClient.cs index eaba8421..833da3e4 100644 --- a/Brightcove.Core/Services/BrightcoveHttpClient.cs +++ b/Brightcove.Core/Services/BrightcoveHttpClient.cs @@ -14,6 +14,7 @@ public static class BrightcoveHttpClient static BrightcoveHttpClient() { Instance = new HttpClient(); + Instance.Timeout = TimeSpan.FromSeconds(30); } } } diff --git a/Brightcove.Core/Services/BrightcoveService.cs b/Brightcove.Core/Services/BrightcoveService.cs index 14b7f4f0..53c9ed8f 100644 --- a/Brightcove.Core/Services/BrightcoveService.cs +++ b/Brightcove.Core/Services/BrightcoveService.cs @@ -16,6 +16,9 @@ public class BrightcoveService { readonly static HttpClient client = BrightcoveHttpClient.Instance; + int retryMax = 3; + int retryAttempt = 0; + readonly string cmsBaseUrl = "https://cms.api.brightcove.com/v1/accounts"; readonly string ingestBaseUrl = "https://ingest.api.brightcove.com/v1/accounts"; readonly string playersBaseUrl = "https://players.api.brightcove.com/v1/accounts"; @@ -306,6 +309,7 @@ public Video UpdateVideo(Video video) Video newVideo = video.ShallowCopy(); newVideo.Id = null; newVideo.Images = null; + newVideo.Variants = null; string content = JsonConvert.SerializeObject(newVideo); @@ -519,12 +523,12 @@ public PlayerList GetPlayers() return players; } - public ExperienceList GetExperiences() + public ExperienceList GetExperiences(string query = "", string sort = "") { HttpRequestMessage request = new HttpRequestMessage(); request.Method = HttpMethod.Get; - request.RequestUri = new Uri($"{experienceBaseUrl}/{accountId}/experiences"); + request.RequestUri = new Uri($"{experienceBaseUrl}/{accountId}/experiences?query={query}&sort={sort}"); HttpResponseMessage response = SendRequest(request); @@ -719,12 +723,12 @@ public bool TryGetPlaylist(string playlistId, out PlayList playlist) return true; } - public int VideosCount() + public int VideosCount(string query = "") { HttpRequestMessage request = new HttpRequestMessage(); request.Method = HttpMethod.Get; - request.RequestUri = new Uri($"{cmsBaseUrl}/{accountId}/counts/videos"); + request.RequestUri = new Uri($"{cmsBaseUrl}/{accountId}/counts/videos?query={query}"); HttpResponseMessage response = SendRequest(request); Count count = JsonConvert.DeserializeObject(response.Content.ReadAsString()); @@ -747,16 +751,30 @@ public int PlayListsCount() private HttpResponseMessage SendRequest(HttpRequestMessage request) { - request.Headers.Authorization = authenticationService.CreateAuthenticationHeader(); + try + { + request.Headers.Authorization = authenticationService.CreateAuthenticationHeader(); - HttpResponseMessage response = client.Send(request); + HttpResponseMessage response = client.Send(request); - if (!response.IsSuccessStatusCode) - { - throw new HttpStatusException(request, response); + if (!response.IsSuccessStatusCode) + { + throw new HttpStatusException(request, response); + } + + return response; } + catch(Exception ex) + { + if(retryAttempt < retryMax) + { + retryAttempt++; + return SendRequest(request); + } - return response; + retryAttempt = 0; + throw ex; + } } } } diff --git a/Brightcove.DataExchangeFramework/Brightcove.DataExchangeFramework.csproj b/Brightcove.DataExchangeFramework/Brightcove.DataExchangeFramework.csproj index 152cf52a..f9677dbb 100644 --- a/Brightcove.DataExchangeFramework/Brightcove.DataExchangeFramework.csproj +++ b/Brightcove.DataExchangeFramework/Brightcove.DataExchangeFramework.csproj @@ -438,10 +438,13 @@ + + - + + @@ -453,6 +456,8 @@ + + @@ -467,18 +472,23 @@ - + + + + + + - - + + diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/ApplyMappingPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ApplyMappingPipelineStepConverter.cs new file mode 100644 index 00000000..d4f5bb4c --- /dev/null +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ApplyMappingPipelineStepConverter.cs @@ -0,0 +1,19 @@ +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data; +using Sitecore.Data.Items; +using Sitecore.DataExchange.ApplyMapping; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Providers.Sc.Converters.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Model; +using System; + +namespace Brightcove.DataExchangeFramework.Converters +{ + [SupportedIds("{AB4AF4DF-D282-4CD1-8268-FA12A9E457A3}")] + public class ApplyMappingPipelineStepConverter : ApplyMappingStepConverter + { + public ApplyMappingPipelineStepConverter(IItemModelRepository repository) : base(repository) { } + } +} \ No newline at end of file diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/ReadAssetItemsPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ReadAssetItemsPipelineStepConverter.cs index 7a25a228..b894ebce 100644 --- a/Brightcove.DataExchangeFramework/Converters/PipelineStep/ReadAssetItemsPipelineStepConverter.cs +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ReadAssetItemsPipelineStepConverter.cs @@ -31,12 +31,16 @@ protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep) if (endpointId != null) { - Item endpointItem = Sitecore.Context.ContentDatabase.GetItem(new ID(endpointId)); + ItemModel endpointItem = ItemModelRepository.Get(endpointId); if(endpointItem != null) { - resolveAssetItemSettings.AcccountItemId = endpointItem["Account"]; + resolveAssetItemSettings.AcccountItemId = this.GetStringValue(endpointItem, "Account") ?? ""; resolveAssetItemSettings.RelativePath = this.GetStringValue(source, "RelativePath") ?? ""; + + Database database = Sitecore.Configuration.Factory.GetDatabase(ItemModelRepository.DatabaseName); + resolveAssetItemSettings.AccountItem = database.GetItem(resolveAssetItemSettings.AcccountItemId); + resolveAssetItemSettings.ParentItem = database.GetItem(resolveAssetItemSettings.AccountItem?.Paths?.Path + "/" + resolveAssetItemSettings.RelativePath); } } diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/ResolveAssetItemPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ResolveAssetItemPipelineStepConverter.cs index ff11c4a6..fda30862 100644 --- a/Brightcove.DataExchangeFramework/Converters/PipelineStep/ResolveAssetItemPipelineStepConverter.cs +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/ResolveAssetItemPipelineStepConverter.cs @@ -24,15 +24,26 @@ protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep) if (endpointId != null) { - Item endpointItem = Sitecore.Context.ContentDatabase.GetItem(new ID(endpointId)); + ItemModel endpointItem = ItemModelRepository.Get(endpointId); if (endpointItem != null) { - resolveAssetItemSettings.AcccountItemId = endpointItem["Account"]; + resolveAssetItemSettings.AcccountItemId = this.GetStringValue(endpointItem, "Account") ?? ""; resolveAssetItemSettings.RelativePath = this.GetStringValue(source, "RelativePath") ?? ""; + + Database database = Sitecore.Configuration.Factory.GetDatabase(ItemModelRepository.DatabaseName); + resolveAssetItemSettings.AccountItem = database.GetItem(resolveAssetItemSettings.AcccountItemId); + resolveAssetItemSettings.ParentItem = database.GetItem(resolveAssetItemSettings.AccountItem?.Paths?.Path + "/" + resolveAssetItemSettings.RelativePath); } } + //We need to store the resolve asset item plugin in the global Sitecore.DataExchangeContext so it + //can be used in the VideoIdsPropertyValueReader + if (Sitecore.DataExchange.Context.GetPlugin() == null) + { + Sitecore.DataExchange.Context.Plugins.Add(resolveAssetItemSettings); + } + pipelineStep.AddPlugin(resolveAssetItemSettings); } } diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/SyncPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/SyncPipelineStepConverter.cs new file mode 100644 index 00000000..a08f4ec5 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/SyncPipelineStepConverter.cs @@ -0,0 +1,44 @@ +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data; +using Sitecore.Data.Items; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Converters.PipelineSteps; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Providers.Sc.Converters.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Brightcove.DataExchangeFramework.Converters +{ + public class SyncPipelineStepConverter : BasePipelineStepConverter + { + public SyncPipelineStepConverter(IItemModelRepository repository) : base(repository) + { + + } + + protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep) + { + Guid endpointId = this.GetGuidValue(source, "EndpointFrom"); + BrightcoveSyncSettings settings = new BrightcoveSyncSettings(); + + if (endpointId != null) + { + ItemModel endpointItem = ItemModelRepository.Get(endpointId); + + if(endpointItem != null) + { + string accountId = this.GetStringValue(endpointItem, "Account") ?? ""; + settings.AccountItem = ItemModelRepository.Get(accountId); + } + } + + pipelineStep.AddPlugin(settings); + } + } +} diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateAssetItemPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateAssetItemPipelineStepConverter.cs new file mode 100644 index 00000000..bbc2b023 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateAssetItemPipelineStepConverter.cs @@ -0,0 +1,19 @@ +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data; +using Sitecore.Data.Items; +using Sitecore.DataExchange.ApplyMapping; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Providers.Sc.Converters.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Model; +using System; + +namespace Brightcove.DataExchangeFramework.Converters +{ + [SupportedIds("{F4AF37B2-F92D-4E49-A017-3D7489E23910}")] + public class UpdateAssetItemPipelineStepConverter : UpdateSitecoreItemStepConverter + { + public UpdateAssetItemPipelineStepConverter(IItemModelRepository repository) : base(repository) { } + } +} \ No newline at end of file diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdatePipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateVideoPipelineStepConverter.cs similarity index 80% rename from Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdatePipelineStepConverter.cs rename to Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateVideoPipelineStepConverter.cs index 8025e7ad..a187cbe3 100644 --- a/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdatePipelineStepConverter.cs +++ b/Brightcove.DataExchangeFramework/Converters/PipelineStep/UpdateVideoPipelineStepConverter.cs @@ -14,9 +14,9 @@ namespace Brightcove.DataExchangeFramework.Converters { [SupportedIds("{D79A5F5C-9A5E-4B1C-B884-8E3B97CACA2D}", "{F598D123-2FE9-45D3-99E2-3E4B5063190A}")] - public class UpdatePipelineStepConverter : BasePipelineStepConverter + public class UpdateVideoPipelineStepConverter : BasePipelineStepConverter { - public UpdatePipelineStepConverter(IItemModelRepository repository) : base(repository) { } + public UpdateVideoPipelineStepConverter(IItemModelRepository repository) : base(repository) { } protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep) { @@ -59,7 +59,7 @@ protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep) return; } - Item accountItem = Sitecore.Context.ContentDatabase.GetItem(new ID(accountItemId)); + ItemModel accountItem = ItemModelRepository.Get(accountItemId); if(accountItem == null) { @@ -70,9 +70,9 @@ protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep) if (accountItem != null) { - webApiSettings.AccountId = accountItem["AccountId"]; - webApiSettings.ClientId = accountItem["ClientId"]; - webApiSettings.ClientSecret = accountItem["ClientSecret"]; + webApiSettings.AccountId = this.GetStringValue(accountItem, "AccountId") ?? ""; + webApiSettings.ClientId = this.GetStringValue(accountItem, "ClientId") ?? ""; + webApiSettings.ClientSecret = this.GetStringValue(accountItem, "ClientSecret") ?? ""; } endpointSettings.BrightcoveEndpoint.AddPlugin(webApiSettings); diff --git a/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs b/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs index 2ef5114d..f69368dc 100644 --- a/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs +++ b/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs @@ -26,16 +26,15 @@ public WebApiEndpointConverter(IItemModelRepository repository) : base(repositor protected override void AddPlugins(ItemModel source, Endpoint endpoint) { Guid accountItemId = this.GetGuidValue(source, TemplateAccount); - Item accountItem = Sitecore.Context.ContentDatabase.GetItem(new ID(accountItemId)); + ItemModel accountItem = ItemModelRepository.Get(accountItemId); WebApiSettings accountSettings = new WebApiSettings(); if(accountItem != null) { - accountSettings.AccountId = accountItem["AccountId"]; - accountSettings.ClientId = accountItem["ClientId"]; - accountSettings.ClientSecret = accountItem["ClientSecret"]; - accountSettings.AccountItem = accountItem; + accountSettings.AccountId = this.GetStringValue(accountItem, "AccountId") ?? ""; + accountSettings.ClientId = this.GetStringValue(accountItem, "ClientId") ?? ""; + accountSettings.ClientSecret = this.GetStringValue(accountItem, "ClientSecret") ?? ""; } endpoint.AddPlugin(accountSettings); diff --git a/Brightcove.DataExchangeFramework/Helpers/BrightcoveSyncSettingsHelper.cs b/Brightcove.DataExchangeFramework/Helpers/BrightcoveSyncSettingsHelper.cs new file mode 100644 index 00000000..02c6bce4 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Helpers/BrightcoveSyncSettingsHelper.cs @@ -0,0 +1,45 @@ +using Brightcove.Core.Exceptions; +using Brightcove.Core.Models; +using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data.Fields; +using Sitecore.Data.Items; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.DataAccess; +using Sitecore.DataExchange.Extensions; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Plugins; +using Sitecore.DataExchange.Processors.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Diagnostics; +using Sitecore.Services.Core.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using Brightcove.Core.Extensions; +using Sitecore.DataExchange.ApplyMapping; +using Sitecore.Data; + +namespace Brightcove.DataExchangeFramework.Helpers +{ + public static class BrightcoveSyncSettingsHelper + { + public static void SetErrorFlag(PipelineContext pipelineContext) + { + if (pipelineContext == null) + { + throw new ArgumentNullException(nameof(pipelineContext)); + } + + var settings = pipelineContext.GetCurrentPipelineBatch().GetPlugin(); + + if(settings == null || settings.ErrorFlag) + { + return; + } + + settings.ErrorFlag = true; + } + } +} diff --git a/Brightcove.DataExchangeFramework/Helpers/IItemModelRepositoryHelper.cs b/Brightcove.DataExchangeFramework/Helpers/IItemModelRepositoryHelper.cs new file mode 100644 index 00000000..5fc93245 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Helpers/IItemModelRepositoryHelper.cs @@ -0,0 +1,38 @@ +using Brightcove.Core.Exceptions; +using Brightcove.Core.Models; +using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data.Fields; +using Sitecore.Data.Items; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.DataAccess; +using Sitecore.DataExchange.Extensions; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Plugins; +using Sitecore.DataExchange.Processors.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Services.Core.Diagnostics; +using Sitecore.Services.Core.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using Brightcove.Core.Extensions; +using Sitecore.DataExchange.ApplyMapping; +using Sitecore.Data; + +namespace Brightcove.DataExchangeFramework.Helpers +{ + public static class IItemModelRepositoryHelper + { + public static string GetIndexName(IItemModelRepository itemModelRepository) + { + if (itemModelRepository == null) + { + throw new ArgumentNullException(nameof(itemModelRepository)); + } + + return $"sitecore_{itemModelRepository.DatabaseName}_index"; + } + } +} diff --git a/Brightcove.DataExchangeFramework/Helpers/ItemUpdater.cs b/Brightcove.DataExchangeFramework/Helpers/ItemUpdater.cs index 93432616..d17a7cf3 100644 --- a/Brightcove.DataExchangeFramework/Helpers/ItemUpdater.cs +++ b/Brightcove.DataExchangeFramework/Helpers/ItemUpdater.cs @@ -76,50 +76,5 @@ private static void FixItemModel(ItemModel itemModel) itemModel[key] = (object)obj.ToString(); } } - - public static SyncStatus GetSyncStatus(ItemModel itemModel) - { - DateTime lastSyncTime = DateTime.MinValue; - DateTime modelLastModifiedTime = DateTime.MinValue; - DateTime itemLastModifiedTime = DateTime.MinValue; - - if (string.IsNullOrWhiteSpace((string)itemModel["LastSyncTime"])) - { - return SyncStatus.NewItem; - } - lastSyncTime = DateTime.Parse((string)itemModel["LastSyncTime"]); - - Item item = Sitecore.Context.ContentDatabase.GetItem(new ID(itemModel.GetItemId())); - - if (item != null) - { - itemLastModifiedTime = ((DateField)item.Fields["__Updated"]).DateTime; - } - - if (itemModel.ContainsKey("LastModifiedTime") && !string.IsNullOrWhiteSpace((string)itemModel["LastModifiedTime"])) - { - modelLastModifiedTime = DateTime.Parse((string)itemModel["LastModifiedTime"]); - } - - if (modelLastModifiedTime > lastSyncTime) - { - return SyncStatus.ModelNewer; - } - - if (itemLastModifiedTime > lastSyncTime) - { - return SyncStatus.ItemNewer; - } - - return SyncStatus.Unmodified; - } - } - - public enum SyncStatus - { - NewItem, - ModelNewer, - ItemNewer, - Unmodified } } diff --git a/Brightcove.DataExchangeFramework/Processors/ApplyMappingPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/ApplyMappingPipelineStepProcessor.cs new file mode 100644 index 00000000..91962954 --- /dev/null +++ b/Brightcove.DataExchangeFramework/Processors/ApplyMappingPipelineStepProcessor.cs @@ -0,0 +1,139 @@ +using Brightcove.Core.Models; +using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Extensions; +using Brightcove.DataExchangeFramework.Helpers; +using Brightcove.DataExchangeFramework.Settings; +using Sitecore.Data.Fields; +using Sitecore.Data.Items; +using Sitecore.DataExchange.ApplyMapping; +using Sitecore.DataExchange.Attributes; +using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.DataAccess; +using Sitecore.DataExchange.Extensions; +using Sitecore.DataExchange.Models; +using Sitecore.DataExchange.Plugins; +using Sitecore.DataExchange.Processors.PipelineSteps; +using Sitecore.DataExchange.Providers.Sc.Processors.PipelineSteps; +using Sitecore.DataExchange.Repositories; +using Sitecore.Globalization; +using Sitecore.Services.Core.Diagnostics; +using Sitecore.Services.Core.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using MappingSettings = Sitecore.DataExchange.ApplyMapping.MappingSettings; + +namespace Brightcove.DataExchangeFramework.Processors +{ + public class ApplyMappingPipelineStepProcessor : ApplyMappingStepProcessor + { + protected override void ProcessPipelineStep(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) + { + try + { + MappingSettings mappingSettings = pipelineStep.GetMappingSettings(); + if (mappingSettings == null) + { + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because the pipeline step is missing a plugin.", new string[1] + { + "plugin: " + typeof (MappingSettings).FullName + }); + } + else + { + IMappingSet mappingSet = mappingSettings.MappingSet; + if (mappingSet == null) + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because the pipeline step has no mapping set assigned.", new string[2] + { + "plugin: " + typeof (MappingSettings).FullName, + "property: MappingSet" + }); + else if (mappingSet.Mappings == null) + { + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because the pipeline step has no mappings assigned.", new string[2] + { + "plugin: " + typeof (MappingSettings).FullName, + "property: MappingSet" + }); + } + else + { + object sourceObject = this.GetSourceObject(mappingSettings, pipelineContext, logger); + if (sourceObject == null) + { + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because no source object could be resolved from the pipeline context.", new string[1] + { + string.Format("source location: {0}", (object) mappingSettings.SourceObjectLocation) + }); + } + else + { + object targetObject = this.GetTargetObject(mappingSettings, pipelineContext, logger); + if (targetObject == null) + { + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because no target object could be resolved from the pipeline context.", new string[1] + { + string.Format("target location: {0}", (object) mappingSettings.TargetObjectLocation) + }); + } + else + { + MappingContext mappingContext = new MappingContext() + { + Source = sourceObject, + Target = targetObject + }; + if (!mappingSet.Run(mappingContext)) + { + this.Log(new Action(logger.Error), pipelineContext, "Pipeline step processing will abort because mapping set failed.", new string[3] + { + string.Format("mappings that succeeded: {0}", (object) mappingContext.RunSuccess.Count), + string.Format("mappings that were not attempted: {0}", (object) mappingContext.RunIgnore.Count), + string.Format("mappings that failed: {0}", (object) mappingContext.RunFail.Count) + }); + this.Log(new Action(logger.Error), pipelineContext, "At least one required value mapping failed.", new string[2] + { + string.Format("mappings that failed: {0}", (object) mappingContext.RunFail.Count), + "value mapping ids: " + string.Join(",", mappingContext.RunFail.Select((Func) (x => x.Identifier)).ToArray()) + }); + + HandleError(pipelineContext); + } + else + { + if (mappingContext.RunFail.Any()) + { + this.Log(new Action(logger.Error), pipelineContext, "At least one value mapping failed.", new string[2] + { + string.Format("mappings that failed: {0}", (object) mappingContext.RunFail.Count), + "value mapping ids: " + string.Join(",", mappingContext.RunFail.Select((Func) (x => x.Identifier)).ToArray()) + }); + + HandleError(pipelineContext); + } + + pipelineContext.GetSynchronizationSettings().IsTargetDirty = this.IsTargetDirty(mappingContext, mappingSettings, pipelineContext, logger); + if (!this.ShouldRunMappingsAppliedActions(mappingContext, mappingSettings, pipelineContext, logger)) + return; + this.RunMappingsAppliedActions(mappingContext, mappingSettings, pipelineContext, logger); + } + } + } + } + } + } + catch(Exception ex) + { + logger.Error("Failed to apply mapping(s) because an unexpected error occured", ex); + HandleError(pipelineContext); + } + } + + protected void HandleError(PipelineContext pipelineContext) + { + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; + pipelineContext.CriticalError = false; + } + } +} diff --git a/Brightcove.DataExchangeFramework/Processors/BasePipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/BasePipelineStepProcessor.cs index f08fffaa..3f611489 100644 --- a/Brightcove.DataExchangeFramework/Processors/BasePipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/BasePipelineStepProcessor.cs @@ -1,5 +1,6 @@ using Brightcove.Core.Models; using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Helpers; using Brightcove.DataExchangeFramework.Settings; using Sitecore.DataExchange; using Sitecore.DataExchange.Attributes; @@ -22,28 +23,34 @@ namespace Brightcove.DataExchangeFramework.Processors { public class BasePipelineStepProcessor : Sitecore.DataExchange.Processors.PipelineSteps.BasePipelineStepProcessor { + protected IItemModelRepository itemModelRepository { get; set; } + protected override void ProcessPipelineStep(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - if (pipelineStep == null) - { - throw new ArgumentNullException(nameof(pipelineStep)); - } - if (pipelineContext == null) - { - throw new ArgumentNullException(nameof(pipelineContext)); - } - if (logger == null) - { - throw new ArgumentNullException(nameof(logger)); - } - try { + if (pipelineStep == null) + { + throw new ArgumentNullException(nameof(pipelineStep)); + } + if (pipelineContext == null) + { + throw new ArgumentNullException(nameof(pipelineContext)); + } + if (logger == null) + { + throw new ArgumentNullException(nameof(logger)); + } + + itemModelRepository = Sitecore.DataExchange.Context.ItemModelRepository; + ProcessPipelineStepInternal(pipelineStep, pipelineContext, logger); } catch(Exception ex) { LogFatal("An unexpected error occured running the internal pipeline step", ex); + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; pipelineContext.CriticalError = false; } } diff --git a/Brightcove.DataExchangeFramework/Processors/BasePipelineStepWithWebApiEndpointProcessor.cs b/Brightcove.DataExchangeFramework/Processors/BasePipelineStepWithWebApiEndpointProcessor.cs index 224fdac4..f2f5e39e 100644 --- a/Brightcove.DataExchangeFramework/Processors/BasePipelineStepWithWebApiEndpointProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/BasePipelineStepWithWebApiEndpointProcessor.cs @@ -1,5 +1,6 @@ using Brightcove.Core.Models; using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Helpers; using Brightcove.DataExchangeFramework.Settings; using Sitecore.Data.Fields; using Sitecore.Data.Items; @@ -10,6 +11,7 @@ using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.DataExchange.Processors.PipelineSteps; +using Sitecore.DataExchange.Providers.Sc.Plugins; using Sitecore.DataExchange.Repositories; using Sitecore.SecurityModel; using Sitecore.Services.Core.Diagnostics; @@ -25,54 +27,71 @@ namespace Brightcove.DataExchangeFramework.Processors public class BasePipelineStepWithWebApiEndpointProcessor : BasePipelineStepProcessor { protected WebApiSettings WebApiSettings { get; set; } + protected Endpoint EndpointFrom { get; set; } + protected BrightcoveService service { get; set; } + protected override void ProcessPipelineStep(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - if (pipelineStep == null) - { - throw new ArgumentNullException(nameof(pipelineStep)); - } - if (pipelineContext == null) - { - throw new ArgumentNullException(nameof(pipelineContext)); - } - if (logger == null) + try { - throw new ArgumentNullException(nameof(logger)); - } + if (pipelineStep == null) + { + throw new ArgumentNullException(nameof(pipelineStep)); + } + if (pipelineContext == null) + { + throw new ArgumentNullException(nameof(pipelineContext)); + } + if (logger == null) + { + throw new ArgumentNullException(nameof(logger)); + } - EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings(); + EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings(); - if (endpointSettings == null) - { - LogFatal("Pipeline step processing will abort because the pipeline step is missing endpoint settings."); - return; - } + if (endpointSettings == null) + { + LogFatal("Pipeline step processing will abort because the pipeline step is missing endpoint settings."); + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; + pipelineContext.CriticalError = false; + return; + } - EndpointFrom = endpointSettings.EndpointFrom; + EndpointFrom = endpointSettings.EndpointFrom; - if (EndpointFrom == null) - { - LogFatal("Pipeline step processing will abort because the pipeline step is missing an endpoint to read from."); - return; - } + if (EndpointFrom == null) + { + LogFatal("Pipeline step processing will abort because the pipeline step is missing an endpoint to read from."); + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; + pipelineContext.CriticalError = false; + return; + } - WebApiSettings = GetPluginOrFail(EndpointFrom); + WebApiSettings = GetPluginOrFail(EndpointFrom); - if(!WebApiSettings.Validate()) - { - LogFatal("Pipeline step processing will abort because the Brightcove web API settings are invalid: "+WebApiSettings.ValidationMessage); - return; - } + if (!WebApiSettings.Validate()) + { + LogFatal("Pipeline step processing will abort because the Brightcove web API settings are invalid: " + WebApiSettings.ValidationMessage); + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; + pipelineContext.CriticalError = false; + return; + } + + itemModelRepository = Sitecore.DataExchange.Context.ItemModelRepository; + service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - try - { ProcessPipelineStepInternal(pipelineStep, pipelineContext, logger); } catch (Exception ex) { LogFatal("An unexpected error occured running the internal pipeline step", ex); + BrightcoveSyncSettingsHelper.SetErrorFlag(pipelineContext); + pipelineContext.Finished = true; pipelineContext.CriticalError = false; } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetExperiencesPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetExperiencesPipelineStepProcessor.cs index 377b0fff..c903e3fe 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetExperiencesPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetExperiencesPipelineStepProcessor.cs @@ -1,8 +1,10 @@ using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Helpers; using Brightcove.DataExchangeFramework.Settings; using Sitecore.Data.Fields; using Sitecore.Data.Items; using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.SecurityModel; @@ -14,25 +16,15 @@ namespace Brightcove.DataExchangeFramework.Processors { class GetExperiencesPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; - protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try - { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - - var data = service.GetExperiences().Items; - var dataSettings = new IterableDataSettings(data); + DateTime lastSyncStartTime = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()).LastSyncStartTime; - LogDebug("Read " + data.Count() + " experience model(s) from web API"); + var data = service.GetExperiences().Items.Where(e => e.LastModifiedDate > lastSyncStartTime); + LogInfo("Identified " + data.Count() + " experience model(s) that have been modified since last sync "+ lastSyncStartTime); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + var dataSettings = new IterableDataSettings(data); + pipelineContext.AddPlugin(dataSettings); } } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetFoldersPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetFoldersPipelineStepProcessor.cs index 89a73471..2446d330 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetFoldersPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetFoldersPipelineStepProcessor.cs @@ -1,9 +1,11 @@ using Brightcove.Core.Models; using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Helpers; using Brightcove.DataExchangeFramework.Settings; using Sitecore.DataExchange.Attributes; using Sitecore.DataExchange.Contexts; using Sitecore.DataExchange.Converters.PipelineSteps; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.DataExchange.Processors.PipelineSteps; @@ -20,31 +22,15 @@ namespace Brightcove.DataExchangeFramework.Processors { public class GetFoldersPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; - protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try - { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - - var folders = service.GetFolders(); - - foreach (Folder folder in folders) - { - folder.LastSyncTime = DateTime.UtcNow; - } - - LogDebug("Read " + folders.Count() + " folder model(s) from web API"); + DateTime lastSyncStartTime = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()).LastSyncStartTime; - var dataSettings = new IterableDataSettings(folders); + var folders = service.GetFolders().Where(f => f.UpdatedDate > lastSyncStartTime); + LogInfo("Identified " + folders.Count() + " folder model(s) that have been modified since last sync " + lastSyncStartTime); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + var dataSettings = new IterableDataSettings(folders); + pipelineContext.AddPlugin(dataSettings); } } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetLabelsPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetLabelsPipelineStepProcessor.cs index 23ad8534..4c8c0627 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetLabelsPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetLabelsPipelineStepProcessor.cs @@ -1,9 +1,11 @@ using Brightcove.Core.Models; using Brightcove.Core.Services; +using Brightcove.DataExchangeFramework.Helpers; using Brightcove.DataExchangeFramework.Settings; using Sitecore.DataExchange.Attributes; using Sitecore.DataExchange.Contexts; using Sitecore.DataExchange.Converters.PipelineSteps; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.DataExchange.Processors.PipelineSteps; @@ -20,31 +22,15 @@ namespace Brightcove.DataExchangeFramework.Processors { public class GetLabelsPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; - protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try - { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - - var labels = service.GetLabels(); - - foreach (Label label in labels) - { - label.LastSyncTime = DateTime.UtcNow; - } - - LogDebug("Read " + labels.Count() + " label model(s) from web API"); + //DateTime lastSyncStartTime = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()).LastSyncStartTime; - var dataSettings = new IterableDataSettings(labels); + var labels = service.GetLabels(); + LogDebug("Identified " + labels.Count() + "label model(s)"); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + var dataSettings = new IterableDataSettings(labels); + pipelineContext.AddPlugin(dataSettings); } } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetPlayListsPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetPlayListsPipelineStepProcessor.cs index 8eaf67b7..9c2166bd 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetPlayListsPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetPlayListsPipelineStepProcessor.cs @@ -5,6 +5,7 @@ using Sitecore.Data.Items; using Sitecore.DataExchange.Attributes; using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.DataExchange.Processors.PipelineSteps; @@ -20,38 +21,32 @@ namespace Brightcove.DataExchangeFramework.Processors { class GetPlayListsPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; + DateTime lastSyncStartTime; int totalCount = 0; protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try - { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - - totalCount = service.PlayListsCount(); - LogDebug("Read " + totalCount + " playlist model(s) from web API"); + lastSyncStartTime = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()).LastSyncStartTime; + totalCount = service.PlayListsCount(); - var data = this.GetIterableData(WebApiSettings, pipelineStep); - var dataSettings = new IterableDataSettings(data); + var data = this.GetIterableData(WebApiSettings, pipelineStep); + var dataSettings = new IterableDataSettings(data); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + pipelineContext.AddPlugin(dataSettings); } protected virtual IEnumerable GetIterableData(WebApiSettings settings, PipelineStep pipelineStep) { - int limit = 100; + IEnumerable playLists; + int limit = 1000; for (int offset = 0; offset < totalCount; offset += limit) { - foreach(PlayList playList in service.GetPlayLists(offset, limit)) + playLists = service.GetPlayLists(offset, limit).Where(p => p.LastModifiedDate > lastSyncStartTime); + LogInfo("Identified " + playLists.Count() + " playlist model(s) that have been modified since last sync " + lastSyncStartTime); + + foreach (PlayList playList in playLists) { - playList.LastSyncTime = DateTime.UtcNow; yield return playList; } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetPlayersPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetPlayersPipelineStepProcessor.cs index 58e69a4e..794385ae 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetPlayersPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetPlayersPipelineStepProcessor.cs @@ -3,6 +3,7 @@ using Sitecore.Data.Fields; using Sitecore.Data.Items; using Sitecore.DataExchange.Contexts; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.SecurityModel; @@ -14,31 +15,15 @@ namespace Brightcove.DataExchangeFramework.Processors { class GetPlayersPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; - protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try - { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); - - var data = service.GetPlayers().Items; - - foreach (var player in data) - { - player.LastSyncTime = DateTime.UtcNow; - } - - var dataSettings = new IterableDataSettings(data); + DateTime lastSyncStartTime = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()).LastSyncStartTime; - LogDebug("Read " + data.Count() + " player model(s) from web API"); + var data = service.GetPlayers().Items.Where(p => p.Branches.Master.UpdatedAt > lastSyncStartTime); + LogInfo("Identified " + data.Count() + " player model(s) that have been modified since last sync " + lastSyncStartTime); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + var dataSettings = new IterableDataSettings(data); + pipelineContext.AddPlugin(dataSettings); } } } diff --git a/Brightcove.DataExchangeFramework/Processors/Get/GetVideosPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/Get/GetVideosPipelineStepProcessor.cs index 152bf2a9..d166b9e4 100644 --- a/Brightcove.DataExchangeFramework/Processors/Get/GetVideosPipelineStepProcessor.cs +++ b/Brightcove.DataExchangeFramework/Processors/Get/GetVideosPipelineStepProcessor.cs @@ -6,6 +6,7 @@ using Sitecore.DataExchange.Attributes; using Sitecore.DataExchange.Contexts; using Sitecore.DataExchange.Converters.PipelineSteps; +using Sitecore.DataExchange.Extensions; using Sitecore.DataExchange.Models; using Sitecore.DataExchange.Plugins; using Sitecore.DataExchange.Processors.PipelineSteps; @@ -22,38 +23,35 @@ namespace Brightcove.DataExchangeFramework.Processors { public class GetVideosPipelineStepProcessor : BasePipelineStepWithWebApiEndpointProcessor { - BrightcoveService service; int totalCount = 0; + string query = ""; protected override void ProcessPipelineStepInternal(PipelineStep pipelineStep = null, PipelineContext pipelineContext = null, ILogger logger = null) { - try + BrightcoveSyncSettings syncSettings = GetPluginOrFail(pipelineContext.GetCurrentPipelineBatch()); + + if (syncSettings.LastSyncStartTime != DateTime.MinValue) { - service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret); + query = $"+updated_at:[{syncSettings.LastSyncStartTime.ToString("s", System.Globalization.CultureInfo.InvariantCulture)}Z TO *]"; + } - totalCount = service.VideosCount(); - LogDebug("Read " + totalCount + " video model(s) from web API"); + totalCount = service.VideosCount(query); + LogInfo("Identified " + totalCount + " video model(s) that have been modified since last sync "+ syncSettings.LastSyncStartTime); - var data = GetIterableData(pipelineStep); - var dataSettings = new IterableDataSettings(data); + var data = GetIterableData(pipelineStep); + var dataSettings = new IterableDataSettings(data); - pipelineContext.AddPlugin(dataSettings); - } - catch (Exception ex) - { - LogError($"Failed to get the brightcove models because an unexpected error has occured", ex); - } + pipelineContext.AddPlugin(dataSettings); } protected virtual IEnumerable