From b545c2c2e4470dc5a8167d296218488775758733 Mon Sep 17 00:00:00 2001
From: Cody Rodgers <codrod63@gmail.com>
Date: Fri, 3 Feb 2023 16:37:00 -0500
Subject: [PATCH] Update the videos/playlists pull pipelines to automatically
 refresh the search index to prevent a race condition (#14)

Co-authored-by: Cody Rodgers <rodgers@rdacorp.com>
---
 .../Converters/WebApiEndpointConverter.cs     |  1 +
 .../GetPlayListsPipelineStepProcessor.cs      | 23 ++++++++++++++--
 .../GetVideosPipelineStepProcessor.cs         | 27 ++++++++++++++++---
 .../Settings/WebApiSettings.cs                |  5 +++-
 4 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs b/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs
index 89f13df7..2ef5114d 100644
--- a/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs
+++ b/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs
@@ -35,6 +35,7 @@ protected override void AddPlugins(ItemModel source, Endpoint endpoint)
                 accountSettings.AccountId = accountItem["AccountId"];
                 accountSettings.ClientId = accountItem["ClientId"];
                 accountSettings.ClientSecret = accountItem["ClientSecret"];
+                accountSettings.AccountItem = accountItem;
             }
 
             endpoint.AddPlugin(accountSettings);
diff --git a/Brightcove.DataExchangeFramework/Processors/GetPlayListsPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/GetPlayListsPipelineStepProcessor.cs
index 9c688568..124b8406 100644
--- a/Brightcove.DataExchangeFramework/Processors/GetPlayListsPipelineStepProcessor.cs
+++ b/Brightcove.DataExchangeFramework/Processors/GetPlayListsPipelineStepProcessor.cs
@@ -1,6 +1,8 @@
 using Brightcove.Core.Models;
 using Brightcove.Core.Services;
 using Brightcove.DataExchangeFramework.Settings;
+using Sitecore.ContentSearch;
+using Sitecore.Data.Items;
 using Sitecore.DataExchange.Attributes;
 using Sitecore.DataExchange.Contexts;
 using Sitecore.DataExchange.Models;
@@ -24,14 +26,15 @@ protected override void ProcessPipelineStep(PipelineStep pipelineStep = null, Pi
         {
             base.ProcessPipelineStep(pipelineStep, pipelineContext, logger);
 
+            //We may need to refresh the search index before syncing
+            RefreshSearchIndex(pipelineStep);
+
             service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret);
 
             var data = this.GetIterableData(WebApiSettings, pipelineStep);
             var dataSettings = new IterableDataSettings(data);
 
             pipelineContext.AddPlugin(dataSettings);
-
-            SetFolderSettings("Playlists");
         }
 
         protected virtual IEnumerable<PlayList> GetIterableData(WebApiSettings settings, PipelineStep pipelineStep)
@@ -51,5 +54,21 @@ protected virtual IEnumerable<PlayList> GetIterableData(WebApiSettings settings,
                 }
             }
         }
+
+        private void RefreshSearchIndex(PipelineStep pipelineStep)
+        {
+            Item videosFolder = Sitecore.Data.Database.GetDatabase("master").GetItem(WebApiSettings.AccountItem.Paths.FullPath + "/Videos");
+
+            Logger.Debug($"Started refreshing the search index before syncing videos... (pipeline step: {pipelineStep.Name})");
+
+            var jobs = Sitecore.ContentSearch.Maintenance.IndexCustodian.RefreshTree((SitecoreIndexableItem)videosFolder);
+
+            foreach (var job in jobs)
+            {
+                job.Wait();
+            }
+
+            Logger.Debug($"Finished refreshing the search index (pipeline step: {pipelineStep.Name})");
+        }
     }
 }
diff --git a/Brightcove.DataExchangeFramework/Processors/GetVideosPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/GetVideosPipelineStepProcessor.cs
index 9396440c..d4669f1d 100644
--- a/Brightcove.DataExchangeFramework/Processors/GetVideosPipelineStepProcessor.cs
+++ b/Brightcove.DataExchangeFramework/Processors/GetVideosPipelineStepProcessor.cs
@@ -1,6 +1,8 @@
 using Brightcove.Core.Models;
 using Brightcove.Core.Services;
 using Brightcove.DataExchangeFramework.Settings;
+using Sitecore.ContentSearch;
+using Sitecore.Data.Items;
 using Sitecore.DataExchange.Attributes;
 using Sitecore.DataExchange.Contexts;
 using Sitecore.DataExchange.Converters.PipelineSteps;
@@ -26,14 +28,15 @@ protected override void ProcessPipelineStep(PipelineStep pipelineStep = null, Pi
         {
             base.ProcessPipelineStep(pipelineStep, pipelineContext, logger);
 
+            //We may need to refresh the search index before syncing
+            RefreshSearchIndex(pipelineStep);
+
             service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret);
 
             var data = GetIterableData(pipelineStep);
             var dataSettings = new IterableDataSettings(data);
 
             pipelineContext.AddPlugin(dataSettings);
-
-            SetFolderSettings("Videos");
         }
 
         protected virtual IEnumerable<Video> GetIterableData(PipelineStep pipelineStep)
@@ -43,7 +46,7 @@ protected virtual IEnumerable<Video> GetIterableData(PipelineStep pipelineStep)
 
             for (int offset = 0; offset < totalVideosCount; offset += limit)
             {
-                foreach(Video video in service.GetVideos(offset, limit))
+                foreach (Video video in service.GetVideos(offset, limit))
                 {
                     Logger.Debug($"Found the brightcove asset {video.Id} (pipeline step: {pipelineStep.Name})");
 
@@ -53,5 +56,23 @@ protected virtual IEnumerable<Video> GetIterableData(PipelineStep pipelineStep)
                 }
             }
         }
+
+        private void RefreshSearchIndex(PipelineStep pipelineStep)
+        {            
+            Item labelsFolder = Sitecore.Data.Database.GetDatabase("master").GetItem(WebApiSettings.AccountItem.Paths.FullPath + "/Labels");
+            Item foldersFolder = Sitecore.Data.Database.GetDatabase("master").GetItem(WebApiSettings.AccountItem.Paths.FullPath + "/Folders");
+
+            Logger.Debug($"Started refreshing the search index before syncing videos... (pipeline step: {pipelineStep.Name})");
+
+            var jobs = Sitecore.ContentSearch.Maintenance.IndexCustodian.RefreshTree((SitecoreIndexableItem)labelsFolder);
+            jobs.Concat(Sitecore.ContentSearch.Maintenance.IndexCustodian.RefreshTree((SitecoreIndexableItem)foldersFolder));
+
+            foreach (var job in jobs)
+            {
+                job.Wait();
+            }
+
+            Logger.Debug($"Finished refreshing the search index (pipeline step: {pipelineStep.Name})");
+        }
     }
 }
diff --git a/Brightcove.DataExchangeFramework/Settings/WebApiSettings.cs b/Brightcove.DataExchangeFramework/Settings/WebApiSettings.cs
index 51b272a4..851d0c7c 100644
--- a/Brightcove.DataExchangeFramework/Settings/WebApiSettings.cs
+++ b/Brightcove.DataExchangeFramework/Settings/WebApiSettings.cs
@@ -1,4 +1,5 @@
-using Sitecore.DataExchange;
+using Sitecore.Data.Items;
+using Sitecore.DataExchange;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -12,5 +13,7 @@ public class WebApiSettings : IPlugin
         public string AccountId { get; set; } = "";
         public string ClientId { get; set; } = "";
         public string ClientSecret { get; set; } = "";
+
+        public Item AccountItem { get; set; } = null;
     }
 }