Massive Speedup in Queue

ShokoAnime · Jan 20, 2024 · 1903433 · 1903433
1 parent 0e107b9
commit 1903433
Show file tree

Hide file tree

Showing 7 changed files with 40 additions and 25 deletions.
diff --git a/Shoko.Server/API/v3/Controllers/QueueController.cs b/Shoko.Server/API/v3/Controllers/QueueController.cs
@@ -12,9 +12,11 @@
 using Shoko.Server.API.v3.Models.Shoko;
 using Shoko.Server.Commands.Generic;
 using Shoko.Server.Databases;
+using Shoko.Server.Databases.NHIbernate;
 using Shoko.Server.Repositories;
 using Shoko.Server.Server;
 using Shoko.Server.Settings;
+using Shoko.Server.Utilities;
 
 namespace Shoko.Server.API.v3.Controllers;
 
@@ -204,7 +206,7 @@ public ActionResult ClearQueueByName([FromRoute] string queueName)
 
         return BaseRepository.Lock(processor, _connectivityService, (commandProcessor, service) =>
         {
-            var session = DatabaseFactory.SessionFactory.OpenSession();
+            using var session = DatabaseFactory.SessionFactory.OpenSession();
             return queueName.ToLowerInvariant() switch
             {
                 "general" => RepoFactory.CommandRequest.GetGeneralCommandsUnsafe(session, showAll)
@@ -235,7 +237,7 @@ public ActionResult<Dictionary<CommandRequestType, int>> GetTypesForItemsInQueue
     {
         return BaseRepository.Lock(() =>
         {
-            var session = DatabaseFactory.SessionFactory.OpenSession();
+            using var session = DatabaseFactory.SessionFactory.OpenSession();
             return queueName.ToLowerInvariant() switch
             {
                 "general" => RepoFactory.CommandRequest.GetGeneralCommandsUnsafe(session, true)

diff --git a/Shoko.Server/API/v3/Helpers/ModelHelper.cs b/Shoko.Server/API/v3/Helpers/ModelHelper.cs
@@ -10,7 +10,6 @@
 using File = Shoko.Server.API.v3.Models.Shoko.File;
 using FileSource = Shoko.Server.API.v3.Models.Shoko.FileSource;
 using GroupSizes = Shoko.Server.API.v3.Models.Shoko.GroupSizes;
-using Series = Shoko.Server.API.v3.Models.Shoko.Series;
 using SeriesSizes = Shoko.Server.API.v3.Models.Shoko.SeriesSizes;
 using SeriesType = Shoko.Server.API.v3.Models.Shoko.SeriesType;
 
@@ -20,30 +19,28 @@ public static class ModelHelper
 {
     public static ListResult<T> ToListResult<T>(this IEnumerable<T> enumerable)
     {
-        var list = enumerable as IReadOnlyList<T> ?? enumerable.ToList();
         return new ListResult<T>
         {
-            Total = list.Count,
-            List = list
+            Total = enumerable.Count(),
+            List = enumerable.ToList()
         };
     }
 
     public static ListResult<T> ToListResult<T>(this IEnumerable<T> enumerable, int page, int pageSize)
     {
-        var list = enumerable as IReadOnlyList<T> ?? enumerable.ToList();
         if (pageSize <= 0)
         {
             return new ListResult<T>
             {
-                Total = list.Count,
-                List = list
+                Total = enumerable.Count(),
+                List = enumerable.ToList()
             };
         }
 
         return new ListResult<T>
         {
-            Total = list.Count,
-            List = list
+            Total = enumerable.Count(),
+            List = enumerable.AsQueryable()
                 .Skip(pageSize * (page - 1))
                 .Take(pageSize)
                 .ToList()
@@ -53,24 +50,24 @@ public static ListResult<T> ToListResult<T>(this IEnumerable<T> enumerable, int
     public static ListResult<U> ToListResult<T, U>(this IEnumerable<T> enumerable, Func<T, U> mapper, int page,
         int pageSize)
     {
-        var list = enumerable as IReadOnlyList<T> ?? enumerable.ToList();
         if (pageSize <= 0)
         {
             return new ListResult<U>
             {
-                Total = list.Count,
-                List = list
+                Total = enumerable.Count(),
+                List = enumerable.ToList()
                     .Select(mapper)
                     .ToList()
             };
         }
 
         return new ListResult<U>
         {
-            Total = list.Count,
-            List = list
+            Total = enumerable.Count(),
+            List = enumerable.AsQueryable()
                 .Skip(pageSize * (page - 1))
                 .Take(pageSize)
+                .ToList()
                 .Select(mapper)
                 .ToList()
         };
@@ -93,7 +90,7 @@ public static ListResult<U> ToListResult<T, U>(this IEnumerable<T> enumerable, F
         return new ListResult<U>
         {
             Total = total,
-            List = enumerable
+            List = enumerable.AsQueryable()
                 .Skip(pageSize * (page - 1))
                 .Take(pageSize)
                 .ToList()
@@ -137,8 +134,7 @@ public static int GetTotalEpisodesForType(List<SVR_AnimeEpisode> episodeList, Ep
     {
         return episodeList
             .Select(episode => episode.AniDB_Episode)
-            .Where(anidbEpisode => anidbEpisode != null && (EpisodeType)anidbEpisode.EpisodeType == episodeType)
-            .Count();
+            .Count(anidbEpisode => anidbEpisode != null && (EpisodeType)anidbEpisode.EpisodeType == episodeType);
     }
 
     public static string ToDataURL(byte[] byteArray, string contentType, string fieldName = "ByteArrayToDataUrl", ModelStateDictionary modelState = null)

diff --git a/Shoko.Server/Commands/CommandProcessorGeneral.cs b/Shoko.Server/Commands/CommandProcessorGeneral.cs
@@ -43,7 +43,7 @@ protected override CommandRequest GetNextCommandRequest()
             return BaseRepository.Lock(() =>
             {
                 using var session = DatabaseFactory.SessionFactory.OpenSession();
-                return RepoFactory.CommandRequest.GetGeneralCommandsUnsafe(session).Take(1).SingleOrDefault();
+                return RepoFactory.CommandRequest.GetGeneralCommandsUnsafe(session).FirstOrDefault();
             });
         }
         catch (Exception e)

diff --git a/Shoko.Server/Commands/CommandProcessorHasher.cs b/Shoko.Server/Commands/CommandProcessorHasher.cs
@@ -43,7 +43,7 @@ protected override CommandRequest GetNextCommandRequest()
             return BaseRepository.Lock(() =>
             {
                 using var session = DatabaseFactory.SessionFactory.OpenSession();
-                return RepoFactory.CommandRequest.GetHasherCommandsUnsafe(session).Take(1).SingleOrDefault();
+                return RepoFactory.CommandRequest.GetHasherCommandsUnsafe(session).FirstOrDefault();
             });
         }
         catch (Exception e)

diff --git a/Shoko.Server/Commands/CommandProcessorImages.cs b/Shoko.Server/Commands/CommandProcessorImages.cs
@@ -7,6 +7,7 @@
 using Shoko.Plugin.Abstractions.Services;
 using Shoko.Server.Commands.Generic;
 using Shoko.Server.Databases;
+using Shoko.Server.Databases.NHIbernate;
 using Shoko.Server.Models;
 using Shoko.Server.Repositories;
 
@@ -43,7 +44,7 @@ protected override CommandRequest GetNextCommandRequest()
             return BaseRepository.Lock(() =>
             {
                 using var session = DatabaseFactory.SessionFactory.OpenSession();
-                return RepoFactory.CommandRequest.GetImageCommandsUnsafe(session).Take(1).SingleOrDefault();
+                return RepoFactory.CommandRequest.GetImageCommandsUnsafe(session).FirstOrDefault();
             });
         }
         catch (Exception e)

diff --git a/Shoko.Server/Databases/NHIbernate/NLogInterceptor.cs b/Shoko.Server/Databases/NHIbernate/NLogInterceptor.cs
@@ -0,0 +1,16 @@
+using NHibernate;
+using NHibernate.SqlCommand;
+using NLog;
+
+namespace Shoko.Server.Databases.NHIbernate;
+
+public class NLogInterceptor : EmptyInterceptor
+{
+    private Logger _logger = LogManager.GetCurrentClassLogger();
+    public override SqlString OnPrepareStatement(SqlString sql)
+    {
+        _logger.Trace($"Executing Query: {sql}");
+
+        return base.OnPrepareStatement(sql);
+    }
+}
diff --git a/Shoko.Server/Repositories/Direct/CommandRequestRepository.cs b/Shoko.Server/Repositories/Direct/CommandRequestRepository.cs
@@ -188,7 +188,7 @@ public bool CheckIfCommandRequestIsDisabled(CommandRequestType type, IConnectivi
         return false;
     }
 
-    public IEnumerable<CommandRequest> GetGeneralCommandsUnsafe(ISession session, bool includeDisabled = false)
+    public IOrderedQueryable<CommandRequest> GetGeneralCommandsUnsafe(ISession session, bool includeDisabled = false)
     {
         try
         {
@@ -205,7 +205,7 @@ public IEnumerable<CommandRequest> GetGeneralCommandsUnsafe(ISession session, bo
             return null;
         }
     }
-    public IEnumerable<CommandRequest> GetHasherCommandsUnsafe(ISession session, bool includeDisabled = false)
+    public IOrderedQueryable<CommandRequest> GetHasherCommandsUnsafe(ISession session, bool includeDisabled = false)
     {
         try
         {
@@ -223,7 +223,7 @@ public IEnumerable<CommandRequest> GetHasherCommandsUnsafe(ISession session, boo
         }
     }
 
-    public IEnumerable<CommandRequest> GetImageCommandsUnsafe(ISession session, bool includeDisabled = false)
+    public IOrderedQueryable<CommandRequest> GetImageCommandsUnsafe(ISession session, bool includeDisabled = false)
     {
         try
         {