Acquisition Filters for More Complex States

like AniDB, Network, Database

Shoko.Server/Scheduling/Acquisition/Attributes/AcquisitionFilterAttribute.cs

@@ -0,0 +1,5 @@
+using System;
+
+namespace Shoko.Server.Scheduling.Acquisition.Attributes;
+
+public abstract class AcquisitionFilterAttribute : Attribute { }

Shoko.Server/Scheduling/Acquisition/Attributes/AniDBHttpRateLimitedAttribute.cs

@@ -0,0 +1,6 @@
+using System;
+
+namespace Shoko.Server.Scheduling.Acquisition.Attributes;
+
+[AttributeUsage(AttributeTargets.Class)]
+public class AniDBHttpRateLimitedAttribute : AcquisitionFilterAttribute { }

Shoko.Server/Scheduling/Acquisition/Attributes/AniDBUdpRateLimitedAttribute.cs

@@ -0,0 +1,6 @@
+using System;
+
+namespace Shoko.Server.Scheduling.Acquisition.Attributes;
+
+[AttributeUsage(AttributeTargets.Class)]
+public class AniDBUdpRateLimitedAttribute : AcquisitionFilterAttribute { }

Shoko.Server/Scheduling/Acquisition/Attributes/DatabaseRequiredAttribute.cs

@@ -0,0 +1,6 @@
+using System;
+
+namespace Shoko.Server.Scheduling.Acquisition.Attributes;
+
+[AttributeUsage(AttributeTargets.Class)]
+public class DatabaseRequiredAttribute : AcquisitionFilterAttribute { }

Shoko.Server/Scheduling/Acquisition/Attributes/NetworkRequiredAttribute.cs

@@ -0,0 +1,6 @@
+using System;
+
+namespace Shoko.Server.Scheduling.Acquisition.Attributes;
+
+[AttributeUsage(AttributeTargets.Class)]
+public class NetworkRequiredAttribute : AcquisitionFilterAttribute { }

Shoko.Server/Scheduling/Acquisition/Filters/AniDBHttpRateLimitedAcquisitionFilter.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Linq;
+using Quartz;
+using Quartz.Util;
+using Shoko.Server.Providers.AniDB.Interfaces;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
+
+namespace Shoko.Server.Scheduling.Acquisition.Filters;
+
+public class AniDBHttpRateLimitedAcquisitionFilter : IAcquisitionFilter
+{
+    private readonly Type[] _types;
+    private readonly IHttpConnectionHandler _connectionHandler;
+
+    public AniDBHttpRateLimitedAcquisitionFilter(IHttpConnectionHandler connectionHandler)
+    {
+        _connectionHandler = connectionHandler;
+        _types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).Where(a =>
+            typeof(IJob).IsAssignableFrom(a) && !a.IsAbstract && ObjectUtils.IsAttributePresent(a, typeof(AniDBHttpRateLimitedAttribute))).ToArray();
+    }
+
+    public Type[] GetTypesToExclude() => _connectionHandler.IsBanned ? _types : Array.Empty<Type>();
+}

Shoko.Server/Scheduling/Acquisition/Filters/AniDBUdpRateLimitedAcquisitionFilter.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Linq;
+using Quartz;
+using Quartz.Util;
+using Shoko.Server.Providers.AniDB.Interfaces;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
+
+namespace Shoko.Server.Scheduling.Acquisition.Filters;
+
+public class AniDBUdpRateLimitedAcquisitionFilter : IAcquisitionFilter
+{
+    private readonly Type[] _types;
+    private readonly IUDPConnectionHandler _connectionHandler;
+
+    public AniDBUdpRateLimitedAcquisitionFilter(IUDPConnectionHandler connectionHandler)
+    {
+        _connectionHandler = connectionHandler;
+        _types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).Where(a =>
+            typeof(IJob).IsAssignableFrom(a) && !a.IsAbstract && ObjectUtils.IsAttributePresent(a, typeof(AniDBUdpRateLimitedAttribute))).ToArray();
+    }
+
+    public Type[] GetTypesToExclude() => _connectionHandler.IsBanned || _connectionHandler.IsInvalidSession ? _types : Array.Empty<Type>();
+}

Shoko.Server/Scheduling/Acquisition/Filters/DatabaseRequiredAcquisitionFilter.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Linq;
+using Quartz;
+using Quartz.Util;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
+using Shoko.Server.Server;
+
+namespace Shoko.Server.Scheduling.Acquisition.Filters;
+
+public class DatabaseRequiredAcquisitionFilter : IAcquisitionFilter
+{
+    private readonly Type[] _types;
+
+    public DatabaseRequiredAcquisitionFilter()
+    {
+        _types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).Where(a =>
+            typeof(IJob).IsAssignableFrom(a) && !a.IsAbstract && ObjectUtils.IsAttributePresent(a, typeof(DatabaseRequiredAttribute))).ToArray();
+    }
+
+    public Type[] GetTypesToExclude() => ServerState.Instance.ServerOnline && !ServerState.Instance.DatabaseBlocked.Blocked ? Array.Empty<Type>() : _types;
+}

Shoko.Server/Scheduling/Acquisition/Filters/IAcquisitionFilter.cs

@@ -0,0 +1,8 @@
+using System;
+
+namespace Shoko.Server.Scheduling.Acquisition.Filters;
+
+public interface IAcquisitionFilter
+{
+    Type[] GetTypesToExclude();
+}

Shoko.Server/Scheduling/Acquisition/Filters/NetworkRequiredAcquisitionFilter.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Quartz;
+using Quartz.Util;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
+using Shoko.Server.Services.Connectivity;
+
+namespace Shoko.Server.Scheduling.Acquisition.Filters;
+
+public class NetworkRequiredAcquisitionFilter : IAcquisitionFilter
+{
+    private readonly Type[] _types;
+    private readonly IConnectivityMonitor[] _connectivityMonitors;
+
+    public NetworkRequiredAcquisitionFilter(IEnumerable<IConnectivityMonitor> connectivityMonitors)
+    {
+        _connectivityMonitors = connectivityMonitors.ToArray();
+        _types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).Where(a =>
+            typeof(IJob).IsAssignableFrom(a) && !a.IsAbstract && ObjectUtils.IsAttributePresent(a, typeof(NetworkRequiredAttribute))).ToArray();
+    }
+
+    public Type[] GetTypesToExclude() => _connectivityMonitors.Any(a => a.HasConnected) ? Array.Empty<Type>() : _types;
+}

Shoko.Server/Scheduling/Jobs/Actions/ImportJob.cs

@@ -7,6 +7,7 @@
 using Quartz;
 using QuartzJobFactory.Attributes;
 using Shoko.Server.Scheduling.Acquisition;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
 
 namespace Shoko.Server.Scheduling.Jobs.Actions;
 

Shoko.Server/Scheduling/Jobs/Actions/MediaInfoJob.cs

@@ -9,6 +9,7 @@
 using Shoko.Server.Commands;
 using Shoko.Server.Repositories;
 using Shoko.Server.Scheduling.Acquisition;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
 
 namespace Shoko.Server.Scheduling.Jobs.Actions;
 

Shoko.Server/Scheduling/Jobs/Actions/RemoveMissingFilesJob.cs

@@ -6,7 +6,7 @@
 using System.Threading.Tasks;
 using Quartz;
 using QuartzJobFactory.Attributes;
-using Shoko.Server.Scheduling.Acquisition;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
 
 namespace Shoko.Server.Scheduling.Jobs.Actions;
 

Shoko.Server/Scheduling/Jobs/Actions/ScanDropFoldersJob.cs

@@ -6,7 +6,7 @@
 using System.Threading.Tasks;
 using Quartz;
 using QuartzJobFactory.Attributes;
-using Shoko.Server.Scheduling.Acquisition;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
 
 namespace Shoko.Server.Scheduling.Jobs.Actions;
 

Shoko.Server/Scheduling/Jobs/AniDB/AniDBGetFileJob.cs

@@ -17,12 +17,13 @@
 using Shoko.Server.Providers.AniDB.UDP.Info;
 using Shoko.Server.Repositories;
 using Shoko.Server.Scheduling.Acquisition;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
 using Shoko.Server.Scheduling.Concurrency;
 using Shoko.Server.Server;
 
 namespace Shoko.Server.Scheduling.Jobs.AniDB;
 
-[DatabaseRequired, NetworkRequired, AniDBUDPRateLimited]
+[DatabaseRequired, NetworkRequired, AniDBUdpRateLimited]
 [DisallowConcurrencyGroup(ConcurrencyGroups.AniDB_UDP)]
 [JobKeyGroup(JobKeyGroup.AniDB)]
 [Command(CommandRequestType.AniDB_GetFileUDP)]

Shoko.Server/Scheduling/Jobs/Shoko/DeleteImportFolderJob.cs

@@ -6,7 +6,7 @@
 using System.Threading.Tasks;
 using Quartz;
 using QuartzJobFactory.Attributes;
-using Shoko.Server.Scheduling.Acquisition;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
 
 namespace Shoko.Server.Scheduling.Jobs.Shoko;
 

Shoko.Server/Scheduling/Jobs/Shoko/DiscoverFileJob.cs

@@ -12,7 +12,7 @@
 using Shoko.Server.Models;
 using Shoko.Server.Repositories;
 using Shoko.Server.Repositories.Cached;
-using Shoko.Server.Scheduling.Acquisition;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
 using Shoko.Server.Server;
 using Shoko.Server.Settings;
 

Shoko.Server/Scheduling/Jobs/Shoko/HashFileJob.cs

@@ -16,7 +16,7 @@
 using Shoko.Server.Models;
 using Shoko.Server.Repositories;
 using Shoko.Server.Repositories.Cached;
-using Shoko.Server.Scheduling.Acquisition;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
 using Shoko.Server.Scheduling.Concurrency;
 using Shoko.Server.Server;
 using Shoko.Server.Settings;

Shoko.Server/Scheduling/Jobs/Shoko/ScanFolderJob.cs

@@ -6,7 +6,7 @@
 using System.Threading.Tasks;
 using Quartz;
 using QuartzJobFactory.Attributes;
-using Shoko.Server.Scheduling.Acquisition;
+using Shoko.Server.Scheduling.Acquisition.Attributes;
 
 namespace Shoko.Server.Scheduling.Jobs.Shoko;
 

Shoko.Server/Scheduling/QuartzStartup.cs

@@ -7,6 +7,7 @@
 using Quartz;
 using Quartz.AspNetCore;
 using QuartzJobFactory;
+using Shoko.Server.Scheduling.Acquisition.Filters;
 using Shoko.Server.Scheduling.Delegates;
 using Shoko.Server.Scheduling.Jobs.Actions;
 using Shoko.Server.Scheduling.Jobs.Shoko;
@@ -20,6 +21,10 @@ public static class QuartzStartup
     internal static void AddQuartz(this IServiceCollection services)
     {
         services.AddSingleton<ThreadPooledJobStore>();
+        services.AddSingleton<IAcquisitionFilter, AniDBHttpRateLimitedAcquisitionFilter>();
+        services.AddSingleton<IAcquisitionFilter, AniDBUdpRateLimitedAcquisitionFilter>();
+        services.AddSingleton<IAcquisitionFilter, DatabaseRequiredAcquisitionFilter>();
+        services.AddSingleton<IAcquisitionFilter, NetworkRequiredAcquisitionFilter>();
         services.AddJobs();
         services.AddQuartz(q =>
         {

Shoko.Server/Scheduling/ThreadPooledJobStore.cs

@@ -9,6 +9,7 @@
 using Quartz.Impl.AdoJobStore;
 using Quartz.Spi;
 using Quartz.Util;
+using Shoko.Server.Scheduling.Acquisition.Filters;
 using Shoko.Server.Scheduling.Concurrency;
 using Shoko.Server.Scheduling.Delegates;
 using Shoko.Server.Utilities;
@@ -20,10 +21,12 @@ public class ThreadPooledJobStore : JobStoreTX
     private readonly ILogger<ThreadPooledJobStore> _logger;
     private ITypeLoadHelper _typeLoadHelper = null!;
     private Dictionary<Type, int> _typeConcurrencyCache;
+    private readonly IAcquisitionFilter[] _acquisitionFilters;
 
-    public ThreadPooledJobStore(ILogger<ThreadPooledJobStore> logger)
+    public ThreadPooledJobStore(ILogger<ThreadPooledJobStore> logger, IEnumerable<IAcquisitionFilter> acquisitionFilters)
     {
         _logger = logger;
+        _acquisitionFilters = acquisitionFilters.ToArray();
         InitConcurrencyCache();
     }
 
@@ -36,6 +39,7 @@ public override async Task Initialize(
         await base.Initialize(loadHelper, signaler, cancellationToken);
     }
 
+    // TODO We may need a way to notify quartz of a state change, or else it waits like 5 seconds to check again (or is notified by new jobs)
     protected override async Task<IReadOnlyCollection<IOperableTrigger>> AcquireNextTrigger(
             ConnectionAndTransactionHolder conn,
             DateTimeOffset noLaterThan,
@@ -163,8 +167,13 @@ protected override async Task<IReadOnlyCollection<IOperableTrigger>> AcquireNext
 
     private Type[] GetTypesToExclude()
     {
-        // TODO We can get the status of things and add things to exclude
-        return Array.Empty<Type>();
+        var result = new List<Type>();
+        foreach (var filter in _acquisitionFilters)
+        {
+            result.AddRange(filter.GetTypesToExclude());
+        }
+
+        return result.Distinct().ToArray();
     }
 
     private bool JobAllowed(TriggerAcquisitionContext context)
@@ -200,7 +209,7 @@ private void InitConcurrencyCache()
     {
         _typeConcurrencyCache = new Dictionary<Type, int>();
         var types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes())
-            .Where(a => typeof(IJob).IsAssignableFrom(a)).ToList();
+            .Where(a => typeof(IJob).IsAssignableFrom(a) && !a.IsAbstract).ToList();
 
         foreach (var type in types)
         {

Shoko.Server/Server/ShokoServer.cs

@@ -359,8 +359,6 @@ private void WorkerSetupDB_DoWork(object sender, DoWorkEventArgs e)
 
             StartWatchingFiles();
 
-            var folders = RepoFactory.ImportFolder.GetAll();
-
             if (settings.Import.ScanDropFoldersOnStart)
             {
                 ScanDropFolders();