diff --git a/Shoko.Server/Scheduling/Acquisition/AniDBHTTPRateLimitedAttribute.cs b/Shoko.Server/Scheduling/Acquisition/AniDBHTTPRateLimitedAttribute.cs deleted file mode 100644 index 627a5bd1e..000000000 --- a/Shoko.Server/Scheduling/Acquisition/AniDBHTTPRateLimitedAttribute.cs +++ /dev/null @@ -1,5 +0,0 @@ -using System; - -namespace Shoko.Server.Scheduling.Acquisition; - -public class AniDBHTTPRateLimitedAttribute : Attribute { } diff --git a/Shoko.Server/Scheduling/Acquisition/AniDBUDPRateLimitedAttribute.cs b/Shoko.Server/Scheduling/Acquisition/AniDBUDPRateLimitedAttribute.cs deleted file mode 100644 index 3f1ec8214..000000000 --- a/Shoko.Server/Scheduling/Acquisition/AniDBUDPRateLimitedAttribute.cs +++ /dev/null @@ -1,5 +0,0 @@ -using System; - -namespace Shoko.Server.Scheduling.Acquisition; - -public class AniDBUDPRateLimitedAttribute : Attribute { } diff --git a/Shoko.Server/Scheduling/Acquisition/Attributes/AcquisitionFilterAttribute.cs b/Shoko.Server/Scheduling/Acquisition/Attributes/AcquisitionFilterAttribute.cs new file mode 100644 index 000000000..e5e68934d --- /dev/null +++ b/Shoko.Server/Scheduling/Acquisition/Attributes/AcquisitionFilterAttribute.cs @@ -0,0 +1,5 @@ +using System; + +namespace Shoko.Server.Scheduling.Acquisition.Attributes; + +public abstract class AcquisitionFilterAttribute : Attribute { } diff --git a/Shoko.Server/Scheduling/Acquisition/Attributes/AniDBHttpRateLimitedAttribute.cs b/Shoko.Server/Scheduling/Acquisition/Attributes/AniDBHttpRateLimitedAttribute.cs new file mode 100644 index 000000000..dfd33c7ac --- /dev/null +++ b/Shoko.Server/Scheduling/Acquisition/Attributes/AniDBHttpRateLimitedAttribute.cs @@ -0,0 +1,6 @@ +using System; + +namespace Shoko.Server.Scheduling.Acquisition.Attributes; + +[AttributeUsage(AttributeTargets.Class)] +public class AniDBHttpRateLimitedAttribute : AcquisitionFilterAttribute { } diff --git a/Shoko.Server/Scheduling/Acquisition/Attributes/AniDBUdpRateLimitedAttribute.cs b/Shoko.Server/Scheduling/Acquisition/Attributes/AniDBUdpRateLimitedAttribute.cs new file mode 100644 index 000000000..03d5911fc --- /dev/null +++ b/Shoko.Server/Scheduling/Acquisition/Attributes/AniDBUdpRateLimitedAttribute.cs @@ -0,0 +1,6 @@ +using System; + +namespace Shoko.Server.Scheduling.Acquisition.Attributes; + +[AttributeUsage(AttributeTargets.Class)] +public class AniDBUdpRateLimitedAttribute : AcquisitionFilterAttribute { } diff --git a/Shoko.Server/Scheduling/Acquisition/Attributes/DatabaseRequiredAttribute.cs b/Shoko.Server/Scheduling/Acquisition/Attributes/DatabaseRequiredAttribute.cs new file mode 100644 index 000000000..9edefbe9b --- /dev/null +++ b/Shoko.Server/Scheduling/Acquisition/Attributes/DatabaseRequiredAttribute.cs @@ -0,0 +1,6 @@ +using System; + +namespace Shoko.Server.Scheduling.Acquisition.Attributes; + +[AttributeUsage(AttributeTargets.Class)] +public class DatabaseRequiredAttribute : AcquisitionFilterAttribute { } diff --git a/Shoko.Server/Scheduling/Acquisition/Attributes/NetworkRequiredAttribute.cs b/Shoko.Server/Scheduling/Acquisition/Attributes/NetworkRequiredAttribute.cs new file mode 100644 index 000000000..f1a1d3b57 --- /dev/null +++ b/Shoko.Server/Scheduling/Acquisition/Attributes/NetworkRequiredAttribute.cs @@ -0,0 +1,6 @@ +using System; + +namespace Shoko.Server.Scheduling.Acquisition.Attributes; + +[AttributeUsage(AttributeTargets.Class)] +public class NetworkRequiredAttribute : AcquisitionFilterAttribute { } diff --git a/Shoko.Server/Scheduling/Acquisition/DatabaseRequiredAttribute.cs b/Shoko.Server/Scheduling/Acquisition/DatabaseRequiredAttribute.cs deleted file mode 100644 index 1a91f5eea..000000000 --- a/Shoko.Server/Scheduling/Acquisition/DatabaseRequiredAttribute.cs +++ /dev/null @@ -1,6 +0,0 @@ -using System; - -namespace Shoko.Server.Scheduling.Acquisition; - -[AttributeUsage(AttributeTargets.Class)] -public class DatabaseRequiredAttribute : Attribute { } diff --git a/Shoko.Server/Scheduling/Acquisition/Filters/AniDBHttpRateLimitedAcquisitionFilter.cs b/Shoko.Server/Scheduling/Acquisition/Filters/AniDBHttpRateLimitedAcquisitionFilter.cs new file mode 100644 index 000000000..369bec98d --- /dev/null +++ b/Shoko.Server/Scheduling/Acquisition/Filters/AniDBHttpRateLimitedAcquisitionFilter.cs @@ -0,0 +1,23 @@ +using System; +using System.Linq; +using Quartz; +using Quartz.Util; +using Shoko.Server.Providers.AniDB.Interfaces; +using Shoko.Server.Scheduling.Acquisition.Attributes; + +namespace Shoko.Server.Scheduling.Acquisition.Filters; + +public class AniDBHttpRateLimitedAcquisitionFilter : IAcquisitionFilter +{ + private readonly Type[] _types; + private readonly IHttpConnectionHandler _connectionHandler; + + public AniDBHttpRateLimitedAcquisitionFilter(IHttpConnectionHandler connectionHandler) + { + _connectionHandler = connectionHandler; + _types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).Where(a => + typeof(IJob).IsAssignableFrom(a) && !a.IsAbstract && ObjectUtils.IsAttributePresent(a, typeof(AniDBHttpRateLimitedAttribute))).ToArray(); + } + + public Type[] GetTypesToExclude() => _connectionHandler.IsBanned ? _types : Array.Empty(); +} diff --git a/Shoko.Server/Scheduling/Acquisition/Filters/AniDBUdpRateLimitedAcquisitionFilter.cs b/Shoko.Server/Scheduling/Acquisition/Filters/AniDBUdpRateLimitedAcquisitionFilter.cs new file mode 100644 index 000000000..38b82ed70 --- /dev/null +++ b/Shoko.Server/Scheduling/Acquisition/Filters/AniDBUdpRateLimitedAcquisitionFilter.cs @@ -0,0 +1,23 @@ +using System; +using System.Linq; +using Quartz; +using Quartz.Util; +using Shoko.Server.Providers.AniDB.Interfaces; +using Shoko.Server.Scheduling.Acquisition.Attributes; + +namespace Shoko.Server.Scheduling.Acquisition.Filters; + +public class AniDBUdpRateLimitedAcquisitionFilter : IAcquisitionFilter +{ + private readonly Type[] _types; + private readonly IUDPConnectionHandler _connectionHandler; + + public AniDBUdpRateLimitedAcquisitionFilter(IUDPConnectionHandler connectionHandler) + { + _connectionHandler = connectionHandler; + _types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).Where(a => + typeof(IJob).IsAssignableFrom(a) && !a.IsAbstract && ObjectUtils.IsAttributePresent(a, typeof(AniDBUdpRateLimitedAttribute))).ToArray(); + } + + public Type[] GetTypesToExclude() => _connectionHandler.IsBanned || _connectionHandler.IsInvalidSession ? _types : Array.Empty(); +} diff --git a/Shoko.Server/Scheduling/Acquisition/Filters/DatabaseRequiredAcquisitionFilter.cs b/Shoko.Server/Scheduling/Acquisition/Filters/DatabaseRequiredAcquisitionFilter.cs new file mode 100644 index 000000000..8d4744811 --- /dev/null +++ b/Shoko.Server/Scheduling/Acquisition/Filters/DatabaseRequiredAcquisitionFilter.cs @@ -0,0 +1,21 @@ +using System; +using System.Linq; +using Quartz; +using Quartz.Util; +using Shoko.Server.Scheduling.Acquisition.Attributes; +using Shoko.Server.Server; + +namespace Shoko.Server.Scheduling.Acquisition.Filters; + +public class DatabaseRequiredAcquisitionFilter : IAcquisitionFilter +{ + private readonly Type[] _types; + + public DatabaseRequiredAcquisitionFilter() + { + _types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).Where(a => + typeof(IJob).IsAssignableFrom(a) && !a.IsAbstract && ObjectUtils.IsAttributePresent(a, typeof(DatabaseRequiredAttribute))).ToArray(); + } + + public Type[] GetTypesToExclude() => ServerState.Instance.ServerOnline && !ServerState.Instance.DatabaseBlocked.Blocked ? Array.Empty() : _types; +} diff --git a/Shoko.Server/Scheduling/Acquisition/Filters/IAcquisitionFilter.cs b/Shoko.Server/Scheduling/Acquisition/Filters/IAcquisitionFilter.cs new file mode 100644 index 000000000..e938d1dbd --- /dev/null +++ b/Shoko.Server/Scheduling/Acquisition/Filters/IAcquisitionFilter.cs @@ -0,0 +1,8 @@ +using System; + +namespace Shoko.Server.Scheduling.Acquisition.Filters; + +public interface IAcquisitionFilter +{ + Type[] GetTypesToExclude(); +} diff --git a/Shoko.Server/Scheduling/Acquisition/Filters/NetworkRequiredAcquisitionFilter.cs b/Shoko.Server/Scheduling/Acquisition/Filters/NetworkRequiredAcquisitionFilter.cs new file mode 100644 index 000000000..df2cf91db --- /dev/null +++ b/Shoko.Server/Scheduling/Acquisition/Filters/NetworkRequiredAcquisitionFilter.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Quartz; +using Quartz.Util; +using Shoko.Server.Scheduling.Acquisition.Attributes; +using Shoko.Server.Services.Connectivity; + +namespace Shoko.Server.Scheduling.Acquisition.Filters; + +public class NetworkRequiredAcquisitionFilter : IAcquisitionFilter +{ + private readonly Type[] _types; + private readonly IConnectivityMonitor[] _connectivityMonitors; + + public NetworkRequiredAcquisitionFilter(IEnumerable connectivityMonitors) + { + _connectivityMonitors = connectivityMonitors.ToArray(); + _types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).Where(a => + typeof(IJob).IsAssignableFrom(a) && !a.IsAbstract && ObjectUtils.IsAttributePresent(a, typeof(NetworkRequiredAttribute))).ToArray(); + } + + public Type[] GetTypesToExclude() => _connectivityMonitors.Any(a => a.HasConnected) ? Array.Empty() : _types; +} diff --git a/Shoko.Server/Scheduling/Acquisition/NetworkRequiredAttribute.cs b/Shoko.Server/Scheduling/Acquisition/NetworkRequiredAttribute.cs deleted file mode 100644 index e1c7c1f18..000000000 --- a/Shoko.Server/Scheduling/Acquisition/NetworkRequiredAttribute.cs +++ /dev/null @@ -1,6 +0,0 @@ -using System; - -namespace Shoko.Server.Scheduling.Acquisition; - -[AttributeUsage(AttributeTargets.Class)] -public class NetworkRequiredAttribute : Attribute { } diff --git a/Shoko.Server/Scheduling/Jobs/Actions/ImportJob.cs b/Shoko.Server/Scheduling/Jobs/Actions/ImportJob.cs index 3ad6975ee..5b6d57988 100644 --- a/Shoko.Server/Scheduling/Jobs/Actions/ImportJob.cs +++ b/Shoko.Server/Scheduling/Jobs/Actions/ImportJob.cs @@ -7,6 +7,7 @@ using Quartz; using QuartzJobFactory.Attributes; using Shoko.Server.Scheduling.Acquisition; +using Shoko.Server.Scheduling.Acquisition.Attributes; namespace Shoko.Server.Scheduling.Jobs.Actions; diff --git a/Shoko.Server/Scheduling/Jobs/Actions/MediaInfoJob.cs b/Shoko.Server/Scheduling/Jobs/Actions/MediaInfoJob.cs index 43b142617..8e089eb31 100644 --- a/Shoko.Server/Scheduling/Jobs/Actions/MediaInfoJob.cs +++ b/Shoko.Server/Scheduling/Jobs/Actions/MediaInfoJob.cs @@ -9,6 +9,7 @@ using Shoko.Server.Commands; using Shoko.Server.Repositories; using Shoko.Server.Scheduling.Acquisition; +using Shoko.Server.Scheduling.Acquisition.Attributes; namespace Shoko.Server.Scheduling.Jobs.Actions; diff --git a/Shoko.Server/Scheduling/Jobs/Actions/RemoveMissingFilesJob.cs b/Shoko.Server/Scheduling/Jobs/Actions/RemoveMissingFilesJob.cs index 17489dd9c..7ef3b76c5 100644 --- a/Shoko.Server/Scheduling/Jobs/Actions/RemoveMissingFilesJob.cs +++ b/Shoko.Server/Scheduling/Jobs/Actions/RemoveMissingFilesJob.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Quartz; using QuartzJobFactory.Attributes; -using Shoko.Server.Scheduling.Acquisition; +using Shoko.Server.Scheduling.Acquisition.Attributes; namespace Shoko.Server.Scheduling.Jobs.Actions; diff --git a/Shoko.Server/Scheduling/Jobs/Actions/ScanDropFoldersJob.cs b/Shoko.Server/Scheduling/Jobs/Actions/ScanDropFoldersJob.cs index 7d15c20a1..7d8fe8d52 100644 --- a/Shoko.Server/Scheduling/Jobs/Actions/ScanDropFoldersJob.cs +++ b/Shoko.Server/Scheduling/Jobs/Actions/ScanDropFoldersJob.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Quartz; using QuartzJobFactory.Attributes; -using Shoko.Server.Scheduling.Acquisition; +using Shoko.Server.Scheduling.Acquisition.Attributes; namespace Shoko.Server.Scheduling.Jobs.Actions; diff --git a/Shoko.Server/Scheduling/Jobs/AniDB/AniDBGetFileJob.cs b/Shoko.Server/Scheduling/Jobs/AniDB/AniDBGetFileJob.cs index 1d6ef180d..2b3fff7cb 100644 --- a/Shoko.Server/Scheduling/Jobs/AniDB/AniDBGetFileJob.cs +++ b/Shoko.Server/Scheduling/Jobs/AniDB/AniDBGetFileJob.cs @@ -17,12 +17,13 @@ using Shoko.Server.Providers.AniDB.UDP.Info; using Shoko.Server.Repositories; using Shoko.Server.Scheduling.Acquisition; +using Shoko.Server.Scheduling.Acquisition.Attributes; using Shoko.Server.Scheduling.Concurrency; using Shoko.Server.Server; namespace Shoko.Server.Scheduling.Jobs.AniDB; -[DatabaseRequired, NetworkRequired, AniDBUDPRateLimited] +[DatabaseRequired, NetworkRequired, AniDBUdpRateLimited] [DisallowConcurrencyGroup(ConcurrencyGroups.AniDB_UDP)] [JobKeyGroup(JobKeyGroup.AniDB)] [Command(CommandRequestType.AniDB_GetFileUDP)] diff --git a/Shoko.Server/Scheduling/Jobs/Shoko/DeleteImportFolderJob.cs b/Shoko.Server/Scheduling/Jobs/Shoko/DeleteImportFolderJob.cs index 48ecdda6a..7e41959fd 100644 --- a/Shoko.Server/Scheduling/Jobs/Shoko/DeleteImportFolderJob.cs +++ b/Shoko.Server/Scheduling/Jobs/Shoko/DeleteImportFolderJob.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Quartz; using QuartzJobFactory.Attributes; -using Shoko.Server.Scheduling.Acquisition; +using Shoko.Server.Scheduling.Acquisition.Attributes; namespace Shoko.Server.Scheduling.Jobs.Shoko; diff --git a/Shoko.Server/Scheduling/Jobs/Shoko/DiscoverFileJob.cs b/Shoko.Server/Scheduling/Jobs/Shoko/DiscoverFileJob.cs index 0bb1255d0..62f1b8b82 100644 --- a/Shoko.Server/Scheduling/Jobs/Shoko/DiscoverFileJob.cs +++ b/Shoko.Server/Scheduling/Jobs/Shoko/DiscoverFileJob.cs @@ -12,7 +12,7 @@ using Shoko.Server.Models; using Shoko.Server.Repositories; using Shoko.Server.Repositories.Cached; -using Shoko.Server.Scheduling.Acquisition; +using Shoko.Server.Scheduling.Acquisition.Attributes; using Shoko.Server.Server; using Shoko.Server.Settings; diff --git a/Shoko.Server/Scheduling/Jobs/Shoko/HashFileJob.cs b/Shoko.Server/Scheduling/Jobs/Shoko/HashFileJob.cs index d05da4902..a1e73dbcb 100644 --- a/Shoko.Server/Scheduling/Jobs/Shoko/HashFileJob.cs +++ b/Shoko.Server/Scheduling/Jobs/Shoko/HashFileJob.cs @@ -16,7 +16,7 @@ using Shoko.Server.Models; using Shoko.Server.Repositories; using Shoko.Server.Repositories.Cached; -using Shoko.Server.Scheduling.Acquisition; +using Shoko.Server.Scheduling.Acquisition.Attributes; using Shoko.Server.Scheduling.Concurrency; using Shoko.Server.Server; using Shoko.Server.Settings; diff --git a/Shoko.Server/Scheduling/Jobs/Shoko/ScanFolderJob.cs b/Shoko.Server/Scheduling/Jobs/Shoko/ScanFolderJob.cs index 12f691801..57b6a7386 100644 --- a/Shoko.Server/Scheduling/Jobs/Shoko/ScanFolderJob.cs +++ b/Shoko.Server/Scheduling/Jobs/Shoko/ScanFolderJob.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Quartz; using QuartzJobFactory.Attributes; -using Shoko.Server.Scheduling.Acquisition; +using Shoko.Server.Scheduling.Acquisition.Attributes; namespace Shoko.Server.Scheduling.Jobs.Shoko; diff --git a/Shoko.Server/Scheduling/QuartzStartup.cs b/Shoko.Server/Scheduling/QuartzStartup.cs index c209e2507..d25413d47 100644 --- a/Shoko.Server/Scheduling/QuartzStartup.cs +++ b/Shoko.Server/Scheduling/QuartzStartup.cs @@ -7,6 +7,7 @@ using Quartz; using Quartz.AspNetCore; using QuartzJobFactory; +using Shoko.Server.Scheduling.Acquisition.Filters; using Shoko.Server.Scheduling.Delegates; using Shoko.Server.Scheduling.Jobs.Actions; using Shoko.Server.Scheduling.Jobs.Shoko; @@ -20,6 +21,10 @@ public static class QuartzStartup internal static void AddQuartz(this IServiceCollection services) { services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddJobs(); services.AddQuartz(q => { diff --git a/Shoko.Server/Scheduling/ThreadPooledJobStore.cs b/Shoko.Server/Scheduling/ThreadPooledJobStore.cs index efa0a789f..d1f775598 100644 --- a/Shoko.Server/Scheduling/ThreadPooledJobStore.cs +++ b/Shoko.Server/Scheduling/ThreadPooledJobStore.cs @@ -9,6 +9,7 @@ using Quartz.Impl.AdoJobStore; using Quartz.Spi; using Quartz.Util; +using Shoko.Server.Scheduling.Acquisition.Filters; using Shoko.Server.Scheduling.Concurrency; using Shoko.Server.Scheduling.Delegates; using Shoko.Server.Utilities; @@ -20,10 +21,12 @@ public class ThreadPooledJobStore : JobStoreTX private readonly ILogger _logger; private ITypeLoadHelper _typeLoadHelper = null!; private Dictionary _typeConcurrencyCache; + private readonly IAcquisitionFilter[] _acquisitionFilters; - public ThreadPooledJobStore(ILogger logger) + public ThreadPooledJobStore(ILogger logger, IEnumerable acquisitionFilters) { _logger = logger; + _acquisitionFilters = acquisitionFilters.ToArray(); InitConcurrencyCache(); } @@ -36,6 +39,7 @@ public override async Task Initialize( await base.Initialize(loadHelper, signaler, cancellationToken); } + // TODO We may need a way to notify quartz of a state change, or else it waits like 5 seconds to check again (or is notified by new jobs) protected override async Task> AcquireNextTrigger( ConnectionAndTransactionHolder conn, DateTimeOffset noLaterThan, @@ -163,8 +167,13 @@ protected override async Task> AcquireNext private Type[] GetTypesToExclude() { - // TODO We can get the status of things and add things to exclude - return Array.Empty(); + var result = new List(); + foreach (var filter in _acquisitionFilters) + { + result.AddRange(filter.GetTypesToExclude()); + } + + return result.Distinct().ToArray(); } private bool JobAllowed(TriggerAcquisitionContext context) @@ -200,7 +209,7 @@ private void InitConcurrencyCache() { _typeConcurrencyCache = new Dictionary(); var types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()) - .Where(a => typeof(IJob).IsAssignableFrom(a)).ToList(); + .Where(a => typeof(IJob).IsAssignableFrom(a) && !a.IsAbstract).ToList(); foreach (var type in types) { diff --git a/Shoko.Server/Server/ShokoServer.cs b/Shoko.Server/Server/ShokoServer.cs index 85e2c8996..c0344d9a0 100644 --- a/Shoko.Server/Server/ShokoServer.cs +++ b/Shoko.Server/Server/ShokoServer.cs @@ -359,8 +359,6 @@ private void WorkerSetupDB_DoWork(object sender, DoWorkEventArgs e) StartWatchingFiles(); - var folders = RepoFactory.ImportFolder.GetAll(); - if (settings.Import.ScanDropFoldersOnStart) { ScanDropFolders();