diff --git a/Shoko.Server/API/v3/Controllers/DuplicateManagementController.cs b/Shoko.Server/API/v3/Controllers/DuplicateManagementController.cs new file mode 100644 index 000000000..64f0b5bdc --- /dev/null +++ b/Shoko.Server/API/v3/Controllers/DuplicateManagementController.cs @@ -0,0 +1,207 @@ +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using Microsoft.AspNetCore.Mvc; +using Shoko.Commons.Extensions; +using Shoko.Server.API.Annotations; +using Shoko.Server.API.ModelBinders; +using Shoko.Server.API.v3.Helpers; +using Shoko.Server.API.v3.Models.Common; +using Shoko.Server.API.v3.Models.Shoko; +using Shoko.Server.Repositories; +using Shoko.Server.Settings; + +#pragma warning disable CA1822 +namespace Shoko.Server.API.v3.Controllers; + +[ApiController] +[Route("/api/v{version:apiVersion}/[controller]")] +[ApiV3] +public class DuplicateManagementController : BaseController +{ + /// + /// Get episodes with duplicate files, with only the files with duplicates for each episode. + /// + /// Include data from selected s. + /// Include media info data. + /// Include file/episode cross-references with the episodes. + /// Limits the number of results per page. Set to 0 to disable the limit. + /// Page number. + /// + [HttpGet("Episodes")] + public ActionResult> GetEpisodes( + [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet includeDataFrom = null, + [FromQuery] bool includeMediaInfo = true, + [FromQuery] bool includeXRefs = false, + [FromQuery, Range(0, 1000)] int pageSize = 100, + [FromQuery, Range(1, int.MaxValue)] int page = 1) + { + var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(); + return enumerable + .ToListResult(episode => + { + var duplicateFiles = episode.VideoLocals + .Select(file => (file, locations: file.Places.ExceptBy((file.FirstValidPlace ?? file.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID).ToList())) + .Where(tuple => tuple.locations.Count > 0) + .ToList(); + var dto = new Episode(HttpContext, episode, includeDataFrom); + dto.Size = duplicateFiles.Count; + dto.Files = duplicateFiles + .Select(tuple => new Models.Shoko.File(HttpContext, tuple.file, includeXRefs, includeDataFrom, includeMediaInfo, true)) + .ToList(); + return dto; + }, page, pageSize); + } + + /// + /// Get the list of file location ids to auto remove across all series. + /// + /// + [HttpGet("FileLocationsToAutoRemove")] + public ActionResult> GetFileIdsWithPreference() + { + var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(); + return enumerable + .SelectMany(episode => + episode.VideoLocals + .SelectMany(a => a.Places.ExceptBy((a.FirstValidPlace ?? a.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID)) + .Select(file => (episode.AnimeSeriesID, episode.AnimeEpisodeID, file.VideoLocalID, file.VideoLocal_Place_ID)) + ) + .GroupBy(tuple => tuple.VideoLocalID, tuple => (tuple.VideoLocal_Place_ID, tuple.AnimeEpisodeID, tuple.AnimeSeriesID)) + .Select(groupBy => new FileIdSet(groupBy)) + .ToList(); + } + + /// + /// Get series with duplicate files. + /// + /// Include data from selected s. + /// Only show finished series. + /// Limits the number of results per page. Set to 0 to disable the limit. + /// Page number. + /// + [HttpGet("Series")] + public ActionResult> GetSeriesWithDuplicateFiles( + [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet includeDataFrom = null, + [FromQuery] bool onlyFinishedSeries = false, + [FromQuery, Range(0, 1000)] int pageSize = 100, + [FromQuery, Range(1, int.MaxValue)] int page = 1) + { + var enumerable = RepoFactory.AnimeSeries.GetWithDuplicateFiles(); + if (onlyFinishedSeries) + enumerable = enumerable.Where(a => a.AniDB_Anime.GetFinishedAiring()); + + return enumerable + .OrderBy(series => series.PreferredTitle) + .ThenBy(series => series.AniDB_ID) + .ToListResult(series => new Series.WithDuplicateFilesResult(series, User.JMMUserID, includeDataFrom), page, pageSize); + } + + /// + /// Get episodes with duplicate files for a series, with only the files with duplicates for each episode. + /// + /// Shoko Series ID + /// Include data from selected s. + /// Include media info data. + /// Include file/episode cross-references with the episodes. + /// Limits the number of results per page. Set to 0 to disable the limit. + /// Page number. + /// + [HttpGet("Series/{seriesID}/Episodes")] + public ActionResult> GetEpisodesForSeries( + [FromRoute, Range(1, int.MaxValue)] int seriesID, + [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet includeDataFrom = null, + [FromQuery] bool includeMediaInfo = true, + [FromQuery] bool includeXRefs = false, + [FromQuery, Range(0, 1000)] int pageSize = 100, + [FromQuery, Range(1, int.MaxValue)] int page = 1) + { + var series = RepoFactory.AnimeSeries.GetByID(seriesID); + if (series == null) + return new ListResult(); + + if (!User.AllowedSeries(series)) + return new ListResult(); + + var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(series.AniDB_ID); + return enumerable + .ToListResult(episode => + { + var duplicateFiles = episode.VideoLocals + .Select(file => (file, locations: file.Places.ExceptBy((file.FirstValidPlace ?? file.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID).ToList())) + .Where(tuple => tuple.locations.Count > 0) + .ToList(); + var dto = new Episode(HttpContext, episode, includeDataFrom); + dto.Size = duplicateFiles.Count; + dto.Files = duplicateFiles + .Select(tuple => new Models.Shoko.File(HttpContext, tuple.file, includeXRefs, includeDataFrom, includeMediaInfo, true)) + .ToList(); + return dto; + }, page, pageSize); + } + + /// + /// Get the list of file location ids to auto remove for the series. + /// + /// Shoko Series ID + /// + [HttpGet("Series/{seriesID}/FileLocationsToAutoRemove")] + public ActionResult> GetFileLocationsIdsAcrossAllEpisodes( + [FromRoute, Range(1, int.MaxValue)] int seriesID + ) + { + var series = RepoFactory.AnimeSeries.GetByID(seriesID); + if (series == null) + return new List(); + + if (!User.AllowedSeries(series)) + return new List(); + + var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(series.AniDB_ID); + return enumerable + .SelectMany(episode => + episode.VideoLocals + .SelectMany(a => a.Places.ExceptBy((a.FirstValidPlace ?? a.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID)) + .Select(file => (episode.AnimeSeriesID, episode.AnimeEpisodeID, file.VideoLocalID, file.VideoLocal_Place_ID)) + ) + .GroupBy(tuple => tuple.VideoLocalID, tuple => (tuple.VideoLocal_Place_ID, tuple.AnimeEpisodeID, tuple.AnimeSeriesID)) + .Select(groupBy => new FileIdSet(groupBy)) + .ToList(); + } + + public DuplicateManagementController(ISettingsProvider settingsProvider) : base(settingsProvider) + { + } + + public class FileIdSet(IGrouping grouping) + { + /// + /// The file ID with duplicates to remove. + /// + public int FileID { get; set; } = grouping.Key; + + /// + /// The series IDs with duplicates to remove. + /// + public List AnimeSeriesIDs { get; set; } = grouping + .Select(tuple => tuple.AnimeSeriesID) + .Distinct() + .ToList(); + + /// + /// The episode IDs with duplicates to remove. + /// + public List AnimeEpisodeIDs { get; set; } = grouping + .Select(tuple => tuple.AnimeEpisodeID) + .Distinct() + .ToList(); + + /// + /// The duplicate locations to remove from the files/episodes. + /// + public List FileLocationIDs { get; set; } = grouping + .Select(tuple => tuple.VideoLocal_Place_ID) + .Distinct() + .ToList(); + } +} diff --git a/Shoko.Server/API/v3/Models/Shoko/Series.cs b/Shoko.Server/API/v3/Models/Shoko/Series.cs index dc69b3297..4d2097a4c 100644 --- a/Shoko.Server/API/v3/Models/Shoko/Series.cs +++ b/Shoko.Server/API/v3/Models/Shoko/Series.cs @@ -851,6 +851,23 @@ public WithMultipleReleasesResult(SVR_AnimeSeries ser, int userId = 0, HashSet + /// An extended model for use with the hard duplicate endpoint. + /// + public class WithDuplicateFilesResult : Series + { + /// + /// Number of episodes in the series which have duplicate files. + /// + public int EpisodeCount { get; set; } + + public WithDuplicateFilesResult(SVR_AnimeSeries ser, int userId = 0, HashSet? includeDataFrom = null) + : base(ser, userId, false, includeDataFrom) + { + EpisodeCount = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(ser.AniDB_ID).Count(); + } + } } public enum SeriesType diff --git a/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs b/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs index dee692e3c..1c10a29db 100644 --- a/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs +++ b/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs @@ -91,13 +91,13 @@ public List GetByHash(string hash) .ToList(); } - private const string IgnoreVariationsWithAnimeQuery = + private const string MultipleReleasesIgnoreVariationsWithAnimeQuery = @"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE ani.AnimeID = :animeID AND vl.IsVariation = 0 AND vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1"; - private const string CountVariationsWithAnimeQuery = + private const string MultipleReleasesCountVariationsWithAnimeQuery = @"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE ani.AnimeID = :animeID AND vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1"; - private const string IgnoreVariationsQuery = + private const string MultipleReleasesIgnoreVariationsQuery = @"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE vl.IsVariation = 0 AND vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1"; - private const string CountVariationsQuery = + private const string MultipleReleasesCountVariationsQuery = @"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1"; public List GetWithMultipleReleases(bool ignoreVariations, int? animeID = null) @@ -107,14 +107,14 @@ public List GetWithMultipleReleases(bool ignoreVariations, int using var session = _databaseFactory.SessionFactory.OpenSession(); if (animeID.HasValue && animeID.Value > 0) { - var animeQuery = ignoreVariations ? IgnoreVariationsWithAnimeQuery : CountVariationsWithAnimeQuery; + var animeQuery = ignoreVariations ? MultipleReleasesIgnoreVariationsWithAnimeQuery : MultipleReleasesCountVariationsWithAnimeQuery; return session.CreateSQLQuery(animeQuery) .AddScalar("EpisodeID", NHibernateUtil.Int32) .SetParameter("animeID", animeID.Value) .List(); } - var query = ignoreVariations ? IgnoreVariationsQuery : CountVariationsQuery; + var query = ignoreVariations ? MultipleReleasesIgnoreVariationsQuery : MultipleReleasesCountVariationsQuery; return session.CreateSQLQuery(query) .AddScalar("EpisodeID", NHibernateUtil.Int32) .List(); @@ -131,6 +131,93 @@ public List GetWithMultipleReleases(bool ignoreVariations, int .ToList(); } + private const string DuplicateFilesWithAnimeQuery = @" +SELECT + ani.EpisodeID +FROM + ( + SELECT + vlp.VideoLocal_Place_ID, + vl.FileSize, + vl.Hash + FROM + VideoLocal AS vl + INNER JOIN + VideoLocal_Place AS vlp + ON vlp.VideoLocalID = vl.VideoLocalID + WHERE + vl.Hash != '' + GROUP BY + vl.VideoLocalID + HAVING + COUNT(vl.VideoLocalID) > 1 + ) AS filtered_vlp +INNER JOIN + CrossRef_File_Episode ani + ON filtered_vlp.Hash = ani.Hash + AND filtered_vlp.FileSize = ani.FileSize +WHERE ani.AnimeID = :animeID +GROUP BY + ani.EpisodeID +"; + + private const string DuplicateFilesQuery = @" +SELECT + ani.EpisodeID +FROM + ( + SELECT + vlp.VideoLocal_Place_ID, + vl.FileSize, + vl.Hash + FROM + VideoLocal AS vl + INNER JOIN + VideoLocal_Place AS vlp + ON vlp.VideoLocalID = vl.VideoLocalID + WHERE + vl.Hash != '' + GROUP BY + vl.VideoLocalID + HAVING + COUNT(vl.VideoLocalID) > 1 + ) AS filtered_vlp +INNER JOIN + CrossRef_File_Episode ani + ON filtered_vlp.Hash = ani.Hash + AND filtered_vlp.FileSize = ani.FileSize +GROUP BY + ani.EpisodeID +"; + + public IEnumerable GetWithDuplicateFiles(int? animeID = null) + { + var ids = Lock(() => + { + using var session = _databaseFactory.SessionFactory.OpenSession(); + if (animeID.HasValue && animeID.Value > 0) + { + return session.CreateSQLQuery(DuplicateFilesWithAnimeQuery) + .AddScalar("EpisodeID", NHibernateUtil.Int32) + .SetParameter("animeID", animeID.Value) + .List(); + } + + return session.CreateSQLQuery(DuplicateFilesQuery) + .AddScalar("EpisodeID", NHibernateUtil.Int32) + .List(); + }); + + return ids + .Select(GetByAniDBEpisodeID) + .Select(episode => (episode, anidbEpisode: episode?.AniDB_Episode)) + .Where(tuple => tuple.anidbEpisode is not null) + .OrderBy(tuple => tuple.anidbEpisode!.AnimeID) + .ThenBy(tuple => tuple.anidbEpisode!.EpisodeTypeEnum) + .ThenBy(tuple => tuple.anidbEpisode!.EpisodeNumber) + .Select(tuple => tuple.episode!); + } + public List GetUnwatchedEpisodes(int seriesid, int userid) { var eps = diff --git a/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs b/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs index cebc9ed50..d4f31c204 100644 --- a/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs +++ b/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs @@ -319,9 +319,9 @@ public List GetMostRecentlyAdded(int maxResults, int userID) .ToList()); } - private const string IgnoreVariationsQuery = + private const string MultipleReleasesIgnoreVariationsQuery = @"SELECT DISTINCT ani.AnimeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE vl.IsVariation = 0 AND vl.Hash != '' GROUP BY ani.AnimeID, ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1"; - private const string CountVariationsQuery = + private const string MultipleReleasesCountVariationsQuery = @"SELECT DISTINCT ani.AnimeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE vl.Hash != '' GROUP BY ani.AnimeID, ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1"; public List GetWithMultipleReleases(bool ignoreVariations) @@ -330,7 +330,7 @@ public List GetWithMultipleReleases(bool ignoreVariations) { using var session = _databaseFactory.SessionFactory.OpenSession(); - var query = ignoreVariations ? IgnoreVariationsQuery : CountVariationsQuery; + var query = ignoreVariations ? MultipleReleasesIgnoreVariationsQuery : MultipleReleasesCountVariationsQuery; return session.CreateSQLQuery(query) .AddScalar("AnimeID", NHibernateUtil.Int32) .List(); @@ -343,6 +343,52 @@ public List GetWithMultipleReleases(bool ignoreVariations) .ToList(); } + private const string DuplicateFilesQuery = @" +SELECT DISTINCT + ani.AnimeID +FROM + ( + SELECT + vlp.VideoLocal_Place_ID, + vl.FileSize, + vl.Hash + FROM + VideoLocal AS vl + INNER JOIN + VideoLocal_Place AS vlp + ON vlp.VideoLocalID = vl.VideoLocalID + WHERE + vl.Hash != '' + GROUP BY + vl.VideoLocalID + HAVING + COUNT(vl.VideoLocalID) > 1 + ) AS vlp_selected +INNER JOIN + CrossRef_File_Episode ani + ON vlp_selected.Hash = ani.Hash + AND vlp_selected.FileSize = ani.FileSize +GROUP BY + ani.AnimeID +"; + + public IEnumerable GetWithDuplicateFiles() + { + var ids = Lock(() => + { + using var session = _databaseFactory.SessionFactory.OpenSession(); + + return session.CreateSQLQuery(DuplicateFilesQuery) + .AddScalar("AnimeID", NHibernateUtil.Int32) + .List(); + }); + + return ids + .Distinct() + .Select(GetByAnimeID) + .WhereNotNull(); + } + public ImageEntityType[] GetAllImageTypes() => [ImageEntityType.Backdrop, ImageEntityType.Banner, ImageEntityType.Logo, ImageEntityType.Poster];