feat: add APIv3 duplicates management controller

Shoko.Server/API/v3/Controllers/DuplicateManagementController.cs

@@ -0,0 +1,207 @@
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.Linq;
+using Microsoft.AspNetCore.Mvc;
+using Shoko.Commons.Extensions;
+using Shoko.Server.API.Annotations;
+using Shoko.Server.API.ModelBinders;
+using Shoko.Server.API.v3.Helpers;
+using Shoko.Server.API.v3.Models.Common;
+using Shoko.Server.API.v3.Models.Shoko;
+using Shoko.Server.Repositories;
+using Shoko.Server.Settings;
+
+#pragma warning disable CA1822
+namespace Shoko.Server.API.v3.Controllers;
+
+[ApiController]
+[Route("/api/v{version:apiVersion}/[controller]")]
+[ApiV3]
+public class DuplicateManagementController : BaseController
+{
+    /// <summary>
+    /// Get episodes with duplicate files, with only the files with duplicates for each episode.
+    /// </summary>
+    /// <param name="includeDataFrom">Include data from selected <see cref="DataSource"/>s.</param>
+    /// <param name="includeMediaInfo">Include media info data.</param>
+    /// <param name="includeXRefs">Include file/episode cross-references with the episodes.</param>
+    /// <param name="pageSize">Limits the number of results per page. Set to 0 to disable the limit.</param>
+    /// <param name="page">Page number.</param>
+    /// <returns></returns>
+    [HttpGet("Episodes")]
+    public ActionResult<ListResult<Episode>> GetEpisodes(
+        [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet<DataSource> includeDataFrom = null,
+        [FromQuery] bool includeMediaInfo = true,
+        [FromQuery] bool includeXRefs = false,
+        [FromQuery, Range(0, 1000)] int pageSize = 100,
+        [FromQuery, Range(1, int.MaxValue)] int page = 1)
+    {
+        var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles();
+        return enumerable
+            .ToListResult(episode =>
+            {
+                var duplicateFiles = episode.VideoLocals
+                    .Select(file => (file, locations: file.Places.ExceptBy((file.FirstValidPlace ?? file.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID).ToList()))
+                    .Where(tuple => tuple.locations.Count > 0)
+                    .ToList();
+                var dto = new Episode(HttpContext, episode, includeDataFrom);
+                dto.Size = duplicateFiles.Count;
+                dto.Files = duplicateFiles
+                    .Select(tuple => new Models.Shoko.File(HttpContext, tuple.file, includeXRefs, includeDataFrom, includeMediaInfo, true))
+                    .ToList();
+                return dto;
+            }, page, pageSize);
+    }
+
+    /// <summary>
+    /// Get the list of file location ids to auto remove across all series.
+    /// </summary>
+    /// <returns></returns>
+    [HttpGet("FileLocationsToAutoRemove")]
+    public ActionResult<List<FileIdSet>> GetFileIdsWithPreference()
+    {
+        var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles();
+        return enumerable
+            .SelectMany(episode =>
+                episode.VideoLocals
+                    .SelectMany(a => a.Places.ExceptBy((a.FirstValidPlace ?? a.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID))
+                    .Select(file => (episode.AnimeSeriesID, episode.AnimeEpisodeID, file.VideoLocalID, file.VideoLocal_Place_ID))
+            )
+            .GroupBy(tuple => tuple.VideoLocalID, tuple => (tuple.VideoLocal_Place_ID, tuple.AnimeEpisodeID, tuple.AnimeSeriesID))
+            .Select(groupBy => new FileIdSet(groupBy))
+            .ToList();
+    }
+
+    /// <summary>
+    /// Get series with duplicate files.
+    /// </summary>
+    /// <param name="includeDataFrom">Include data from selected <see cref="DataSource"/>s.</param>
+    /// <param name="onlyFinishedSeries">Only show finished series.</param>
+    /// <param name="pageSize">Limits the number of results per page. Set to 0 to disable the limit.</param>
+    /// <param name="page">Page number.</param>
+    /// <returns></returns>
+    [HttpGet("Series")]
+    public ActionResult<ListResult<Series.WithDuplicateFilesResult>> GetSeriesWithDuplicateFiles(
+        [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet<DataSource> includeDataFrom = null,
+        [FromQuery] bool onlyFinishedSeries = false,
+        [FromQuery, Range(0, 1000)] int pageSize = 100,
+        [FromQuery, Range(1, int.MaxValue)] int page = 1)
+    {
+        var enumerable = RepoFactory.AnimeSeries.GetWithDuplicateFiles();
+        if (onlyFinishedSeries)
+            enumerable = enumerable.Where(a => a.AniDB_Anime.GetFinishedAiring());
+
+        return enumerable
+            .OrderBy(series => series.PreferredTitle)
+            .ThenBy(series => series.AniDB_ID)
+            .ToListResult(series => new Series.WithDuplicateFilesResult(series, User.JMMUserID, includeDataFrom), page, pageSize);
+    }
+
+    /// <summary>
+    /// Get episodes with duplicate files for a series, with only the files with duplicates for each episode.
+    /// </summary>
+    /// <param name="seriesID">Shoko Series ID</param>
+    /// <param name="includeDataFrom">Include data from selected <see cref="DataSource"/>s.</param>
+    /// <param name="includeMediaInfo">Include media info data.</param>
+    /// <param name="includeXRefs">Include file/episode cross-references with the episodes.</param>
+    /// <param name="pageSize">Limits the number of results per page. Set to 0 to disable the limit.</param>
+    /// <param name="page">Page number.</param>
+    /// <returns></returns>
+    [HttpGet("Series/{seriesID}/Episodes")]
+    public ActionResult<ListResult<Episode>> GetEpisodesForSeries(
+        [FromRoute, Range(1, int.MaxValue)] int seriesID,
+        [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet<DataSource> includeDataFrom = null,
+        [FromQuery] bool includeMediaInfo = true,
+        [FromQuery] bool includeXRefs = false,
+        [FromQuery, Range(0, 1000)] int pageSize = 100,
+        [FromQuery, Range(1, int.MaxValue)] int page = 1)
+    {
+        var series = RepoFactory.AnimeSeries.GetByID(seriesID);
+        if (series == null)
+            return new ListResult<Episode>();
+
+        if (!User.AllowedSeries(series))
+            return new ListResult<Episode>();
+
+        var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(series.AniDB_ID);
+        return enumerable
+            .ToListResult(episode =>
+            {
+                var duplicateFiles = episode.VideoLocals
+                    .Select(file => (file, locations: file.Places.ExceptBy((file.FirstValidPlace ?? file.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID).ToList()))
+                    .Where(tuple => tuple.locations.Count > 0)
+                    .ToList();
+                var dto = new Episode(HttpContext, episode, includeDataFrom);
+                dto.Size = duplicateFiles.Count;
+                dto.Files = duplicateFiles
+                    .Select(tuple => new Models.Shoko.File(HttpContext, tuple.file, includeXRefs, includeDataFrom, includeMediaInfo, true))
+                    .ToList();
+                return dto;
+            }, page, pageSize);
+    }
+
+    /// <summary>
+    /// Get the list of file location ids to auto remove for the series.
+    /// </summary>
+    /// <param name="seriesID">Shoko Series ID</param>
+    /// <returns></returns>
+    [HttpGet("Series/{seriesID}/FileLocationsToAutoRemove")]
+    public ActionResult<List<FileIdSet>> GetFileLocationsIdsAcrossAllEpisodes(
+        [FromRoute, Range(1, int.MaxValue)] int seriesID
+    )
+    {
+        var series = RepoFactory.AnimeSeries.GetByID(seriesID);
+        if (series == null)
+            return new List<FileIdSet>();
+
+        if (!User.AllowedSeries(series))
+            return new List<FileIdSet>();
+
+        var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(series.AniDB_ID);
+        return enumerable
+            .SelectMany(episode =>
+                episode.VideoLocals
+                    .SelectMany(a => a.Places.ExceptBy((a.FirstValidPlace ?? a.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID))
+                    .Select(file => (episode.AnimeSeriesID, episode.AnimeEpisodeID, file.VideoLocalID, file.VideoLocal_Place_ID))
+            )
+            .GroupBy(tuple => tuple.VideoLocalID, tuple => (tuple.VideoLocal_Place_ID, tuple.AnimeEpisodeID, tuple.AnimeSeriesID))
+            .Select(groupBy => new FileIdSet(groupBy))
+            .ToList();
+    }
+
+    public DuplicateManagementController(ISettingsProvider settingsProvider) : base(settingsProvider)
+    {
+    }
+
+    public class FileIdSet(IGrouping<int, (int VideoLocal_Place_ID, int AnimeEpisodeID, int AnimeSeriesID)> grouping)
+    {
+        /// <summary>
+        /// The file ID with duplicates to remove.
+        /// </summary>
+        public int FileID { get; set; } = grouping.Key;
+
+        /// <summary>
+        /// The series IDs with duplicates to remove.
+        /// </summary>
+        public List<int> AnimeSeriesIDs { get; set; } = grouping
+            .Select(tuple => tuple.AnimeSeriesID)
+            .Distinct()
+            .ToList();
+
+        /// <summary>
+        /// The episode IDs with duplicates to remove.
+        /// </summary>
+        public List<int> AnimeEpisodeIDs { get; set; } = grouping
+            .Select(tuple => tuple.AnimeEpisodeID)
+            .Distinct()
+            .ToList();
+
+        /// <summary>
+        /// The duplicate locations to remove from the files/episodes.
+        /// </summary>
+        public List<int> FileLocationIDs { get; set; } = grouping
+            .Select(tuple => tuple.VideoLocal_Place_ID)
+            .Distinct()
+            .ToList();
+    }
+}

Shoko.Server/API/v3/Models/Shoko/Series.cs

@@ -851,6 +851,23 @@ public WithMultipleReleasesResult(SVR_AnimeSeries ser, int userId = 0, HashSet<D
             EpisodeCount = RepoFactory.AnimeEpisode.GetWithMultipleReleases(ignoreVariations, ser.AniDB_ID).Count;
         }
     }
+
+    /// <summary>
+    /// An extended model for use with the hard duplicate endpoint.
+    /// </summary>
+    public class WithDuplicateFilesResult : Series
+    {
+        /// <summary>
+        /// Number of episodes in the series which have duplicate files.
+        /// </summary>
+        public int EpisodeCount { get; set; }
+
+        public WithDuplicateFilesResult(SVR_AnimeSeries ser, int userId = 0, HashSet<DataSource>? includeDataFrom = null)
+            : base(ser, userId, false, includeDataFrom)
+        {
+            EpisodeCount = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(ser.AniDB_ID).Count();
+        }
+    }
 }
 
 public enum SeriesType

Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs

@@ -91,13 +91,13 @@ public List<SVR_AnimeEpisode> GetByHash(string hash)
             .ToList();
     }
 
-    private const string IgnoreVariationsWithAnimeQuery =
+    private const string MultipleReleasesIgnoreVariationsWithAnimeQuery =
         @"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE ani.AnimeID = :animeID AND vl.IsVariation = 0 AND vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1";
-    private const string CountVariationsWithAnimeQuery =
+    private const string MultipleReleasesCountVariationsWithAnimeQuery =
         @"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE ani.AnimeID = :animeID AND vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1";
-    private const string IgnoreVariationsQuery =
+    private const string MultipleReleasesIgnoreVariationsQuery =
         @"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE vl.IsVariation = 0 AND vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1";
-    private const string CountVariationsQuery =
+    private const string MultipleReleasesCountVariationsQuery =
         @"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1";
 
     public List<SVR_AnimeEpisode> GetWithMultipleReleases(bool ignoreVariations, int? animeID = null)
@@ -107,14 +107,14 @@ public List<SVR_AnimeEpisode> GetWithMultipleReleases(bool ignoreVariations, int
             using var session = _databaseFactory.SessionFactory.OpenSession();
             if (animeID.HasValue && animeID.Value > 0)
             {
-                var animeQuery = ignoreVariations ? IgnoreVariationsWithAnimeQuery : CountVariationsWithAnimeQuery;
+                var animeQuery = ignoreVariations ? MultipleReleasesIgnoreVariationsWithAnimeQuery : MultipleReleasesCountVariationsWithAnimeQuery;
                 return session.CreateSQLQuery(animeQuery)
                     .AddScalar("EpisodeID", NHibernateUtil.Int32)
                     .SetParameter("animeID", animeID.Value)
                     .List<int>();
             }
 
-            var query = ignoreVariations ? IgnoreVariationsQuery : CountVariationsQuery;
+            var query = ignoreVariations ? MultipleReleasesIgnoreVariationsQuery : MultipleReleasesCountVariationsQuery;
             return session.CreateSQLQuery(query)
                 .AddScalar("EpisodeID", NHibernateUtil.Int32)
                 .List<int>();
@@ -131,6 +131,93 @@ public List<SVR_AnimeEpisode> GetWithMultipleReleases(bool ignoreVariations, int
             .ToList();
     }
 
+    private const string DuplicateFilesWithAnimeQuery = @"
+SELECT 
+    ani.EpisodeID
+FROM 
+    (
+        SELECT 
+            vlp.VideoLocal_Place_ID, 
+            vl.FileSize, 
+            vl.Hash
+        FROM 
+            VideoLocal AS vl
+        INNER JOIN 
+            VideoLocal_Place AS vlp 
+            ON vlp.VideoLocalID = vl.VideoLocalID
+        WHERE 
+            vl.Hash != ''
+        GROUP BY 
+            vl.VideoLocalID
+        HAVING 
+            COUNT(vl.VideoLocalID) > 1
+    ) AS filtered_vlp
+INNER JOIN 
+    CrossRef_File_Episode ani 
+    ON filtered_vlp.Hash = ani.Hash 
+       AND filtered_vlp.FileSize = ani.FileSize
+WHERE ani.AnimeID = :animeID
+GROUP BY 
+    ani.EpisodeID
+";
+
+    private const string DuplicateFilesQuery = @"
+SELECT 
+    ani.EpisodeID
+FROM 
+    (
+        SELECT 
+            vlp.VideoLocal_Place_ID, 
+            vl.FileSize, 
+            vl.Hash
+        FROM 
+            VideoLocal AS vl
+        INNER JOIN 
+            VideoLocal_Place AS vlp 
+            ON vlp.VideoLocalID = vl.VideoLocalID
+        WHERE 
+            vl.Hash != ''
+        GROUP BY 
+            vl.VideoLocalID
+        HAVING 
+            COUNT(vl.VideoLocalID) > 1
+    ) AS filtered_vlp
+INNER JOIN 
+    CrossRef_File_Episode ani 
+    ON filtered_vlp.Hash = ani.Hash 
+       AND filtered_vlp.FileSize = ani.FileSize
+GROUP BY 
+    ani.EpisodeID
+";
+
+    public IEnumerable<SVR_AnimeEpisode> GetWithDuplicateFiles(int? animeID = null)
+    {
+        var ids = Lock(() =>
+        {
+            using var session = _databaseFactory.SessionFactory.OpenSession();
+            if (animeID.HasValue && animeID.Value > 0)
+            {
+                return session.CreateSQLQuery(DuplicateFilesWithAnimeQuery)
+                    .AddScalar("EpisodeID", NHibernateUtil.Int32)
+                    .SetParameter("animeID", animeID.Value)
+                    .List<int>();
+            }
+
+            return session.CreateSQLQuery(DuplicateFilesQuery)
+                .AddScalar("EpisodeID", NHibernateUtil.Int32)
+                .List<int>();
+        });
+
+        return ids
+            .Select(GetByAniDBEpisodeID)
+            .Select(episode => (episode, anidbEpisode: episode?.AniDB_Episode))
+            .Where(tuple => tuple.anidbEpisode is not null)
+            .OrderBy(tuple => tuple.anidbEpisode!.AnimeID)
+            .ThenBy(tuple => tuple.anidbEpisode!.EpisodeTypeEnum)
+            .ThenBy(tuple => tuple.anidbEpisode!.EpisodeNumber)
+            .Select(tuple => tuple.episode!);
+    }
+
     public List<SVR_AnimeEpisode> GetUnwatchedEpisodes(int seriesid, int userid)
     {
         var eps =