diff --git a/Shoko.Server/API/v3/Controllers/DuplicateManagementController.cs b/Shoko.Server/API/v3/Controllers/DuplicateManagementController.cs
new file mode 100644
index 000000000..64f0b5bdc
--- /dev/null
+++ b/Shoko.Server/API/v3/Controllers/DuplicateManagementController.cs
@@ -0,0 +1,207 @@
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.Linq;
+using Microsoft.AspNetCore.Mvc;
+using Shoko.Commons.Extensions;
+using Shoko.Server.API.Annotations;
+using Shoko.Server.API.ModelBinders;
+using Shoko.Server.API.v3.Helpers;
+using Shoko.Server.API.v3.Models.Common;
+using Shoko.Server.API.v3.Models.Shoko;
+using Shoko.Server.Repositories;
+using Shoko.Server.Settings;
+
+#pragma warning disable CA1822
+namespace Shoko.Server.API.v3.Controllers;
+
+[ApiController]
+[Route("/api/v{version:apiVersion}/[controller]")]
+[ApiV3]
+public class DuplicateManagementController : BaseController
+{
+ ///
+ /// Get episodes with duplicate files, with only the files with duplicates for each episode.
+ ///
+ /// Include data from selected s.
+ /// Include media info data.
+ /// Include file/episode cross-references with the episodes.
+ /// Limits the number of results per page. Set to 0 to disable the limit.
+ /// Page number.
+ ///
+ [HttpGet("Episodes")]
+ public ActionResult> GetEpisodes(
+ [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet includeDataFrom = null,
+ [FromQuery] bool includeMediaInfo = true,
+ [FromQuery] bool includeXRefs = false,
+ [FromQuery, Range(0, 1000)] int pageSize = 100,
+ [FromQuery, Range(1, int.MaxValue)] int page = 1)
+ {
+ var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles();
+ return enumerable
+ .ToListResult(episode =>
+ {
+ var duplicateFiles = episode.VideoLocals
+ .Select(file => (file, locations: file.Places.ExceptBy((file.FirstValidPlace ?? file.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID).ToList()))
+ .Where(tuple => tuple.locations.Count > 0)
+ .ToList();
+ var dto = new Episode(HttpContext, episode, includeDataFrom);
+ dto.Size = duplicateFiles.Count;
+ dto.Files = duplicateFiles
+ .Select(tuple => new Models.Shoko.File(HttpContext, tuple.file, includeXRefs, includeDataFrom, includeMediaInfo, true))
+ .ToList();
+ return dto;
+ }, page, pageSize);
+ }
+
+ ///
+ /// Get the list of file location ids to auto remove across all series.
+ ///
+ ///
+ [HttpGet("FileLocationsToAutoRemove")]
+ public ActionResult> GetFileIdsWithPreference()
+ {
+ var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles();
+ return enumerable
+ .SelectMany(episode =>
+ episode.VideoLocals
+ .SelectMany(a => a.Places.ExceptBy((a.FirstValidPlace ?? a.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID))
+ .Select(file => (episode.AnimeSeriesID, episode.AnimeEpisodeID, file.VideoLocalID, file.VideoLocal_Place_ID))
+ )
+ .GroupBy(tuple => tuple.VideoLocalID, tuple => (tuple.VideoLocal_Place_ID, tuple.AnimeEpisodeID, tuple.AnimeSeriesID))
+ .Select(groupBy => new FileIdSet(groupBy))
+ .ToList();
+ }
+
+ ///
+ /// Get series with duplicate files.
+ ///
+ /// Include data from selected s.
+ /// Only show finished series.
+ /// Limits the number of results per page. Set to 0 to disable the limit.
+ /// Page number.
+ ///
+ [HttpGet("Series")]
+ public ActionResult> GetSeriesWithDuplicateFiles(
+ [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet includeDataFrom = null,
+ [FromQuery] bool onlyFinishedSeries = false,
+ [FromQuery, Range(0, 1000)] int pageSize = 100,
+ [FromQuery, Range(1, int.MaxValue)] int page = 1)
+ {
+ var enumerable = RepoFactory.AnimeSeries.GetWithDuplicateFiles();
+ if (onlyFinishedSeries)
+ enumerable = enumerable.Where(a => a.AniDB_Anime.GetFinishedAiring());
+
+ return enumerable
+ .OrderBy(series => series.PreferredTitle)
+ .ThenBy(series => series.AniDB_ID)
+ .ToListResult(series => new Series.WithDuplicateFilesResult(series, User.JMMUserID, includeDataFrom), page, pageSize);
+ }
+
+ ///
+ /// Get episodes with duplicate files for a series, with only the files with duplicates for each episode.
+ ///
+ /// Shoko Series ID
+ /// Include data from selected s.
+ /// Include media info data.
+ /// Include file/episode cross-references with the episodes.
+ /// Limits the number of results per page. Set to 0 to disable the limit.
+ /// Page number.
+ ///
+ [HttpGet("Series/{seriesID}/Episodes")]
+ public ActionResult> GetEpisodesForSeries(
+ [FromRoute, Range(1, int.MaxValue)] int seriesID,
+ [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet includeDataFrom = null,
+ [FromQuery] bool includeMediaInfo = true,
+ [FromQuery] bool includeXRefs = false,
+ [FromQuery, Range(0, 1000)] int pageSize = 100,
+ [FromQuery, Range(1, int.MaxValue)] int page = 1)
+ {
+ var series = RepoFactory.AnimeSeries.GetByID(seriesID);
+ if (series == null)
+ return new ListResult();
+
+ if (!User.AllowedSeries(series))
+ return new ListResult();
+
+ var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(series.AniDB_ID);
+ return enumerable
+ .ToListResult(episode =>
+ {
+ var duplicateFiles = episode.VideoLocals
+ .Select(file => (file, locations: file.Places.ExceptBy((file.FirstValidPlace ?? file.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID).ToList()))
+ .Where(tuple => tuple.locations.Count > 0)
+ .ToList();
+ var dto = new Episode(HttpContext, episode, includeDataFrom);
+ dto.Size = duplicateFiles.Count;
+ dto.Files = duplicateFiles
+ .Select(tuple => new Models.Shoko.File(HttpContext, tuple.file, includeXRefs, includeDataFrom, includeMediaInfo, true))
+ .ToList();
+ return dto;
+ }, page, pageSize);
+ }
+
+ ///
+ /// Get the list of file location ids to auto remove for the series.
+ ///
+ /// Shoko Series ID
+ ///
+ [HttpGet("Series/{seriesID}/FileLocationsToAutoRemove")]
+ public ActionResult> GetFileLocationsIdsAcrossAllEpisodes(
+ [FromRoute, Range(1, int.MaxValue)] int seriesID
+ )
+ {
+ var series = RepoFactory.AnimeSeries.GetByID(seriesID);
+ if (series == null)
+ return new List();
+
+ if (!User.AllowedSeries(series))
+ return new List();
+
+ var enumerable = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(series.AniDB_ID);
+ return enumerable
+ .SelectMany(episode =>
+ episode.VideoLocals
+ .SelectMany(a => a.Places.ExceptBy((a.FirstValidPlace ?? a.FirstResolvedPlace) is { } fileLocation ? [fileLocation.VideoLocal_Place_ID] : [], b => b.VideoLocal_Place_ID))
+ .Select(file => (episode.AnimeSeriesID, episode.AnimeEpisodeID, file.VideoLocalID, file.VideoLocal_Place_ID))
+ )
+ .GroupBy(tuple => tuple.VideoLocalID, tuple => (tuple.VideoLocal_Place_ID, tuple.AnimeEpisodeID, tuple.AnimeSeriesID))
+ .Select(groupBy => new FileIdSet(groupBy))
+ .ToList();
+ }
+
+ public DuplicateManagementController(ISettingsProvider settingsProvider) : base(settingsProvider)
+ {
+ }
+
+ public class FileIdSet(IGrouping grouping)
+ {
+ ///
+ /// The file ID with duplicates to remove.
+ ///
+ public int FileID { get; set; } = grouping.Key;
+
+ ///
+ /// The series IDs with duplicates to remove.
+ ///
+ public List AnimeSeriesIDs { get; set; } = grouping
+ .Select(tuple => tuple.AnimeSeriesID)
+ .Distinct()
+ .ToList();
+
+ ///
+ /// The episode IDs with duplicates to remove.
+ ///
+ public List AnimeEpisodeIDs { get; set; } = grouping
+ .Select(tuple => tuple.AnimeEpisodeID)
+ .Distinct()
+ .ToList();
+
+ ///
+ /// The duplicate locations to remove from the files/episodes.
+ ///
+ public List FileLocationIDs { get; set; } = grouping
+ .Select(tuple => tuple.VideoLocal_Place_ID)
+ .Distinct()
+ .ToList();
+ }
+}
diff --git a/Shoko.Server/API/v3/Models/Shoko/Series.cs b/Shoko.Server/API/v3/Models/Shoko/Series.cs
index dc69b3297..4d2097a4c 100644
--- a/Shoko.Server/API/v3/Models/Shoko/Series.cs
+++ b/Shoko.Server/API/v3/Models/Shoko/Series.cs
@@ -851,6 +851,23 @@ public WithMultipleReleasesResult(SVR_AnimeSeries ser, int userId = 0, HashSet
+ /// An extended model for use with the hard duplicate endpoint.
+ ///
+ public class WithDuplicateFilesResult : Series
+ {
+ ///
+ /// Number of episodes in the series which have duplicate files.
+ ///
+ public int EpisodeCount { get; set; }
+
+ public WithDuplicateFilesResult(SVR_AnimeSeries ser, int userId = 0, HashSet? includeDataFrom = null)
+ : base(ser, userId, false, includeDataFrom)
+ {
+ EpisodeCount = RepoFactory.AnimeEpisode.GetWithDuplicateFiles(ser.AniDB_ID).Count();
+ }
+ }
}
public enum SeriesType
diff --git a/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs b/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs
index dee692e3c..1c10a29db 100644
--- a/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs
+++ b/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs
@@ -91,13 +91,13 @@ public List GetByHash(string hash)
.ToList();
}
- private const string IgnoreVariationsWithAnimeQuery =
+ private const string MultipleReleasesIgnoreVariationsWithAnimeQuery =
@"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE ani.AnimeID = :animeID AND vl.IsVariation = 0 AND vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1";
- private const string CountVariationsWithAnimeQuery =
+ private const string MultipleReleasesCountVariationsWithAnimeQuery =
@"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE ani.AnimeID = :animeID AND vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1";
- private const string IgnoreVariationsQuery =
+ private const string MultipleReleasesIgnoreVariationsQuery =
@"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE vl.IsVariation = 0 AND vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1";
- private const string CountVariationsQuery =
+ private const string MultipleReleasesCountVariationsQuery =
@"SELECT ani.EpisodeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE vl.Hash != '' GROUP BY ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1";
public List GetWithMultipleReleases(bool ignoreVariations, int? animeID = null)
@@ -107,14 +107,14 @@ public List GetWithMultipleReleases(bool ignoreVariations, int
using var session = _databaseFactory.SessionFactory.OpenSession();
if (animeID.HasValue && animeID.Value > 0)
{
- var animeQuery = ignoreVariations ? IgnoreVariationsWithAnimeQuery : CountVariationsWithAnimeQuery;
+ var animeQuery = ignoreVariations ? MultipleReleasesIgnoreVariationsWithAnimeQuery : MultipleReleasesCountVariationsWithAnimeQuery;
return session.CreateSQLQuery(animeQuery)
.AddScalar("EpisodeID", NHibernateUtil.Int32)
.SetParameter("animeID", animeID.Value)
.List();
}
- var query = ignoreVariations ? IgnoreVariationsQuery : CountVariationsQuery;
+ var query = ignoreVariations ? MultipleReleasesIgnoreVariationsQuery : MultipleReleasesCountVariationsQuery;
return session.CreateSQLQuery(query)
.AddScalar("EpisodeID", NHibernateUtil.Int32)
.List();
@@ -131,6 +131,93 @@ public List GetWithMultipleReleases(bool ignoreVariations, int
.ToList();
}
+ private const string DuplicateFilesWithAnimeQuery = @"
+SELECT
+ ani.EpisodeID
+FROM
+ (
+ SELECT
+ vlp.VideoLocal_Place_ID,
+ vl.FileSize,
+ vl.Hash
+ FROM
+ VideoLocal AS vl
+ INNER JOIN
+ VideoLocal_Place AS vlp
+ ON vlp.VideoLocalID = vl.VideoLocalID
+ WHERE
+ vl.Hash != ''
+ GROUP BY
+ vl.VideoLocalID
+ HAVING
+ COUNT(vl.VideoLocalID) > 1
+ ) AS filtered_vlp
+INNER JOIN
+ CrossRef_File_Episode ani
+ ON filtered_vlp.Hash = ani.Hash
+ AND filtered_vlp.FileSize = ani.FileSize
+WHERE ani.AnimeID = :animeID
+GROUP BY
+ ani.EpisodeID
+";
+
+ private const string DuplicateFilesQuery = @"
+SELECT
+ ani.EpisodeID
+FROM
+ (
+ SELECT
+ vlp.VideoLocal_Place_ID,
+ vl.FileSize,
+ vl.Hash
+ FROM
+ VideoLocal AS vl
+ INNER JOIN
+ VideoLocal_Place AS vlp
+ ON vlp.VideoLocalID = vl.VideoLocalID
+ WHERE
+ vl.Hash != ''
+ GROUP BY
+ vl.VideoLocalID
+ HAVING
+ COUNT(vl.VideoLocalID) > 1
+ ) AS filtered_vlp
+INNER JOIN
+ CrossRef_File_Episode ani
+ ON filtered_vlp.Hash = ani.Hash
+ AND filtered_vlp.FileSize = ani.FileSize
+GROUP BY
+ ani.EpisodeID
+";
+
+ public IEnumerable GetWithDuplicateFiles(int? animeID = null)
+ {
+ var ids = Lock(() =>
+ {
+ using var session = _databaseFactory.SessionFactory.OpenSession();
+ if (animeID.HasValue && animeID.Value > 0)
+ {
+ return session.CreateSQLQuery(DuplicateFilesWithAnimeQuery)
+ .AddScalar("EpisodeID", NHibernateUtil.Int32)
+ .SetParameter("animeID", animeID.Value)
+ .List();
+ }
+
+ return session.CreateSQLQuery(DuplicateFilesQuery)
+ .AddScalar("EpisodeID", NHibernateUtil.Int32)
+ .List();
+ });
+
+ return ids
+ .Select(GetByAniDBEpisodeID)
+ .Select(episode => (episode, anidbEpisode: episode?.AniDB_Episode))
+ .Where(tuple => tuple.anidbEpisode is not null)
+ .OrderBy(tuple => tuple.anidbEpisode!.AnimeID)
+ .ThenBy(tuple => tuple.anidbEpisode!.EpisodeTypeEnum)
+ .ThenBy(tuple => tuple.anidbEpisode!.EpisodeNumber)
+ .Select(tuple => tuple.episode!);
+ }
+
public List GetUnwatchedEpisodes(int seriesid, int userid)
{
var eps =
diff --git a/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs b/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs
index cebc9ed50..d4f31c204 100644
--- a/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs
+++ b/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs
@@ -319,9 +319,9 @@ public List GetMostRecentlyAdded(int maxResults, int userID)
.ToList());
}
- private const string IgnoreVariationsQuery =
+ private const string MultipleReleasesIgnoreVariationsQuery =
@"SELECT DISTINCT ani.AnimeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE vl.IsVariation = 0 AND vl.Hash != '' GROUP BY ani.AnimeID, ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1";
- private const string CountVariationsQuery =
+ private const string MultipleReleasesCountVariationsQuery =
@"SELECT DISTINCT ani.AnimeID FROM VideoLocal AS vl JOIN CrossRef_File_Episode ani ON vl.Hash = ani.Hash WHERE vl.Hash != '' GROUP BY ani.AnimeID, ani.EpisodeID HAVING COUNT(ani.EpisodeID) > 1";
public List GetWithMultipleReleases(bool ignoreVariations)
@@ -330,7 +330,7 @@ public List GetWithMultipleReleases(bool ignoreVariations)
{
using var session = _databaseFactory.SessionFactory.OpenSession();
- var query = ignoreVariations ? IgnoreVariationsQuery : CountVariationsQuery;
+ var query = ignoreVariations ? MultipleReleasesIgnoreVariationsQuery : MultipleReleasesCountVariationsQuery;
return session.CreateSQLQuery(query)
.AddScalar("AnimeID", NHibernateUtil.Int32)
.List();
@@ -343,6 +343,52 @@ public List GetWithMultipleReleases(bool ignoreVariations)
.ToList();
}
+ private const string DuplicateFilesQuery = @"
+SELECT DISTINCT
+ ani.AnimeID
+FROM
+ (
+ SELECT
+ vlp.VideoLocal_Place_ID,
+ vl.FileSize,
+ vl.Hash
+ FROM
+ VideoLocal AS vl
+ INNER JOIN
+ VideoLocal_Place AS vlp
+ ON vlp.VideoLocalID = vl.VideoLocalID
+ WHERE
+ vl.Hash != ''
+ GROUP BY
+ vl.VideoLocalID
+ HAVING
+ COUNT(vl.VideoLocalID) > 1
+ ) AS vlp_selected
+INNER JOIN
+ CrossRef_File_Episode ani
+ ON vlp_selected.Hash = ani.Hash
+ AND vlp_selected.FileSize = ani.FileSize
+GROUP BY
+ ani.AnimeID
+";
+
+ public IEnumerable GetWithDuplicateFiles()
+ {
+ var ids = Lock(() =>
+ {
+ using var session = _databaseFactory.SessionFactory.OpenSession();
+
+ return session.CreateSQLQuery(DuplicateFilesQuery)
+ .AddScalar("AnimeID", NHibernateUtil.Int32)
+ .List();
+ });
+
+ return ids
+ .Distinct()
+ .Select(GetByAnimeID)
+ .WhereNotNull();
+ }
+
public ImageEntityType[] GetAllImageTypes()
=> [ImageEntityType.Backdrop, ImageEntityType.Banner, ImageEntityType.Logo, ImageEntityType.Poster];