From a54ce97815671352e1c30605c58b7b6c83f015c0 Mon Sep 17 00:00:00 2001 From: Jordan Fearnley Date: Mon, 18 Nov 2024 23:46:24 +0000 Subject: [PATCH 01/10] Misc: Add endpoint to download missing TMDB People Optionally allows for the complete removal of the person/cast/crew entry if unable to update it. --- .../API/v3/Controllers/TmdbController.cs | 42 +++++++++++++++++++ .../Providers/TMDB/TmdbMetadataService.cs | 4 +- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/Shoko.Server/API/v3/Controllers/TmdbController.cs b/Shoko.Server/API/v3/Controllers/TmdbController.cs index 60fc6e1ca..430d24625 100644 --- a/Shoko.Server/API/v3/Controllers/TmdbController.cs +++ b/Shoko.Server/API/v3/Controllers/TmdbController.cs @@ -1478,6 +1478,48 @@ public async Task DownloadImagesForTmdbShowByShowID( return NoContent(); } + /// + /// Download any missing TMDB Persons. + /// + /// Removes all references to the bad TMDB Person if unable to successfully download the person + /// with a summary of found/updated/skipped/deleted actions. + [HttpGet("Person/DownloadMissing")] + public async Task RepairMissingTmdbPersons([FromQuery] bool removeErrors = false) + { + var personIds = new HashSet(); + var (errorCount, removedCount) = (0, 0); + + var people = RepoFactory.TMDB_Person.GetAll().Select(person => person.TmdbPersonID).ToList(); + + RepoFactory.TMDB_Episode_Cast.GetAll().AsParallel().Where(p => + !people.Contains(p.TmdbPersonID)).ForEach(p => personIds.Add(p.TmdbPersonID)); + RepoFactory.TMDB_Episode_Crew.GetAll().AsParallel().Where(p => + !people.Contains(p.TmdbPersonID)).ForEach(p => personIds.Add(p.TmdbPersonID)); + + RepoFactory.TMDB_Movie_Cast.GetAll().AsParallel().Where(p => + !people.Contains(p.TmdbPersonID)).ForEach(p => personIds.Add(p.TmdbPersonID)); + RepoFactory.TMDB_Movie_Crew.GetAll().AsParallel().Where(p => + !people.Contains(p.TmdbPersonID)).ForEach(p => personIds.Add(p.TmdbPersonID)); + + foreach (var id in personIds) + try + { + await _tmdbMetadataService.UpdatePerson(id, forceRefresh: true); + } + catch (NullReferenceException) + { + _logger.LogInformation("Unable to find TMDB Person record {@Id} on TMDB", id); + errorCount++; + if (removeErrors && await _tmdbMetadataService.PurgePerson(id, forceRemoval: true)) + removedCount++; + } + + var updateCount = personIds.Count - errorCount; + var skippedCount = errorCount - removedCount; + + return Ok($"(Found/Updated/Skipped/Deleted) ({personIds.Count}/{updateCount}/{skippedCount}/{removedCount}) missing TMDB person(s)."); + } + #endregion #region Online (Search / Bulk / Single) diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index c1223e89f..2de0034b4 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -2072,11 +2072,11 @@ private async Task DownloadPersonImages(int personId, ProfileImages images, bool await _imageService.DownloadImagesByType(images.Profiles, ImageEntityType.Person, ForeignEntityType.Person, personId, settings.TMDB.MaxAutoStaffImages, [], forceDownload); } - public async Task PurgePerson(int personId, bool removeImageFiles = true) + public async Task PurgePerson(int personId, bool removeImageFiles = true, bool forceRemoval = false) { using (await GetLockForEntity(ForeignEntityType.Person, personId, "metadata & images", "Purge")) { - if (IsPersonLinkedToOtherEntities(personId)) + if (!forceRemoval && IsPersonLinkedToOtherEntities(personId)) return false; var person = _tmdbPeople.GetByTmdbPersonID(personId); From 01045f11a5fa3024c9f6fee0c43cb36ffcaf210d Mon Sep 17 00:00:00 2001 From: Jordan Fearnley Date: Tue, 19 Nov 2024 23:46:05 +0000 Subject: [PATCH 02/10] Refactor: Move TMDB Person repair logic to the TMDB Metadata Service --- .../API/v3/Controllers/TmdbController.cs | 34 +------------- .../Providers/TMDB/TmdbMetadataService.cs | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/Shoko.Server/API/v3/Controllers/TmdbController.cs b/Shoko.Server/API/v3/Controllers/TmdbController.cs index 430d24625..24a4be35d 100644 --- a/Shoko.Server/API/v3/Controllers/TmdbController.cs +++ b/Shoko.Server/API/v3/Controllers/TmdbController.cs @@ -1486,38 +1486,8 @@ public async Task DownloadImagesForTmdbShowByShowID( [HttpGet("Person/DownloadMissing")] public async Task RepairMissingTmdbPersons([FromQuery] bool removeErrors = false) { - var personIds = new HashSet(); - var (errorCount, removedCount) = (0, 0); - - var people = RepoFactory.TMDB_Person.GetAll().Select(person => person.TmdbPersonID).ToList(); - - RepoFactory.TMDB_Episode_Cast.GetAll().AsParallel().Where(p => - !people.Contains(p.TmdbPersonID)).ForEach(p => personIds.Add(p.TmdbPersonID)); - RepoFactory.TMDB_Episode_Crew.GetAll().AsParallel().Where(p => - !people.Contains(p.TmdbPersonID)).ForEach(p => personIds.Add(p.TmdbPersonID)); - - RepoFactory.TMDB_Movie_Cast.GetAll().AsParallel().Where(p => - !people.Contains(p.TmdbPersonID)).ForEach(p => personIds.Add(p.TmdbPersonID)); - RepoFactory.TMDB_Movie_Crew.GetAll().AsParallel().Where(p => - !people.Contains(p.TmdbPersonID)).ForEach(p => personIds.Add(p.TmdbPersonID)); - - foreach (var id in personIds) - try - { - await _tmdbMetadataService.UpdatePerson(id, forceRefresh: true); - } - catch (NullReferenceException) - { - _logger.LogInformation("Unable to find TMDB Person record {@Id} on TMDB", id); - errorCount++; - if (removeErrors && await _tmdbMetadataService.PurgePerson(id, forceRemoval: true)) - removedCount++; - } - - var updateCount = personIds.Count - errorCount; - var skippedCount = errorCount - removedCount; - - return Ok($"(Found/Updated/Skipped/Deleted) ({personIds.Count}/{updateCount}/{skippedCount}/{removedCount}) missing TMDB person(s)."); + var result = await _tmdbMetadataService.RepairMissingPersons(); + return Ok($"(Found/Updated/Skipped/Deleted) ({result.Found}/{result.Updated}/{result.Skipped}/{result.Removed}) missing TMDB person(s)."); } #endregion diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index 2de0034b4..952b63867 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -2031,6 +2031,50 @@ private void PurgeCompany(int companyId, bool removeImageFiles = true) #region People + public async Task<(int Found, int Updated, int Skipped, int Removed)> RepairMissingPersons(bool removeErrors = false) + { + var missingIds = new HashSet(); + var (updateCount, skippedCount, removedCount) = (0, 0, 0); + + var people = _tmdbPeople.GetAll().Select(person => person.TmdbPersonID).ToList(); + + _tmdbEpisodeCast.GetAll().AsParallel() + .Where(p => !people.Contains(p.TmdbPersonID)) + .ForAll(p => missingIds.Add(p.TmdbPersonID)); + _tmdbEpisodeCrew.GetAll().AsParallel() + .Where(p => !people.Contains(p.TmdbPersonID)) + .ForAll(p => missingIds.Add(p.TmdbPersonID)); + + _tmdbMovieCast.GetAll().AsParallel() + .Where(p => !people.Contains(p.TmdbPersonID)) + .ForAll(p => missingIds.Add(p.TmdbPersonID)); + _tmdbMovieCrew.GetAll().AsParallel() + .Where(p => !people.Contains(p.TmdbPersonID)) + .ForAll(p => missingIds.Add(p.TmdbPersonID)); + + foreach (var personId in missingIds) + { + try + { + await UpdatePerson(personId, forceRefresh: true); + updateCount++; + } + catch (NullReferenceException) + { + _logger.LogDebug("Unable to update TMDB Person ({@PersonId})", personId); + + if (removeErrors && await PurgePerson(personId, removeImageFiles: true, forceRemoval: true)) + { + removedCount++; + continue; + } + skippedCount++; + } + } + + return (missingIds.Count, updateCount, skippedCount, removedCount); + } + public async Task<(bool added, bool updated)> UpdatePerson(int personId, bool forceRefresh = false, bool downloadImages = false) { using (await GetLockForEntity(ForeignEntityType.Person, personId, "metadata & images", "Update").ConfigureAwait(false)) From 519163b15210f4b4f2a8bafa9206c3b1a44d8811 Mon Sep 17 00:00:00 2001 From: Jordan Fearnley Date: Tue, 19 Nov 2024 23:48:24 +0000 Subject: [PATCH 03/10] Misc: Add DB migration to ensure TMDB Cast/Crew are backed by a Person --- Shoko.Server/Databases/DatabaseFixes.cs | 6 ++++++ Shoko.Server/Databases/MySQL.cs | 3 ++- Shoko.Server/Databases/SQLServer.cs | 3 ++- Shoko.Server/Databases/SQLite.cs | 3 ++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Shoko.Server/Databases/DatabaseFixes.cs b/Shoko.Server/Databases/DatabaseFixes.cs index 80007142e..60bebc6a0 100644 --- a/Shoko.Server/Databases/DatabaseFixes.cs +++ b/Shoko.Server/Databases/DatabaseFixes.cs @@ -837,4 +837,10 @@ public static void ClearQuartzQueue() var queueHandler = Utils.ServiceContainer.GetRequiredService(); queueHandler.Clear().ConfigureAwait(false).GetAwaiter().GetResult(); } + + public static void RepairMissingTMDBPersons() + { + var service = Utils.ServiceContainer.GetRequiredService(); + service.RepairMissingPersons(true).ConfigureAwait(false).GetAwaiter().GetResult(); + } } diff --git a/Shoko.Server/Databases/MySQL.cs b/Shoko.Server/Databases/MySQL.cs index 588499569..16a986f09 100644 --- a/Shoko.Server/Databases/MySQL.cs +++ b/Shoko.Server/Databases/MySQL.cs @@ -27,7 +27,7 @@ namespace Shoko.Server.Databases; public class MySQL : BaseDatabase { public override string Name { get; } = "MySQL"; - public override int RequiredVersion { get; } = 139; + public override int RequiredVersion { get; } = 140; private List createVersionTable = new() { @@ -849,6 +849,7 @@ public class MySQL : BaseDatabase new(139, 10, "ALTER TABLE Trakt_Show ADD COLUMN TmdbShowID INT NULL;"), new(139, 11, DatabaseFixes.CleanupAfterRemovingTvDB), new(139, 12, DatabaseFixes.ClearQuartzQueue), + new (140, 1, DatabaseFixes.RepairMissingTMDBPersons) }; private DatabaseCommand linuxTableVersionsFix = new("RENAME TABLE versions TO Versions;"); diff --git a/Shoko.Server/Databases/SQLServer.cs b/Shoko.Server/Databases/SQLServer.cs index 45aa05dc7..855cf5afd 100644 --- a/Shoko.Server/Databases/SQLServer.cs +++ b/Shoko.Server/Databases/SQLServer.cs @@ -28,7 +28,7 @@ namespace Shoko.Server.Databases; public class SQLServer : BaseDatabase { public override string Name { get; } = "SQLServer"; - public override int RequiredVersion { get; } = 131; + public override int RequiredVersion { get; } = 132; public override void BackupDatabase(string fullfilename) { @@ -779,6 +779,7 @@ public override bool HasVersionsTable() new DatabaseCommand(131, 10, "ALTER TABLE Trakt_Show ADD TmdbShowID INT NULL;"), new DatabaseCommand(131, 11, DatabaseFixes.CleanupAfterRemovingTvDB), new DatabaseCommand(131, 12, DatabaseFixes.ClearQuartzQueue), + new DatabaseCommand(132, 1, DatabaseFixes.RepairMissingTMDBPersons) }; private static void AlterImdbMovieIDType() diff --git a/Shoko.Server/Databases/SQLite.cs b/Shoko.Server/Databases/SQLite.cs index fa3661948..90fd55248 100644 --- a/Shoko.Server/Databases/SQLite.cs +++ b/Shoko.Server/Databases/SQLite.cs @@ -28,7 +28,7 @@ public class SQLite : BaseDatabase { public override string Name => "SQLite"; - public override int RequiredVersion => 123; + public override int RequiredVersion => 124; public override void BackupDatabase(string fullfilename) { @@ -774,6 +774,7 @@ public override void CreateDatabase() new(123, 10, "ALTER TABLE Trakt_Show ADD COLUMN TmdbShowID INTEGER NULL;"), new(123, 11, DatabaseFixes.CleanupAfterRemovingTvDB), new(123, 12, DatabaseFixes.ClearQuartzQueue), + new (124, 1, DatabaseFixes.RepairMissingTMDBPersons) }; private static Tuple MigrateRenamers(object connection) From 874ee3a810c7232848971d8f3f47a8ea401cea84 Mon Sep 17 00:00:00 2001 From: Jordan Fearnley Date: Wed, 20 Nov 2024 18:55:56 +0000 Subject: [PATCH 04/10] Refactor: Allow `UpdatePerson` to gracefully handle null provided values. Misc: Remove parallelization in `/TMDB/Person/DownloadMissing` endpoint --- .../API/v3/Controllers/TmdbController.cs | 12 ++- Shoko.Server/Databases/DatabaseFixes.cs | 2 +- .../Providers/TMDB/TmdbMetadataService.cs | 83 +++++++++++-------- 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/Shoko.Server/API/v3/Controllers/TmdbController.cs b/Shoko.Server/API/v3/Controllers/TmdbController.cs index 24a4be35d..d3be1fb74 100644 --- a/Shoko.Server/API/v3/Controllers/TmdbController.cs +++ b/Shoko.Server/API/v3/Controllers/TmdbController.cs @@ -1479,17 +1479,15 @@ public async Task DownloadImagesForTmdbShowByShowID( } /// - /// Download any missing TMDB Persons. + /// Download any missing TMDB Person records. /// - /// Removes all references to the bad TMDB Person if unable to successfully download the person - /// with a summary of found/updated/skipped/deleted actions. [HttpGet("Person/DownloadMissing")] - public async Task RepairMissingTmdbPersons([FromQuery] bool removeErrors = false) + public ActionResult RepairMissingTmdbPersonRecords() { - var result = await _tmdbMetadataService.RepairMissingPersons(); - return Ok($"(Found/Updated/Skipped/Deleted) ({result.Found}/{result.Updated}/{result.Skipped}/{result.Removed}) missing TMDB person(s)."); + Task.Run(() => _tmdbMetadataService.RepairMissingPersonRecords()).ConfigureAwait(false); + return Ok(); } - + #endregion #region Online (Search / Bulk / Single) diff --git a/Shoko.Server/Databases/DatabaseFixes.cs b/Shoko.Server/Databases/DatabaseFixes.cs index 60bebc6a0..4f6bfef6f 100644 --- a/Shoko.Server/Databases/DatabaseFixes.cs +++ b/Shoko.Server/Databases/DatabaseFixes.cs @@ -841,6 +841,6 @@ public static void ClearQuartzQueue() public static void RepairMissingTMDBPersons() { var service = Utils.ServiceContainer.GetRequiredService(); - service.RepairMissingPersons(true).ConfigureAwait(false).GetAwaiter().GetResult(); + service.RepairMissingPersonRecords().ConfigureAwait(false).GetAwaiter().GetResult(); } } diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index 952b63867..bb8adf7fb 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -2031,48 +2031,36 @@ private void PurgeCompany(int companyId, bool removeImageFiles = true) #region People - public async Task<(int Found, int Updated, int Skipped, int Removed)> RepairMissingPersons(bool removeErrors = false) + public async Task RepairMissingPersonRecords() { var missingIds = new HashSet(); - var (updateCount, skippedCount, removedCount) = (0, 0, 0); + var (updateCount, skippedCount) = (0, 0); - var people = _tmdbPeople.GetAll().Select(person => person.TmdbPersonID).ToList(); - - _tmdbEpisodeCast.GetAll().AsParallel() - .Where(p => !people.Contains(p.TmdbPersonID)) - .ForAll(p => missingIds.Add(p.TmdbPersonID)); - _tmdbEpisodeCrew.GetAll().AsParallel() - .Where(p => !people.Contains(p.TmdbPersonID)) - .ForAll(p => missingIds.Add(p.TmdbPersonID)); + var peopleIds = _tmdbPeople.GetAll().Select(person => person.TmdbPersonID).ToHashSet(); + + foreach (var person in _tmdbEpisodeCast.GetAll()) + if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID); + foreach (var person in _tmdbEpisodeCrew.GetAll()) + if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID); - _tmdbMovieCast.GetAll().AsParallel() - .Where(p => !people.Contains(p.TmdbPersonID)) - .ForAll(p => missingIds.Add(p.TmdbPersonID)); - _tmdbMovieCrew.GetAll().AsParallel() - .Where(p => !people.Contains(p.TmdbPersonID)) - .ForAll(p => missingIds.Add(p.TmdbPersonID)); + foreach (var person in _tmdbMovieCast.GetAll()) + if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID); + foreach (var person in _tmdbMovieCrew.GetAll()) + if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID); + _logger.LogDebug("Found {@Count} unique missing TMDB Person record(s) for Episode & Movie staff", missingIds.Count); + foreach (var personId in missingIds) { - try - { - await UpdatePerson(personId, forceRefresh: true); + var (_, updated) = await UpdatePerson(personId, forceRefresh: true); + if (updated) updateCount++; - } - catch (NullReferenceException) - { - _logger.LogDebug("Unable to update TMDB Person ({@PersonId})", personId); - - if (removeErrors && await PurgePerson(personId, removeImageFiles: true, forceRemoval: true)) - { - removedCount++; - continue; - } + else skippedCount++; - } } - return (missingIds.Count, updateCount, skippedCount, removedCount); + _logger.LogInformation("Updated missing TMDB People: Found/Updated/Skipped {@Found}/{@Updated}/{@Skipped}", + missingIds.Count, updateCount, skippedCount); } public async Task<(bool added, bool updated)> UpdatePerson(int personId, bool forceRefresh = false, bool downloadImages = false) @@ -2092,6 +2080,13 @@ private void PurgeCompany(int companyId, bool removeImageFiles = true) methods |= PersonMethods.Images; var newlyAdded = tmdbPerson.TMDB_PersonID is 0; var person = await UseClient(c => c.GetPersonAsync(personId, methods), $"Get person {personId}"); + if (person is null) + { + _logger.LogDebug("Unable to update staff; Refreshing related links. (Person={PersonId})", personId); + // Defer the update to avoid recursively locking the person... + _ = Task.Run(() => UpdateMoviesAndShowsByPerson(personId)).ConfigureAwait(false); + return (newlyAdded, false); + } var updated = tmdbPerson.Populate(person); if (updated) { @@ -2105,6 +2100,28 @@ private void PurgeCompany(int companyId, bool removeImageFiles = true) return (newlyAdded, updated); } } + + private async Task UpdateMoviesAndShowsByPerson(int tmdbPersonId) + { + var showIds = new HashSet(); + var movieIds = new HashSet(); + + foreach (var staff in _tmdbEpisodeCast.GetByTmdbPersonID(tmdbPersonId)) + showIds.Add(staff.TmdbShowID); + foreach (var staff in _tmdbEpisodeCrew.GetByTmdbPersonID(tmdbPersonId)) + showIds.Add(staff.TmdbShowID); + + foreach (var staff in _tmdbMovieCast.GetByTmdbPersonID(tmdbPersonId)) + movieIds.Add(staff.TmdbMovieID); + foreach (var staff in _tmdbMovieCrew.GetByTmdbPersonID(tmdbPersonId)) + movieIds.Add(staff.TmdbMovieID); + + foreach (var showId in showIds) + await UpdateShow(showId, downloadCrewAndCast: true); + + foreach (var movieId in movieIds) + await UpdateMovie(movieId, downloadCrewAndCast: true); + } private async Task DownloadPersonImages(int personId, ProfileImages images, bool forceDownload = false) { @@ -2116,11 +2133,11 @@ private async Task DownloadPersonImages(int personId, ProfileImages images, bool await _imageService.DownloadImagesByType(images.Profiles, ImageEntityType.Person, ForeignEntityType.Person, personId, settings.TMDB.MaxAutoStaffImages, [], forceDownload); } - public async Task PurgePerson(int personId, bool removeImageFiles = true, bool forceRemoval = false) + public async Task PurgePerson(int personId, bool removeImageFiles = true) { using (await GetLockForEntity(ForeignEntityType.Person, personId, "metadata & images", "Purge")) { - if (!forceRemoval && IsPersonLinkedToOtherEntities(personId)) + if (IsPersonLinkedToOtherEntities(personId)) return false; var person = _tmdbPeople.GetByTmdbPersonID(personId); From 11dd8846a43240d61e1b334ca895261d691eb6e5 Mon Sep 17 00:00:00 2001 From: Jordan Fearnley Date: Wed, 20 Nov 2024 19:16:34 +0000 Subject: [PATCH 05/10] Misc: Rename `PersonRecords` to `People` --- Shoko.Server/API/v3/Controllers/TmdbController.cs | 6 +++--- Shoko.Server/Databases/DatabaseFixes.cs | 2 +- Shoko.Server/Providers/TMDB/TmdbMetadataService.cs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Shoko.Server/API/v3/Controllers/TmdbController.cs b/Shoko.Server/API/v3/Controllers/TmdbController.cs index d3be1fb74..98884cf55 100644 --- a/Shoko.Server/API/v3/Controllers/TmdbController.cs +++ b/Shoko.Server/API/v3/Controllers/TmdbController.cs @@ -1479,12 +1479,12 @@ public async Task DownloadImagesForTmdbShowByShowID( } /// - /// Download any missing TMDB Person records. + /// Download any missing TMDB People. /// [HttpGet("Person/DownloadMissing")] - public ActionResult RepairMissingTmdbPersonRecords() + public ActionResult RepairMissingTmdbPeople() { - Task.Run(() => _tmdbMetadataService.RepairMissingPersonRecords()).ConfigureAwait(false); + Task.Run(() => _tmdbMetadataService.RepairMissingPeople()).ConfigureAwait(false); return Ok(); } diff --git a/Shoko.Server/Databases/DatabaseFixes.cs b/Shoko.Server/Databases/DatabaseFixes.cs index 4f6bfef6f..02b52083f 100644 --- a/Shoko.Server/Databases/DatabaseFixes.cs +++ b/Shoko.Server/Databases/DatabaseFixes.cs @@ -841,6 +841,6 @@ public static void ClearQuartzQueue() public static void RepairMissingTMDBPersons() { var service = Utils.ServiceContainer.GetRequiredService(); - service.RepairMissingPersonRecords().ConfigureAwait(false).GetAwaiter().GetResult(); + service.RepairMissingPeople().ConfigureAwait(false).GetAwaiter().GetResult(); } } diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index bb8adf7fb..491657095 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -2031,7 +2031,7 @@ private void PurgeCompany(int companyId, bool removeImageFiles = true) #region People - public async Task RepairMissingPersonRecords() + public async Task RepairMissingPeople() { var missingIds = new HashSet(); var (updateCount, skippedCount) = (0, 0); @@ -2048,7 +2048,7 @@ public async Task RepairMissingPersonRecords() foreach (var person in _tmdbMovieCrew.GetAll()) if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID); - _logger.LogDebug("Found {@Count} unique missing TMDB Person record(s) for Episode & Movie staff", missingIds.Count); + _logger.LogDebug("Found {@Count} unique missing TMDB People for Episode & Movie staff", missingIds.Count); foreach (var personId in missingIds) { From bfce02671d5b5954e546627819db51e3fed9619d Mon Sep 17 00:00:00 2001 From: Jordan Fearnley Date: Wed, 20 Nov 2024 19:23:29 +0000 Subject: [PATCH 06/10] Refactor: Schedules the update of Movies/Shows linked to People failing to update. --- .../Providers/TMDB/TmdbMetadataService.cs | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index 491657095..b5f940d3a 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -2082,10 +2082,9 @@ public async Task RepairMissingPeople() var person = await UseClient(c => c.GetPersonAsync(personId, methods), $"Get person {personId}"); if (person is null) { - _logger.LogDebug("Unable to update staff; Refreshing related links. (Person={PersonId})", personId); - // Defer the update to avoid recursively locking the person... - _ = Task.Run(() => UpdateMoviesAndShowsByPerson(personId)).ConfigureAwait(false); - return (newlyAdded, false); + _logger.LogDebug("Unable to update staff; Scheduling refresh of related links. (Person={PersonId})", personId); + await ScheduleUpdateMoviesAndShowsByPerson(personId); + return (false, false); } var updated = tmdbPerson.Populate(person); if (updated) @@ -2100,27 +2099,27 @@ public async Task RepairMissingPeople() return (newlyAdded, updated); } } - - private async Task UpdateMoviesAndShowsByPerson(int tmdbPersonId) + + private async Task ScheduleUpdateMoviesAndShowsByPerson(int tmdbPersonId) { var showIds = new HashSet(); var movieIds = new HashSet(); - + foreach (var staff in _tmdbEpisodeCast.GetByTmdbPersonID(tmdbPersonId)) showIds.Add(staff.TmdbShowID); foreach (var staff in _tmdbEpisodeCrew.GetByTmdbPersonID(tmdbPersonId)) showIds.Add(staff.TmdbShowID); - + foreach (var staff in _tmdbMovieCast.GetByTmdbPersonID(tmdbPersonId)) movieIds.Add(staff.TmdbMovieID); foreach (var staff in _tmdbMovieCrew.GetByTmdbPersonID(tmdbPersonId)) movieIds.Add(staff.TmdbMovieID); - + foreach (var showId in showIds) - await UpdateShow(showId, downloadCrewAndCast: true); + await ScheduleUpdateOfShow(showId, downloadCrewAndCast: true, forceRefresh: true); foreach (var movieId in movieIds) - await UpdateMovie(movieId, downloadCrewAndCast: true); + await ScheduleUpdateOfMovie(movieId, downloadCrewAndCast: true, forceRefresh: true); } private async Task DownloadPersonImages(int personId, ProfileImages images, bool forceDownload = false) From fca85bd89d6f77cb5b2b288524e29ec70369e870 Mon Sep 17 00:00:00 2001 From: Jordan Fearnley Date: Wed, 20 Nov 2024 19:35:47 +0000 Subject: [PATCH 07/10] =?UTF-8?q?Misc:=20Whitespace=20changes=20?= =?UTF-8?q?=F0=9F=98=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Shoko.Server/Providers/TMDB/TmdbMetadataService.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index b5f940d3a..43bf4d207 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -2104,20 +2104,20 @@ private async Task ScheduleUpdateMoviesAndShowsByPerson(int tmdbPersonId) { var showIds = new HashSet(); var movieIds = new HashSet(); - + foreach (var staff in _tmdbEpisodeCast.GetByTmdbPersonID(tmdbPersonId)) showIds.Add(staff.TmdbShowID); foreach (var staff in _tmdbEpisodeCrew.GetByTmdbPersonID(tmdbPersonId)) showIds.Add(staff.TmdbShowID); - + foreach (var staff in _tmdbMovieCast.GetByTmdbPersonID(tmdbPersonId)) movieIds.Add(staff.TmdbMovieID); foreach (var staff in _tmdbMovieCrew.GetByTmdbPersonID(tmdbPersonId)) movieIds.Add(staff.TmdbMovieID); - + foreach (var showId in showIds) await ScheduleUpdateOfShow(showId, downloadCrewAndCast: true, forceRefresh: true); - + foreach (var movieId in movieIds) await ScheduleUpdateOfMovie(movieId, downloadCrewAndCast: true, forceRefresh: true); } From 7f400937a1d0a0ef71dd0399530c75fdace93562 Mon Sep 17 00:00:00 2001 From: Jordan Fearnley Date: Thu, 21 Nov 2024 10:15:47 +0000 Subject: [PATCH 08/10] Refactor: Avoid tuple destructuring to assign count variables --- Shoko.Server/Providers/TMDB/TmdbMetadataService.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index 43bf4d207..aae3d2d39 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -2034,7 +2034,8 @@ private void PurgeCompany(int companyId, bool removeImageFiles = true) public async Task RepairMissingPeople() { var missingIds = new HashSet(); - var (updateCount, skippedCount) = (0, 0); + var updateCount = 0; + var skippedCount = 0; var peopleIds = _tmdbPeople.GetAll().Select(person => person.TmdbPersonID).ToHashSet(); From d1a6e4be00e42b43297aea242642170a4636d526 Mon Sep 17 00:00:00 2001 From: Jordan Fearnley Date: Thu, 21 Nov 2024 10:21:28 +0000 Subject: [PATCH 09/10] Misc: Move new endpoint to the ActionController --- Shoko.Server/API/v3/Controllers/ActionController.cs | 10 ++++++++++ Shoko.Server/API/v3/Controllers/TmdbController.cs | 10 ---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Shoko.Server/API/v3/Controllers/ActionController.cs b/Shoko.Server/API/v3/Controllers/ActionController.cs index 137702115..6b31cbf96 100644 --- a/Shoko.Server/API/v3/Controllers/ActionController.cs +++ b/Shoko.Server/API/v3/Controllers/ActionController.cs @@ -192,6 +192,16 @@ public ActionResult UpdateAllTmdbShows() return Ok(); } + /// + /// Download any missing TMDB People. + /// + [HttpGet("DownloadMissingTmdbPeople")] + public ActionResult DownloadMissingTmdbPeople() + { + Task.Factory.StartNew(() => _tmdbService.RepairMissingPeople()); + return Ok(); + } + /// /// Purge all unused TMDB Shows that are not linked to any AniDB anime. /// diff --git a/Shoko.Server/API/v3/Controllers/TmdbController.cs b/Shoko.Server/API/v3/Controllers/TmdbController.cs index 98884cf55..60fc6e1ca 100644 --- a/Shoko.Server/API/v3/Controllers/TmdbController.cs +++ b/Shoko.Server/API/v3/Controllers/TmdbController.cs @@ -1478,16 +1478,6 @@ public async Task DownloadImagesForTmdbShowByShowID( return NoContent(); } - /// - /// Download any missing TMDB People. - /// - [HttpGet("Person/DownloadMissing")] - public ActionResult RepairMissingTmdbPeople() - { - Task.Run(() => _tmdbMetadataService.RepairMissingPeople()).ConfigureAwait(false); - return Ok(); - } - #endregion #region Online (Search / Bulk / Single) From a37b46dba6bdacaed4274e3fffde03231400583f Mon Sep 17 00:00:00 2001 From: "Mikal S." <7761729+revam@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:01:21 +0100 Subject: [PATCH 10/10] misc: remove extra whitespace in `new()` expressions --- Shoko.Server/Databases/MySQL.cs | 2 +- Shoko.Server/Databases/SQLite.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Shoko.Server/Databases/MySQL.cs b/Shoko.Server/Databases/MySQL.cs index 16a986f09..c981f79b7 100644 --- a/Shoko.Server/Databases/MySQL.cs +++ b/Shoko.Server/Databases/MySQL.cs @@ -849,7 +849,7 @@ public class MySQL : BaseDatabase new(139, 10, "ALTER TABLE Trakt_Show ADD COLUMN TmdbShowID INT NULL;"), new(139, 11, DatabaseFixes.CleanupAfterRemovingTvDB), new(139, 12, DatabaseFixes.ClearQuartzQueue), - new (140, 1, DatabaseFixes.RepairMissingTMDBPersons) + new(140, 1, DatabaseFixes.RepairMissingTMDBPersons) }; private DatabaseCommand linuxTableVersionsFix = new("RENAME TABLE versions TO Versions;"); diff --git a/Shoko.Server/Databases/SQLite.cs b/Shoko.Server/Databases/SQLite.cs index 90fd55248..2a9f76f9c 100644 --- a/Shoko.Server/Databases/SQLite.cs +++ b/Shoko.Server/Databases/SQLite.cs @@ -774,7 +774,7 @@ public override void CreateDatabase() new(123, 10, "ALTER TABLE Trakt_Show ADD COLUMN TmdbShowID INTEGER NULL;"), new(123, 11, DatabaseFixes.CleanupAfterRemovingTvDB), new(123, 12, DatabaseFixes.ClearQuartzQueue), - new (124, 1, DatabaseFixes.RepairMissingTMDBPersons) + new(124, 1, DatabaseFixes.RepairMissingTMDBPersons) }; private static Tuple MigrateRenamers(object connection)