Skip to content

Commit

Permalink
Added NexusModsModToModuleInfoHistoryEntity that will store the whole…
Browse files Browse the repository at this point in the history 
… ModuleInfo entry of NexusMods mods

Will allow us to check for mod updates
Also, batched savings for Artcle and File parsers
  • Loading branch information
@Aragas
Aragas committed Oct 6, 2023
1 parent 54faca4 commit 539dcac
Show file tree
Hide file tree
Showing 20 changed files with 1,886 additions and 151 deletions.
3 changes: 3 additions & 0 deletions src/BUTR.Site.NexusMods.Server/BUTR.Site.NexusMods.Server.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,13 @@

<GenerateDocumentationFile>true</GenerateDocumentationFile>
<NoWarn>$(NoWarn);1591</NoWarn>

<DefineConstants>$(DefineConstants);BANNERLORDBUTRMODULEMANAGER_NULLABLE</DefineConstants>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Aragas.Extensions.Options.FluentValidation" Version="1.1.0.14" />
<PackageReference Include="Bannerlord.ModuleManager" Version="5.0.211" />
<PackageReference Include="BUTR.Authentication.NexusMods" Version="1.0.0.6" />
<PackageReference Include="BUTR.CrashReport" Version="13.0.0.22" />
<PackageReference Include="BUTR.CrashReport.Bannerlord.Parser" Version="13.0.0.22" />
Expand Down
2 changes: 2 additions & 0 deletions src/BUTR.Site.NexusMods.Server/Contexts/BaseAppDbContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public class BaseAppDbContext : DbContext
public required DbSet<NexusModsModToNameEntity> NexusModsModName { get; set; }
public required DbSet<NexusModsModToModuleEntity> NexusModsModModules { get; set; }
public required DbSet<NexusModsModToFileUpdateEntity> NexusModsModToFileUpdates { get; set; }
public required DbSet<NexusModsModToModuleInfoHistoryEntity> NexusModsModToModuleInfoHistory { get; set; }

public required DbSet<StatisticsTopExceptionsTypeEntity> StatisticsTopExceptionsTypes { get; set; }
public required DbSet<StatisticsCrashScoreInvolvedEntity> StatisticsCrashScoreInvolveds { get; set; }
Expand Down Expand Up @@ -105,6 +106,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
_entityConfigurationFactory.ApplyConfigurationWithTenant<NexusModsModToFileUpdateEntity>(modelBuilder);
_entityConfigurationFactory.ApplyConfigurationWithTenant<NexusModsModToModuleEntity>(modelBuilder);
_entityConfigurationFactory.ApplyConfigurationWithTenant<NexusModsModToNameEntity>(modelBuilder);
_entityConfigurationFactory.ApplyConfigurationWithTenant<NexusModsModToModuleInfoHistoryEntity>(modelBuilder);

_entityConfigurationFactory.ApplyConfiguration<NexusModsUserEntity>(modelBuilder);
_entityConfigurationFactory.ApplyConfigurationWithTenant<NexusModsUserToCrashReportEntity>(modelBuilder);
Expand Down
38 changes: 38 additions & 0 deletions ...e.NexusMods.Server/Contexts/Configs/NexusModsModToModuleInfoHistoryEntityConfiguration.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
using BUTR.Site.NexusMods.Server.Models;
using BUTR.Site.NexusMods.Server.Models.Database;

using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;

namespace BUTR.Site.NexusMods.Server.Contexts.Configs;

public class NexusModsModToModuleInfoHistoryEntityConfiguration : BaseEntityConfigurationWithTenant<NexusModsModToModuleInfoHistoryEntity>
{
public NexusModsModToModuleInfoHistoryEntityConfiguration(ITenantContextAccessor tenantContextAccessor) : base(tenantContextAccessor) { }

protected override void ConfigureModel(EntityTypeBuilder<NexusModsModToModuleInfoHistoryEntity> builder)
{
builder.Property<NexusModsModId>(nameof(NexusModsModEntity.NexusModsModId)).HasColumnName("nexusmods_mod_name_id").HasConversion<NexusModsModId.EfCoreValueConverter>().ValueGeneratedNever();
builder.Property<ModuleId>(nameof(ModuleEntity.ModuleId)).HasColumnName("module_id").HasConversion<ModuleId.EfCoreValueConverter>();
builder.Property(x => x.ModuleVersion).HasColumnName("module_version").HasConversion<ModuleVersion.EfCoreValueConverter>();
builder.Property(x => x.ModuleInfo).HasColumnName("module_info").HasColumnType("jsonb");
builder.ToTable("nexusmods_mod_module_info_history", "nexusmods_mod").HasKey(nameof(NexusModsModToModuleInfoHistoryEntity.TenantId), nameof(NexusModsModEntity.NexusModsModId), nameof(ModuleEntity.ModuleId), nameof(NexusModsModToModuleInfoHistoryEntity.ModuleVersion));

builder.HasOne(x => x.NexusModsMod)
.WithMany()
.HasForeignKey(nameof(NexusModsModToModuleInfoHistoryEntity.TenantId), nameof(NexusModsModEntity.NexusModsModId))
.HasPrincipalKey(x => new { x.TenantId, x.NexusModsModId })
.OnDelete(DeleteBehavior.Cascade);

builder.HasOne(x => x.Module)
.WithMany()
.HasForeignKey(nameof(NexusModsModToModuleInfoHistoryEntity.TenantId), nameof(ModuleEntity.ModuleId))
.HasPrincipalKey(x => new { x.TenantId, x.ModuleId })
.OnDelete(DeleteBehavior.Cascade);

builder.Navigation(x => x.NexusModsMod).AutoInclude();
builder.Navigation(x => x.Module).AutoInclude();

base.ConfigureModel(builder);
}
}
1 change: 1 addition & 0 deletions src/BUTR.Site.NexusMods.Server/Contexts/IAppDbContextRead.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public interface IAppDbContextRead
DbSet<NexusModsModToNameEntity> NexusModsModName { get; }
DbSet<NexusModsModToModuleEntity> NexusModsModModules { get; }
DbSet<NexusModsModToFileUpdateEntity> NexusModsModToFileUpdates { get; }
DbSet<NexusModsModToModuleInfoHistoryEntity> NexusModsModToModuleInfoHistory { get; }

DbSet<StatisticsTopExceptionsTypeEntity> StatisticsTopExceptionsTypes { get; }
DbSet<StatisticsCrashScoreInvolvedEntity> StatisticsCrashScoreInvolveds { get; }
Expand Down
10 changes: 5 additions & 5 deletions src/BUTR.Site.NexusMods.Server/Controllers/NexusModsUserController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ public sealed record NexusModsUserToNexusModsModManualLinkModel(NexusModsModId N

private readonly ILogger _logger;
private readonly NexusModsAPIClient _nexusModsAPIClient;
private readonly NexusModsInfo _nexusModsInfo;
private readonly NexusModsModFileParser _nexusModsModFileParser;
private readonly IAppDbContextWrite _dbContextWrite;
private readonly IAppDbContextRead _dbContextRead;

public NexusModsUserController(ILogger<NexusModsUserController> logger, NexusModsAPIClient nexusModsAPIClient, NexusModsInfo nexusModsInfo, IAppDbContextWrite dbContextWrite, IAppDbContextRead dbContextRead)
public NexusModsUserController(ILogger<NexusModsUserController> logger, NexusModsAPIClient nexusModsAPIClient, NexusModsModFileParser nexusModsModFileParser, IAppDbContextWrite dbContextWrite, IAppDbContextRead dbContextRead)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_nexusModsAPIClient = nexusModsAPIClient ?? throw new ArgumentNullException(nameof(nexusModsAPIClient));
_nexusModsInfo = nexusModsInfo ?? throw new ArgumentNullException(nameof(nexusModsInfo));
_nexusModsModFileParser = nexusModsModFileParser ?? throw new ArgumentNullException(nameof(nexusModsModFileParser));
_dbContextWrite = dbContextWrite ?? throw new ArgumentNullException(nameof(dbContextWrite));
_dbContextRead = dbContextRead ?? throw new ArgumentNullException(nameof(dbContextRead));
}
Expand Down Expand Up @@ -178,12 +178,12 @@ public NexusModsUserController(ILogger<NexusModsUserController> logger, NexusMod

if (HttpContext.GetIsPremium()) // Premium is needed for API based downloading
{
var exposedModIds = await _nexusModsInfo.GetModIdsAsync(gameDomain, modInfo.Id, apiKey, ct).Distinct().ToImmutableArrayAsync(ct);
var exposedModIds = await _nexusModsModFileParser.GetModuleInfosAsync(gameDomain, modInfo.Id, apiKey, ct).Distinct().ToImmutableArrayAsync(ct);
var entities = exposedModIds.Select(y => new NexusModsModToModuleEntity
{
TenantId = tenant,
NexusModsMod = entityFactory.GetOrCreateNexusModsMod(query.NexusModsModId),
Module = entityFactory.GetOrCreateModule(y),
Module = entityFactory.GetOrCreateModule(ModuleId.From(y.Id)),
LinkType = NexusModsModToModuleLinkType.ByUnverifiedFileExposure,
LastUpdateDate = DateTime.UtcNow
}).ToList();
Expand Down
98 changes: 52 additions & 46 deletions src/BUTR.Site.NexusMods.Server/Jobs/NexusModsArticleProcessorJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,64 +54,70 @@ private static async Task HandleTenantAsync(TenantId tenant, IServiceProvider se
var client = serviceProvider.GetRequiredService<NexusModsClient>();
var dbContextWrite = serviceProvider.GetRequiredService<IAppDbContextWrite>();
var entityFactory = dbContextWrite.CreateEntityFactory();
await using var _ = dbContextWrite.CreateSaveScope();

var gameDomain = tenant.ToGameDomain();

var articles = new List<NexusModsArticleEntity>();

var articleIdRaw = 0;
var notFoundArticles = 0;
while (!ct.IsCancellationRequested)
var @break = false;
while (!ct.IsCancellationRequested && !@break)
{
var articleId = NexusModsArticleId.From(articleIdRaw);
await using var _ = dbContextWrite.CreateSaveScope();
var articles = new List<NexusModsArticleEntity>();

var articleDocument = await client.GetArticleAsync(gameDomain, articleId, ct);
if (articleDocument is null) continue;
for (var i = 0; i < 50; i++)
{
var articleId = NexusModsArticleId.From(articleIdRaw);

if (await client.GetArticleAsync(gameDomain, articleId, ct) is not { } articleDocument)
{
articleIdRaw++;
continue;
}

var errorElement = articleDocument.GetElementbyId($"{articleId}-title");
if (errorElement is not null)
{
notFoundArticles++;
articleIdRaw++;
if (notFoundArticles >= notFoundArticlesTreshold)
if (articleDocument.GetElementbyId($"{articleId}-title") is not null)
{
break;
notFoundArticles++;
articleIdRaw++;
if (notFoundArticles >= notFoundArticlesTreshold)
{
@break = true;
break;
}
continue;
}
continue;
notFoundArticles = 0;

var pagetitleElement = articleDocument.GetElementbyId("pagetitle");
var titleElement = pagetitleElement.ChildNodes.FindFirst("h1");
var title = titleElement.InnerText;

var authorElement = articleDocument.GetElementbyId("image-author-name");
var authorUrl = authorElement.GetAttributeValue("href", "0");
var authorUrlSplit = authorUrl.Split('/', StringSplitOptions.RemoveEmptyEntries);
var authorIdText = authorUrlSplit.LastOrDefault() ?? string.Empty;
var authorId = NexusModsUserId.TryParse(authorIdText, out var val) ? val : throw new Exception("Author Id invalid");
var authorText = authorElement.InnerText;

var fileinfoElement = articleDocument.GetElementbyId("fileinfo");
var dateTimeText1 = fileinfoElement.ChildNodes.FindFirst("div");
var dateTimeText2 = dateTimeText1?.ChildNodes.FindFirst("time");
var dateTimeText = dateTimeText2?.GetAttributeValue("datetime", "");
var dateTime = DateTimeOffset.TryParse(dateTimeText, out var dateTimeVal) ? dateTimeVal.UtcDateTime : DateTimeOffset.MinValue.UtcDateTime;

articles.Add(new()
{
TenantId = tenant,
Title = title,
NexusModsArticleId = articleId,
NexusModsUser = entityFactory.GetOrCreateNexusModsUserWithName(authorId, NexusModsUserName.From(authorText)),
CreateDate = dateTime
});
articleIdRaw++;
}
notFoundArticles = 0;

var pagetitleElement = articleDocument.GetElementbyId("pagetitle");
var titleElement = pagetitleElement.ChildNodes.FindFirst("h1");
var title = titleElement.InnerText;

var authorElement = articleDocument.GetElementbyId("image-author-name");
var authorUrl = authorElement.GetAttributeValue("href", "0");
var authorUrlSplit = authorUrl.Split('/', StringSplitOptions.RemoveEmptyEntries);
var authorIdText = authorUrlSplit.LastOrDefault() ?? string.Empty;
var authorId = NexusModsUserId.TryParse(authorIdText, out var val) ? val : throw new Exception("Author Id invalid");
var authorText = authorElement.InnerText;

var fileinfoElement = articleDocument.GetElementbyId("fileinfo");
var dateTimeText1 = fileinfoElement.ChildNodes.FindFirst("div");
var dateTimeText2 = dateTimeText1?.ChildNodes.FindFirst("time");
var dateTimeText = dateTimeText2?.GetAttributeValue("datetime", "");
var dateTime = DateTimeOffset.TryParse(dateTimeText, out var dateTimeVal) ? dateTimeVal.UtcDateTime : DateTimeOffset.MinValue.UtcDateTime;

articles.Add(new()
{
TenantId = tenant,
Title = title,
NexusModsArticleId = articleId,
NexusModsUser = entityFactory.GetOrCreateNexusModsUserWithName(authorId, NexusModsUserName.From(authorText)),
CreateDate = dateTime
});
articleIdRaw++;
}

dbContextWrite.FutureUpsert(x => x.NexusModsArticles, articles);
// Disposing the DBContext will save the data
dbContextWrite.FutureUpsert(x => x.NexusModsArticles, articles);
// Disposing the DBContext will save the data
}
}
}
11 changes: 6 additions & 5 deletions src/BUTR.Site.NexusMods.Server/Jobs/NexusModsArticleUpdatesProcessorJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,13 @@ private static async Task<int> HandleTenantAsync(TenantId tenant, IServiceProvid
{
var articleId = NexusModsArticleId.From(articleIdRaw);

var articleDocument = await client.GetArticleAsync(gameDomain, articleId, ct);
if (articleDocument is null) continue;

if (await client.GetArticleAsync(gameDomain, articleId, ct) is not { } articleDocument)
{
articleIdRaw++;
continue;
}

var errorElement = articleDocument.GetElementbyId($"{articleId}-title");
if (errorElement is not null)
if (articleDocument.GetElementbyId($"{articleId}-title") is not null)
{
notFoundArticles++;
articleIdRaw++;
Expand Down
Loading

0 comments on commit 539dcac

Please sign in to comment.