Skip to content

Commit

Permalink
Better module parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@Aragas
Aragas committed Jul 3, 2024
1 parent 322e740 commit 3cc39df
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 16 deletions.
11 changes: 7 additions & 4 deletions src/BUTR.Site.NexusMods.Server/Jobs/NexusModsModFileProcessorJob.cs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using Bannerlord.ModuleManager;

using BUTR.Site.NexusMods.Server.Extensions;
using BUTR.Site.NexusMods.Server.Models;
using BUTR.Site.NexusMods.Server.Models.Database;
Expand Down Expand Up @@ -77,7 +79,6 @@ public async Task Execute(IJobExecutionContext context)
var modIdRaw = 1; //2907, 5090
while (!ct.IsCancellationRequested)
{

var modId = NexusModsModId.From(modIdRaw);

var nexusModsModModuleEntities = ImmutableArray.CreateBuilder<NexusModsModToModuleEntity>();
Expand All @@ -100,7 +101,9 @@ public async Task Execute(IJobExecutionContext context)
var infos = await _nexusModsModFileParser.GetModuleInfosAsync(gameDomain, modId, response.Files, _nexusModsOptions.ApiKey, ct).ToArrayAsync(ct);
var latestFileUpdate = DateTimeOffset.FromUnixTimeSeconds(response.Files.Select(x => x.UploadedTimestamp).Where(x => x is not null).Max() ?? 0).ToUniversalTime();

nexusModsModModuleEntities.AddRange(infos.Select(x => x.ModuleInfo).DistinctBy(x => x.Id).Select(x => new NexusModsModToModuleEntity
exceptions.AddRange(infos.Select(x => x.Exception).OfType<Exception>());

nexusModsModModuleEntities.AddRange(infos.Select(x => x.ModuleInfo).OfType<ModuleInfoExtended>().DistinctBy(x => x.Id).Select(x => new NexusModsModToModuleEntity
{
TenantId = tenant,
NexusModsModId = modId,
Expand All @@ -117,13 +120,13 @@ public async Task Execute(IJobExecutionContext context)
NexusModsMod = unitOfWrite.UpsertEntityFactory.GetOrCreateNexusModsMod(modId),
LastCheckedDate = latestFileUpdate,
});
nexusModsModToModuleInfoHistoryEntities.AddRange(infos.DistinctBy(x => new { x.ModuleInfo.Id, x.ModuleInfo.Version, x.FileId }).Select(x => new NexusModsModToModuleInfoHistoryEntity
nexusModsModToModuleInfoHistoryEntities.AddRange(infos.Where(x => x.ModuleInfo is not null).DistinctBy(x => new { x.ModuleInfo!.Id, x.ModuleInfo.Version, x.FileId }).Select(x => new NexusModsModToModuleInfoHistoryEntity
{
TenantId = tenant,
NexusModsFileId = x.FileId,
NexusModsModId = modId,
NexusModsMod = unitOfWrite.UpsertEntityFactory.GetOrCreateNexusModsMod(modId),
ModuleId = ModuleId.From(x.ModuleInfo.Id),
ModuleId = ModuleId.From(x.ModuleInfo!.Id),
Module = unitOfWrite.UpsertEntityFactory.GetOrCreateModule(ModuleId.From(x.ModuleInfo.Id)),
ModuleVersion = ModuleVersion.From(x.ModuleInfo.Version.ToString()),
ModuleInfo = ModuleInfoModel.Create(x.ModuleInfo),
Expand Down
10 changes: 7 additions & 3 deletions src/BUTR.Site.NexusMods.Server/Jobs/NexusModsModFileUpdatesProcessorJob.cs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using Bannerlord.ModuleManager;

using BUTR.Site.NexusMods.Server.Extensions;
using BUTR.Site.NexusMods.Server.Models;
using BUTR.Site.NexusMods.Server.Models.Database;
Expand Down Expand Up @@ -107,7 +109,9 @@ public async Task Execute(IJobExecutionContext context)
var infos = await _nexusModsModFileParser.GetModuleInfosAsync(gameDomain, modUpdate.Id, response.Files.Where(x => updates.Any(y => y.NewId == x.FileId)), _nexusModsOptions.ApiKey, ct).ToArrayAsync(ct);
var lastUpdateTime = DateTimeOffset.FromUnixTimeSeconds(modUpdate.LatestFileUpdateTimestamp).ToUniversalTime();

unitOfWrite.NexusModsModModules.UpsertRange(infos.Select(x => x.ModuleInfo).DistinctBy(x => x.Id).Select(x => new NexusModsModToModuleEntity
exceptions.AddRange(infos.Select(x => x.Exception).OfType<Exception>());

unitOfWrite.NexusModsModModules.UpsertRange(infos.Select(x => x.ModuleInfo).OfType<ModuleInfoExtended>().DistinctBy(x => x.Id).Select(x => new NexusModsModToModuleEntity
{
TenantId = tenant,
NexusModsModId = modUpdate.Id,
Expand All @@ -124,13 +128,13 @@ public async Task Execute(IJobExecutionContext context)
NexusModsMod = unitOfWrite.UpsertEntityFactory.GetOrCreateNexusModsMod(modUpdate.Id),
LastCheckedDate = lastUpdateTime
});
unitOfWrite.NexusModsModToModuleInfoHistory.UpsertRange(infos.DistinctBy(x => new { x.ModuleInfo.Id, x.ModuleInfo.Version, x.FileId }).Select(x => new NexusModsModToModuleInfoHistoryEntity
unitOfWrite.NexusModsModToModuleInfoHistory.UpsertRange(infos.Where(x => x.ModuleInfo is not null).DistinctBy(x => new { x.ModuleInfo!.Id, x.ModuleInfo.Version, x.FileId }).Select(x => new NexusModsModToModuleInfoHistoryEntity
{
TenantId = tenant,
NexusModsFileId = x.FileId,
NexusModsModId = modUpdate.Id,
NexusModsMod = unitOfWrite.UpsertEntityFactory.GetOrCreateNexusModsMod(modUpdate.Id),
ModuleId = ModuleId.From(x.ModuleInfo.Id),
ModuleId = ModuleId.From(x.ModuleInfo!.Id),
Module = unitOfWrite.UpsertEntityFactory.GetOrCreateModule(ModuleId.From(x.ModuleInfo.Id)),
ModuleVersion = ModuleVersion.From(x.ModuleInfo.Version.ToString()),
ModuleInfo = ModuleInfoModel.Create(x.ModuleInfo),
Expand Down
67 changes: 58 additions & 9 deletions src/BUTR.Site.NexusMods.Server/Services/General/INexusModsModFileParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ namespace BUTR.Site.NexusMods.Server.Services;

public sealed record NexusModsModFileParserResult
{
public required ModuleInfoExtended ModuleInfo { get; init; }
public required ModuleInfoExtended? ModuleInfo { get; init; }
public required GameVersion[] GameVersions { get; init; }
public required NexusModsFileId FileId { get; init; }
public required DateTimeOffset Uploaded { get; init; }
public required Exception? Exception { get; init; }
}

public interface INexusModsModFileParser
Expand Down Expand Up @@ -77,20 +78,53 @@ public async IAsyncEnumerable<NexusModsModFileParserResult> GetModuleInfosAsync(
if (await SubModuleXmlCountAsync(fileInfo) is var subModuleCount && false || subModuleCount == 0) continue;

var uploadedTimestamp = DateTimeOffset.FromUnixTimeSeconds(uploadedTimestampRaw).ToUniversalTime();
var downloadLinks = await _apiClient.GetModFileLinksAsync(gameDomain, modId, fileId, apiKey, ct) ?? Array.Empty<NexusModsDownloadLinkResponse>();
var downloadLinks = await _apiClient.GetModFileLinksAsync(gameDomain, modId, fileId, apiKey, ct) ?? [];
if (downloadLinks.Length == 0) continue;

await using var httpStream = HttpRangeStream.CreateOrDefault(downloadLinks.OrderByDescending(x => x.Url.Contains("cf-files")).Select(x => new Uri(x.Url)).ToArray(), _httpClient, new HttpRangeOptions { BufferSize = DefaultBufferSize });
if (httpStream is null) throw new InvalidOperationException($"Failed to get HttpStream for file '{fileInfo.FileName}'");
if (httpStream is null)
{
yield return new()
{
Exception = new InvalidOperationException($"Failed to get HttpStream for file '{fileInfo.FileName}'"),
FileId = fileId,
Uploaded = uploadedTimestamp,
ModuleInfo = null,
GameVersions = []
};
continue;
}

using var archive = ArchiveExtensions.OpenOrDefault(httpStream, new ReaderOptions { LeaveStreamOpen = true });
if (archive is null) throw new InvalidOperationException($"Failed to get Archive for file '{fileInfo.FileName}'");
if (archive is null)
{
yield return new()
{
Exception = new InvalidOperationException($"Failed to get Archive for file '{fileInfo.FileName}'"),
FileId = fileId,
Uploaded = uploadedTimestamp,
ModuleInfo = null,
GameVersions = []
};
continue;
}

if (archive is { IsSolid: true, Type: ArchiveType.Rar })
{
httpStream.SetBufferSize(LargeBufferSize);
using var reader = ReaderExtensions.OpenOrDefault(httpStream, new ReaderOptions { LeaveStreamOpen = true });
if (reader is null) throw new InvalidOperationException($"Failed to get Reader for file '{fileInfo.FileName}'");
if (reader is null)
{
yield return new()
{
Exception = new InvalidOperationException($"Failed to get Reader for file '{fileInfo.FileName}'"),
FileId = fileId,
Uploaded = uploadedTimestamp,
ModuleInfo = null,
GameVersions = []
};
continue;
}

var moduleInfosReader = await GetModuleInfosFromReaderAsync(reader, subModuleCount).ToListAsync(ct);
var gameVersions = await GetGameVersionsFromReaderAsync(reader, moduleInfosReader).ToListAsync(ct);
Expand All @@ -101,7 +135,8 @@ public async IAsyncEnumerable<NexusModsModFileParserResult> GetModuleInfosAsync(
ModuleInfo = grouping.Select(x => x.Item1).First(),
FileId = fileId,
Uploaded = uploadedTimestamp,
GameVersions = grouping.Select(x => GameVersion.From(x.Item3)).ToArray()
GameVersions = grouping.Select(x => GameVersion.From(x.Item3)).ToArray(),
Exception = null,
};
}
continue;
Expand All @@ -114,7 +149,18 @@ public async IAsyncEnumerable<NexusModsModFileParserResult> GetModuleInfosAsync(
httpStream.SetBufferSize(LargeBufferSize);

var moduleInfosArchive = await GetModuleInfosFromArchiveAsync(archive, subModuleCount).ToListAsync(ct);
if (moduleInfosArchive.Count != subModuleCount) throw new InvalidOperationException($"Failed to get all ModuleInfos for file '{fileInfo.FileName}'");
if (moduleInfosArchive.Count != subModuleCount)
{
yield return new()
{
Exception = new InvalidOperationException($"Failed to get all ModuleInfos for file '{fileInfo.FileName}'"),
FileId = fileId,
Uploaded = uploadedTimestamp,
ModuleInfo = null,
GameVersions = []
};
continue;
}

var dataArchive = await GetGameVersionsFromArchiveAsync(archive, moduleInfosArchive).ToListAsync(ct);
foreach (var grouping in dataArchive.GroupBy(x => new { x.Item1.Id, x.Item1.Version }))
Expand All @@ -124,7 +170,8 @@ public async IAsyncEnumerable<NexusModsModFileParserResult> GetModuleInfosAsync(
ModuleInfo = grouping.Select(x => x.Item1).First(),
FileId = fileId,
Uploaded = uploadedTimestamp,
GameVersions = grouping.Select(x => GameVersion.From(x.Item3)).ToArray()
GameVersions = grouping.Select(x => GameVersion.From(x.Item3)).ToArray(),
Exception = null,
};
}
}
Expand Down Expand Up @@ -283,8 +330,10 @@ static int ContainsSubModuleFile(IReadOnlyList<NexusModsModFileContentResponse.C
{
try
{
// No idea why stream based reader is not working sometimes
using var reader = new StreamReader(stream);
var document = new XmlDocument();
document.Load(stream);
document.LoadXml(reader.ReadToEnd());

return ModuleInfoExtended.FromXml(document);
}
Expand Down

0 comments on commit 3cc39df

Please sign in to comment.