From 597bf54d423de6d2df2a2e0858dc703d4d2f9429 Mon Sep 17 00:00:00 2001 From: Vitalii Mikhailov Date: Sat, 24 Feb 2024 23:06:50 +0200 Subject: [PATCH] Switching to postgres --- .../BUTR.CrashReportServer.csproj | 1 + .../Config/JsonEntityConfiguration.cs | 2 +- .../Contexts/OldAppDbContext.cs | 69 +++++++++++ .../Controllers/CrashUploadController.cs | 23 ++-- .../Controllers/ReportController.cs | 38 ++++-- .../Extensions/HostExtensions.cs | 2 +- .../Models/Database/JsonEntity.cs | 2 +- .../Models/Database/OldJsonEntity.cs | 7 ++ .../Services/DatabaseMigrator.cs | 108 ++++++------------ src/BUTR.CrashReportServer/Startup.cs | 3 +- 10 files changed, 159 insertions(+), 96 deletions(-) create mode 100644 src/BUTR.CrashReportServer/Contexts/OldAppDbContext.cs create mode 100644 src/BUTR.CrashReportServer/Models/Database/OldJsonEntity.cs diff --git a/src/BUTR.CrashReportServer/BUTR.CrashReportServer.csproj b/src/BUTR.CrashReportServer/BUTR.CrashReportServer.csproj index fcf9f83..7310021 100644 --- a/src/BUTR.CrashReportServer/BUTR.CrashReportServer.csproj +++ b/src/BUTR.CrashReportServer/BUTR.CrashReportServer.csproj @@ -28,6 +28,7 @@ + diff --git a/src/BUTR.CrashReportServer/Contexts/Config/JsonEntityConfiguration.cs b/src/BUTR.CrashReportServer/Contexts/Config/JsonEntityConfiguration.cs index 6647f07..a6eb895 100644 --- a/src/BUTR.CrashReportServer/Contexts/Config/JsonEntityConfiguration.cs +++ b/src/BUTR.CrashReportServer/Contexts/Config/JsonEntityConfiguration.cs @@ -10,7 +10,7 @@ public class JsonEntityConfiguration : BaseEntityConfiguration protected override void ConfigureModel(EntityTypeBuilder builder) { builder.Property(nameof(IdEntity.FileId)).HasColumnName("file_id"); - builder.Property(p => p.CrashReportCompressed).HasColumnName("data_compressed"); + builder.Property(p => p.CrashReport).HasColumnName("data").HasColumnType("jsonb"); builder.ToTable("json_entity").HasKey(nameof(IdEntity.FileId)); builder.HasOne(x => x.Id) diff --git a/src/BUTR.CrashReportServer/Contexts/OldAppDbContext.cs b/src/BUTR.CrashReportServer/Contexts/OldAppDbContext.cs new file mode 100644 index 0000000..f6e2a72 --- /dev/null +++ b/src/BUTR.CrashReportServer/Contexts/OldAppDbContext.cs @@ -0,0 +1,69 @@ +using BUTR.CrashReportServer.Contexts.Config; +using BUTR.CrashReportServer.Models.Database; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace BUTR.CrashReportServer.Contexts; + +public class FileEntityConfiguration : BaseEntityConfiguration +{ + protected override void ConfigureModel(EntityTypeBuilder builder) + { + builder.Property(nameof(IdEntity.FileId)).HasColumnName("file_id"); + builder.Property(p => p.DataCompressed).HasColumnName("data_compressed"); + builder.ToTable("file_entity").HasKey(nameof(IdEntity.FileId)).HasName("file_entity_pkey"); + + builder.HasOne(x => x.Id) + .WithOne() + .HasForeignKey(nameof(IdEntity.FileId)) + .HasPrincipalKey(x => x.FileId) + .OnDelete(DeleteBehavior.Cascade); + + builder.Navigation(x => x.Id).AutoInclude(); + } +} +public class IdEntityConfiguration : BaseEntityConfiguration +{ + protected override void ConfigureModel(EntityTypeBuilder builder) + { + builder.Property(x => x.FileId).HasColumnName("file_id").HasDefaultValueSql("hex(randomblob(3))"); + builder.Property(x => x.CrashReportId).HasColumnName("crash_report_id"); + builder.Property(x => x.Version).HasColumnName("version"); + builder.Property(x => x.Created).HasColumnName("created"); + builder.ToTable("id_entity").HasKey(x => x.FileId); + + builder.HasIndex(x => x.CrashReportId).IsUnique(false); + } +} +public class OldJsonEntityConfiguration : BaseEntityConfiguration +{ + protected override void ConfigureModel(EntityTypeBuilder builder) + { + builder.Property(nameof(IdEntity.FileId)).HasColumnName("file_id"); + builder.Property(p => p.CrashReportCompressed).HasColumnName("data_compressed"); + builder.ToTable("json_entity").HasKey(nameof(IdEntity.FileId)); + + builder.HasOne(x => x.Id) + .WithOne() + .HasForeignKey(nameof(IdEntity.FileId)) + .HasPrincipalKey(x => x.FileId) + .OnDelete(DeleteBehavior.Cascade); + + builder.Navigation(x => x.Id).AutoInclude(); + } +} + +public class OldAppDbContext : DbContext +{ + public OldAppDbContext(DbContextOptions options) : base(options) { } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + modelBuilder.ApplyConfiguration(new IdEntityConfiguration()); + modelBuilder.ApplyConfiguration(new FileEntityConfiguration()); + modelBuilder.ApplyConfiguration(new OldJsonEntityConfiguration()); + } +} \ No newline at end of file diff --git a/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs b/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs index ccbfd1a..662d70e 100644 --- a/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs +++ b/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs @@ -36,6 +36,7 @@ public sealed record CrashReportUploadBody(CrashReportModel CrashReport, ICollec private readonly CrashUploadOptions _options; private readonly JsonSerializerOptions _jsonSerializerOptions; private readonly AppDbContext _dbContext; + private readonly OldAppDbContext _dbContextOld; private readonly GZipCompressor _gZipCompressor; private readonly HexGenerator _hexGenerator; @@ -48,7 +49,7 @@ public CrashUploadController( AppDbContext dbContext, GZipCompressor gZipCompressor, HexGenerator hexGenerator, - IMeterFactory meterFactory) + IMeterFactory meterFactory, OldAppDbContext dbContextOld) { var meter = meterFactory.Create("BUTR.CrashReportServer.Controllers.CrashUploadController", "1.0.0"); @@ -58,6 +59,7 @@ public CrashUploadController( _jsonSerializerOptions = jsonSerializerOptions.Value ?? throw new ArgumentNullException(nameof(jsonSerializerOptions)); _options = options.Value ?? throw new ArgumentNullException(nameof(options)); _dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); + _dbContextOld = dbContextOld ?? throw new ArgumentNullException(nameof(dbContextOld)); _gZipCompressor = gZipCompressor ?? throw new ArgumentNullException(nameof(gZipCompressor)); _hexGenerator = hexGenerator ?? throw new ArgumentNullException(nameof(hexGenerator)); } @@ -70,9 +72,13 @@ private string GenerateFileId(CancellationToken ct) { var fileIds = _hexGenerator.GetHex(count, 3); var existing = _dbContext.Set().Select(x => x.FileId).Where(x => fileIds.Contains(x)).ToHashSet(); - if (existing.Count == count) continue; + var existing2 = _dbContextOld.Set().Select(x => x.FileId).Where(x => fileIds.Contains(x)).ToHashSet(); + //if (existing.Count == count) continue; fileId = existing.First(x => !fileIds.Contains(x)); - break; + fileIds.ExceptWith(existing); + fileIds.ExceptWith(existing2); + if (fileIds.Count == 0) break; + return fileIds.First(); } return fileId; } @@ -88,16 +94,16 @@ private async Task UploadHtmlAsync(CancellationToken ct) if (await _dbContext.Set().FirstOrDefaultAsync(x => x.CrashReportId == id, ct) is { } idEntity) return Ok($"{_options.BaseUri}/{idEntity.FileId}"); - await using var compressedHtmlStream = await _gZipCompressor.CompressAsync(Request.Body, ct); - await using var compressedJsonStream = await _gZipCompressor.CompressAsync(JsonSerializer.SerializeToUtf8Bytes(crashReportModel, new JsonSerializerOptions(JsonSerializerDefaults.Web) + var json = JsonSerializer.Serialize(crashReportModel, new JsonSerializerOptions(JsonSerializerDefaults.Web) { Converters = { new JsonStringEnumConverter() } - }), ct); + }); + await using var compressedHtmlStream = await _gZipCompressor.CompressAsync(Request.Body, ct); idEntity = new IdEntity { FileId = GenerateFileId(ct), CrashReportId = id, Version = version, Created = DateTime.UtcNow, }; await _dbContext.Set().AddAsync(idEntity, ct); await _dbContext.Set().AddAsync(new FileEntity { Id = idEntity, DataCompressed = compressedHtmlStream.ToArray(), }, ct); - if (version >= 13) await _dbContext.Set().AddAsync(new JsonEntity { Id = idEntity, CrashReportCompressed = compressedJsonStream.ToArray(), }, ct); + if (version >= 13) await _dbContext.Set().AddAsync(new JsonEntity { Id = idEntity, CrashReport = json, }, ct); await _dbContext.SaveChangesAsync(ct); _reportVersion.Add(1, new[] { new KeyValuePair("Version", version) }); @@ -120,11 +126,10 @@ private async Task UploadJsonAsync(CancellationToken ct) var html = CrashReportHtmlRenderer.AddData(CrashReportHtmlRenderer.Build(crashReport, logSources), json); await using var compressedHtmlStream = await _gZipCompressor.CompressAsync(html.AsStream(), ct); - await using var compressedJsonStream = await _gZipCompressor.CompressAsync(json.AsStream(), ct); idEntity = new IdEntity { FileId = GenerateFileId(ct), CrashReportId = crashReport.Id, Version = crashReport.Version, Created = DateTime.UtcNow, }; await _dbContext.Set().AddAsync(idEntity, ct); - await _dbContext.Set().AddAsync(new JsonEntity { Id = idEntity, CrashReportCompressed = compressedJsonStream.ToArray(), }, ct); + await _dbContext.Set().AddAsync(new JsonEntity { Id = idEntity, CrashReport = json, }, ct); await _dbContext.Set().AddAsync(new FileEntity { Id = idEntity, DataCompressed = compressedHtmlStream.ToArray(), }, ct); await _dbContext.SaveChangesAsync(ct); diff --git a/src/BUTR.CrashReportServer/Controllers/ReportController.cs b/src/BUTR.CrashReportServer/Controllers/ReportController.cs index 71e63f4..c1ad66b 100644 --- a/src/BUTR.CrashReportServer/Controllers/ReportController.cs +++ b/src/BUTR.CrashReportServer/Controllers/ReportController.cs @@ -21,6 +21,7 @@ using System.Net; using System.Runtime.CompilerServices; using System.Security.Authentication; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -38,13 +39,15 @@ public sealed record GetNewCrashReportsBody private readonly ILogger _logger; private readonly ReportOptions _options; private readonly AppDbContext _dbContext; + private readonly OldAppDbContext _dbContextOld; private readonly GZipCompressor _gZipCompressor; - public ReportController(ILogger logger, IOptionsSnapshot options, AppDbContext dbContext, GZipCompressor gZipCompressor) + public ReportController(ILogger logger, IOptionsSnapshot options, AppDbContext dbContext, OldAppDbContext dbContextOld, GZipCompressor gZipCompressor) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _options = options.Value ?? throw new ArgumentNullException(nameof(options)); _dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); + _dbContextOld = dbContextOld ?? throw new ArgumentNullException(nameof(dbContextOld)); _gZipCompressor = gZipCompressor ?? throw new ArgumentNullException(nameof(gZipCompressor)); } @@ -75,15 +78,25 @@ private async Task GetHtml(string filename, CancellationToken ct) if (ValidateRequest(ref filename) is { } errorResponse) return errorResponse; - if (await _dbContext.Set().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct) is not { } file) + var file = await _dbContext.Set().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct); + var fileOld = await _dbContextOld.Set().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct); + if (file is null && fileOld is null) return StatusCode(StatusCodes.Status404NotFound); if (Request.GetTypedHeaders().AcceptEncoding.Any(x => x.Value.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))) { Response.Headers.ContentEncoding = "gzip"; - return File(file.DataCompressed, "text/html; charset=utf-8", true); + if (file is not null) + return File(file.DataCompressed, "text/html; charset=utf-8", true); + if (fileOld is not null) + return File(fileOld.DataCompressed, "text/html; charset=utf-8", true); } - return File(await _gZipCompressor.DecompressAsync(file.DataCompressed, ct), "text/html; charset=utf-8", true); + if (file is not null) + return File(await _gZipCompressor.DecompressAsync(file.DataCompressed, ct), "text/html; charset=utf-8", true); + if (fileOld is not null) + return File(await _gZipCompressor.DecompressAsync(fileOld.DataCompressed, ct), "text/html; charset=utf-8", true); + + return StatusCode(StatusCodes.Status500InternalServerError); } private async Task GetJson(string filename, CancellationToken ct) @@ -91,15 +104,26 @@ private async Task GetJson(string filename, CancellationToken ct) if (ValidateRequest(ref filename) is { } errorResponse) return errorResponse; - if (await _dbContext.Set().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct) is not { } file) + var file = await _dbContext.Set().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct); + var fileOld = await _dbContextOld.Set().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct); + if (file is null && fileOld is null) return StatusCode(StatusCodes.Status404NotFound); if (Request.GetTypedHeaders().AcceptEncoding.Any(x => x.Value.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))) { Response.Headers.ContentEncoding = "gzip"; - return File(file.CrashReportCompressed, "application/json; charset=utf-8", true); + if (file is not null) + return File(await _gZipCompressor.CompressAsync(new MemoryStream(Encoding.UTF8.GetBytes(file.CrashReport)), ct), "application/json; charset=utf-8", true); + if (fileOld is not null) + return File(fileOld.CrashReportCompressed, "application/json; charset=utf-8", true); + } - return File(await _gZipCompressor.DecompressAsync(file.CrashReportCompressed, ct), "application/json; charset=utf-8", true); + if (file is not null) + return File(new MemoryStream(Encoding.UTF8.GetBytes(file.CrashReport)), "application/json; charset=utf-8", true); + if (fileOld is not null) + return File(await _gZipCompressor.DecompressAsync(fileOld.CrashReportCompressed, ct), "application/json; charset=utf-8", true); + + return StatusCode(StatusCodes.Status500InternalServerError); } [AllowAnonymous] diff --git a/src/BUTR.CrashReportServer/Extensions/HostExtensions.cs b/src/BUTR.CrashReportServer/Extensions/HostExtensions.cs index 431c55d..158c135 100644 --- a/src/BUTR.CrashReportServer/Extensions/HostExtensions.cs +++ b/src/BUTR.CrashReportServer/Extensions/HostExtensions.cs @@ -20,7 +20,7 @@ public static async Task SeedDbContextAsync(this IHost host) { var migrations = (await dbContext.Database.GetPendingMigrationsAsync()).Count(); await dbContext.Database.MigrateAsync(); - if (migrations > 0) await dbContext.Database.ExecuteSqlRawAsync("VACUUM;"); + //if (migrations > 0) await dbContext.Database.ExecuteSqlRawAsync("VACUUM;"); } catch (Exception ex) { diff --git a/src/BUTR.CrashReportServer/Models/Database/JsonEntity.cs b/src/BUTR.CrashReportServer/Models/Database/JsonEntity.cs index 0976669..5e6b9d1 100644 --- a/src/BUTR.CrashReportServer/Models/Database/JsonEntity.cs +++ b/src/BUTR.CrashReportServer/Models/Database/JsonEntity.cs @@ -3,5 +3,5 @@ public sealed record JsonEntity : IEntity { public required IdEntity Id { get; set; } - public required byte[] CrashReportCompressed { get; set; } + public required string CrashReport { get; set; } } \ No newline at end of file diff --git a/src/BUTR.CrashReportServer/Models/Database/OldJsonEntity.cs b/src/BUTR.CrashReportServer/Models/Database/OldJsonEntity.cs new file mode 100644 index 0000000..a08a511 --- /dev/null +++ b/src/BUTR.CrashReportServer/Models/Database/OldJsonEntity.cs @@ -0,0 +1,7 @@ +namespace BUTR.CrashReportServer.Models.Database; + +public sealed record OldJsonEntity : IEntity +{ + public required IdEntity Id { get; set; } + public required byte[] CrashReportCompressed { get; set; } +} \ No newline at end of file diff --git a/src/BUTR.CrashReportServer/Services/DatabaseMigrator.cs b/src/BUTR.CrashReportServer/Services/DatabaseMigrator.cs index e49d91c..e5d345c 100644 --- a/src/BUTR.CrashReportServer/Services/DatabaseMigrator.cs +++ b/src/BUTR.CrashReportServer/Services/DatabaseMigrator.cs @@ -1,17 +1,13 @@ using BUTR.CrashReportServer.Contexts; using BUTR.CrashReportServer.Models.Database; -using BUTR.CrashReportServer.Utils; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using System; -using System.Collections.Concurrent; using System.IO; -using System.IO.Pipelines; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -19,90 +15,50 @@ namespace BUTR.CrashReportServer.Services; public sealed class DatabaseMigrator : BackgroundService { - private class ObjectPool - { - private readonly ConcurrentBag _objects = new(); - private readonly Func> _objectGenerator; - - public ObjectPool(Func> objectGenerator) - { - _objectGenerator = objectGenerator ?? throw new ArgumentNullException(nameof(objectGenerator)); - } - - public async Task GetAsync(CancellationToken ct) => _objects.TryTake(out var item) ? item : await _objectGenerator(ct); - - public void Return(T item) => _objects.Add(item); - } - - private readonly IServiceScopeFactory _scopeFactory; - private readonly GZipCompressor _gZipCompressor; + private readonly GZipCompressor _compressor; - public DatabaseMigrator(IServiceScopeFactory scopeFactory, GZipCompressor gZipCompressor) + public DatabaseMigrator(IServiceScopeFactory scopeFactory, GZipCompressor compressor) { _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); - _gZipCompressor = gZipCompressor ?? throw new ArgumentNullException(nameof(gZipCompressor)); + _compressor = compressor ?? throw new ArgumentNullException(nameof(compressor)); } protected override async Task ExecuteAsync(CancellationToken ct) { await using var scope = _scopeFactory.CreateAsyncScope(); var dbContextFactory = scope.ServiceProvider.GetRequiredService>(); - var dbContextPool = new ObjectPool(dbContextFactory.CreateDbContextAsync); - - var options = new ParallelOptions { CancellationToken = ct }; - await Parallel.ForEachAsync(Enumerable.Range(0, 4), options, async (_, ct2) => - { - var dbContext = await dbContextPool.GetAsync(ct2); + var dbContextFactoryOld = scope.ServiceProvider.GetRequiredService>(); - var wrong = dbContext.Set().AsNoTracking().Where(x => x.Id.Version == 0).Take(1000); - while (true) - { - var entities = wrong.ToArray(); - if (entities.Length == 0) break; + var postgres = await dbContextFactory.CreateDbContextAsync(ct); + var sqlite = await dbContextFactoryOld.CreateDbContextAsync(ct); - var sb = new StringBuilder(); - sb.AppendLine("BEGIN TRANSACTION;"); - foreach (var entity in entities) + const int take = 10000; + + var idDataCount = sqlite.Set().Count(); + for (var i = 0; i < idDataCount % take; i+= take) + { + var data = await sqlite.Set().Skip(i * take).Take(take).ToArrayAsync(ct); + await postgres.Set().AddRangeAsync(data, ct); + } + + var fileDataCount = sqlite.Set().Count(); + for (var i = 0; i < fileDataCount % take; i+= take) + { + var data = await sqlite.Set().Skip(i * take).Take(take).ToArrayAsync(ct); + await postgres.Set().AddRangeAsync(data, ct); + } + + var jsonDataCount = sqlite.Set().Count(); + for (var i = 0; i < jsonDataCount % take; i+= take) + { + var data = await sqlite.Set().Skip(i * take).Take(take).AsAsyncEnumerable() + .SelectAwait(async (x) => new JsonEntity { - await using var decompressed = await _gZipCompressor.DecompressAsync(entity.DataCompressed, ct); - decompressed.Seek(0, SeekOrigin.Begin); - decompressed.Seek(0, SeekOrigin.Begin); - - var valid = false; - var version = 0; - try - { - var (valid2, id, version2, json) = await CrashReportRawParser.TryReadCrashReportDataAsync(PipeReader.Create(decompressed)); - valid = valid2; - version = version2; - } - catch (Exception) { } - - if (valid) - { - sb.AppendLine($""" - UPDATE id_entity - SET version = '{version}' - WHERE file_id = '{entity.Id.FileId}'; - """); - } - else - { - sb.AppendLine($""" - DELETE FROM id_entity - WHERE file_id = '{entity.Id.FileId}'; - DELETE FROM file_entity - WHERE file_id = '{entity.Id.FileId}'; - DELETE FROM json_entity - WHERE file_id = '{entity.Id.FileId}'; - """); - } - } - sb.AppendLine("COMMIT;"); - - await dbContext.Database.ExecuteSqlRawAsync(sb.ToString(), ct); - } - }); + Id = x.Id, + CrashReport = await new StreamReader(await _compressor.DecompressAsync(x.CrashReportCompressed, ct)).ReadToEndAsync(ct), + }).ToArrayAsync(ct); + await postgres.Set().AddRangeAsync(data, ct); + } } } \ No newline at end of file diff --git a/src/BUTR.CrashReportServer/Startup.cs b/src/BUTR.CrashReportServer/Startup.cs index 3e641d9..d41b4da 100644 --- a/src/BUTR.CrashReportServer/Startup.cs +++ b/src/BUTR.CrashReportServer/Startup.cs @@ -49,7 +49,8 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); services.AddHostedService(); - services.AddDbContextFactory(x => x.UseSqlite(_configuration.GetConnectionString("Main"))); + services.AddDbContextFactory(x => x.UseNpgsql(_configuration.GetConnectionString("Main"))); + services.AddDbContextFactory(x => x.UseSqlite(_configuration.GetConnectionString("Old"))); services.AddSwaggerGen(opt => {