Switching to postgres

BUTR · Feb 24, 2024 · 597bf54 · 597bf54
1 parent e3b0346
commit 597bf54
Show file tree

Hide file tree

Showing 10 changed files with 159 additions and 96 deletions.
diff --git a/src/BUTR.CrashReportServer/BUTR.CrashReportServer.csproj b/src/BUTR.CrashReportServer/BUTR.CrashReportServer.csproj
@@ -28,6 +28,7 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.2" />
     <PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.0" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.6" />
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.2" />
     <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.7.0" />
     <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.7.0" />
     <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.7.1" />

diff --git a/src/BUTR.CrashReportServer/Contexts/Config/JsonEntityConfiguration.cs b/src/BUTR.CrashReportServer/Contexts/Config/JsonEntityConfiguration.cs
@@ -10,7 +10,7 @@ public class JsonEntityConfiguration : BaseEntityConfiguration<JsonEntity>
     protected override void ConfigureModel(EntityTypeBuilder<JsonEntity> builder)
     {
         builder.Property<string>(nameof(IdEntity.FileId)).HasColumnName("file_id");
-        builder.Property(p => p.CrashReportCompressed).HasColumnName("data_compressed");
+        builder.Property(p => p.CrashReport).HasColumnName("data").HasColumnType("jsonb");
         builder.ToTable("json_entity").HasKey(nameof(IdEntity.FileId));
 
         builder.HasOne(x => x.Id)

diff --git a/src/BUTR.CrashReportServer/Contexts/OldAppDbContext.cs b/src/BUTR.CrashReportServer/Contexts/OldAppDbContext.cs
@@ -0,0 +1,69 @@
+using BUTR.CrashReportServer.Contexts.Config;
+using BUTR.CrashReportServer.Models.Database;
+
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace BUTR.CrashReportServer.Contexts;
+
+public class FileEntityConfiguration : BaseEntityConfiguration<FileEntity>
+{
+    protected override void ConfigureModel(EntityTypeBuilder<FileEntity> builder)
+    {
+        builder.Property<string>(nameof(IdEntity.FileId)).HasColumnName("file_id");
+        builder.Property(p => p.DataCompressed).HasColumnName("data_compressed");
+        builder.ToTable("file_entity").HasKey(nameof(IdEntity.FileId)).HasName("file_entity_pkey");
+
+        builder.HasOne(x => x.Id)
+            .WithOne()
+            .HasForeignKey<FileEntity>(nameof(IdEntity.FileId))
+            .HasPrincipalKey<IdEntity>(x => x.FileId)
+            .OnDelete(DeleteBehavior.Cascade);
+
+        builder.Navigation(x => x.Id).AutoInclude();
+    }
+}
+public class IdEntityConfiguration : BaseEntityConfiguration<IdEntity>
+{
+    protected override void ConfigureModel(EntityTypeBuilder<IdEntity> builder)
+    {
+        builder.Property(x => x.FileId).HasColumnName("file_id").HasDefaultValueSql("hex(randomblob(3))");
+        builder.Property(x => x.CrashReportId).HasColumnName("crash_report_id");
+        builder.Property(x => x.Version).HasColumnName("version");
+        builder.Property(x => x.Created).HasColumnName("created");
+        builder.ToTable("id_entity").HasKey(x => x.FileId);
+
+        builder.HasIndex(x => x.CrashReportId).IsUnique(false);
+    }
+}
+public class OldJsonEntityConfiguration : BaseEntityConfiguration<OldJsonEntity>
+{
+    protected override void ConfigureModel(EntityTypeBuilder<OldJsonEntity> builder)
+    {
+        builder.Property<string>(nameof(IdEntity.FileId)).HasColumnName("file_id");
+        builder.Property(p => p.CrashReportCompressed).HasColumnName("data_compressed");
+        builder.ToTable("json_entity").HasKey(nameof(IdEntity.FileId));
+
+        builder.HasOne(x => x.Id)
+            .WithOne()
+            .HasForeignKey<OldJsonEntity>(nameof(IdEntity.FileId))
+            .HasPrincipalKey<IdEntity>(x => x.FileId)
+            .OnDelete(DeleteBehavior.Cascade);
+
+        builder.Navigation(x => x.Id).AutoInclude();
+    }
+}
+
+public class OldAppDbContext : DbContext
+{
+    public OldAppDbContext(DbContextOptions<OldAppDbContext> options) : base(options) { }
+
+    protected override void OnModelCreating(ModelBuilder modelBuilder)
+    {
+        base.OnModelCreating(modelBuilder);
+
+        modelBuilder.ApplyConfiguration(new IdEntityConfiguration());
+        modelBuilder.ApplyConfiguration(new FileEntityConfiguration());
+        modelBuilder.ApplyConfiguration(new OldJsonEntityConfiguration());
+    }
+}
diff --git a/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs b/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs
@@ -36,6 +36,7 @@ public sealed record CrashReportUploadBody(CrashReportModel CrashReport, ICollec
     private readonly CrashUploadOptions _options;
     private readonly JsonSerializerOptions _jsonSerializerOptions;
     private readonly AppDbContext _dbContext;
+    private readonly OldAppDbContext _dbContextOld;
     private readonly GZipCompressor _gZipCompressor;
     private readonly HexGenerator _hexGenerator;
 
@@ -48,7 +49,7 @@ public CrashUploadController(
         AppDbContext dbContext,
         GZipCompressor gZipCompressor,
         HexGenerator hexGenerator,
-        IMeterFactory meterFactory)
+        IMeterFactory meterFactory, OldAppDbContext dbContextOld)
     {
         var meter = meterFactory.Create("BUTR.CrashReportServer.Controllers.CrashUploadController", "1.0.0");
 
@@ -58,6 +59,7 @@ public CrashUploadController(
         _jsonSerializerOptions = jsonSerializerOptions.Value ?? throw new ArgumentNullException(nameof(jsonSerializerOptions));
         _options = options.Value ?? throw new ArgumentNullException(nameof(options));
         _dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext));
+        _dbContextOld = dbContextOld ?? throw new ArgumentNullException(nameof(dbContextOld));
         _gZipCompressor = gZipCompressor ?? throw new ArgumentNullException(nameof(gZipCompressor));
         _hexGenerator = hexGenerator ?? throw new ArgumentNullException(nameof(hexGenerator));
     }
@@ -70,9 +72,13 @@ private string GenerateFileId(CancellationToken ct)
         {
             var fileIds = _hexGenerator.GetHex(count, 3);
             var existing = _dbContext.Set<IdEntity>().Select(x => x.FileId).Where(x => fileIds.Contains(x)).ToHashSet();
-            if (existing.Count == count) continue;
+            var existing2 = _dbContextOld.Set<IdEntity>().Select(x => x.FileId).Where(x => fileIds.Contains(x)).ToHashSet();
+            //if (existing.Count == count) continue;
             fileId = existing.First(x => !fileIds.Contains(x));
-            break;
+            fileIds.ExceptWith(existing);
+            fileIds.ExceptWith(existing2);
+            if (fileIds.Count == 0) break;
+            return fileIds.First();
         }
         return fileId;
     }
@@ -88,16 +94,16 @@ private async Task<IActionResult> UploadHtmlAsync(CancellationToken ct)
         if (await _dbContext.Set<IdEntity>().FirstOrDefaultAsync(x => x.CrashReportId == id, ct) is { } idEntity)
             return Ok($"{_options.BaseUri}/{idEntity.FileId}");
 
-        await using var compressedHtmlStream = await _gZipCompressor.CompressAsync(Request.Body, ct);
-        await using var compressedJsonStream = await _gZipCompressor.CompressAsync(JsonSerializer.SerializeToUtf8Bytes(crashReportModel, new JsonSerializerOptions(JsonSerializerDefaults.Web)
+        var json = JsonSerializer.Serialize(crashReportModel, new JsonSerializerOptions(JsonSerializerDefaults.Web)
         {
             Converters = { new JsonStringEnumConverter() }
-        }), ct);
+        });
+        await using var compressedHtmlStream = await _gZipCompressor.CompressAsync(Request.Body, ct);
 
         idEntity = new IdEntity { FileId = GenerateFileId(ct), CrashReportId = id, Version = version, Created = DateTime.UtcNow, };
         await _dbContext.Set<IdEntity>().AddAsync(idEntity, ct);
         await _dbContext.Set<FileEntity>().AddAsync(new FileEntity { Id = idEntity, DataCompressed = compressedHtmlStream.ToArray(), }, ct);
-        if (version >= 13) await _dbContext.Set<JsonEntity>().AddAsync(new JsonEntity { Id = idEntity, CrashReportCompressed = compressedJsonStream.ToArray(), }, ct);
+        if (version >= 13) await _dbContext.Set<JsonEntity>().AddAsync(new JsonEntity { Id = idEntity, CrashReport = json, }, ct);
         await _dbContext.SaveChangesAsync(ct);
 
         _reportVersion.Add(1, new[] { new KeyValuePair<string, object?>("Version", version) });
@@ -120,11 +126,10 @@ private async Task<IActionResult> UploadJsonAsync(CancellationToken ct)
         var html = CrashReportHtmlRenderer.AddData(CrashReportHtmlRenderer.Build(crashReport, logSources), json);
 
         await using var compressedHtmlStream = await _gZipCompressor.CompressAsync(html.AsStream(), ct);
-        await using var compressedJsonStream = await _gZipCompressor.CompressAsync(json.AsStream(), ct);
 
         idEntity = new IdEntity { FileId = GenerateFileId(ct), CrashReportId = crashReport.Id, Version = crashReport.Version, Created = DateTime.UtcNow, };
         await _dbContext.Set<IdEntity>().AddAsync(idEntity, ct);
-        await _dbContext.Set<JsonEntity>().AddAsync(new JsonEntity { Id = idEntity, CrashReportCompressed = compressedJsonStream.ToArray(), }, ct);
+        await _dbContext.Set<JsonEntity>().AddAsync(new JsonEntity { Id = idEntity, CrashReport = json, }, ct);
         await _dbContext.Set<FileEntity>().AddAsync(new FileEntity { Id = idEntity, DataCompressed = compressedHtmlStream.ToArray(), }, ct);
         await _dbContext.SaveChangesAsync(ct);
 

diff --git a/src/BUTR.CrashReportServer/Controllers/ReportController.cs b/src/BUTR.CrashReportServer/Controllers/ReportController.cs
@@ -21,6 +21,7 @@
 using System.Net;
 using System.Runtime.CompilerServices;
 using System.Security.Authentication;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -38,13 +39,15 @@ public sealed record GetNewCrashReportsBody
     private readonly ILogger _logger;
     private readonly ReportOptions _options;
     private readonly AppDbContext _dbContext;
+    private readonly OldAppDbContext _dbContextOld;
     private readonly GZipCompressor _gZipCompressor;
 
-    public ReportController(ILogger<ReportController> logger, IOptionsSnapshot<ReportOptions> options, AppDbContext dbContext, GZipCompressor gZipCompressor)
+    public ReportController(ILogger<ReportController> logger, IOptionsSnapshot<ReportOptions> options, AppDbContext dbContext, OldAppDbContext dbContextOld, GZipCompressor gZipCompressor)
     {
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         _options = options.Value ?? throw new ArgumentNullException(nameof(options));
         _dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext));
+        _dbContextOld = dbContextOld ?? throw new ArgumentNullException(nameof(dbContextOld));
         _gZipCompressor = gZipCompressor ?? throw new ArgumentNullException(nameof(gZipCompressor));
     }
 
@@ -75,31 +78,52 @@ private async Task<IActionResult> GetHtml(string filename, CancellationToken ct)
         if (ValidateRequest(ref filename) is { } errorResponse)
             return errorResponse;
 
-        if (await _dbContext.Set<FileEntity>().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct) is not { } file)
+        var file = await _dbContext.Set<FileEntity>().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct);
+        var fileOld = await _dbContextOld.Set<FileEntity>().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct);
+        if (file is null && fileOld is null)
             return StatusCode(StatusCodes.Status404NotFound);
 
         if (Request.GetTypedHeaders().AcceptEncoding.Any(x => x.Value.Equals("gzip", StringComparison.InvariantCultureIgnoreCase)))
         {
             Response.Headers.ContentEncoding = "gzip";
-            return File(file.DataCompressed, "text/html; charset=utf-8", true);
+            if (file is not null)
+                return File(file.DataCompressed, "text/html; charset=utf-8", true);
+            if (fileOld is not null)
+                return File(fileOld.DataCompressed, "text/html; charset=utf-8", true);
         }
-        return File(await _gZipCompressor.DecompressAsync(file.DataCompressed, ct), "text/html; charset=utf-8", true);
+        if (file is not null)
+            return File(await _gZipCompressor.DecompressAsync(file.DataCompressed, ct), "text/html; charset=utf-8", true);
+        if (fileOld is not null)
+            return File(await _gZipCompressor.DecompressAsync(fileOld.DataCompressed, ct), "text/html; charset=utf-8", true);
+
+        return StatusCode(StatusCodes.Status500InternalServerError);
     }
 
     private async Task<IActionResult> GetJson(string filename, CancellationToken ct)
     {
         if (ValidateRequest(ref filename) is { } errorResponse)
             return errorResponse;
 
-        if (await _dbContext.Set<JsonEntity>().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct) is not { } file)
+        var file = await _dbContext.Set<JsonEntity>().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct);
+        var fileOld = await _dbContextOld.Set<OldJsonEntity>().FirstOrDefaultAsync(x => x.Id.FileId == filename, ct);
+        if (file is null && fileOld is null)
             return StatusCode(StatusCodes.Status404NotFound);
 
         if (Request.GetTypedHeaders().AcceptEncoding.Any(x => x.Value.Equals("gzip", StringComparison.InvariantCultureIgnoreCase)))
         {
             Response.Headers.ContentEncoding = "gzip";
-            return File(file.CrashReportCompressed, "application/json; charset=utf-8", true);
+            if (file is not null)
+                return File(await _gZipCompressor.CompressAsync(new MemoryStream(Encoding.UTF8.GetBytes(file.CrashReport)), ct), "application/json; charset=utf-8", true);
+            if (fileOld is not null)
+                return File(fileOld.CrashReportCompressed, "application/json; charset=utf-8", true);
+
         }
-        return File(await _gZipCompressor.DecompressAsync(file.CrashReportCompressed, ct), "application/json; charset=utf-8", true);
+        if (file is not null)
+            return File(new MemoryStream(Encoding.UTF8.GetBytes(file.CrashReport)), "application/json; charset=utf-8", true);
+        if (fileOld is not null)
+            return File(await _gZipCompressor.DecompressAsync(fileOld.CrashReportCompressed, ct), "application/json; charset=utf-8", true);
+
+        return StatusCode(StatusCodes.Status500InternalServerError);
     }
 
     [AllowAnonymous]

diff --git a/src/BUTR.CrashReportServer/Extensions/HostExtensions.cs b/src/BUTR.CrashReportServer/Extensions/HostExtensions.cs
@@ -20,7 +20,7 @@ public static async Task<IHost> SeedDbContextAsync<TDbContext>(this IHost host)
         {
             var migrations = (await dbContext.Database.GetPendingMigrationsAsync()).Count();
             await dbContext.Database.MigrateAsync();
-            if (migrations > 0) await dbContext.Database.ExecuteSqlRawAsync("VACUUM;");
+            //if (migrations > 0) await dbContext.Database.ExecuteSqlRawAsync("VACUUM;");
         }
         catch (Exception ex)
         {

diff --git a/src/BUTR.CrashReportServer/Models/Database/JsonEntity.cs b/src/BUTR.CrashReportServer/Models/Database/JsonEntity.cs
@@ -3,5 +3,5 @@
 public sealed record JsonEntity : IEntity
 {
     public required IdEntity Id { get; set; }
-    public required byte[] CrashReportCompressed { get; set; }
+    public required string CrashReport { get; set; }
 }
diff --git a/src/BUTR.CrashReportServer/Models/Database/OldJsonEntity.cs b/src/BUTR.CrashReportServer/Models/Database/OldJsonEntity.cs
@@ -0,0 +1,7 @@
+namespace BUTR.CrashReportServer.Models.Database;
+
+public sealed record OldJsonEntity : IEntity
+{
+    public required IdEntity Id { get; set; }
+    public required byte[] CrashReportCompressed { get; set; }
+}