Added back postgres

BUTR · Feb 25, 2024 · cec75ab · cec75ab
1 parent d55db81
commit cec75ab
Show file tree

Hide file tree

Showing 30 changed files with 368 additions and 790 deletions.
diff --git a/src/BUTR.CrashReportServer/BUTR.CrashReportServer.csproj b/src/BUTR.CrashReportServer/BUTR.CrashReportServer.csproj
@@ -28,6 +28,7 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.2" />
     <PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.0" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.6" />
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.2" />
     <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.7.0" />
     <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.7.0" />
     <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.7.1" />

diff --git a/src/BUTR.CrashReportServer/Contexts/AppDbContext.cs b/src/BUTR.CrashReportServer/Contexts/AppDbContext.cs
@@ -1,11 +1,16 @@
 using BUTR.CrashReportServer.Contexts.Config;
+using BUTR.CrashReportServer.Models.Database;
 
 using Microsoft.EntityFrameworkCore;
 
 namespace BUTR.CrashReportServer.Contexts;
 
 public class AppDbContext : DbContext
 {
+    public DbSet<IdEntity> IdEntities { get; set; }
+    public DbSet<FileEntity> FileEntities { get; set; }
+    public DbSet<JsonEntity> JsonEntities { get; set; }
+
     public AppDbContext(DbContextOptions<AppDbContext> options) : base(options) { }
 
     protected override void OnModelCreating(ModelBuilder modelBuilder)

diff --git a/src/BUTR.CrashReportServer/Contexts/Config/FileEntityConfiguration.cs b/src/BUTR.CrashReportServer/Contexts/Config/FileEntityConfiguration.cs
@@ -9,13 +9,13 @@ public class FileEntityConfiguration : BaseEntityConfiguration<FileEntity>
 {
     protected override void ConfigureModel(EntityTypeBuilder<FileEntity> builder)
     {
-        builder.Property<string>(nameof(IdEntity.FileId)).HasColumnName("file_id");
-        builder.Property(p => p.DataCompressed).HasColumnName("data_compressed");
-        builder.ToTable("file_entity").HasKey(nameof(IdEntity.FileId)).HasName("file_entity_pkey");
+        builder.Property(x => x.FileId).HasColumnName("file_id");
+        builder.Property(x => x.DataCompressed).HasColumnName("data_compressed");
+        builder.ToTable("file_entity").HasKey(x => x.FileId).HasName("file_entity_pkey");
 
         builder.HasOne(x => x.Id)
             .WithOne()
-            .HasForeignKey<FileEntity>(nameof(IdEntity.FileId))
+            .HasForeignKey<FileEntity>(x => x.FileId)
             .HasPrincipalKey<IdEntity>(x => x.FileId)
             .OnDelete(DeleteBehavior.Cascade);
 

diff --git a/src/BUTR.CrashReportServer/Contexts/Config/IdEntityConfiguration.cs b/src/BUTR.CrashReportServer/Contexts/Config/IdEntityConfiguration.cs
@@ -9,7 +9,7 @@ public class IdEntityConfiguration : BaseEntityConfiguration<IdEntity>
 {
     protected override void ConfigureModel(EntityTypeBuilder<IdEntity> builder)
     {
-        builder.Property(x => x.FileId).HasColumnName("file_id").HasDefaultValueSql("hex(randomblob(3))");
+        builder.Property(x => x.FileId).HasColumnName("file_id");
         builder.Property(x => x.CrashReportId).HasColumnName("crash_report_id");
         builder.Property(x => x.Version).HasColumnName("version");
         builder.Property(x => x.Created).HasColumnName("created");

diff --git a/src/BUTR.CrashReportServer/Contexts/Config/JsonEntityConfiguration.cs b/src/BUTR.CrashReportServer/Contexts/Config/JsonEntityConfiguration.cs
@@ -9,13 +9,13 @@ public class JsonEntityConfiguration : BaseEntityConfiguration<JsonEntity>
 {
     protected override void ConfigureModel(EntityTypeBuilder<JsonEntity> builder)
     {
-        builder.Property<string>(nameof(IdEntity.FileId)).HasColumnName("file_id");
-        builder.Property(p => p.CrashReportCompressed).HasColumnName("data_compressed");
-        builder.ToTable("json_entity").HasKey(nameof(IdEntity.FileId));
+        builder.Property(x => x.FileId).HasColumnName("file_id");
+        builder.Property(x => x.CrashReport).HasColumnName("data").HasColumnType("jsonb");
+        builder.ToTable("json_entity").HasKey(x => x.FileId);
 
         builder.HasOne(x => x.Id)
             .WithOne()
-            .HasForeignKey<JsonEntity>(nameof(IdEntity.FileId))
+            .HasForeignKey<JsonEntity>(x => x.FileId)
             .HasPrincipalKey<IdEntity>(x => x.FileId)
             .OnDelete(DeleteBehavior.Cascade);
 

diff --git a/src/BUTR.CrashReportServer/Contexts/OldAppDbContext.cs b/src/BUTR.CrashReportServer/Contexts/OldAppDbContext.cs
@@ -0,0 +1,76 @@
+using BUTR.CrashReportServer.Contexts.Config;
+using BUTR.CrashReportServer.Models.Database;
+
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace BUTR.CrashReportServer.Contexts;
+
+public class OldFileEntityConfiguration : BaseEntityConfiguration<OldFileEntity>
+{
+    protected override void ConfigureModel(EntityTypeBuilder<OldFileEntity> builder)
+    {
+        builder.Property<string>(nameof(IdEntity.FileId)).HasColumnName("file_id");
+        builder.Property(p => p.DataCompressed).HasColumnName("data_compressed");
+        builder.ToTable("file_entity").HasKey(x => x.RowId);
+        builder.HasAlternateKey(nameof(IdEntity.FileId)).HasName("file_entity_pkey");
+
+        builder.HasOne(x => x.Id)
+            .WithOne()
+            .HasForeignKey<OldFileEntity>(nameof(IdEntity.FileId))
+            .HasPrincipalKey<OldIdEntity>(x => x.FileId)
+            .OnDelete(DeleteBehavior.Cascade);
+
+        builder.Navigation(x => x.Id).AutoInclude();
+    }
+}
+public class OldIdEntityConfiguration : BaseEntityConfiguration<OldIdEntity>
+{
+    protected override void ConfigureModel(EntityTypeBuilder<OldIdEntity> builder)
+    {
+        builder.Property(x => x.FileId).HasColumnName("file_id");
+        builder.Property(x => x.CrashReportId).HasColumnName("crash_report_id");
+        builder.Property(x => x.Version).HasColumnName("version");
+        builder.Property(x => x.Created).HasColumnName("created");
+        builder.ToTable("id_entity").HasKey(x => x.RowId);
+        builder.HasAlternateKey(x => x.FileId);
+
+        builder.HasIndex(x => x.CrashReportId).IsUnique(false);
+    }
+}
+public class OldJsonEntityConfiguration : BaseEntityConfiguration<OldJsonEntity>
+{
+    protected override void ConfigureModel(EntityTypeBuilder<OldJsonEntity> builder)
+    {
+        builder.Property<string>(nameof(IdEntity.FileId)).HasColumnName("file_id");
+        builder.Property(p => p.CrashReportCompressed).HasColumnName("data_compressed");
+        builder.ToTable("json_entity").HasKey(x => x.RowId);
+        builder.HasAlternateKey(nameof(IdEntity.FileId));
+
+        builder.HasOne(x => x.Id)
+            .WithOne()
+            .HasForeignKey<OldJsonEntity>(nameof(IdEntity.FileId))
+            .HasPrincipalKey<OldIdEntity>(x => x.FileId)
+            .OnDelete(DeleteBehavior.Cascade);
+
+        builder.Navigation(x => x.Id).AutoInclude();
+    }
+}
+
+public class OldAppDbContext : DbContext
+{
+    public DbSet<OldIdEntity> IdEntities { get; set; }
+    public DbSet<OldFileEntity> FileEntities { get; set; }
+    public DbSet<OldJsonEntity> JsonEntities { get; set; }
+
+    public OldAppDbContext(DbContextOptions<OldAppDbContext> options) : base(options) { }
+
+    protected override void OnModelCreating(ModelBuilder modelBuilder)
+    {
+        base.OnModelCreating(modelBuilder);
+
+        modelBuilder.ApplyConfiguration(new OldIdEntityConfiguration());
+        modelBuilder.ApplyConfiguration(new OldFileEntityConfiguration());
+        modelBuilder.ApplyConfiguration(new OldJsonEntityConfiguration());
+    }
+}
diff --git a/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs b/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs
@@ -17,7 +17,6 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.Metrics;
-using System.IO.Pipelines;
 using System.Linq;
 using System.Text.Json;
 using System.Text.Json.Serialization;
@@ -36,6 +35,7 @@ public sealed record CrashReportUploadBody(CrashReportModel CrashReport, ICollec
     private readonly CrashUploadOptions _options;
     private readonly JsonSerializerOptions _jsonSerializerOptions;
     private readonly AppDbContext _dbContext;
+    private readonly OldAppDbContext _dbContextOld;
     private readonly GZipCompressor _gZipCompressor;
     private readonly HexGenerator _hexGenerator;
 
@@ -48,7 +48,7 @@ public CrashUploadController(
         AppDbContext dbContext,
         GZipCompressor gZipCompressor,
         HexGenerator hexGenerator,
-        IMeterFactory meterFactory)
+        IMeterFactory meterFactory, OldAppDbContext dbContextOld)
     {
         var meter = meterFactory.Create("BUTR.CrashReportServer.Controllers.CrashUploadController", "1.0.0");
 
@@ -58,6 +58,7 @@ public CrashUploadController(
         _jsonSerializerOptions = jsonSerializerOptions.Value ?? throw new ArgumentNullException(nameof(jsonSerializerOptions));
         _options = options.Value ?? throw new ArgumentNullException(nameof(options));
         _dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext));
+        _dbContextOld = dbContextOld ?? throw new ArgumentNullException(nameof(dbContextOld));
         _gZipCompressor = gZipCompressor ?? throw new ArgumentNullException(nameof(gZipCompressor));
         _hexGenerator = hexGenerator ?? throw new ArgumentNullException(nameof(hexGenerator));
     }
@@ -69,10 +70,14 @@ private string GenerateFileId(CancellationToken ct)
         while (!ct.IsCancellationRequested)
         {
             var fileIds = _hexGenerator.GetHex(count, 3);
-            var existing = _dbContext.Set<IdEntity>().Select(x => x.FileId).Where(x => fileIds.Contains(x)).ToHashSet();
-            if (existing.Count == count) continue;
-            if (existing.Count == 0) return fileIds.First();
-            return existing.First(x => !fileIds.Contains(x));
+            var existing = _dbContext.IdEntities.Select(x => x.FileId).Where(x => fileIds.Contains(x)).ToHashSet();
+            var existing2 = _dbContextOld.IdEntities.Select(x => x.FileId).Where(x => fileIds.Contains(x)).ToHashSet();
+            //if (existing.Count == count) continue;
+            //fileId = existing.First(x => !fileIds.Contains(x));
+            fileIds.ExceptWith(existing);
+            fileIds.ExceptWith(existing2);
+            if (fileIds.Count == 0) continue;
+            return fileIds.First();
         }
         return fileId;
     }
@@ -81,23 +86,23 @@ private async Task<IActionResult> UploadHtmlAsync(CancellationToken ct)
     {
         Request.EnableBuffering();
 
-        var (valid, id, version, crashReportModel) = await CrashReportRawParser.TryReadCrashReportDataAsync(PipeReader.Create(Request.Body));
+        var (valid, id, version, crashReportModel) = await CrashReportRawParser.TryReadCrashReportDataAsync(Request.BodyReader, ct);
         if (!valid)
             return StatusCode(StatusCodes.Status500InternalServerError);
 
-        if (await _dbContext.Set<IdEntity>().FirstOrDefaultAsync(x => x.CrashReportId == id, ct) is { } idEntity)
+        if (await _dbContext.IdEntities.FirstOrDefaultAsync(x => x.CrashReportId == id, ct) is { } idEntity)
             return Ok($"{_options.BaseUri}/{idEntity.FileId}");
 
-        await using var compressedHtmlStream = await _gZipCompressor.CompressAsync(Request.Body, ct);
-        await using var compressedJsonStream = await _gZipCompressor.CompressAsync(JsonSerializer.SerializeToUtf8Bytes(crashReportModel, new JsonSerializerOptions(JsonSerializerDefaults.Web)
+        var json = JsonSerializer.Serialize(crashReportModel, new JsonSerializerOptions(JsonSerializerDefaults.Web)
         {
             Converters = { new JsonStringEnumConverter() }
-        }), ct);
+        });
+        await using var compressedHtmlStream = await _gZipCompressor.CompressAsync(Request.Body, ct);
 
         idEntity = new IdEntity { FileId = GenerateFileId(ct), CrashReportId = id, Version = version, Created = DateTime.UtcNow, };
-        await _dbContext.Set<IdEntity>().AddAsync(idEntity, ct);
-        await _dbContext.Set<FileEntity>().AddAsync(new FileEntity { Id = idEntity, DataCompressed = compressedHtmlStream.ToArray(), }, ct);
-        if (version >= 13) await _dbContext.Set<JsonEntity>().AddAsync(new JsonEntity { Id = idEntity, CrashReportCompressed = compressedJsonStream.ToArray(), }, ct);
+        await _dbContext.IdEntities.AddAsync(idEntity, ct);
+        await _dbContext.FileEntities.AddAsync(new FileEntity { FileId = idEntity.FileId, DataCompressed = compressedHtmlStream.ToArray(), }, ct);
+        if (version >= 13) await _dbContext.JsonEntities.AddAsync(new JsonEntity { FileId = idEntity.FileId, CrashReport = json, }, ct);
         await _dbContext.SaveChangesAsync(ct);
 
         _reportVersion.Add(1, new[] { new KeyValuePair<string, object?>("Version", version) });
@@ -110,7 +115,7 @@ private async Task<IActionResult> UploadJsonAsync(CancellationToken ct)
         if (await HttpContext.Request.ReadFromJsonAsync<CrashReportUploadBody>(_jsonSerializerOptions, ct) is not { CrashReport: { } crashReport, LogSources: { } logSources })
             return StatusCode(StatusCodes.Status500InternalServerError);
 
-        if (await _dbContext.Set<IdEntity>().FirstOrDefaultAsync(x => x.CrashReportId == crashReport.Id, ct) is { } idEntity)
+        if (await _dbContext.IdEntities.FirstOrDefaultAsync(x => x.CrashReportId == crashReport.Id, ct) is { } idEntity)
             return Ok($"{_options.BaseUri}/{idEntity.FileId}");
 
         var json = JsonSerializer.Serialize(crashReport, new JsonSerializerOptions(JsonSerializerDefaults.Web)
@@ -120,12 +125,11 @@ private async Task<IActionResult> UploadJsonAsync(CancellationToken ct)
         var html = CrashReportHtmlRenderer.AddData(CrashReportHtmlRenderer.Build(crashReport, logSources), json);
 
         await using var compressedHtmlStream = await _gZipCompressor.CompressAsync(html.AsStream(), ct);
-        await using var compressedJsonStream = await _gZipCompressor.CompressAsync(json.AsStream(), ct);
 
         idEntity = new IdEntity { FileId = GenerateFileId(ct), CrashReportId = crashReport.Id, Version = crashReport.Version, Created = DateTime.UtcNow, };
-        await _dbContext.Set<IdEntity>().AddAsync(idEntity, ct);
-        await _dbContext.Set<JsonEntity>().AddAsync(new JsonEntity { Id = idEntity, CrashReportCompressed = compressedJsonStream.ToArray(), }, ct);
-        await _dbContext.Set<FileEntity>().AddAsync(new FileEntity { Id = idEntity, DataCompressed = compressedHtmlStream.ToArray(), }, ct);
+        await _dbContext.IdEntities.AddAsync(idEntity, ct);
+        await _dbContext.JsonEntities.AddAsync(new JsonEntity { FileId = idEntity.FileId, CrashReport = json, }, ct);
+        await _dbContext.FileEntities.AddAsync(new FileEntity { FileId = idEntity.FileId, DataCompressed = compressedHtmlStream.ToArray(), }, ct);
         await _dbContext.SaveChangesAsync(ct);
 
         _reportVersion.Add(1, new[] {new KeyValuePair<string, object?>("Version", crashReport.Version)});