diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a7f7dd7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/scripts/*.csv +/scripts/*.log \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..32cfc61 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.watcherExclude": { + "**/target": true + } +} \ No newline at end of file diff --git a/extension/Cargo.toml b/extension/Cargo.toml index c5fe703..9093a29 100644 --- a/extension/Cargo.toml +++ b/extension/Cargo.toml @@ -20,8 +20,8 @@ pg_test = [] # pallas = "0.21" pallas = { git = "https://github.com/txpipe/pallas.git" } pgrx = "=0.11.3" -serde_json = "1.0.114" -serde = "1.0.197" +serde_json = "1.0.128" +serde = "1.0.209" hex = "0.4.3" bech32 = "0.9.1" chrono = "0.4.38" diff --git a/extension/src/lib.rs b/extension/src/lib.rs index a674d54..01a3301 100644 --- a/extension/src/lib.rs +++ b/extension/src/lib.rs @@ -197,7 +197,7 @@ fn tx_outputs( 'static, ( name!(output_index, i32), - name!(address, String), + name!(address, Option), name!(lovelace, pgrx::AnyNumeric), name!(assets, pgrx::Json), name!(datum, pgrx::Json), @@ -215,10 +215,7 @@ fn tx_outputs( .map(|(i, o)| { ( i as i32, - match o.address() { - Ok(addr) => addr.to_string(), - Err(_) => "ERROR PARSING ADDRESS".to_string(), - }, + output_to_address_string(o), AnyNumeric::from(o.lovelace_amount()), pgrx::Json( serde_json::to_value( @@ -270,10 +267,7 @@ fn tx_outputs_json(tx_cbor: &[u8]) -> pgrx::JsonB { .map(|(i, o)| { serde_json::json!({ "output_index": i as i32, - "address": match o.address() { - Ok(addr) => addr.to_string(), - Err(_) => "ERROR PARSING ADDRESS".to_string(), - }, + "address": output_to_address_string(o), "lovelace": o.lovelace_amount().to_string(), "assets": o.non_ada_assets() .iter() @@ -314,7 +308,7 @@ fn tx_addresses(tx_cbor: &[u8]) -> Vec> { let outputs_data = tx .outputs() .iter() - .map(|o| o.address().ok().map(|addr| addr.to_string())) + .map(|o| output_to_address_string(o)) .collect::>(); outputs_data @@ -362,10 +356,10 @@ fn tx_fee(tx_cbor: &[u8]) -> pgrx::AnyNumeric { } #[pg_extern(immutable)] -fn tx_mint(tx_cbor: &[u8]) -> pgrx::JsonB { +fn tx_mint(tx_cbor: &[u8]) -> Option { let tx = match MultiEraTx::decode(tx_cbor) { Ok(x) => x, - Err(_) => return pgrx::JsonB(serde_json::json!(null)), + Err(_) => return None, }; let mints = tx.mints(); @@ -384,7 +378,11 @@ fn tx_mint(tx_cbor: &[u8]) -> pgrx::JsonB { }) .collect(); - pgrx::JsonB(serde_json::json!(mint_data)) + if mint_data.is_empty() { + return None; + } + + Some(pgrx::JsonB(serde_json::json!(mint_data))) } #[pg_extern(immutable)] @@ -615,7 +613,7 @@ fn address_payment_part(address: &[u8]) -> Vec { let payment_part = match address { Address::Shelley(a) => a.payment().to_vec(), - Address::Byron(a) => { + Address::Byron(_) => { vec![] } _ => return vec![], @@ -633,7 +631,7 @@ fn address_stake_part(address: &[u8]) -> Vec { let stake_part = match address { Address::Shelley(a) => a.delegation().to_vec(), - Address::Byron(a) => { + Address::Byron(_) => { vec![] } _ => return vec![], @@ -903,25 +901,19 @@ fn utxo_subject_amount(era: i32, utxo_cbor: &[u8], subject: &[u8]) -> pgrx::AnyN } #[pg_extern(immutable)] -fn utxo_plutus_data(era: i32, utxo_cbor: &[u8]) -> pgrx::Json { - let era_enum = match pallas::ledger::traverse::Era::from_int(era) { - Some(x) => x, - None => return pgrx::Json(serde_json::json!(null)), - }; +fn utxo_plutus_data(era: i32, utxo_cbor: &[u8]) -> Option { + let era_enum = pallas::ledger::traverse::Era::from_int(era)?; - let output = match MultiEraOutput::decode(era_enum, utxo_cbor) { - Ok(x) => x, - Err(_) => return pgrx::Json(serde_json::json!(null)), - }; + let output = MultiEraOutput::decode(era_enum, utxo_cbor).ok()?; - match output.datum().unwrap() { - pallas::ledger::primitives::conway::PseudoDatumOption::Hash(_) => { - pgrx::Json(serde_json::json!(null)) - } - pallas::ledger::primitives::conway::PseudoDatumOption::Data(d) => { - pgrx::Json(d.unwrap().deref().to_json()) + output.datum().and_then(|datum_option| { + match datum_option { + pallas::ledger::primitives::conway::PseudoDatumOption::Hash(_) => None, + pallas::ledger::primitives::conway::PseudoDatumOption::Data(d) => { + Some(pgrx::Json(d.deref().to_json())) + } } - } + }) } #[pg_extern(immutable)] @@ -940,6 +932,21 @@ fn from_bech32(bech32: &str) -> Vec { } } +fn output_to_address_string(output: &MultiEraOutput) -> Option { + match output.address() { + Ok(addr) => { + let addr_str = addr.to_string(); + // Truncate the address to 103 characters typical HEADER, PAYMENT PART and DELEGATION PART encoding in bech32 + if addr_str.len() > 103 { + Some(addr_str[..103].to_string()) + } else { + Some(addr_str) + } + } + Err(_) => None, + } +} + #[cfg(any(test, feature = "pg_test"))] #[pg_schema] mod tests { diff --git a/indexer/.gitignore b/indexer/.gitignore new file mode 100644 index 0000000..8d4a6c0 --- /dev/null +++ b/indexer/.gitignore @@ -0,0 +1,2 @@ +bin +obj \ No newline at end of file diff --git a/indexer/Data/MumakDbContext.cs b/indexer/Data/MumakDbContext.cs new file mode 100644 index 0000000..42f01ed --- /dev/null +++ b/indexer/Data/MumakDbContext.cs @@ -0,0 +1,39 @@ +using Cardano.Sync.Data; +using Cardano.Sync.Data.Models; +using Cardano.Sync.Reducers; +using Microsoft.EntityFrameworkCore; + +namespace Mumak.Indexer.Data; + +public class MumakDbContext +( + DbContextOptions options, + IConfiguration configuration +) : CardanoDbContext(options, configuration) +{ + public DbSet Utxos { get; set; } + + override protected void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + modelBuilder.Entity(entity => + { + entity.ToTable("utxos"); + + entity.HasKey(e => new { e.Slot, e.Id, e.TxIndex }); + + entity.Property(e => e.Id) + .HasColumnName("id"); + + entity.Property(e => e.TxIndex) + .HasColumnName("tx_index"); + + entity.Property(e => e.Slot) + .HasColumnName("slot"); + + entity.Property(e => e.Raw) + .HasColumnName("raw"); + }); + } +} diff --git a/indexer/Data/Utxo.cs b/indexer/Data/Utxo.cs new file mode 100644 index 0000000..c946dc6 --- /dev/null +++ b/indexer/Data/Utxo.cs @@ -0,0 +1,10 @@ +using Cardano.Sync.Data.Models; + +namespace Mumak.Indexer.Data; + +public record Utxo( + string Id, + ulong TxIndex, + ulong Slot, + byte[] Raw +) : IReducerModel; \ No newline at end of file diff --git a/indexer/Migrations/20240910100707_InitialCreate.Designer.cs b/indexer/Migrations/20240910100707_InitialCreate.Designer.cs new file mode 100644 index 0000000..7090fee --- /dev/null +++ b/indexer/Migrations/20240910100707_InitialCreate.Designer.cs @@ -0,0 +1,71 @@ +// +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Mumak.Indexer.Data; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Mumak.Indexer.Migrations +{ + [DbContext(typeof(MumakDbContext))] + [Migration("20240910100707_InitialCreate")] + partial class InitialCreate + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("public") + .HasAnnotation("ProductVersion", "8.0.8") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Cardano.Sync.Data.Models.ReducerState", b => + { + b.Property("Name") + .HasColumnType("text"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Slot") + .HasColumnType("numeric(20,0)"); + + b.HasKey("Name"); + + b.ToTable("ReducerStates", "public"); + }); + + modelBuilder.Entity("Mumak.Indexer.Data.Utxo", b => + { + b.Property("Slot") + .HasColumnType("numeric(20,0)") + .HasColumnName("slot"); + + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("TxIndex") + .HasColumnType("numeric(20,0)") + .HasColumnName("tx_index"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("raw"); + + b.HasKey("Slot", "Id", "TxIndex"); + + b.ToTable("utxos", "public"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/indexer/Migrations/20240910100707_InitialCreate.cs b/indexer/Migrations/20240910100707_InitialCreate.cs new file mode 100644 index 0000000..551fe1b --- /dev/null +++ b/indexer/Migrations/20240910100707_InitialCreate.cs @@ -0,0 +1,58 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Mumak.Indexer.Migrations +{ + /// + public partial class InitialCreate : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.EnsureSchema( + name: "public"); + + migrationBuilder.CreateTable( + name: "ReducerStates", + schema: "public", + columns: table => new + { + Name = table.Column(type: "text", nullable: false), + Slot = table.Column(type: "numeric(20,0)", nullable: false), + Hash = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ReducerStates", x => x.Name); + }); + + migrationBuilder.CreateTable( + name: "utxos", + schema: "public", + columns: table => new + { + id = table.Column(type: "text", nullable: false), + tx_index = table.Column(type: "numeric(20,0)", nullable: false), + slot = table.Column(type: "numeric(20,0)", nullable: false), + raw = table.Column(type: "bytea", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_utxos", x => new { x.slot, x.id, x.tx_index }); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "ReducerStates", + schema: "public"); + + migrationBuilder.DropTable( + name: "utxos", + schema: "public"); + } + } +} diff --git a/indexer/Migrations/MumakDbContextModelSnapshot.cs b/indexer/Migrations/MumakDbContextModelSnapshot.cs new file mode 100644 index 0000000..a256b83 --- /dev/null +++ b/indexer/Migrations/MumakDbContextModelSnapshot.cs @@ -0,0 +1,68 @@ +// +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Mumak.Indexer.Data; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Mumak.Indexer.Migrations +{ + [DbContext(typeof(MumakDbContext))] + partial class MumakDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("public") + .HasAnnotation("ProductVersion", "8.0.8") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Cardano.Sync.Data.Models.ReducerState", b => + { + b.Property("Name") + .HasColumnType("text"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Slot") + .HasColumnType("numeric(20,0)"); + + b.HasKey("Name"); + + b.ToTable("ReducerStates", "public"); + }); + + modelBuilder.Entity("Mumak.Indexer.Data.Utxo", b => + { + b.Property("Slot") + .HasColumnType("numeric(20,0)") + .HasColumnName("slot"); + + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("TxIndex") + .HasColumnType("numeric(20,0)") + .HasColumnName("tx_index"); + + b.Property("Raw") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("raw"); + + b.HasKey("Slot", "Id", "TxIndex"); + + b.ToTable("utxos", "public"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/indexer/Mumak.Indexer.csproj b/indexer/Mumak.Indexer.csproj new file mode 100644 index 0000000..b2de58e --- /dev/null +++ b/indexer/Mumak.Indexer.csproj @@ -0,0 +1,20 @@ + + + + net8.0 + enable + enable + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + diff --git a/indexer/Mumak.Indexer.sln b/indexer/Mumak.Indexer.sln new file mode 100644 index 0000000..5ed5cb3 --- /dev/null +++ b/indexer/Mumak.Indexer.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Mumak.Indexer", "Mumak.Indexer.csproj", "{A97E6506-13AC-4225-A72C-2D17DA62E44B}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {A97E6506-13AC-4225-A72C-2D17DA62E44B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A97E6506-13AC-4225-A72C-2D17DA62E44B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A97E6506-13AC-4225-A72C-2D17DA62E44B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A97E6506-13AC-4225-A72C-2D17DA62E44B}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {9AB7B189-9B67-4292-9C6D-25BCF25F4D02} + EndGlobalSection +EndGlobal diff --git a/indexer/Program.cs b/indexer/Program.cs new file mode 100644 index 0000000..abe0691 --- /dev/null +++ b/indexer/Program.cs @@ -0,0 +1,27 @@ +using Mumak.Indexer.Data; +using Cardano.Sync.Extensions; +using Cardano.Sync.Reducers; +using Cardano.Sync.Data.Models; +using Mumak.Indexer.Reducers; + +var builder = WebApplication.CreateBuilder(args); + +// Add services to the container. +// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddSwaggerGen(); + +builder.Services.AddCardanoIndexer(builder.Configuration); +builder.Services.AddSingleton, UtxoReducer>(); + +var app = builder.Build(); + +// Configure the HTTP request pipeline. +if (app.Environment.IsDevelopment()) +{ + app.UseSwagger(); + app.UseSwaggerUI(); +} + + +app.Run(); \ No newline at end of file diff --git a/indexer/Properties/launchSettings.json b/indexer/Properties/launchSettings.json new file mode 100644 index 0000000..e7c311b --- /dev/null +++ b/indexer/Properties/launchSettings.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:53008", + "sslPort": 44346 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "http://localhost:5262", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "https://localhost:7222;http://localhost:5262", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/indexer/Reducers/UtxoReducer.cs b/indexer/Reducers/UtxoReducer.cs new file mode 100644 index 0000000..9a25eed --- /dev/null +++ b/indexer/Reducers/UtxoReducer.cs @@ -0,0 +1,54 @@ +using System.Linq.Expressions; +using Cardano.Sync.Extensions; +using Cardano.Sync.Reducers; +using Microsoft.EntityFrameworkCore; +using Mumak.Indexer.Data; +using PallasDotnet.Models; + +namespace Mumak.Indexer.Reducers; + +public class UtxoReducer(IDbContextFactory dbContextFactory) : IReducer +{ + public async Task RollBackwardAsync(NextResponse response) + { + using MumakDbContext dbContext = await dbContextFactory.CreateDbContextAsync(); + dbContext.Utxos.RemoveRange(dbContext.Utxos.AsNoTracking().Where(b => b.Slot > response.Block.Slot)); + await dbContext.SaveChangesAsync(); + dbContext.Dispose(); + } + + public async Task RollForwardAsync(NextResponse response) + { + using MumakDbContext dbContext = await dbContextFactory.CreateDbContextAsync(); + + Expression> predicate = PredicateBuilder.False(); + + var inputs = response.Block.TransactionBodies + .SelectMany(tx => tx.Inputs) + .Select(input => new { input.Id, input.Index }) + .ToList(); + + inputs.ForEach(input => + { + predicate = predicate.Or(u => u.Id == input.Id.ToHex() && u.TxIndex == input.Index); + }); + + dbContext.Utxos.RemoveRange(dbContext.Utxos.AsNoTracking().Where(predicate)); + + response.Block.TransactionBodies.ToList() + .ForEach(tx => tx.Outputs. + ToList() + .ForEach(output => + { + dbContext.Utxos.Add(new Utxo( + tx.Id.ToHex(), + output.Index, + response.Block.Slot, + output.Raw + )); + }) + ); + + await dbContext.SaveChangesAsync(); + } +} \ No newline at end of file diff --git a/indexer/appsettings.json b/indexer/appsettings.json new file mode 100644 index 0000000..3c691bd --- /dev/null +++ b/indexer/appsettings.json @@ -0,0 +1,17 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "ConnectionStrings": { + "CardanoContext": "Host=localhost;Database=postgres;Username=postgres;Password=Test1234;Port=5432", + "CardanoContextSchema":"public" + }, + "CardanoNodeSocketPath": "/home/rawriclark/.dmtr/tmp/freezing-reaction-6db5b4/mainnet-stable.socket", + "CardanoNetworkMagic": 764824073, + "CardanoIndexStartSlot": 134314450, + "CardanoIndexStartHash": "515a8b1f3b4f133ba2a4fdaea4cddae031a2b5e7727d0c3957d0c8d2008a5911" +}