diff --git a/Cargo.lock b/Cargo.lock index d38ec075..2927d1ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,15 +364,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "bip39-dict" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f1d227703899f704884cb6dfe1dc6a1bd447ea9f91418539989618ebf01685" -dependencies = [ - "cryptoxide", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -498,21 +489,6 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" -[[package]] -name = "cardano-net" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e272ed83035e509fa204c1cf59833c9384cbd4d15d4e303a5abd529b1bd024" -dependencies = [ - "cardano-sdk", - "cbored", - "cryptoxide", - "thiserror", - "tokio", - "tracing", - "trust-dns-resolver", -] - [[package]] name = "cardano-projected-nft" version = "0.1.0" @@ -529,22 +505,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "cardano-sdk" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67795439ca871ba2b0d67f7805248f00fd0e6f6b40014a07a9693568e9c54a6" -dependencies = [ - "bech32 0.9.1", - "bip39-dict", - "cbored", - "cryptoxide", - "ed25519-bip32", - "hex", - "strum 0.24.1", - "thiserror", -] - [[package]] name = "carp" version = "3.2.1" @@ -585,9 +545,6 @@ name = "cbored" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54198fe95600b062b9f0129b871b8e8df90b33a41cf2ccbae64b4ae08dbba9c1" -dependencies = [ - "cbored-derive", -] [[package]] name = "cbored-derive" @@ -764,8 +721,7 @@ dependencies = [ [[package]] name = "cml-chain" version = "6.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "148c24010a72a0d8cb4f65ddfe9e38bc3a31ddb5699a7b7ca90408d9c858aaa5" +source = "git+https://github.com/dcSpark/cardano-multiplatform-lib?rev=b7acbd3634f5ba8402a9704f0413bd434ed157c3#b7acbd3634f5ba8402a9704f0413bd434ed157c3" dependencies = [ "base64 0.21.7", "bech32 0.7.3", @@ -845,8 +801,7 @@ dependencies = [ [[package]] name = "cml-core" version = "6.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0275489215ce6de487151721b7fe79c52a52d0d9e8d75fd7609b7e818b85b4a0" +source = "git+https://github.com/dcSpark/cardano-multiplatform-lib?rev=b7acbd3634f5ba8402a9704f0413bd434ed157c3#b7acbd3634f5ba8402a9704f0413bd434ed157c3" dependencies = [ "base64 0.13.1", "bech32 0.7.3", @@ -917,8 +872,7 @@ dependencies = [ [[package]] name = "cml-crypto" version = "6.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebc51fce7179a5a272a589fb41bd5a8c029b204182640d651d4beb06e65fd64" +source = "git+https://github.com/dcSpark/cardano-multiplatform-lib?rev=b7acbd3634f5ba8402a9704f0413bd434ed157c3#b7acbd3634f5ba8402a9704f0413bd434ed157c3" dependencies = [ "base64 0.21.7", "bech32 0.7.3", @@ -960,8 +914,7 @@ dependencies = [ [[package]] name = "cml-multi-era" version = "6.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f250aedb4c53625d6c3c5cb80340c7a4b78383821b97854097eab7a84c8e79" +source = "git+https://github.com/dcSpark/cardano-multiplatform-lib?rev=b7acbd3634f5ba8402a9704f0413bd434ed157c3#b7acbd3634f5ba8402a9704f0413bd434ed157c3" dependencies = [ "bech32 0.7.3", "cbor_event", @@ -1157,21 +1110,13 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "data-encoding" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" - [[package]] name = "dcspark-blockchain-source" version = "0.1.0" -source = "git+https://github.com/dcSpark/dcspark-core.git?rev=176ea4830d7c3d00eca1c0a4246e9b364b889851#176ea4830d7c3d00eca1c0a4246e9b364b889851" +source = "git+https://github.com/dcSpark/dcspark-core.git?rev=ebb245ac047f9d45dba97f07a5bb525ffd81a539#ebb245ac047f9d45dba97f07a5bb525ffd81a539" dependencies = [ "anyhow", "async-trait", - "cardano-net", - "cardano-sdk", "cbored", "cbored-derive", "cml-chain 6.0.0", @@ -1182,6 +1127,7 @@ dependencies = [ "deps", "hex", "multiverse", + "pallas-network", "serde", "thiserror", "tokio", @@ -1191,7 +1137,7 @@ dependencies = [ [[package]] name = "dcspark-core" version = "0.1.0" -source = "git+https://github.com/dcSpark/dcspark-core.git?rev=176ea4830d7c3d00eca1c0a4246e9b364b889851#176ea4830d7c3d00eca1c0a4246e9b364b889851" +source = "git+https://github.com/dcSpark/dcspark-core.git?rev=ebb245ac047f9d45dba97f07a5bb525ffd81a539#ebb245ac047f9d45dba97f07a5bb525ffd81a539" dependencies = [ "anyhow", "async-trait", @@ -1208,7 +1154,7 @@ dependencies = [ [[package]] name = "deps" version = "0.1.0" -source = "git+https://github.com/dcSpark/dcspark-core.git?rev=176ea4830d7c3d00eca1c0a4246e9b364b889851#176ea4830d7c3d00eca1c0a4246e9b364b889851" +source = "git+https://github.com/dcSpark/dcspark-core.git?rev=ebb245ac047f9d45dba97f07a5bb525ffd81a539#ebb245ac047f9d45dba97f07a5bb525ffd81a539" dependencies = [ "bigdecimal", "serde_json", @@ -1320,18 +1266,6 @@ dependencies = [ "serde", ] -[[package]] -name = "enum-as-inner" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "env_logger" version = "0.10.2" @@ -1711,17 +1645,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "hostname" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" -dependencies = [ - "libc", - "match_cfg", - "winapi", -] - [[package]] name = "humantime" version = "2.1.0" @@ -1751,17 +1674,6 @@ dependencies = [ "cc", ] -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.5.0" @@ -1833,24 +1745,6 @@ dependencies = [ "ghost", ] -[[package]] -name = "ipconfig" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" -dependencies = [ - "socket2", - "widestring", - "windows-sys 0.48.0", - "winreg", -] - -[[package]] -name = "ipnet" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" - [[package]] name = "is-terminal" version = "0.4.13" @@ -1963,26 +1857,11 @@ dependencies = [ "value-bag", ] -[[package]] -name = "lru-cache" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "markdown-gen" version = "1.2.1" source = "git+https://github.com/dcSpark/markdown-gen-rs?branch=hbina-add-ability-to-write-raw-str#06342df71812111825ab8030b77fe8726e082252" -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - [[package]] name = "matchers" version = "0.1.0" @@ -1992,12 +1871,6 @@ dependencies = [ "regex-automata 0.1.10", ] -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "md-5" version = "0.10.6" @@ -2123,7 +1996,7 @@ checksum = "a785740271256c230f57462d3b83e52f998433a7062fc18f96d5999474a9f915" [[package]] name = "multiverse" version = "0.1.0" -source = "git+https://github.com/dcSpark/dcspark-core.git?rev=176ea4830d7c3d00eca1c0a4246e9b364b889851#176ea4830d7c3d00eca1c0a4246e9b364b889851" +source = "git+https://github.com/dcSpark/dcspark-core.git?rev=ebb245ac047f9d45dba97f07a5bb525ffd81a539#ebb245ac047f9d45dba97f07a5bb525ffd81a539" dependencies = [ "dcspark-core", "deps", @@ -2493,6 +2366,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "pallas-network" +version = "0.30.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c081efb304917ec0025182e11295c6e1727b779888b83cc93d89533c71db50c" +dependencies = [ + "byteorder", + "hex", + "itertools 0.13.0", + "pallas-codec 0.30.2", + "pallas-crypto", + "rand", + "socket2", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "pallas-primitives" version = "0.30.2" @@ -2760,12 +2651,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quote" version = "1.0.37" @@ -2941,16 +2826,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "resolv-conf" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" -dependencies = [ - "hostname", - "quick-error", -] - [[package]] name = "ring" version = "0.16.20" @@ -3708,9 +3583,6 @@ name = "strum" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" -dependencies = [ - "strum_macros 0.24.3", -] [[package]] name = "strum" @@ -4141,51 +4013,6 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "trust-dns-proto" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d" -dependencies = [ - "async-trait", - "cfg-if 1.0.0", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna 0.2.3", - "ipnet", - "lazy_static", - "log", - "rand", - "smallvec", - "thiserror", - "tinyvec", - "tokio", - "url", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558" -dependencies = [ - "cfg-if 1.0.0", - "futures-util", - "ipconfig", - "lazy_static", - "log", - "lru-cache", - "parking_lot 0.12.3", - "resolv-conf", - "smallvec", - "thiserror", - "tokio", - "trust-dns-proto", -] - [[package]] name = "tynm" version = "0.1.10" @@ -4274,7 +4101,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna", "percent-encoding", ] @@ -4444,12 +4271,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "widestring" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" - [[package]] name = "winapi" version = "0.3.9" @@ -4647,16 +4468,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if 1.0.0", - "windows-sys 0.48.0", -] - [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index f1e3faa4..8084624e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,12 @@ members = [ [workspace.dependencies] -cml-chain = { version = "6.0.0" } -cml-core = { version = "6.0.0" } -cml-crypto = { version = "6.0.0" } -cml-multi-era = { version = "6.0.0" } +# cml-chain = { version = "6.0.0" } +# cml-core = { version = "6.0.0" } +# cml-crypto = { version = "6.0.0" } +# cml-multi-era = { version = "6.0.0" } + +cml-chain = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", rev = "b7acbd3634f5ba8402a9704f0413bd434ed157c3" } +cml-core = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", rev = "b7acbd3634f5ba8402a9704f0413bd434ed157c3" } +cml-crypto = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", rev = "b7acbd3634f5ba8402a9704f0413bd434ed157c3" } +cml-multi-era = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", rev = "b7acbd3634f5ba8402a9704f0413bd434ed157c3" } diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index a62916d3..c350d6b1 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -9,9 +9,9 @@ strip = true [dependencies] # [core] -dcspark-core = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "176ea4830d7c3d00eca1c0a4246e9b364b889851" } -dcspark-blockchain-source = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "176ea4830d7c3d00eca1c0a4246e9b364b889851" } -multiverse = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "176ea4830d7c3d00eca1c0a4246e9b364b889851" } +dcspark-core = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "ebb245ac047f9d45dba97f07a5bb525ffd81a539" } +dcspark-blockchain-source = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "ebb245ac047f9d45dba97f07a5bb525ffd81a539" } +multiverse = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "ebb245ac047f9d45dba97f07a5bb525ffd81a539" } # [local] entity = { path = "entity" } diff --git a/indexer/configs/custom.yml b/indexer/configs/custom.yml index 9ab0870f..ee14d5ef 100644 --- a/indexer/configs/custom.yml +++ b/indexer/configs/custom.yml @@ -18,21 +18,6 @@ sink: relay: - "localhost" - 3001 - from: - BlockHeader: - slot_nb: 1 - hash: "ba8066f73eb9cf4ad9adf38051c54d3a51d92cb98561cffc1f202b1b97739cd5" - genesis_parent: "0ded594a3411f6d3236228abc1e2ef8c2a21e09d859ea23bfc2182f92853cba8" - genesis: - BlockHeader: - slot_nb: 0 - hash: "7a32184d9e0068b0fa75fd0ecaad798f9bc573d4921c519b12968e26ff0747a3" - shelley_era_config: - first_slot: 0 - start_epoch: 0 - known_time: 1722355346 - slot_length: 1 - epoch_length_seconds: 500 start_block: diff --git a/indexer/entity/src/block.rs b/indexer/entity/src/block.rs index 7b04d664..6e34327e 100644 --- a/indexer/entity/src/block.rs +++ b/indexer/entity/src/block.rs @@ -69,3 +69,17 @@ impl TryFrom for EraValue { } } } + +impl EraValue { + pub fn to_str(&self) -> &'static str { + match self { + EraValue::Byron => "byron", + EraValue::Shelley => "shelley", + EraValue::Allegra => "allegra", + EraValue::Mary => "mary", + EraValue::Alonzo => "alonzo", + EraValue::Babbage => "babbage", + EraValue::Conway => "conway", + } + } +} diff --git a/indexer/entity/src/genesis.rs b/indexer/entity/src/genesis.rs new file mode 100644 index 00000000..1e0aa831 --- /dev/null +++ b/indexer/entity/src/genesis.rs @@ -0,0 +1,32 @@ +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Deserialize, Serialize)] +#[sea_orm(table_name = "Era")] +pub struct Model { + #[sea_orm(primary_key)] + pub era: i32, + pub block_id: i32, + pub block_height: i32, + pub first_slot: i64, + pub start_epoch: i64, + pub epoch_length_seconds: i64, +} + +#[derive(Copy, Clone, Debug, DeriveRelation, EnumIter)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::block::Entity", + from = "Column::BlockId", + to = "super::block::Column::Id" + )] + Block, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Block.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/indexer/entity/src/lib.rs b/indexer/entity/src/lib.rs index 136153ef..059a7d50 100644 --- a/indexer/entity/src/lib.rs +++ b/indexer/entity/src/lib.rs @@ -13,12 +13,12 @@ pub mod asset_mint; pub mod asset_utxos; pub mod cip25_entry; pub mod dex_swap; +pub mod genesis; pub mod governance_votes; pub mod native_asset; pub mod plutus_data; pub mod plutus_data_hash; pub mod projected_nft; -// todo: rename to pool? pub mod stake_delegation; pub mod stake_delegation_drep; pub mod transaction_metadata; diff --git a/indexer/entity/src/prelude.rs b/indexer/entity/src/prelude.rs index ee2e1a58..d33377be 100644 --- a/indexer/entity/src/prelude.rs +++ b/indexer/entity/src/prelude.rs @@ -27,6 +27,10 @@ pub use super::dex_swap::{ ActiveModel as DexSwapActiveModel, Column as DexSwapColumn, Entity as DexSwap, Model as DexSwapModel, PrimaryKey as DexSwapPrimaryKey, Relation as DexSwapRelation, }; +pub use super::genesis::{ + ActiveModel as GenesisActiveModel, Column as GenesisColumn, Entity as Genesis, + Model as GenesisModel, PrimaryKey as GenesisPrimaryKey, Relation as GenesisRelation, +}; pub use super::governance_votes::{ ActiveModel as GovernanceVoteActiveModel, Column as GovernanceVoteColumn, Entity as GovernanceVote, Model as GovernanceVoteModel, PrimaryKey as GovernanceVotePrimaryKey, diff --git a/indexer/execution_plans/default.toml b/indexer/execution_plans/default.toml index 0a08c3ca..d50b81c7 100644 --- a/indexer/execution_plans/default.toml +++ b/indexer/execution_plans/default.toml @@ -14,6 +14,8 @@ include_payload=false [GenesisTransactionTask] include_payload=true +[ShelleyGenesisBlockTask] + [ByronBlockTask] readonly=false include_payload=false diff --git a/indexer/migration/src/lib.rs b/indexer/migration/src/lib.rs index ddde20b0..5a8e8543 100644 --- a/indexer/migration/src/lib.rs +++ b/indexer/migration/src/lib.rs @@ -23,6 +23,7 @@ mod m20231220_000018_asset_utxo_table; mod m20240229_000019_add_block_tx_count_column; mod m20240326_000020_create_drep_delegation_table; mod m20240326_000021_create_governance_voting_table; +mod m20240920_000022_create_genesis_tracking_table; pub struct Migrator; @@ -53,6 +54,7 @@ impl MigratorTrait for Migrator { Box::new(m20240229_000019_add_block_tx_count_column::Migration), Box::new(m20240326_000020_create_drep_delegation_table::Migration), Box::new(m20240326_000021_create_governance_voting_table::Migration), + Box::new(m20240920_000022_create_genesis_tracking_table::Migration), ] } } diff --git a/indexer/migration/src/m20240920_000022_create_genesis_tracking_table.rs b/indexer/migration/src/m20240920_000022_create_genesis_tracking_table.rs new file mode 100644 index 00000000..c566e2ff --- /dev/null +++ b/indexer/migration/src/m20240920_000022_create_genesis_tracking_table.rs @@ -0,0 +1,56 @@ +use sea_schema::migration::prelude::*; + +use entity::genesis::*; +use entity::prelude::{Block, BlockColumn}; + +pub struct Migration; + +impl MigrationName for Migration { + fn name(&self) -> &str { + "m20240920_000022_create_genesis_tracking_table" + } +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Entity) + .if_not_exists() + .col( + ColumnDef::new(Column::Era) + .integer() + .not_null() + .primary_key(), + ) + .col(ColumnDef::new(Column::BlockId).integer().not_null()) + .col(ColumnDef::new(Column::BlockHeight).integer().not_null()) + .col(ColumnDef::new(Column::FirstSlot).big_integer().not_null()) + .col(ColumnDef::new(Column::StartEpoch).big_integer().not_null()) + .col( + ColumnDef::new(Column::EpochLengthSeconds) + .big_integer() + .not_null(), + ) + .foreign_key( + ForeignKey::create() + .name("fk-transaction-block_id") + .from(Entity, Column::BlockId) + .to(Block, BlockColumn::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(Entity).to_owned()) + .await + } +} diff --git a/indexer/src/engine.rs b/indexer/src/engine.rs index d8b74fc1..27bb4316 100644 --- a/indexer/src/engine.rs +++ b/indexer/src/engine.rs @@ -3,7 +3,6 @@ use crate::sink::Sink; use crate::types::StoppableService; use async_trait::async_trait; use dcspark_blockchain_source::{GetNextFrom, PullFrom, Source}; -use entity::block::EraValue; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::SeqCst; use std::sync::Arc; @@ -45,9 +44,6 @@ impl< let mut perf_aggregator = PerfAggregator::new(); - // TODO: fetch from DB - let mut latest_era: Option = None; - while self.running.load(SeqCst) { let event_fetch_start = std::time::Instant::now(); let event = self.source.pull(&pull_from).await?; @@ -59,9 +55,7 @@ impl< }; perf_aggregator.block_fetch += event_fetch_start.elapsed(); let new_from = event.next_from().unwrap_or(pull_from); - self.sink - .process(event, &mut perf_aggregator, &mut latest_era) - .await?; + self.sink.process(event, &mut perf_aggregator).await?; pull_from = new_from; } diff --git a/indexer/src/genesis.rs b/indexer/src/genesis.rs index 53314dda..54e18e7a 100644 --- a/indexer/src/genesis.rs +++ b/indexer/src/genesis.rs @@ -1,4 +1,4 @@ -use anyhow::anyhow; +use anyhow::{anyhow, Context as _}; use entity::block::EraValue; use std::fs; use std::sync::{Arc, Mutex}; @@ -23,7 +23,7 @@ pub async fn process_byron_genesis( tracing::info!("Parsing genesis file..."); let mut time_counter = std::time::Instant::now(); - let file = fs::File::open(genesis_file).expect("Failed to open genesis file"); + let file = fs::File::open(genesis_file).context("Failed to open genesis file")?; let genesis_file: Box = Box::new( parse_genesis_data(file).map_err(|err| anyhow!("can't parse genesis data: {:?}", err))?, ); @@ -85,76 +85,3 @@ pub async fn insert_byron_genesis( Ok(()) } - -pub async fn process_shelley_genesis( - conn: &DatabaseConnection, - genesis_file: &str, - exec_plan: Arc, -) -> anyhow::Result<()> { - let task_perf_aggregator = Arc::new(Mutex::new(TaskPerfAggregator::default())); - - tracing::info!("Parsing genesis file..."); - let mut time_counter = std::time::Instant::now(); - - let file = fs::File::open(genesis_file).expect("Failed to open genesis file"); - let genesis_file: Box = Box::new( - parse_genesis_data(file).map_err(|err| anyhow!("can't parse genesis data: {:?}", err))?, - ); - - tracing::info!( - "Finished parsing genesis file after {:?}", - time_counter.elapsed() - ); - time_counter = std::time::Instant::now(); - - tracing::info!("Inserting Shelley genesis data into database..."); - conn.transaction(|txn| { - Box::pin(insert_shelley_genesis( - txn, - genesis_file, - exec_plan.clone(), - task_perf_aggregator.clone(), - )) - }) - .await?; - - tracing::info!( - "Finished inserting genesis data after {:?}", - time_counter.elapsed() - ); - tracing::trace!( - "Genesis task-wise time spent:\n{:#?}", - task_perf_aggregator.lock().unwrap() - ); - - Ok(()) -} - -pub async fn insert_shelley_genesis( - txn: &DatabaseTransaction, - genesis_file: Box, - exec_plan: Arc, - task_perf_aggregator: Arc>, -) -> Result<(), DbErr> { - let genesis_hash = genesis_file.genesis_prev.to_raw_bytes(); - tracing::info!( - "Starting sync based on genesis hash {}", - hex::encode(genesis_hash) - ); - - let block_global_info = BlockGlobalInfo { - era: EraValue::Shelley, - epoch: None, - epoch_slot: None, - }; - - process_genesis_block( - txn, - ("", &genesis_file, &block_global_info), - &exec_plan, - task_perf_aggregator.clone(), - ) - .await?; - - Ok(()) -} diff --git a/indexer/src/main.rs b/indexer/src/main.rs index 43024167..6de33882 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -8,7 +8,6 @@ use dcspark_blockchain_source::{GetNextFrom, Source}; use migration::async_std::path::PathBuf; use oura::sources::BearerKind; use serde::Deserialize; -use std::borrow::Cow; use std::fs::File; use std::process::exit; use std::sync::atomic::{AtomicBool, Ordering}; @@ -65,14 +64,12 @@ pub enum SinkConfig { }, } -pub enum Network {} - #[derive(Debug, Clone, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] #[serde(deny_unknown_fields)] pub enum SourceConfig { Oura { socket: String, bearer: BearerKind }, - CardanoNet { relay: (Cow<'static, str>, u16) }, + CardanoNet { relay: (String, u16) }, } #[derive(Debug, Clone, Deserialize)] @@ -239,8 +236,11 @@ async fn main() -> anyhow::Result<()> { .ok_or_else(|| anyhow!("Starting points list is empty"))?; let network_config = dcspark_blockchain_source::cardano::NetworkConfiguration { - relay: relay.clone(), - from: start_from.clone(), + relay: dcspark_blockchain_source::cardano::Relay::UrlPort( + relay.clone().0, + relay.clone().1, + ), + from: None, ..base_config }; diff --git a/indexer/src/sink.rs b/indexer/src/sink.rs index 8d148b01..aef56941 100644 --- a/indexer/src/sink.rs +++ b/indexer/src/sink.rs @@ -1,7 +1,6 @@ use crate::perf_aggregator::PerfAggregator; use async_trait::async_trait; use dcspark_blockchain_source::{EventObject, PullFrom}; -use entity::block::EraValue; #[async_trait] pub trait Sink { @@ -13,6 +12,5 @@ pub trait Sink { &mut self, event: Self::Event, perf_aggregator: &mut PerfAggregator, - latest_era: &mut Option, ) -> anyhow::Result<()>; } diff --git a/indexer/src/sinks/cardano.rs b/indexer/src/sinks/cardano.rs index 5e881673..1cb88248 100644 --- a/indexer/src/sinks/cardano.rs +++ b/indexer/src/sinks/cardano.rs @@ -3,9 +3,9 @@ use crate::perf_aggregator::PerfAggregator; use crate::sink::Sink; use crate::types::{MultiEraBlock, StoppableService}; use crate::{genesis, DbConfig, SinkConfig}; -use anyhow::anyhow; +use anyhow::{anyhow, Context as _}; use async_trait::async_trait; - +use dcspark_blockchain_source::cardano::time::Era; use dcspark_blockchain_source::cardano::Point; use dcspark_core::{BlockId, SlotNumber}; use entity::sea_orm::Database; @@ -18,6 +18,8 @@ use entity::{ prelude::{Block, BlockColumn}, sea_orm::{DatabaseConnection, EntityTrait, QueryOrder, QuerySelect}, }; +use std::io::Cursor; +use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; use tasks::byron::byron_executor::process_byron_block; @@ -25,16 +27,53 @@ use tasks::dsl::database_task::BlockGlobalInfo; use tasks::execution_plan::ExecutionPlan; use tasks::multiera::multiera_executor::process_multiera_block; use tasks::utils::TaskPerfAggregator; +use tokio::io::AsyncReadExt; + +#[derive(Clone)] +pub enum Network { + Mainnet, + Preview, + Preprod, + Sanchonet, + Custom { genesis_files: PathBuf }, +} + +impl Network { + pub fn genesis_filename(&self, era: EraValue) -> String { + match self { + Network::Mainnet | Network::Preview | Network::Preprod | Network::Sanchonet => { + format!("{}-{}-genesis.json", self.to_str(), era.to_str()) + } + Network::Custom { genesis_files: _ } => format!("{}-genesis.json", era.to_str()), + } + } + + pub fn to_str(&self) -> &'static str { + match self { + Network::Mainnet => "mainnet", + Network::Preview => "preview", + Network::Preprod => "preprod", + Network::Sanchonet => "sanchonet", + Network::Custom { genesis_files: _ } => "custom", + } + } +} + +impl std::fmt::Display for Network { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.to_str()) + } +} pub struct CardanoSink { db: DatabaseConnection, - network: String, - genesis_folder: Option, + network: Network, exec_plan: Arc, last_epoch: i128, epoch_start_time: std::time::Instant, task_perf_aggregator: Arc>, + shelley_era: Option, } impl CardanoSink { @@ -49,18 +88,43 @@ impl CardanoSink { } => (db, network, genesis_folder), _ => todo!("Invalid sink config provided"), }; + + let network = if network == "custom" { + Network::Custom { + genesis_files: genesis_folder + .ok_or(anyhow!( + "genesis_folder should be specified for custom networks" + ))? + .into(), + } + } else { + match network.as_ref() { + "mainnet" => Network::Mainnet, + "preview" => Network::Preview, + "preprod" => Network::Preprod, + "sanchonet" => Network::Sanchonet, + unknown => { + anyhow::bail!( + "{unknown} is invalid. NETWORK must be either mainnet/preview/preprod or a 'custom' network", + ) + } + } + }; + match db_config { DbConfig::Postgres { database_url } => { let conn = Database::connect(&database_url).await?; + let shelley_era = get_shelley_era_data_from_db(&conn).await?; + Ok(Self { db: conn, network, - genesis_folder, exec_plan, last_epoch: -1, epoch_start_time: std::time::Instant::now(), task_perf_aggregator: Arc::new(Mutex::new(TaskPerfAggregator::default())), + shelley_era, }) } _ => todo!("Only postgres is supported atm"), @@ -141,9 +205,13 @@ impl Sink for CardanoSink { if start.is_empty() { // https://github.com/txpipe/oura/blob/67b01e8739ed2927ced270e08daea74b03bcc7f7/src/sources/common.rs#L91 - let genesis_file: String = - get_genesis_file(&self.network, &self.genesis_folder, "byron")?; - genesis::process_byron_genesis(&self.db, &genesis_file, self.exec_plan.clone()).await?; + let genesis_file: PathBuf = get_genesis_file(&self.network, EraValue::Byron)?; + genesis::process_byron_genesis( + &self.db, + &genesis_file.to_string_lossy(), + self.exec_plan.clone(), + ) + .await?; return self.get_latest_point().await; } @@ -154,17 +222,24 @@ impl Sink for CardanoSink { &mut self, event: Self::Event, perf_aggregator: &mut PerfAggregator, - latest_era: &mut Option, ) -> anyhow::Result<()> { match event { CardanoEventType::Block { cbor_hex, - epoch, + mut epoch, epoch_slot, block_number, block_hash, block_slot: _block_slot, } => { + // this won't work for the first block in the shelley era, since + // the shelley genesis is processed after this however, this + // probably doesn't matter that much for the perf aggregator + // since it's only one block and it only happens once. + if let Some(shelley_era) = &self.shelley_era { + epoch = shelley_era.absolute_slot_to_epoch(epoch_slot.unwrap()); + } + match epoch { Some(epoch) if epoch as i128 > self.last_epoch => { let epoch_duration = self.epoch_start_time.elapsed(); @@ -206,23 +281,21 @@ impl Sink for CardanoSink { } _ => (), }; - *latest_era = Some( - self.db - .transaction::<_, EraValue, DbErr>(|txn| { - Box::pin(insert_block( - cbor_hex, - epoch, - epoch_slot, - txn, - self.exec_plan.clone(), - self.task_perf_aggregator.clone(), - *latest_era, - self.genesis_folder.clone(), - self.network.clone(), - )) - }) - .await?, - ); + self.shelley_era = self + .db + .transaction::<_, Option, DbErr>(|txn| { + Box::pin(insert_block( + cbor_hex, + epoch, + epoch_slot, + txn, + self.exec_plan.clone(), + self.task_perf_aggregator.clone(), + self.network.clone(), + self.shelley_era.clone(), + )) + }) + .await?; } CardanoEventType::RollBack { block_slot, @@ -251,20 +324,19 @@ impl Sink for CardanoSink { "Rollback destination did not exist. Maybe you're stuck on a fork?" ); } - *latest_era = None; } Some(point) => { Block::delete_many() .filter(BlockColumn::Id.gt(point.id)) .exec(&self.db) .await?; - let latest_block = Block::find() - .order_by_desc(BlockColumn::Id) - .one(&self.db) - .await? - .unwrap(); - *latest_era = - Some(EraValue::try_from(latest_block.era).expect("Unknown era")); + + // the table that keeps track of the shelley genesis + // has a foreign key to the block in which we triggered + // that update, this means the entry will get deleted if + // we rollback to a point before that, in which case we + // re-fetch it just to be sure. + self.shelley_era = get_shelley_era_data_from_db(&self.db).await?; } } @@ -294,6 +366,7 @@ fn to_era_value(x: &MultiEraBlock) -> EraValue { } } +#[allow(clippy::too_many_arguments)] async fn insert_block( cbor_hex: String, epoch: Option, @@ -301,10 +374,9 @@ async fn insert_block( txn: &DatabaseTransaction, exec_plan: Arc, task_perf_aggregator: Arc>, - previous_era: Option, - custom_genesis_folder: Option, - network: String, -) -> Result { + network: Network, + mut shelley_era: Option, +) -> Result, DbErr> { let mut perf_aggregator = PerfAggregator::new(); let block_parse_counter = std::time::Instant::now(); @@ -313,39 +385,59 @@ async fn insert_block( let multi_block = MultiEraBlock::from_explicit_network_cbor_bytes(&block_payload).unwrap(); let era = to_era_value(&multi_block); - let block_global_info = BlockGlobalInfo { + let mut block_global_info = BlockGlobalInfo { era, epoch, epoch_slot, }; - match previous_era { - None => { - let genesis_file = get_genesis_file(&network, &custom_genesis_folder, "byron"); - tasks::genesis::genesis_executor::process_genesis_block( - txn, - ("", &genesis_file, &block_global_info), - &exec_plan, - task_perf_aggregator.clone(), - ) - .await? - } - Some(prev_era) if era > prev_era => { - let genesis_file = get_genesis_file( - &network, - &custom_genesis_folder, - &format!("{:?}", era).to_lowercase(), - ); - process_genesis_block( - txn, - ("", &genesis_file, &block_global_info), - &exec_plan, - task_perf_aggregator.clone(), - ) + if era > EraValue::Byron && shelley_era.is_none() { + // we don't have the code to parse the other genesis blocks (alonzo, conway). + let genesis_file_path = get_genesis_file(&network, EraValue::Shelley) + .context("Couldn't get the shelley genesis file from the filesystem") + .unwrap(); + + let mut buffer = Vec::new(); + + tokio::fs::File::open(genesis_file_path) + .await + .unwrap() + .read_to_end(&mut buffer) + .await + .unwrap(); + + let genesis = cml_chain::genesis::shelley::parse::parse_genesis_data(Cursor::new(buffer)) + .expect("Failed to parse genesis"); + + tasks::genesis::genesis_executor::process_shelley_genesis_block( + txn, + ("", &genesis, &block_global_info), + &exec_plan, + task_perf_aggregator.clone(), + ) + .await?; + + shelley_era = entity::genesis::Entity::find() + .filter(entity::genesis::Column::Era.eq(i32::from(EraValue::Shelley))) + .limit(1) + .one(txn) .await? - } - _ => {} - }; + .map(|model| Era { + first_slot: model.first_slot.try_into().unwrap(), + start_epoch: model.start_epoch.try_into().unwrap(), + epoch_length_seconds: model.epoch_length_seconds.try_into().unwrap(), + // we don't need to know these since we don't compute timestamps + known_time: 0, + slot_length: 0, + }); + } + + // in the byron era the epoch it's in the header, so we only need to compute + // this if we already transitioned to shelley. + if let Some(shelley_era) = &shelley_era { + block_global_info.epoch = + shelley_era.absolute_slot_to_epoch(block_global_info.epoch_slot.unwrap()); + } perf_aggregator.block_parse += block_parse_counter.elapsed(); @@ -370,35 +462,43 @@ async fn insert_block( } } - Ok(era) + Ok(shelley_era) } -fn get_genesis_folder<'a>( - network: &str, - custom_genesis_folder: &'a Option, -) -> anyhow::Result<&'a str> { - match &network[..] { - "mainnet" | "preview" | "preprod" | "sanchonet" => Ok(KNOWN_GENESIS_FOLDER), - "custom" => Ok(custom_genesis_folder - .as_ref() - .expect("genesis_folder should be specified for custom networks")), - rest => { - return Err(anyhow!( - "{} is invalid. NETWORK must be either mainnet/preview/preprod or a 'custom' network", - rest - )) +fn get_genesis_file(network: &Network, era: EraValue) -> anyhow::Result { + let mut path = PathBuf::new(); + + let known_genesis_folder = PathBuf::from(KNOWN_GENESIS_FOLDER); + let genesis_folder = match network { + Network::Mainnet | Network::Preview | Network::Preprod | Network::Sanchonet => { + &known_genesis_folder } - } + Network::Custom { genesis_files } => genesis_files, + }; + + path.push(genesis_folder); + path.push(network.genesis_filename(era)); + + Ok(path) } -fn get_genesis_file<'a>( - network: &str, - custom_genesis_folder: &'a Option, - era: &str, -) -> anyhow::Result { - let genesis_folder = get_genesis_folder(network, custom_genesis_folder)?; - Ok(format!( - "{}/{}-{}-genesis.json", - genesis_folder, network, era - )) +async fn get_shelley_era_data_from_db( + conn: &DatabaseConnection, +) -> Result, anyhow::Error> { + let shelley_era = entity::genesis::Entity::find() + .filter(entity::genesis::Column::Era.eq(i32::from(EraValue::Shelley))) + .limit(1) + .one(conn) + .await? + .map(|model| { + Era { + first_slot: model.first_slot.try_into().unwrap(), + start_epoch: model.start_epoch.try_into().unwrap(), + epoch_length_seconds: model.epoch_length_seconds.try_into().unwrap(), + // we don't need to know these since we don't compute timestamps + known_time: 0, + slot_length: 0, + } + }); + Ok(shelley_era) } diff --git a/indexer/src/sources/cardano.rs b/indexer/src/sources/cardano.rs index 5ce6c539..8763fb33 100644 --- a/indexer/src/sources/cardano.rs +++ b/indexer/src/sources/cardano.rs @@ -62,7 +62,7 @@ impl Source for CardanoSource { tracing::debug!(id = %block_event.id, "block event received"); Ok(Some(CardanoEventType::Block { cbor_hex: hex::encode(block_event.raw_block), - epoch: Some(block_event.epoch), + epoch: block_event.epoch, epoch_slot: Some(block_event.slot_number.into()), block_number: block_event.block_number.into(), block_hash: block_event.id.to_string(), @@ -88,7 +88,7 @@ impl Source for CardanoSource { impl CardanoSource { pub async fn new(configuration: NetworkConfiguration) -> anyhow::Result { - WrappedCardanoSource::connect(&configuration, Duration::from_millis(5000)) + WrappedCardanoSource::connect(&configuration, Duration::from_millis(5000), true) .await .and_then(|cardano_source| { Multiverse::temporary() diff --git a/indexer/tasks/src/dsl/database_task.rs b/indexer/tasks/src/dsl/database_task.rs index 0cb81b13..093124d8 100644 --- a/indexer/tasks/src/dsl/database_task.rs +++ b/indexer/tasks/src/dsl/database_task.rs @@ -1,5 +1,5 @@ use crate::utils::TaskPerfAggregator; -use cml_chain::genesis::byron::config::GenesisData; +use cml_chain::genesis::{byron::config::GenesisData, shelley::config::ShelleyGenesisData}; use entity::{block::EraValue, prelude::*, sea_orm::DatabaseTransaction}; use shred::DispatcherBuilder; use std::sync::{Arc, Mutex}; @@ -55,6 +55,7 @@ pub trait TaskBuilder<'a, BlockType, BlockExtraType> { #[derive(Copy, Clone)] pub enum TaskRegistryEntry { Genesis(GenesisTaskRegistryEntry), + ShelleyGenesis(ShelleyGenesisTaskRegistryEntry), Byron(ByronTaskRegistryEntry), Multiera(MultieraTaskRegistryEntry), } @@ -64,6 +65,11 @@ pub struct GenesisTaskRegistryEntry { pub builder: &'static (dyn for<'a> TaskBuilder<'a, GenesisData, BlockGlobalInfo> + Sync), } +#[derive(Copy, Clone)] +pub struct ShelleyGenesisTaskRegistryEntry { + pub builder: &'static (dyn for<'a> TaskBuilder<'a, ShelleyGenesisData, BlockGlobalInfo> + Sync), +} + #[derive(Copy, Clone)] pub struct ByronTaskRegistryEntry { pub builder: &'static (dyn for<'a> TaskBuilder<'a, cml_multi_era::MultiEraBlock, BlockGlobalInfo> diff --git a/indexer/tasks/src/dsl/task_macro.rs b/indexer/tasks/src/dsl/task_macro.rs index 8ec1f8c8..ce9370c6 100644 --- a/indexer/tasks/src/dsl/task_macro.rs +++ b/indexer/tasks/src/dsl/task_macro.rs @@ -3,7 +3,8 @@ pub use crate::utils::find_task_registry_entry; pub use crate::{ dsl::database_task::{ BlockGlobalInfo, BlockInfo, ByronTaskRegistryEntry, DatabaseTaskMeta, - GenesisTaskRegistryEntry, MultieraTaskRegistryEntry, TaskBuilder, TaskRegistryEntry, + GenesisTaskRegistryEntry, MultieraTaskRegistryEntry, ShelleyGenesisTaskRegistryEntry, + TaskBuilder, TaskRegistryEntry, }, era_common::AddressInBlock, utils::TaskPerfAggregator, @@ -17,6 +18,9 @@ macro_rules! era_to_block { (genesis) => { GenesisData }; + (shelley_genesis) => { + ShelleyGenesisData + }; (byron) => { cml_multi_era::MultiEraBlock }; @@ -29,6 +33,9 @@ macro_rules! era_to_block_info { (genesis) => { BlockGlobalInfo }; + (shelley_genesis) => { + BlockGlobalInfo + }; (byron) => { BlockGlobalInfo }; @@ -45,6 +52,11 @@ cfg_if::cfg_if! { builder: &$task_builder, }) }; + (shelley_genesis $task_builder:expr) => { + TaskMarkdownRegistryEntry::Genesis(GenesisTaskMarkdownRegistryEntry { + builder: &$task_builder, + }) + }; (byron $task_builder:expr) => { TaskMarkdownRegistryEntry::Byron(ByronTaskMarkdownRegistryEntry { builder: &$task_builder, @@ -204,6 +216,11 @@ cfg_if::cfg_if! { builder: &$task_builder, }) }; + (shelley_genesis $task_builder:expr) => { + TaskRegistryEntry::ShelleyGenesis(ShelleyGenesisTaskRegistryEntry { + builder: &$task_builder, + }) + }; (byron $task_builder:expr) => { TaskRegistryEntry::Byron(ByronTaskRegistryEntry { builder: &$task_builder, diff --git a/indexer/tasks/src/genesis/genesis_executor.rs b/indexer/tasks/src/genesis/genesis_executor.rs index 6c9f7c7e..1af6d6b7 100644 --- a/indexer/tasks/src/genesis/genesis_executor.rs +++ b/indexer/tasks/src/genesis/genesis_executor.rs @@ -1,13 +1,13 @@ -use std::sync::{Arc, Mutex}; - use crate::dsl::database_task::TaskRegistryEntry; use crate::dsl::database_task::{BlockGlobalInfo, BlockInfo}; use crate::execution_plan::ExecutionPlan; use crate::utils::find_task_registry_entry; use crate::utils::TaskPerfAggregator; use cml_chain::genesis::byron::config::GenesisData; +use cml_chain::genesis::shelley::config::ShelleyGenesisData; use entity::sea_orm::{prelude::*, DatabaseTransaction}; use shred::{DispatcherBuilder, World}; +use std::sync::{Arc, Mutex}; use tokio::runtime::Handle; pub async fn process_genesis_block( @@ -60,3 +60,54 @@ pub async fn process_genesis_block( Ok(()) } + +pub async fn process_shelley_genesis_block( + txn: &DatabaseTransaction, + block: BlockInfo<'_, ShelleyGenesisData, BlockGlobalInfo>, + exec_plan: &ExecutionPlan, + perf_aggregator: Arc>, +) -> Result<(), DbErr> { + let ep_start_time = std::time::Instant::now(); + + let handle = Handle::current(); + + let mut world = World::empty(); + + let mut dispatcher_builder = DispatcherBuilder::new(); + + for (task_name, val) in exec_plan.0.iter() { + if let toml::value::Value::Table(_task_props) = val { + let entry = find_task_registry_entry(task_name); + match &entry { + None => { + panic!("Could not find task named {task_name}"); + } + Some(task) => { + if let TaskRegistryEntry::ShelleyGenesis(entry) = task { + entry.builder.maybe_add_task( + &mut dispatcher_builder, + txn, + block, + &handle, + perf_aggregator.clone(), + val, + ); + } + } + } + } + } + + if !dispatcher_builder.is_empty() { + let mut dispatcher = dispatcher_builder.build(); + dispatcher.setup(&mut world); + dispatcher.dispatch(&world); + } + + perf_aggregator + .lock() + .unwrap() + .add_to_total(&ep_start_time.elapsed()); + + Ok(()) +} diff --git a/indexer/tasks/src/genesis/mod.rs b/indexer/tasks/src/genesis/mod.rs index 34cb2652..8a7b2072 100644 --- a/indexer/tasks/src/genesis/mod.rs +++ b/indexer/tasks/src/genesis/mod.rs @@ -1,3 +1,4 @@ pub mod genesis_block; pub mod genesis_executor; pub mod genesis_txs; +pub mod shelley_genesis; diff --git a/indexer/tasks/src/genesis/shelley_genesis.rs b/indexer/tasks/src/genesis/shelley_genesis.rs new file mode 100644 index 00000000..83b34670 --- /dev/null +++ b/indexer/tasks/src/genesis/shelley_genesis.rs @@ -0,0 +1,238 @@ +use crate::config::EmptyConfig::EmptyConfig; +use crate::dsl::task_macro::*; +use cml_chain::genesis::shelley::config::ShelleyGenesisData; +use cml_chain::transaction::AlonzoFormatTxOut; +use cml_chain::Serialize as _; +use cml_core::serialization::ToBytes; +use cml_crypto::{blake2b256, RawBytesEncoding}; +use entity::block::{self, EraValue}; +use entity::stake_credential; +use entity::{ + prelude::*, + sea_orm::{entity::*, prelude::*, Condition, DatabaseTransaction}, +}; +use hex::ToHex; +use sea_orm::{QueryOrder, QuerySelect as _}; + +carp_task! { + name ShelleyGenesisBlockTask; + configuration EmptyConfig; + doc "Adds the block to the database"; + era shelley_genesis; + dependencies []; + read []; + write [genesis_block]; + should_add_task |_block, _properties| { + true + }; + execute |_previous_data, task| handle_block( + task.db_tx, + task.block + ); + merge_result |_previous_data, _result| { + }; +} + +async fn handle_block( + db_tx: &DatabaseTransaction, + block: BlockInfo<'_, ShelleyGenesisData, BlockGlobalInfo>, +) -> Result<(), DbErr> { + if Genesis::find() + .filter(GenesisColumn::Era.eq(i32::from(EraValue::Shelley))) + .limit(1) + .one(db_tx) + .await? + .is_some() + { + // There are two cases where we need to run the code in this task + // 1. We have a new block with the Shelley era. + // 2. We skipped the Shelley era and we got a block with a newer era. + // + // However, if we get a new era directly (like Conway), we need to know + // if we've seen Shelley before or not. That's the reason we need this + // condition. + // + // Note: Remember that the genesis file for each era is different. + return Ok(()); + } + + let (latest_block_height, latest_block_epoch) = Block::find() + .order_by_desc(block::Column::Height) + .limit(1) + .one(db_tx) + .await? + .map(|block| (block.height, block.epoch)) + .unwrap_or_default(); + + // assuming that hard forks can only happen at epoch boundaries? + let start_epoch = latest_block_epoch + 1; + + // TODO: these values should come from the byron genesis, but for the + // existing networks the shelley and byron epoch lenghts are the same, and + // for all of them the slot duration is 20 seconds too. + // potentially we may want to add an entry in the era table with values for these though? + // or we could read the genesis file here. + let byron_slot_duration = 20; + let epoch_length_in_byron_slots = block.1.epoch_length / byron_slot_duration; + + let first_slot = (block.2.epoch_slot.unwrap() / epoch_length_in_byron_slots + * epoch_length_in_byron_slots) as i64; + + let inserted_block = Block::insert(BlockActiveModel { + era: Set(EraValue::Shelley.into()), + height: Set(latest_block_height + 1), + epoch: Set(start_epoch), + payload: Set(None), + tx_count: Set(block.1.initial_funds.len().try_into().unwrap()), + // TODO: what should we hash? + hash: Set(b"shelley-genesis".to_vec()), + slot: Set(first_slot.try_into().unwrap()), + ..Default::default() + }) + .exec_with_returning(true, db_tx) + .await? + .unwrap(); + + Genesis::insert(GenesisActiveModel { + era: Set(i32::from(EraValue::Shelley)), + block_id: Set(inserted_block.id), + block_height: Set(latest_block_height + 1), + first_slot: Set(first_slot), + start_epoch: Set(start_epoch.into()), + epoch_length_seconds: Set(block.1.epoch_length as i64), + }) + .exec(db_tx) + .await?; + + let stake_credentials = handle_initial_funds(block, inserted_block, db_tx).await?; + + if let Some(staking) = block + .1 + .staking + .as_ref() + .filter(|staking| !staking.stake.is_empty()) + { + entity::stake_delegation::Entity::insert_many(staking.stake.iter().map( + |(stake_credential, pool)| { + let stake_credential_entry = stake_credentials + .iter() + .find(|inserted_credential| { + inserted_credential.credential + == cml_chain::certs::StakeCredential::new_pub_key(*stake_credential) + .to_cbor_bytes() + }) + .unwrap(); + + entity::stake_delegation::ActiveModel { + pool_credential: Set(Some(pool.to_raw_bytes().to_vec())), + previous_pool: Set(None), + stake_credential: Set(stake_credential_entry.id), + // Note: this is not really the tx of the delegation, but the tx + // of the utxo with the initial funds. However there is no other + // tx to assign this to otherwise. + tx_id: Set(stake_credential_entry.first_tx), + ..Default::default() + } + }, + )) + .exec(db_tx) + .await?; + } + + Ok(()) +} + +async fn handle_initial_funds( + block: (&str, &ShelleyGenesisData, &BlockGlobalInfo), + inserted_block: BlockModel, + db_tx: &DatabaseTransaction, +) -> Result, DbErr> { + if block.1.initial_funds.is_empty() { + return Ok(vec![]); + } + + let inserted_transactions = + Transaction::insert_many(block.1.initial_funds.keys().map(|address| { + let tx_id = blake2b256(&address.to_raw_bytes()); + + TransactionActiveModel { + block_id: Set(inserted_block.id), + hash: Set(tx_id.to_vec()), + is_valid: Set(true), + payload: Set(vec![]), + tx_index: Set(0), + ..Default::default() + } + })) + .exec_many_with_returning(db_tx) + .await?; + + let inserted_addresses = Address::insert_many( + block + .1 + .initial_funds + .iter() + .zip(inserted_transactions.iter()) + .map(|((address, _), inserted_tx)| AddressActiveModel { + payload: Set(address.to_raw_bytes()), + first_tx: Set(inserted_tx.id), + ..Default::default() + }), + ) + .exec_many_with_returning(db_tx) + .await?; + + TransactionOutput::insert_many(block.1.initial_funds.iter().zip(inserted_addresses).map( + |((address, coin), address_model)| { + TransactionOutputActiveModel { + address_id: Set(address_model.id), + tx_id: Set(address_model.first_tx), + payload: Set( + cml_chain::transaction::TransactionOutput::AlonzoFormatTxOut( + AlonzoFormatTxOut::new( + address.clone(), + cml_chain::Value::new(*coin, Default::default()), + ), + ) + .to_cbor_bytes(), + ), + output_index: Set(0), + ..Default::default() + } + }, + )) + .exec_many_with_returning(db_tx) + .await?; + + let inserted_credentials = StakeCredential::insert_many( + block + .1 + .initial_funds + .iter() + .zip(inserted_transactions) + .filter_map(|((address, _), inserted_tx)| { + let stake_credential = match address { + cml_chain::address::Address::Base(base) => Some(base.stake.clone()), + // TODO: this doesn't seem possible? + cml_chain::address::Address::Ptr(_) => todo!(), + cml_chain::address::Address::Enterprise(_) => None, + cml_chain::address::Address::Reward(_) => None, + cml_chain::address::Address::Byron(_) => None, + }; + + if let Some(stake_credential) = stake_credential { + Some(StakeCredentialActiveModel { + credential: Set(stake_credential.to_cbor_bytes()), + first_tx: Set(inserted_tx.id), + ..Default::default() + }) + } else { + None + } + }), + ) + .exec_many_with_returning(db_tx) + .await?; + + Ok(inserted_credentials) +} diff --git a/indexer/tasks/src/utils.rs b/indexer/tasks/src/utils.rs index 65f4434c..99785fe0 100644 --- a/indexer/tasks/src/utils.rs +++ b/indexer/tasks/src/utils.rs @@ -58,6 +58,11 @@ pub fn find_task_registry_entry(task_name: &str) -> Option { return Some(*registry_entry); } } + TaskRegistryEntry::ShelleyGenesis(entry) => { + if entry.builder.get_name() == task_name { + return Some(*registry_entry); + } + } } } None