Update CML to regenerated version and use instead of Pallas (#159)

* Update CML to regenerated version

* Tmp fix commit

* feat!: update CML to newest version

* fix: cargo rust linters

* fix: config cardano node

* Small improvements

* feat: update deps

* fix: to_canonical_cbor_bytes -> to_cbor_bytes

* feat!: remove pallas completely

* docs: update pitfalls

* fix: redundant copy

* feat: upgrade CML to v5.0.0

docs/docs/pitfalls.md

@@ -23,7 +23,5 @@ sidebar_position: 1
 
 ## Risks with using this codebase:
 
-- **Missing functionality in binary data parsing libraries**: This codebase parses raw CBOR (binary data) from the Cardano blockchain using two libraries: Pallas and CML. It has happened in the past that one of these libraries is missing some feature of the Cardano blockchain which could cause Carp to fail if these ever appear on-chain
-- **Incompatibility bugs between parsing libraries**: This project uses both Pallas and CML. Although both libraries implement some overlapping features, they are occasionally implemented differently. This should not cause any issues, but subtle implementation differences may cause issues leading to bugs in Carp
-- **Cardano ledger bugs** (yes, this happens): There has been multiple occasions where the Cardano node itself has a bug in it causing the data generated not to be parsable by Pallas/CML until patched.
-- **Byron-era limitations**: CML (at the time of writing) doesn't support most of Byron-era structures. We use Pallas inside Carp so they should appear in the SQL database properly, but they won't necessarily be parsable if you're reading from the database using CML. There is also some Byron-era features we didn't expose like "Epoch Boundary Blocks" (EBBs)
+- **Missing functionality in binary data parsing libraries**: This codebase parses raw CBOR (binary data) from the Cardano blockchain using CML. It has happened in the past that the library is missing some feature of the Cardano blockchain which could cause Carp to fail if these ever appear on-chain.
+- **Cardano ledger bugs** (yes, this happens): There has been multiple occasions where the Cardano node itself has a bug in it causing the data generated not to be parsable by CML until patched.

indexer/Cargo.toml

@@ -21,13 +21,15 @@ tasks = { path = "tasks" }
 # [indexer]
 anyhow = { version = "1.0.69" }
 async-trait = { version = "0.1.64" }
-cardano-multiplatform-lib = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", branch = "metadata-and-addr" }
+cml-chain = { version = "5.0.0" }
+cml-core = { version = "5.0.0" }
+cml-crypto = { version = "5.0.0" }
+cml-multi-era = { version = "5.0.0" }
 clap = { version = "3.1", features = ["derive"] }
 ctrlc = { version = "3.2.4", features = ["termination"] }
 dotenv = { version = "0.15.0" }
 hex = { version = "0.4.3" }
 oura = { git = "https://github.com/txpipe/oura.git", rev = "27fb7e876471b713841d96e292ede40101b151d7" }
-pallas = { version = "0.18.2" }
 serde = { version = "1.0.152", features = ["derive", "rc"] }
 serde_json = { version = "1.0.92" }
 serde_yaml = { version = "0.9.17" }

indexer/configs/cardano_node.yml

@@ -1,12 +1,12 @@
 source:
   type: cardano_net
-  relay: 
+  relay:
     - relays-new.cardano-mainnet.iohk.io
     - 3001
       # - preview-node.world.dev.cardano.org
       # - 30002
       # - preprod-node.world.dev.cardano.org
-      # - 30000
+    # - 30000
 
 sink:
   type: cardano

indexer/entity/Cargo.toml

@@ -6,7 +6,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-pallas = { version = "0.18.2" }
 sea-orm = { git = "https://github.com/dcSpark/sea-orm", branch = "insert-many-returning", features = [
     "runtime-tokio-rustls",
     "sqlx-postgres",

indexer/entity/src/block.rs

@@ -1,4 +1,3 @@
-use pallas::ledger::traverse::Era;
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
@@ -31,6 +30,7 @@ pub enum EraValue {
     Mary,
     Alonzo,
     Babbage,
+    Conway,
 }
 
 impl From<EraValue> for i32 {
@@ -42,6 +42,7 @@ impl From<EraValue> for i32 {
             EraValue::Mary => 3,
             EraValue::Alonzo => 4,
             EraValue::Babbage => 5,
+            EraValue::Conway => 6,
         }
     }
 }
@@ -57,20 +58,8 @@ impl TryFrom<i32> for EraValue {
             3 => Ok(EraValue::Mary),
             4 => Ok(EraValue::Alonzo),
             5 => Ok(EraValue::Babbage),
+            6 => Ok(EraValue::Conway),
             _ => Err(()),
         }
     }
 }
-
-impl From<EraValue> for Era {
-    fn from(item: EraValue) -> Self {
-        match item {
-            EraValue::Byron => Era::Byron,
-            EraValue::Shelley => Era::Shelley,
-            EraValue::Allegra => Era::Allegra,
-            EraValue::Mary => Era::Mary,
-            EraValue::Alonzo => Era::Alonzo,
-            EraValue::Babbage => Era::Babbage,
-        }
-    }
-}

indexer/plan-visualizer/Cargo.toml

@@ -12,8 +12,8 @@ tasks = { path = "../tasks" }
 anyhow = { version = "1.0.69" }
 clap = { version = "3.1", features = ["derive"] }
 dot = { git = "https://github.com/dcSpark/dot-rust", branch = "subgraph" }
-strum = { version = "0.24" }
-strum_macros = { version = "0.24" }
-toml = { version = "0.5", features = ["preserve_order"] }
+strum = { version = "0.25.0" }
+strum_macros = { version = "0.25.3" }
+toml = { version = "0.8.6", features = ["preserve_order"] }
 tracing = { version = "0.1.37" }
 tracing-subscriber = { version = "0.3.16" }

indexer/reparse/Cargo.toml

@@ -10,7 +10,10 @@ entity = { path = "../entity" }
 
 # [reparse]
 anyhow = { version = "1.0.69" }
-cardano-multiplatform-lib = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", branch = "metadata-and-addr" }
+cml-chain = { version = "5.0.0" }
+cml-core = { version = "5.0.0" }
+cml-crypto = { version = "5.0.0" }
+cml-multi-era = { version = "5.0.0" }
 dotenv = { version = "0.15.0" }
 futures = { version = "0.3.21" }
 hex = { version = "0.4.0" }

indexer/reparse/src/reparse.rs

@@ -1,3 +1,4 @@
+use cml_core::serialization::FromBytes;
 use entity::{
     prelude::*,
     sea_orm::{prelude::*, JoinType, QueryOrder, QuerySelect},
@@ -34,8 +35,7 @@ async fn reparse_txs(conn: &DatabaseConnection, start_index: u64) -> Result<(),
         for tx in txs {
             // TODO: this will fail on all Byron txs
             // https://github.com/dcSpark/cardano-multiplatform-lib/issues/61
-            if let Err(e) = &cardano_multiplatform_lib::Transaction::from_bytes(tx.payload.clone())
-            {
+            if let Err(e) = &cml_chain::transaction::Transaction::from_bytes(tx.payload.clone()) {
                 println!(
                     "\nFailed tx at tx hash {}. {:?} {}\n",
                     hex::encode(&tx.hash),
@@ -65,9 +65,7 @@ async fn reparse_addresses(conn: &DatabaseConnection, start_index: u64) -> Resul
             (100.0 * addresses.first().unwrap().id as f64) / (address_count as f64)
         );
         for addr in addresses {
-            if let Err(e) =
-                &cardano_multiplatform_lib::address::Address::from_bytes(addr.payload.clone())
-            {
+            if let Err(e) = &cml_chain::address::Address::from_bytes(addr.payload.clone()) {
                 let bad_tx = Transaction::find()
                     .join(
                         JoinType::InnerJoin,
@@ -119,7 +117,7 @@ async fn reparse_tx_out(conn: &DatabaseConnection, start_index: u64) -> Result<(
             // TODO: this will fail on all Byron txs
             // https://github.com/dcSpark/cardano-multiplatform-lib/issues/61
             if let Err(e) =
-                &cardano_multiplatform_lib::TransactionOutput::from_bytes(tx_out.payload.clone())
+                &cml_chain::transaction::TransactionOutput::from_bytes(tx_out.payload.clone())
             {
                 let bad_tx = Transaction::find()
                     .join(

indexer/src/genesis.rs

@@ -4,7 +4,10 @@ use std::fs;
 use std::sync::{Arc, Mutex};
 use tasks::dsl::database_task::BlockGlobalInfo;
 
-use cardano_multiplatform_lib::genesis::byron::{config::GenesisData, parse::parse};
+use cml_chain::genesis::byron::config::GenesisData;
+use cml_chain::genesis::byron::parse::parse_genesis_data;
+
+use cml_crypto::RawBytesEncoding;
 use entity::sea_orm::{DatabaseConnection, DatabaseTransaction, TransactionTrait};
 use migration::DbErr;
 use tasks::utils::TaskPerfAggregator;
@@ -40,7 +43,9 @@ pub async fn process_genesis(
     let mut time_counter = std::time::Instant::now();
 
     let file = fs::File::open(genesis_path).expect("Failed to open genesis file");
-    let genesis_file: Box<GenesisData> = Box::new(parse(file));
+    let genesis_file: Box<GenesisData> = Box::new(
+        parse_genesis_data(file).map_err(|err| anyhow!("can't parse genesis data: {:?}", err))?,
+    );
 
     tracing::info!(
         "Finished parsing genesis file after {:?}",
@@ -77,10 +82,10 @@ pub async fn insert_genesis(
     exec_plan: Arc<ExecutionPlan>,
     task_perf_aggregator: Arc<Mutex<TaskPerfAggregator>>,
 ) -> Result<(), DbErr> {
-    let genesis_hash = genesis_file.genesis_prev.to_bytes();
+    let genesis_hash = genesis_file.genesis_prev.to_raw_bytes();
     tracing::info!(
         "Starting sync based on genesis hash {}",
-        hex::encode(genesis_hash.clone())
+        hex::encode(genesis_hash)
     );
 
     let block_global_info = BlockGlobalInfo {

indexer/src/sinks/cardano.rs

@@ -4,6 +4,7 @@ use crate::sink::Sink;
 use crate::types::{MultiEraBlock, StoppableService};
 use crate::{genesis, DbConfig, SinkConfig};
 use async_trait::async_trait;
+
 use dcspark_blockchain_source::cardano::Point;
 use dcspark_core::{BlockId, SlotNumber};
 use entity::sea_orm::Database;
@@ -254,14 +255,14 @@ impl StoppableService for CardanoSink {
     }
 }
 
-fn to_era_value(x: pallas::ledger::traverse::Era) -> EraValue {
+fn to_era_value(x: &MultiEraBlock) -> EraValue {
     match x {
-        pallas::ledger::traverse::Era::Byron => EraValue::Byron,
-        pallas::ledger::traverse::Era::Shelley => EraValue::Shelley,
-        pallas::ledger::traverse::Era::Allegra => EraValue::Allegra,
-        pallas::ledger::traverse::Era::Mary => EraValue::Mary,
-        pallas::ledger::traverse::Era::Alonzo => EraValue::Alonzo,
-        pallas::ledger::traverse::Era::Babbage => EraValue::Babbage,
+        MultiEraBlock::Byron(_) => EraValue::Byron,
+        MultiEraBlock::Shelley(_) => EraValue::Shelley,
+        MultiEraBlock::Allegra(_) => EraValue::Allegra,
+        MultiEraBlock::Mary(_) => EraValue::Mary,
+        MultiEraBlock::Alonzo(_) => EraValue::Alonzo,
+        MultiEraBlock::Babbage(_) => EraValue::Babbage,
         _ => unreachable!("all known eras are handled"),
     }
 }
@@ -279,18 +280,18 @@ async fn insert_block(
     let block_parse_counter = std::time::Instant::now();
 
     let block_payload = hex::decode(cbor_hex.clone()).unwrap();
-    let multi_block = MultiEraBlock::decode(&block_payload).unwrap();
+    let multi_block = MultiEraBlock::from_explicit_network_cbor_bytes(&block_payload).unwrap();
 
     let block_global_info = BlockGlobalInfo {
-        era: to_era_value(multi_block.era()),
+        era: to_era_value(&multi_block),
         epoch,
         epoch_slot,
     };
 
     perf_aggregator.block_parse += block_parse_counter.elapsed();
 
-    match &multi_block.era() {
-        pallas::ledger::traverse::Era::Byron => {
+    match &multi_block {
+        MultiEraBlock::Byron(_byron) => {
             process_byron_block(
                 txn,
                 (&cbor_hex, &multi_block, &block_global_info),

indexer/src/types.rs

@@ -1,6 +1,6 @@
 use async_trait::async_trait;
 
-pub type MultiEraBlock<'b> = pallas::ledger::traverse::MultiEraBlock<'b>;
+pub type MultiEraBlock = cml_multi_era::MultiEraBlock;
 
 #[async_trait]
 pub trait StoppableService {

indexer/tasks/Cargo.toml

@@ -12,24 +12,25 @@ entity = { path = "../entity" }
 
 # [tasks]
 anyhow = { version = "1.0.69" }
-cardano-multiplatform-lib = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", branch = "metadata-and-addr" }
-cml-chain  = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", rev = "acca172633d0570a7432058aa5b0717ad2f0c6d7" }
-cml-core  = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", rev = "acca172633d0570a7432058aa5b0717ad2f0c6d7" }
-cml-crypto  = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", rev = "acca172633d0570a7432058aa5b0717ad2f0c6d7" }
-projected-nft-sdk = { git = "https://github.com/dcSpark/projected-nft-whirlpool.git", rev = "a350ccfcbdef22b73c93561c6816bd42b13f00ff" }
-cfg-if = { version = "0.1.10" }
+cml-chain = { version = "5.0.0" }
+cml-core = { version = "5.0.0" }
+cml-crypto = { version = "5.0.0" }
+cml-multi-era = { version = "5.0.0" }
+cardano-projected-nft = { git = "https://github.com/dcSpark/projected-nft-whirlpool.git", rev = "13f81e8666743fefd14c5e1affb1cd828d8c473b" }
+cfg-if = { version = "1.0.0" }
 cryptoxide = { version = "0.4.2" }
 hex = { version = "0.4.3" }
 inventory = { version = "0.2.3" }
 markdown-gen = { git = "https://github.com/dcSpark/markdown-gen-rs", branch = "hbina-add-ability-to-write-raw-str" }
 nameof = { version = "1.2.2" }
-pallas = { version = "0.18.2" }
 paste = { version = "1.0.11" }
 sea-orm = { git = "https://github.com/dcSpark/sea-orm", branch = "insert-many-returning", features = [ "macros" ], default-features = false }
 serde = { version = "1.0.152", features = ["derive"] }
+serde_json = { version = "1.0.108" }
+schemars = { version = "0.8.8" }
 shred = { git = "https://github.com/dcSpark/shred", branch = "builder-getters" }
 tokio = { version = "1.16.1", features = ["full"] }
-toml = { version = "0.5", features = ["preserve_order"] }
+toml = { version = "0.8.6", features = ["preserve_order"] }
 urlencoding = { version = "2.1.2" }
 tracing-subscriber = { version = "0.3.16" }
 tracing = { version = "0.1.37" }