Merge pull request input-output-hk#1494 from input-output-hk/djo/1492…

…/mithril-db-crate

Extract persistence related code to a new shared crate

.github/workflows/ci.yml

@@ -701,8 +701,8 @@ jobs:
         run: |
           # Force `--lib` to avoid a collision between the client lib and the client cli binary who share
           # the same name (we only want to document those anyway)
-          cargo doc --no-deps --lib -p mithril-stm -p mithril-common -p mithril-aggregator \
-                    -p mithril-signer -p mithril-client -p mithril-client-cli \
+          cargo doc --no-deps --lib -p mithril-stm -p mithril-common -p mithril-persistence \
+                    -p mithril-aggregator -p mithril-signer -p mithril-client -p mithril-client-cli \
                     --all-features --message-format=json \
                     | clippy-sarif | tee rust-cargo-doc-results.sarif | sarif-fmt
           

Cargo.toml

@@ -11,6 +11,7 @@ members = [
   "mithril-client-cli",
   "mithril-client-wasm",
   "mithril-common",
+  "mithril-persistence",
   "mithril-relay",
   "mithril-signer",
   "mithril-stm",

Makefile

@@ -1,4 +1,4 @@
-COMPONENTS = mithril-common mithril-stm mithril-aggregator mithril-client mithril-client-cli mithril-signer demo/protocol-demo mithril-test-lab/mithril-end-to-end
+COMPONENTS = mithril-common mithril-persistence mithril-stm mithril-aggregator mithril-client mithril-client-cli mithril-signer demo/protocol-demo mithril-test-lab/mithril-end-to-end
 GOALS := $(or $(MAKECMDGOALS),all)
 
 .PHONY: $(GOALS) $(COMPONENTS)

README.md

@@ -63,6 +63,8 @@ This repository consists of the following parts:
 * [**Mithril client WASM**](./mithril-client-wasm): the WASM compatible library used for retrieving the certified artifacts produced by the **Mithril network**.
 
 * [**Mithril common**](./mithril-common): this is the **common** library that is used by the **Mithril network** nodes.
+
+* [**Mithril persistence**](./mithril-persistence): the **persistence** library that is used by the **Mithril network** nodes.
 
 * [**Mithril STM**](./mithril-stm): the **core** library that implements **Mithril** protocol cryptographic engine.
 

docs/website/root/manual/developer-docs/references.md

@@ -27,6 +27,7 @@ To learn more about the **Mithril protocol**, please refer to the [about Mithril
 | Dependency | Description | Source repository | Rust documentation | REST API
 |------------|-------------|:-----------------:|:------------------:|:------------:|
 | **Mithril common** | The **common** library used by **Mithril network** nodes. | [:arrow_upper_right:](https://github.com/input-output-hk/mithril/tree/main/mithril-common) | [:arrow_upper_right:](https://mithril.network/rust-doc/mithril_common/index.html) | -
+| **Mithril persistence** | The **persistence** library used by **Mithril network** nodes. | [:arrow_upper_right:](https://github.com/input-output-hk/mithril/tree/main/mithril-persistence) | [:arrow_upper_right:](https://mithril.network/rust-doc/mithril_persistence/index.html) | -
 | **Mithril STM** | The **core** library that implements the cryptographic engine for the **Mithril** protocol. | [:arrow_upper_right:](https://github.com/input-output-hk/mithril/tree/main/mithril-stm) | [:arrow_upper_right:](https://mithril.network/rust-doc/mithril_stm/index.html) | -
 | **Mithril aggregator** | The node within the **Mithril network** responsible for collecting individual signatures from the **Mithril signers** and aggregating them into a multi-signature. This capability enables the **Mithril aggregator** to provide certified snapshots of the **Cardano** blockchain. | [:arrow_upper_right:](https://github.com/input-output-hk/mithril/tree/main/mithril-aggregator) | [:arrow_upper_right:](https://mithril.network/rust-doc/mithril_aggregator/index.html) | [:arrow_upper_right:](/doc/aggregator-api)
 | **Mithril client CLI** | The node within the **Mithril network** responsible for restoring the **Cardano** blockchain on an empty node from a certified snapshot. | [:arrow_upper_right:](https://github.com/input-output-hk/mithril/tree/main/mithril-client-cli) | [:arrow_upper_right:](https://mithril.network/rust-doc/mithril_client_cli/index.html) | -

flake.nix

@@ -108,6 +108,7 @@
           mithril-client-cli = buildPackage ./mithril-client-cli/Cargo.toml mithril.cargoArtifacts {};
           mithril-aggregator = buildPackage ./mithril-aggregator/Cargo.toml mithril.cargoArtifacts {};
           mithril-signer = buildPackage ./mithril-signer/Cargo.toml mithril.cargoArtifacts {};
+          mithril-persistence = buildPackage ./mithril-persistence/Cargo.toml mithril.cargoArtifacts {};
           mithrildemo = buildPackage ./demo/protocol-demo/Cargo.toml mithril.cargoArtifacts {};
           mithril-end-to-end = buildPackage ./mithril-test-lab/mithril-end-to-end/Cargo.toml null {};
         };

mithril-aggregator/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.4.35"
+version = "0.4.36"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }
@@ -19,6 +19,7 @@ config = "0.13.4"
 flate2 = "1.0.28"
 hex = "0.4.3"
 mithril-common = { path = "../mithril-common", features = ["full"] }
+mithril-persistence = { path = "../mithril-persistence" }
 openssl = { version = "0.10.63", features = ["vendored"], optional = true }
 openssl-probe = { version = "0.1.5", optional = true }
 reqwest = { version = "0.11.23", features = ["json"] }

mithril-aggregator/src/commands/tools_command.rs

@@ -1,8 +1,8 @@
 use anyhow::Context;
 use clap::{Parser, Subcommand};
 use config::{builder::DefaultState, ConfigBuilder};
-use mithril_common::sqlite::vacuum_database;
 use mithril_common::StdResult;
+use mithril_persistence::sqlite::vacuum_database;
 use slog_scope::debug;
 use std::sync::Arc;
 

mithril-aggregator/src/database/cardano_transaction_migration.rs

@@ -1,6 +1,6 @@
 //! Migration module for cardano transactions store
 //!
-use mithril_common::database::SqlMigration;
+use mithril_persistence::database::SqlMigration;
 
 /// Get all the migrations required by this version of the software.
 /// There shall be one migration per database version. There could be several

mithril-aggregator/src/database/migration.rs

@@ -1,6 +1,6 @@
 //! Migration module
 //!
-use mithril_common::database::SqlMigration;
+use mithril_persistence::database::SqlMigration;
 
 /// Get all the migrations required by this version of the software.
 /// There shall be one migration per database version. There could be several

mithril-aggregator/src/database/provider/cardano_transaction.rs

@@ -1,12 +1,12 @@
 use mithril_common::{
     entities::{Beacon, BlockNumber, CardanoTransaction, ImmutableFileNumber, TransactionHash},
     signable_builder::TransactionStore,
-    sqlite::{
-        HydrationError, Projection, Provider, SourceAlias, SqLiteEntity, SqliteConnection,
-        WhereCondition,
-    },
     StdResult,
 };
+use mithril_persistence::sqlite::{
+    HydrationError, Projection, Provider, SourceAlias, SqLiteEntity, SqliteConnection,
+    WhereCondition,
+};
 
 use anyhow::Context;
 use async_trait::async_trait;
@@ -295,7 +295,7 @@ impl TransactionsRetriever for CardanoTransactionRepository {
 
 #[cfg(test)]
 mod tests {
-    use mithril_common::sqlite::SourceAlias;
+    use mithril_persistence::sqlite::SourceAlias;
     use sqlite::Connection;
 
     use crate::{dependency_injection::DependenciesBuilder, Configuration};

mithril-aggregator/src/database/provider/certificate.rs

@@ -15,12 +15,11 @@ use mithril_common::{
         CertificateListItemMessage, CertificateListItemMessageMetadata, CertificateMessage,
         CertificateMetadataMessagePart,
     },
-    sqlite::{
-        EntityCursor, HydrationError, Projection, Provider, SourceAlias, SqLiteEntity,
-        WhereCondition,
-    },
     StdResult,
 };
+use mithril_persistence::sqlite::{
+    EntityCursor, HydrationError, Projection, Provider, SourceAlias, SqLiteEntity, WhereCondition,
+};
 
 #[cfg(test)]
 use mithril_common::test_utils::fake_keys;

mithril-aggregator/src/database/provider/epoch_setting.rs

@@ -5,13 +5,13 @@ use std::sync::Arc;
 
 use mithril_common::{
     entities::{Epoch, ProtocolParameters},
-    sqlite::{
-        EntityCursor, HydrationError, Projection, Provider, SourceAlias, SqLiteEntity,
-        SqliteConnection, WhereCondition,
-    },
-    store::adapter::AdapterError,
     StdResult,
 };
+use mithril_persistence::sqlite::{
+    EntityCursor, HydrationError, Projection, Provider, SourceAlias, SqLiteEntity,
+    SqliteConnection, WhereCondition,
+};
+use mithril_persistence::store::adapter::AdapterError;
 
 use crate::ProtocolParametersStorer;
 

mithril-aggregator/src/database/provider/open_message.rs

@@ -1,12 +1,13 @@
 use anyhow::Context;
 use mithril_common::{
     entities::{Epoch, ProtocolMessage, SignedEntityType, SingleSignatures},
-    sqlite::{
-        HydrationError, Projection, Provider, SourceAlias, SqLiteEntity, SqliteConnection,
-        WhereCondition,
-    },
     StdResult,
 };
+use mithril_persistence::database::SignedEntityTypeHydrator;
+use mithril_persistence::sqlite::{
+    HydrationError, Projection, Provider, SourceAlias, SqLiteEntity, SqliteConnection,
+    WhereCondition,
+};
 
 use chrono::{DateTime, Utc};
 use sqlite::{Row, Value};
@@ -117,7 +118,8 @@ impl SqLiteEntity for OpenMessageRecord {
                 "Integer field open_message.signed_entity_type_id cannot be turned into usize: {e}"
             )
         })?;
-        let signed_entity_type = SignedEntityType::hydrate(signed_entity_type_id, &beacon_str)?;
+        let signed_entity_type =
+            SignedEntityTypeHydrator::hydrate(signed_entity_type_id, &beacon_str)?;
         let is_certified = row.read::<i64, _>(5) != 0;
         let datetime = &row.read::<&str, _>(7);
         let created_at =
@@ -624,7 +626,8 @@ impl OpenMessageRepository {
 
 #[cfg(test)]
 mod tests {
-    use mithril_common::{entities::Beacon, sqlite::SourceAlias};
+    use mithril_common::entities::Beacon;
+    use mithril_persistence::sqlite::SourceAlias;
     use sqlite::Connection;
 
     use crate::database::provider::{

mithril-aggregator/src/database/provider/signed_entity.rs

@@ -16,12 +16,15 @@ use mithril_common::{
         SignerWithStakeMessagePart, SnapshotListItemMessage, SnapshotMessage,
     },
     signable_builder::Artifact,
+    StdError, StdResult,
+};
+use mithril_persistence::{
+    database::SignedEntityTypeHydrator,
     sqlite::{
         EntityCursor, HydrationError, Projection, Provider, SourceAlias, SqLiteEntity,
         SqliteConnection, WhereCondition,
     },
     store::adapter::AdapterError,
-    StdError, StdResult,
 };
 
 #[cfg(test)]
@@ -224,7 +227,7 @@ impl SqLiteEntity for SignedEntityRecord {
 
         let signed_entity_record = Self {
             signed_entity_id,
-            signed_entity_type: SignedEntityType::hydrate(
+            signed_entity_type: SignedEntityTypeHydrator::hydrate(
                 signed_entity_type_id_int.try_into().map_err(|e| {
                     HydrationError::InvalidData(format!(
                         "Could not cast i64 ({signed_entity_type_id_int}) to u64. Error: '{e}'"

mithril-aggregator/src/database/provider/signer.rs

@@ -5,10 +5,11 @@ use std::collections::HashMap;
 use std::iter::repeat;
 use std::sync::Arc;
 
-use mithril_common::sqlite::{
-    EntityCursor, HydrationError, Projection, Provider, SourceAlias, SqLiteEntity, WhereCondition,
+use mithril_common::StdResult;
+use mithril_persistence::sqlite::{
+    EntityCursor, HydrationError, Projection, Provider, SourceAlias, SqLiteEntity,
+    SqliteConnection, WhereCondition,
 };
-use mithril_common::{sqlite::SqliteConnection, StdResult};
 
 use crate::signer_registerer::SignerRecorder;