From 0227a3b3e07f0beca6a124e8222c02fedc16c5f0 Mon Sep 17 00:00:00 2001 From: Joel Nordell <94570446+joel-u410@users.noreply.github.com> Date: Tue, 3 Dec 2024 02:37:13 -0600 Subject: [PATCH 01/14] [enhancement] store balances at block heights - implementation (#81) * [chain] Query balances at block_height * [enhancement] balances -> balance_changes, record balance history by height * [chain/test] Update balance conflict tests to be more valid * [chain/test] Add test for behavior when inserting new balance amount at the same height * [chain] Retry initial_query due to race condition between getting last block & querying for all balances * [logging] Clean up logging, make info level less verbose --- chain/Cargo.toml | 1 + chain/src/config.rs | 10 +- chain/src/main.rs | 142 ++++++++++++++---- chain/src/repository/balance.rs | 129 +++++++++++++--- chain/src/services/namada.rs | 37 +++-- chain/src/services/utils.rs | 11 +- .../2024-04-18-102935_init_balances/down.sql | 4 +- .../2024-04-18-102935_init_balances/up.sql | 28 +++- orm/src/balances.rs | 22 ++- orm/src/lib.rs | 1 + orm/src/schema.rs | 7 +- orm/src/views.rs | 9 ++ seeder/src/main.rs | 14 +- shared/src/balance.rs | 3 + transactions/src/main.rs | 27 +++- webserver/src/repository/balance.rs | 2 +- 16 files changed, 355 insertions(+), 92 deletions(-) create mode 100644 orm/src/views.rs diff --git a/chain/Cargo.toml b/chain/Cargo.toml index ccc3d1613..afe4694fc 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" [dependencies] test_helpers.workspace = true tokio.workspace = true +tokio-retry.workspace = true tracing.workspace = true tracing-subscriber.workspace = true serde_json.workspace = true diff --git a/chain/src/config.rs b/chain/src/config.rs index 745dd46f7..925c03666 100644 --- a/chain/src/config.rs +++ b/chain/src/config.rs @@ -23,9 +23,17 @@ pub struct AppConfig { #[clap(long, env)] pub database_url: String, - #[clap(long, env)] + #[clap( + long, + env, + default_value = "100", + help = "Time between retry attempts in milliseconds" + )] pub initial_query_retry_time: u64, + #[clap(long, env, default_value = "5")] + pub initial_query_retry_attempts: usize, + #[command(flatten)] pub verbosity: Verbosity, } diff --git a/chain/src/main.rs b/chain/src/main.rs index 097da4259..b55d50c4e 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -29,6 +29,8 @@ use shared::id::Id; use shared::token::Token; use shared::validator::ValidatorSet; use tendermint_rpc::HttpClient; +use tokio_retry::strategy::{jitter, ExponentialBackoff}; +use tokio_retry::Retry; use tracing::Level; use tracing_subscriber::FmtSubscriber; @@ -73,7 +75,13 @@ async fn main() -> Result<(), MainError> { .context_db_interact_error() .into_db_error()?; - initial_query(&client, &conn).await?; + initial_query( + &client, + &conn, + config.initial_query_retry_time, + config.initial_query_retry_attempts, + ) + .await?; let crawler_state = db_service::get_chain_crawler_state(&conn) .await @@ -106,22 +114,26 @@ async fn crawling_fn( let timestamp = Utc::now().naive_utc(); update_crawler_timestamp(&conn, timestamp).await?; - tracing::warn!("Block {} was not processed, retry...", block_height); + tracing::trace!( + block = block_height, + "Block does not exist yet, waiting...", + ); return Err(MainError::NoAction); } - tracing::info!("Query block..."); + tracing::debug!(block = block_height, "Query block..."); let tm_block_response = tendermint_service::query_raw_block_at_height(&client, block_height) .await .into_rpc_error()?; - tracing::info!( + tracing::debug!( + block = block_height, "Raw block contains {} txs...", tm_block_response.block.data.len() ); - tracing::info!("Query block results..."); + tracing::debug!(block = block_height, "Query block results..."); let tm_block_results_response = tendermint_service::query_raw_block_results_at_height( &client, @@ -131,13 +143,13 @@ async fn crawling_fn( .into_rpc_error()?; let block_results = BlockResult::from(tm_block_results_response); - tracing::info!("Query epoch..."); + tracing::debug!(block = block_height, "Query epoch..."); let epoch = namada_service::get_epoch_at_block_height(&client, block_height) .await .into_rpc_error()?; - tracing::info!("Query first block in epoch..."); + tracing::debug!(block = block_height, "Query first block in epoch..."); let first_block_in_epoch = namada_service::get_first_block_in_epoch(&client) .await @@ -150,19 +162,34 @@ async fn crawling_fn( epoch, block_height, ); - tracing::info!("Deserialized {} txs...", block.transactions.len()); + tracing::debug!( + block = block_height, + txs = block.transactions.len(), + "Deserialized {} txs...", + block.transactions.len() + ); let native_token = namada_service::get_native_token(&client) .await .into_rpc_error()?; - let ibc_tokens = block.ibc_tokens().into_iter().map(Token::Ibc).collect(); + let ibc_tokens = block + .ibc_tokens() + .into_iter() + .map(Token::Ibc) + .collect::>(); let addresses = block.addresses_with_balance_change(native_token); - let balances = namada_service::query_balance(&client, &addresses) - .await - .into_rpc_error()?; - tracing::info!("Updating balance for {} addresses...", addresses.len()); + + let balances = + namada_service::query_balance(&client, &addresses, block_height) + .await + .into_rpc_error()?; + tracing::debug!( + block = block_height, + "Updating balance for {} addresses...", + addresses.len() + ); let next_governance_proposal_id = namada_service::query_next_governance_id(&client, block_height) @@ -170,7 +197,11 @@ async fn crawling_fn( .into_rpc_error()?; let proposals = block.governance_proposal(next_governance_proposal_id); - tracing::info!("Creating {} governance proposals...", proposals.len()); + tracing::debug!( + block = block_height, + "Creating {} governance proposals...", + proposals.len() + ); let proposals_with_tally = namada_service::query_tallies(&client, proposals) @@ -178,7 +209,11 @@ async fn crawling_fn( .into_rpc_error()?; let proposals_votes = block.governance_votes(); - tracing::info!("Creating {} governance votes...", proposals_votes.len()); + tracing::debug!( + block = block_height, + "Creating {} governance votes...", + proposals_votes.len() + ); let validators = block.validators(); let validator_set = ValidatorSet { @@ -188,7 +223,11 @@ async fn crawling_fn( let addresses = block.bond_addresses(); let bonds = query_bonds(&client, addresses).await.into_rpc_error()?; - tracing::info!("Updating bonds for {} addresses", bonds.len()); + tracing::debug!( + block = block_height, + "Updating bonds for {} addresses", + bonds.len() + ); let bonds_updates = bonds .iter() @@ -206,12 +245,17 @@ async fn crawling_fn( let unbonds = namada_service::query_unbonds(&client, addresses) .await .into_rpc_error()?; - tracing::info!("Updating unbonds for {} addresses", unbonds.len()); + tracing::debug!( + block = block_height, + "Updating unbonds for {} addresses", + unbonds.len() + ); let withdraw_addreses = block.withdraw_addresses(); let revealed_pks = block.revealed_pks(); - tracing::info!( + tracing::debug!( + block = block_height, "Updating revealed pks for {} addresses", revealed_pks.len() ); @@ -229,6 +273,24 @@ async fn crawling_fn( timestamp: timestamp_in_sec, }; + tracing::info!( + txs = block.transactions.len(), + ibc_tokens = ibc_tokens.len(), + balance_changes = balances.len(), + proposals = proposals_with_tally.len(), + votes = proposals_votes.len(), + validators = validators.len(), + bonds = bonds_updates.len(), + unbonds = unbonds.len(), + withdraws = withdraw_addreses.len(), + claimed_rewards = reward_claimers.len(), + revealed_pks = revealed_pks.len(), + epoch = epoch, + first_block_in_epoch = first_block_in_epoch, + block = block_height, + "Queried block successfully", + ); + conn.interact(move |conn| { conn.build_transaction() .read_write() @@ -238,7 +300,7 @@ async fn crawling_fn( ibc_tokens, )?; - repository::balance::insert_balance( + repository::balance::insert_balance_in_chunks( transaction_conn, balances, )?; @@ -298,14 +360,30 @@ async fn crawling_fn( .context_db_interact_error() .into_db_error()? .context("Commit block db transaction error") - .into_db_error() + .into_db_error()?; + + tracing::info!(block = block_height, "Inserted block into database",); + + Ok(()) } async fn initial_query( client: &HttpClient, conn: &Object, + retry_time: u64, + retry_attempts: usize, +) -> Result<(), MainError> { + let retry_strategy = ExponentialBackoff::from_millis(retry_time) + .map(jitter) + .take(retry_attempts); + Retry::spawn(retry_strategy, || try_initial_query(client, conn)).await +} + +async fn try_initial_query( + client: &HttpClient, + conn: &Object, ) -> Result<(), MainError> { - tracing::info!("Querying initial data..."); + tracing::debug!("Querying initial data..."); let block_height = query_last_block_height(client).await.into_rpc_error()?; let epoch = namada_service::get_epoch_at_block_height(client, block_height) @@ -317,9 +395,14 @@ async fn initial_query( let tokens = query_tokens(client).await.into_rpc_error()?; - let balances = query_all_balances(client).await.into_rpc_error()?; + // This can sometimes fail if the last block height in the node has moved forward after we queried + // for it. In that case, query_all_balances returns an Err indicating that it can only be used for + // the last block. This function will be retried in that case. + let balances = query_all_balances(client, block_height) + .await + .into_rpc_error()?; - tracing::info!("Querying validators set..."); + tracing::debug!(block = block_height, "Querying validators set..."); let pipeline_length = namada_service::query_pipeline_length(client) .await .into_rpc_error()?; @@ -332,12 +415,12 @@ async fn initial_query( .await .into_rpc_error()?; - tracing::info!("Querying bonds and unbonds..."); + tracing::debug!(block = block_height, "Querying bonds and unbonds...",); let (bonds, unbonds) = query_all_bonds_and_unbonds(client, None, None) .await .into_rpc_error()?; - tracing::info!("Querying proposals..."); + tracing::debug!(block = block_height, "Querying proposals..."); let proposals = query_all_proposals(client).await.into_rpc_error()?; let proposals_with_tally = namada_service::query_tallies(client, proposals.clone()) @@ -360,7 +443,7 @@ async fn initial_query( timestamp, }; - tracing::info!("Inserting initial data... "); + tracing::info!(block = block_height, "Inserting initial data..."); conn.interact(move |conn| { conn.build_transaction() @@ -368,6 +451,11 @@ async fn initial_query( .run(|transaction_conn| { repository::balance::insert_tokens(transaction_conn, tokens)?; + tracing::debug!( + block = block_height, + "Inserting {} balances...", + balances.len() + ); repository::balance::insert_balance_in_chunks( transaction_conn, balances, @@ -409,8 +497,6 @@ async fn can_process( block_height: u32, client: Arc, ) -> Result { - tracing::info!("Attempting to process block: {}...", block_height); - let last_block_height = namada_service::query_last_block_height(&client) .await .map_err(|e| { diff --git a/chain/src/repository/balance.rs b/chain/src/repository/balance.rs index a0595ead0..cfd3ee59f 100644 --- a/chain/src/repository/balance.rs +++ b/chain/src/repository/balance.rs @@ -1,11 +1,8 @@ use anyhow::Context; use diesel::sql_types::BigInt; -use diesel::upsert::excluded; -use diesel::{ - sql_query, ExpressionMethods, PgConnection, QueryableByName, RunQueryDsl, -}; -use orm::balances::BalancesInsertDb; -use orm::schema::{balances, ibc_token, token}; +use diesel::{sql_query, PgConnection, QueryableByName, RunQueryDsl}; +use orm::balances::BalanceChangesInsertDb; +use orm::schema::{balance_changes, ibc_token, token}; use orm::token::{IbcTokenInsertDb, TokenInsertDb}; use shared::balance::Balances; use shared::token::Token; @@ -21,19 +18,19 @@ pub fn insert_balance( transaction_conn: &mut PgConnection, balances: Balances, ) -> anyhow::Result<()> { - diesel::insert_into(balances::table) - .values::<&Vec>( + diesel::insert_into(balance_changes::table) + .values::<&Vec>( &balances .into_iter() - .map(BalancesInsertDb::from_balance) + .map(BalanceChangesInsertDb::from_balance) .collect::>(), ) - .on_conflict((balances::columns::owner, balances::columns::token)) - .do_update() - .set( - balances::columns::raw_amount - .eq(excluded(balances::columns::raw_amount)), - ) + .on_conflict(( + balance_changes::columns::owner, + balance_changes::columns::token, + balance_changes::columns::height, + )) + .do_nothing() .execute(transaction_conn) .context("Failed to update balances in db")?; @@ -99,12 +96,16 @@ pub fn insert_tokens( mod tests { use anyhow::Context; - use diesel::{BoolExpressionMethods, QueryDsl, SelectableHelper}; + use diesel::{ + BoolExpressionMethods, ExpressionMethods, QueryDsl, SelectableHelper, + }; use namada_sdk::token::Amount as NamadaAmount; use namada_sdk::uint::MAX_SIGNED_VALUE; use orm::balances::BalanceDb; + use orm::views::balances; use shared::balance::{Amount, Balance}; use shared::id::Id; + use shared::token::IbcToken; use test_helpers::db::TestDb; use super::*; @@ -140,11 +141,13 @@ mod tests { "tnam1q87wtaqqtlwkw927gaff34hgda36huk0kgry692a".to_string(), )); let amount = Amount::from(NamadaAmount::from_u64(100)); + let height = 42; let balance = Balance { owner: owner.clone(), token: token.clone(), amount: amount.clone(), + height, }; insert_tokens(conn, vec![token.clone()])?; @@ -162,7 +165,7 @@ mod tests { } /// Test that the function updates existing balances when there is a - /// conflict. + /// later height. #[tokio::test] async fn test_insert_balance_with_existing_balances_update() { let db = TestDb::new(); @@ -174,19 +177,23 @@ mod tests { "tnam1q87wtaqqtlwkw927gaff34hgda36huk0kgry692a".to_string(), )); let amount = Amount::from(NamadaAmount::from_u64(100)); + let height = 42; let balance = Balance { owner: owner.clone(), token: token.clone(), amount: amount.clone(), + height, }; db.run_test(move |conn| { seed_balance(conn, vec![balance.clone()])?; let new_amount = Amount::from(NamadaAmount::from_u64(200)); + let new_height = 43; let new_balance = Balance { amount: new_amount.clone(), + height: new_height, ..(balance.clone()) }; @@ -204,7 +211,7 @@ mod tests { } /// Test the function's behavior when inserting balances that cause a - /// conflict. + /// conflict (same owner, different token). #[tokio::test] async fn test_insert_balance_with_conflicting_owners() { let db = TestDb::new(); @@ -216,19 +223,32 @@ mod tests { "tnam1qxfj3sf6a0meahdu9t6znp05g8zx4dkjtgyn9gfu".to_string(), )); let amount = Amount::from(NamadaAmount::from_u64(100)); + let height = 42; let balance = Balance { owner: owner.clone(), token: token.clone(), amount: amount.clone(), + height, }; db.run_test(move |conn| { seed_balance(conn, vec![balance.clone()])?; + // this is probably not a valid way to construct an IbcToken + // but seems to be sufficient for testing purposes here. + let new_token = Token::Ibc(IbcToken { + address: Id::Account( + "tnam1q9rhgyv3ydq0zu3whnftvllqnvhvhm270qxay5tn".to_string(), + ), + trace: Id::Account( + "tnam1q9rhgyv3ydq0zu3whnftvllqnvhvhm270qxay5tn".to_string(), + ), + }); + let new_amount = Amount::from(NamadaAmount::from_u64(200)); let new_balance = Balance { - token: token.clone(), + token: new_token.clone(), amount: new_amount.clone(), ..(balance.clone()) }; @@ -240,7 +260,17 @@ mod tests { let queried_balance = query_balance_by_address(conn, owner.clone(), token.clone())?; - assert_eq!(Amount::from(queried_balance.raw_amount), new_amount); + let queried_balance_new = query_balance_by_address( + conn, + owner.clone(), + new_token.clone(), + )?; + + assert_eq!(Amount::from(queried_balance.raw_amount), amount); + assert_eq!( + Amount::from(queried_balance_new.raw_amount), + new_amount + ); anyhow::Ok(()) }) @@ -248,7 +278,7 @@ mod tests { .expect("Failed to run test"); } /// Test the function's behavior when inserting balances that cause a - /// conflict. + /// conflict. (same token, different owner) #[tokio::test] async fn test_insert_balance_with_conflicting_tokens() { let db = TestDb::new(); @@ -260,11 +290,13 @@ mod tests { "tnam1qxfj3sf6a0meahdu9t6znp05g8zx4dkjtgyn9gfu".to_string(), )); let amount = Amount::from(NamadaAmount::from_u64(100)); + let height = 42; let balance = Balance { owner: owner.clone(), token: token.clone(), amount: amount.clone(), + height, }; db.run_test(move |conn| { @@ -303,6 +335,53 @@ mod tests { .expect("Failed to run test"); } + /// Test the function's behavior when inserting balances that cause a + /// conflict (same owner, same token, same height, different amount). + #[tokio::test] + async fn test_insert_balance_with_conflicting_heights() { + let db = TestDb::new(); + + let owner = Id::Account( + "tnam1qqshvryx9pngpk7mmzpzkjkm6klelgusuvmkc0uz".to_string(), + ); + let token = Token::Native(Id::Account( + "tnam1qxfj3sf6a0meahdu9t6znp05g8zx4dkjtgyn9gfu".to_string(), + )); + let amount = Amount::from(NamadaAmount::from_u64(100)); + let height = 42; + + let balance = Balance { + owner: owner.clone(), + token: token.clone(), + amount: amount.clone(), + height, + }; + + db.run_test(move |conn| { + seed_balance(conn, vec![balance.clone()])?; + + let new_amount = Amount::from(NamadaAmount::from_u64(200)); + let new_balance = Balance { + amount: new_amount.clone(), + ..(balance.clone()) + }; + + let res = insert_balance(conn, vec![new_balance]); + + // Conflicting insert succeeds, but is ignored + assert!(res.is_ok()); + + // Balance is not updated when height is the same + let queried_balance = + query_balance_by_address(conn, owner.clone(), token.clone())?; + assert_eq!(Amount::from(queried_balance.raw_amount), amount); + + anyhow::Ok(()) + }) + .await + .expect("Failed to run test"); + } + /// Test the function's ability to handle a large number of balance inserts /// efficiently. #[tokio::test] @@ -342,11 +421,13 @@ mod tests { "tnam1q87wtaqqtlwkw927gaff34hgda36huk0kgry692a".to_string(), )); let max_amount = Amount::from(NamadaAmount::from(MAX_SIGNED_VALUE)); + let height = 42; let balance = Balance { owner: owner.clone(), token: token.clone(), amount: max_amount.clone(), + height, }; insert_tokens(conn, vec![token.clone()])?; @@ -447,11 +528,11 @@ mod tests { ) -> anyhow::Result<()> { seed_tokens_from_balance(conn, balances.clone())?; - diesel::insert_into(balances::table) - .values::<&Vec>( + diesel::insert_into(balance_changes::table) + .values::<&Vec>( &balances .into_iter() - .map(BalancesInsertDb::from_balance) + .map(BalanceChangesInsertDb::from_balance) .collect::>(), ) .execute(conn) diff --git a/chain/src/services/namada.rs b/chain/src/services/namada.rs index 54f5de09e..1ef4774c8 100644 --- a/chain/src/services/namada.rs +++ b/chain/src/services/namada.rs @@ -73,6 +73,7 @@ pub async fn get_epoch_at_block_height( pub async fn query_balance( client: &HttpClient, balance_changes: &HashSet, + block_height: BlockHeight, ) -> anyhow::Result { Ok(futures::stream::iter(balance_changes) .filter_map(|balance_change| async move { @@ -92,15 +93,20 @@ pub async fn query_balance( } .into(); - let amount = - rpc::get_token_balance(client, &token_addr, &owner, None) - .await - .unwrap_or_default(); + let amount = rpc::get_token_balance( + client, + &token_addr, + &owner, + Some(to_block_height(block_height)), + ) + .await + .unwrap_or_default(); Some(Balance { owner: balance_change.address.clone(), token: balance_change.token.clone(), amount: Amount::from(amount), + height: block_height, }) }) .map(futures::future::ready) @@ -137,7 +143,8 @@ async fn query_ibc_tokens( let prefix = ibc_trace_key_prefix(None); let mut tokens: HashSet = HashSet::new(); - let ibc_traces = query_storage_prefix::(client, &prefix).await?; + let ibc_traces = + query_storage_prefix::(client, &prefix, None).await?; if let Some(ibc_traces) = ibc_traces { for (key, ibc_trace) in ibc_traces { if let Some((_, hash)) = is_ibc_trace_key(&key) { @@ -162,12 +169,13 @@ async fn query_ibc_tokens( pub async fn query_all_balances( client: &HttpClient, + height: BlockHeight, ) -> anyhow::Result { let tokens = query_tokens(client).await?; let mut all_balances: Balances = vec![]; for token in tokens.into_iter() { - let balances = add_balance(client, token).await?; + let balances = add_balance(client, token, height).await?; all_balances.extend(balances); } @@ -177,6 +185,7 @@ pub async fn query_all_balances( async fn add_balance( client: &HttpClient, token: Token, + height: BlockHeight, ) -> anyhow::Result> { let mut all_balances: Vec = vec![]; let token_addr = match token { @@ -188,10 +197,13 @@ async fn add_balance( &NamadaSdkAddress::from(token_addr), ); - let balances = - query_storage_prefix::(client, &balance_prefix) - .await - .context("Failed to query all balances")?; + let balances = query_storage_prefix::( + client, + &balance_prefix, + Some(height), + ) + .await + .context("Failed to query all balances")?; if let Some(balances) = balances { for (key, balance) in balances { @@ -206,6 +218,7 @@ async fn add_balance( owner: Id::from(o), token: token.clone(), amount: Amount::from(b), + height, }) } } @@ -688,6 +701,8 @@ pub async fn query_pipeline_length(client: &HttpClient) -> anyhow::Result { Ok(pos_parameters.pipeline_len) } -fn to_block_height(block_height: u32) -> NamadaSdkBlockHeight { +pub(super) fn to_block_height( + block_height: BlockHeight, +) -> NamadaSdkBlockHeight { NamadaSdkBlockHeight::from(block_height as u64) } diff --git a/chain/src/services/utils.rs b/chain/src/services/utils.rs index 4e1311587..64dcfc57e 100644 --- a/chain/src/services/utils.rs +++ b/chain/src/services/utils.rs @@ -3,19 +3,28 @@ use namada_sdk::queries::RPC; use namada_sdk::storage::{self, PrefixValue}; use tendermint_rpc::HttpClient; +use shared::block::BlockHeight; + /// Query a range of storage values with a matching prefix and decode them with /// [`BorshDeserialize`]. Returns an iterator of the storage keys paired with /// their associated values. pub async fn query_storage_prefix( client: &HttpClient, key: &storage::Key, + height: Option, ) -> anyhow::Result>> where T: BorshDeserialize, { let values = RPC .shell() - .storage_prefix(client, None, None, false, key) + .storage_prefix( + client, + None, + height.map(super::namada::to_block_height), + false, + key, + ) .await?; let decode = |PrefixValue { key, value }: PrefixValue| { diff --git a/orm/migrations/2024-04-18-102935_init_balances/down.sql b/orm/migrations/2024-04-18-102935_init_balances/down.sql index a0016856d..4591f9dc1 100644 --- a/orm/migrations/2024-04-18-102935_init_balances/down.sql +++ b/orm/migrations/2024-04-18-102935_init_balances/down.sql @@ -1,3 +1,5 @@ -- This file should undo anything in `up.sql` -DROP TABLE IF EXISTS balances; +DROP VIEW IF EXISTS balances; + +DROP TABLE IF EXISTS balance_changes; diff --git a/orm/migrations/2024-04-18-102935_init_balances/up.sql b/orm/migrations/2024-04-18-102935_init_balances/up.sql index e276648ea..ad2b14c1c 100644 --- a/orm/migrations/2024-04-18-102935_init_balances/up.sql +++ b/orm/migrations/2024-04-18-102935_init_balances/up.sql @@ -1,13 +1,35 @@ -- Your SQL goes here -CREATE TABLE balances ( +CREATE TABLE balance_changes ( id SERIAL PRIMARY KEY, + height INTEGER NOT NULL, owner VARCHAR NOT NULL, token VARCHAR(64) NOT NULL, raw_amount NUMERIC(78, 0) NOT NULL, CONSTRAINT fk_balances_token FOREIGN KEY(token) REFERENCES token(address) ON DELETE CASCADE ); -ALTER TABLE balances ADD UNIQUE (owner, token); +ALTER TABLE balance_changes ADD UNIQUE (owner, token, height); -CREATE INDEX index_balances_owner ON balances (owner, token); +CREATE INDEX index_balance_changes_owner_token_height ON balance_changes (owner, token, height); + +CREATE VIEW balances AS +SELECT + bc.id, + bc.owner, + bc.token, + bc.raw_amount +FROM + balance_changes bc + JOIN ( + SELECT + owner, + token, + MAX(height) AS max_height + FROM + balance_changes + GROUP BY + owner, + token) max_heights ON bc.owner = max_heights.owner + AND bc.token = max_heights.token + AND bc.height = max_heights.max_height; diff --git a/orm/src/balances.rs b/orm/src/balances.rs index 191ebeef1..046f69dd8 100644 --- a/orm/src/balances.rs +++ b/orm/src/balances.rs @@ -5,20 +5,31 @@ use diesel::{Insertable, Queryable, Selectable}; use shared::balance::Balance; use shared::token::Token; -use crate::schema::balances; +use crate::schema::balance_changes; +use crate::views::balances; #[derive(Insertable, Clone, Queryable, Selectable, Debug)] -#[diesel(table_name = balances)] +#[diesel(table_name = balance_changes)] #[diesel(check_for_backend(diesel::pg::Pg))] -pub struct BalancesInsertDb { +pub struct BalanceChangesInsertDb { pub owner: String, pub token: String, pub raw_amount: BigDecimal, + pub height: i32, } -pub type BalanceDb = BalancesInsertDb; +pub type BalanceChangeDb = BalanceChangesInsertDb; + +#[derive(Clone, Queryable, Selectable, Debug)] +#[diesel(table_name = balances)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct BalanceDb { + pub owner: String, + pub token: String, + pub raw_amount: BigDecimal, +} -impl BalancesInsertDb { +impl BalanceChangesInsertDb { pub fn from_balance(balance: Balance) -> Self { let token = match balance.token { Token::Native(token) => token.to_string(), @@ -30,6 +41,7 @@ impl BalancesInsertDb { token, raw_amount: BigDecimal::from_str(&balance.amount.to_string()) .expect("Invalid amount"), + height: balance.height as i32, } } } diff --git a/orm/src/lib.rs b/orm/src/lib.rs index 2386e0d57..24d01b10b 100644 --- a/orm/src/lib.rs +++ b/orm/src/lib.rs @@ -15,3 +15,4 @@ pub mod token; pub mod transactions; pub mod unbond; pub mod validators; +pub mod views; diff --git a/orm/src/schema.rs b/orm/src/schema.rs index 55a579edd..854563909 100644 --- a/orm/src/schema.rs +++ b/orm/src/schema.rs @@ -75,12 +75,13 @@ pub mod sql_types { } diesel::table! { - balances (id) { + balance_changes (id) { id -> Int4, owner -> Varchar, #[max_length = 64] token -> Varchar, raw_amount -> Numeric, + height -> Int4, } } @@ -278,7 +279,7 @@ diesel::table! { } } -diesel::joinable!(balances -> token (token)); +diesel::joinable!(balance_changes -> token (token)); diesel::joinable!(bonds -> validators (validator_id)); diesel::joinable!(governance_votes -> governance_proposals (proposal_id)); diesel::joinable!(ibc_token -> token (address)); @@ -287,7 +288,7 @@ diesel::joinable!(pos_rewards -> validators (validator_id)); diesel::joinable!(unbonds -> validators (validator_id)); diesel::allow_tables_to_appear_in_same_query!( - balances, + balance_changes, bonds, chain_parameters, crawler_state, diff --git a/orm/src/views.rs b/orm/src/views.rs new file mode 100644 index 000000000..b44b5c4d3 --- /dev/null +++ b/orm/src/views.rs @@ -0,0 +1,9 @@ +// Manually create schema for views - see also https://github.com/diesel-rs/diesel/issues/1482 +diesel::table! { + balances (id) { + id -> Int4, + owner -> Varchar, + token -> Varchar, + raw_amount -> Numeric, + } +} diff --git a/seeder/src/main.rs b/seeder/src/main.rs index 17a8865da..d763cd144 100644 --- a/seeder/src/main.rs +++ b/seeder/src/main.rs @@ -2,7 +2,7 @@ use anyhow::Context; use clap::Parser; use clap_verbosity_flag::LevelFilter; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper}; -use orm::balances::BalancesInsertDb; +use orm::balances::BalanceChangesInsertDb; use orm::bond::BondInsertDb; use orm::governance_proposal::{ GovernanceProposalInsertDb, GovernanceProposalUpdateStatusDb, @@ -10,8 +10,8 @@ use orm::governance_proposal::{ use orm::governance_votes::GovernanceProposalVoteInsertDb; use orm::pos_rewards::PosRewardInsertDb; use orm::schema::{ - balances, bonds, governance_proposals, governance_votes, pos_rewards, - unbonds, validators, + balance_changes, bonds, governance_proposals, governance_votes, + pos_rewards, unbonds, validators, }; use orm::unbond::UnbondInsertDb; use orm::validators::{ValidatorDb, ValidatorInsertDb}; @@ -138,7 +138,7 @@ async fn main() -> anyhow::Result<(), MainError> { .execute(transaction_conn) .context("Failed to remove all validators")?; - diesel::delete(balances::table) + diesel::delete(balance_changes::table) .execute(transaction_conn) .context("Failed to remove all validators")?; @@ -201,12 +201,12 @@ async fn main() -> anyhow::Result<(), MainError> { .execute(transaction_conn) .context("Failed to insert pos rewards in db")?; - diesel::insert_into(balances::table) - .values::<&Vec>( + diesel::insert_into(balance_changes::table) + .values::<&Vec>( &balances .into_iter() .map(|balance| { - BalancesInsertDb::from_balance(balance) + BalanceChangesInsertDb::from_balance(balance) }) .collect::>(), ) diff --git a/shared/src/balance.rs b/shared/src/balance.rs index ce9ac6039..9912f5f95 100644 --- a/shared/src/balance.rs +++ b/shared/src/balance.rs @@ -82,6 +82,7 @@ pub struct Balance { pub owner: Id, pub token: Token, pub amount: Amount, + pub height: u32, } pub type Balances = Vec; @@ -97,6 +98,7 @@ impl Balance { owner: Id::Account(address.to_string()), token: Token::Native(Id::Account(token_address.to_string())), amount: Amount::fake(), + height: (0..10000).fake::(), } } @@ -108,6 +110,7 @@ impl Balance { owner: Id::Account(address.to_string()), token, amount: Amount::fake(), + height: (0..10000).fake::(), } } } diff --git a/transactions/src/main.rs b/transactions/src/main.rs index dabadbe6d..961e6c348 100644 --- a/transactions/src/main.rs +++ b/transactions/src/main.rs @@ -100,22 +100,26 @@ async fn crawling_fn( let timestamp = Utc::now().naive_utc(); update_crawler_timestamp(&conn, timestamp).await?; - tracing::warn!("Block {} was not processed, retry...", block_height); + tracing::trace!( + block = block_height, + "Block does not exist yet, waiting...", + ); return Err(MainError::NoAction); } - tracing::info!("Query block..."); + tracing::debug!(block = block_height, "Query block..."); let tm_block_response = tendermint_service::query_raw_block_at_height(&client, block_height) .await .into_rpc_error()?; - tracing::info!( + tracing::debug!( + block = block_height, "Raw block contains {} txs...", tm_block_response.block.data.len() ); - tracing::info!("Query block results..."); + tracing::debug!(block = block_height, "Query block results..."); let tm_block_results_response = tendermint_service::query_raw_block_results_at_height( &client, @@ -136,7 +140,9 @@ async fn crawling_fn( let inner_txs = block.inner_txs(); let wrapper_txs = block.wrapper_txs(); - tracing::info!( + tracing::debug!( + block = block_height, + txs = inner_txs.len(), "Deserialized {} txs...", wrapper_txs.len() + inner_txs.len() ); @@ -149,6 +155,13 @@ async fn crawling_fn( last_processed_block: block_height, }; + tracing::info!( + wrapper_txs = wrapper_txs.len(), + inner_txs = inner_txs.len(), + block = block_height, + "Queried block successfully", + ); + conn.interact(move |conn| { conn.build_transaction() .read_write() @@ -174,6 +187,8 @@ async fn crawling_fn( .and_then(identity) .into_db_error()?; + tracing::info!(block = block_height, "Inserted block into database",); + Ok(()) } @@ -181,8 +196,6 @@ async fn can_process( block_height: u32, client: Arc, ) -> Result { - tracing::info!("Attempting to process block: {}...", block_height); - let last_block_height = namada_service::get_last_block(&client).await.map_err(|e| { tracing::error!( diff --git a/webserver/src/repository/balance.rs b/webserver/src/repository/balance.rs index 702fe6459..f428b2120 100644 --- a/webserver/src/repository/balance.rs +++ b/webserver/src/repository/balance.rs @@ -1,7 +1,7 @@ use axum::async_trait; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper}; use orm::balances::BalanceDb; -use orm::schema::balances; +use orm::views::balances; use crate::appstate::AppState; From 5ec3619ef699e23e15865a5566e2ebfb3671b3cd Mon Sep 17 00:00:00 2001 From: Fraccaroli Gianmarco Date: Wed, 4 Dec 2024 11:31:47 +0100 Subject: [PATCH 02/14] improve ci (#174) --- .github/workflows/deploy.yml | 59 +++++++++++++++------ .github/workflows/scripts/update-package.py | 17 +++--- .gitignore | 4 +- swagger-codegen.json | 5 -- 4 files changed, 52 insertions(+), 33 deletions(-) delete mode 100644 swagger-codegen.json diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index cca1a9195..8b344b15c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -5,22 +5,42 @@ on: tags: - "v[0-9]+.[0-9]+.[0-9]+" - "v[0-9]+.[0-9]+.[0-9]+-[a-z]+" + branches: + - "v?[0-9]+.[0-9]+.[0-9]+-rc" + - "v?[0-9]+.[0-9]+.[0-9]+-[a-z]+-rc" + +permissions: + contents: read + pages: write + id-token: write jobs: swagger-ui: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest steps: + - name: Checkout repo + uses: actions/checkout@v4 - name: Generate Swagger UI uses: Legion2/swagger-ui-action@v1 with: output: swagger-ui spec-file: swagger.yml GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v3 + - name: Setup Pages + if: startsWith(github.ref, 'refs/tags/v') + uses: actions/configure-pages@v5 + - name: Upload artifact + if: startsWith(github.ref, 'refs/tags/v') + uses: actions/upload-pages-artifact@v3 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: swagger-ui + path: 'swagger-ui' + - name: Deploy to GitHub Pages + if: startsWith(github.ref, 'refs/tags/v') + id: deployment + uses: actions/deploy-pages@v4 docker: name: Docker @@ -34,14 +54,14 @@ jobs: matrix: docker: [ - { image: namada-indexer-chain, context: chain }, - { image: namada-indexer-governance, context: governance }, - { image: namada-indexer-pos, context: pos }, - { image: namada-indexer-rewards, context: rewards }, - { image: namada-indexer-seeder, context: seeder }, - { image: namada-indexer-webserver, context: webserver }, - { image: namada-indexer-parameters, context: parameters }, - { image: namada-indexer-transactions, context: transactions }, + { image: chain, package: chain }, + { image: governance, package: governance }, + { image: pos, package: pos }, + { image: rewards, package: rewards }, + { image: seeder, package: seeder }, + { image: webserver, package: webserver }, + { image: parameters, package: parameters }, + { image: transactions, package: transactions }, ] steps: @@ -62,8 +82,9 @@ jobs: uses: docker/build-push-action@v6 with: context: . - file: ${{ matrix.docker.context }}/Dockerfile - push: ${{ github.ref == 'refs/heads/main' }} + file: Dockerfile + build-args: PACKAGE=${{ matrix.docker.package }} + push: startsWith(github.ref, 'refs/tags/v') tags: ${{ env.REGISTRY_URL }}/anoma/namada-indexer:${{ matrix.docker.image }}-${{ steps.get_version.outputs.version-without-v }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha @@ -72,6 +93,8 @@ jobs: swagger-npm-package: runs-on: ubuntu-latest steps: + - name: Checkout repo + uses: actions/checkout@v4 - id: get_version uses: battila7/get-version-action@v2 - uses: actions/setup-node@v4 @@ -79,12 +102,14 @@ jobs: node-version: 20 - name: Authenticate with private NPM package run: echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > ~/.npmrc + - name: Generate openapi client configuration file + run: python3 .github/workflows/scripts/update-package.py ${{ steps.get_version.outputs.version-without-v }} - name: Generate Typescript Client uses: openapi-generators/openapitools-generator-action@v1 with: generator: typescript-axios openapi-file: swagger.yml - - name: Update package.json - run: python3 .github/workflows/scripts/update-package.py typescript-axios/package.json ${{ steps.get_version.outputs.version-without-v }} + command-args: -c swagger-codegen.json -o client - name: Publish package - run: cd typescript-axios && npm publish --access public --verbose \ No newline at end of file + if: startsWith(github.ref, 'refs/tags/v') + run: cd client && npm install && npm run build && npm publish --access public --verbose \ No newline at end of file diff --git a/.github/workflows/scripts/update-package.py b/.github/workflows/scripts/update-package.py index 24d6ce295..a283c98b3 100644 --- a/.github/workflows/scripts/update-package.py +++ b/.github/workflows/scripts/update-package.py @@ -1,15 +1,12 @@ import json import sys -package_json_path = sys.argv[1] -package_version = sys.argv[2] +package_version = sys.argv[1] -package_json = json.load(open(package_json_path)) +configuration = { + "npmName": "@namada/indexer-client", + "npmVersion": package_version +} -package_json['name'] = "namada-indexer-client" -package_json['version'] = package_version -package_json['description'] = "Set of API to interact with a namada indexer." -package_json['license'] = "GPL-3.0 license" - -with open(package_json_path, 'w', encoding='utf-8') as f: - json.dump(package_json, f, ensure_ascii=False, indent=4) \ No newline at end of file +with open("swagger-codegen.json", 'w+', encoding='utf-8') as f: + json.dump(configuration, f, ensure_ascii=False, indent=4) \ No newline at end of file diff --git a/.gitignore b/.gitignore index d45bedcaf..e9111ef7f 100644 --- a/.gitignore +++ b/.gitignore @@ -85,4 +85,6 @@ rust-project.json # End of https://www.toptal.com/developers/gitignore/api/osx,git,macos,rust-analyzer,rust -.vscode \ No newline at end of file +.vscode + +swagger-codegen.json \ No newline at end of file diff --git a/swagger-codegen.json b/swagger-codegen.json deleted file mode 100644 index dfb784369..000000000 --- a/swagger-codegen.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "npmName": "@namada/indexer-client", - "npmVersion": "0.0.30" -} - From 838301dc7e85900ba02ec192320482da2d6b4fa4 Mon Sep 17 00:00:00 2001 From: Joel Nordell <94570446+joel-u410@users.noreply.github.com> Date: Wed, 4 Dec 2024 07:11:12 -0600 Subject: [PATCH 03/14] feat: add json logging with LOG_FORMAT option (#173) * feat: add json logging with LOG_FORMAT option * refactor: make common log config into a shared struct --- Cargo.toml | 2 +- chain/Cargo.toml | 2 -- chain/src/config.rs | 6 ++--- chain/src/main.rs | 17 +------------- governance/Cargo.toml | 2 -- governance/src/config.rs | 6 ++--- governance/src/main.rs | 18 +------------- parameters/Cargo.toml | 2 -- parameters/src/config.rs | 8 +++---- parameters/src/main.rs | 17 +------------- pos/Cargo.toml | 2 -- pos/src/config.rs | 6 ++--- pos/src/main.rs | 17 +------------- rewards/src/config.rs | 6 ++--- rewards/src/main.rs | 18 +------------- seeder/Cargo.toml | 2 -- seeder/src/config.rs | 6 ++--- seeder/src/main.rs | 18 +------------- shared/Cargo.toml | 3 +++ shared/src/lib.rs | 1 + shared/src/log_config.rs | 48 ++++++++++++++++++++++++++++++++++++++ transactions/Cargo.toml | 2 -- transactions/src/config.rs | 6 ++--- transactions/src/main.rs | 17 +------------- 24 files changed, 82 insertions(+), 150 deletions(-) create mode 100644 shared/src/log_config.rs diff --git a/Cargo.toml b/Cargo.toml index 040261e1b..0d4e558d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,7 @@ tower-http = { version = "0.5.0", features = [ ] } tower-layer = "0.3.2" tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } serde = { version = "1.0.138", features = ["derive"] } serde_json = "1.0" clap = { version = "4.4.2", features = ["derive", "env"] } diff --git a/chain/Cargo.toml b/chain/Cargo.toml index afe4694fc..313cf1c1b 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -17,7 +17,6 @@ test_helpers.workspace = true tokio.workspace = true tokio-retry.workspace = true tracing.workspace = true -tracing-subscriber.workspace = true serde_json.workspace = true chrono.workspace = true clap.workspace = true @@ -32,7 +31,6 @@ shared.workspace = true deadpool-diesel.workspace = true diesel.workspace = true orm.workspace = true -clap-verbosity-flag.workspace = true futures.workspace = true [build-dependencies] diff --git a/chain/src/config.rs b/chain/src/config.rs index 925c03666..c6bcb878a 100644 --- a/chain/src/config.rs +++ b/chain/src/config.rs @@ -1,7 +1,7 @@ use core::fmt; use std::fmt::Display; -use clap_verbosity_flag::{InfoLevel, Verbosity}; +use shared::log_config::LogConfig; #[derive(clap::ValueEnum, Clone, Debug, Copy)] pub enum CargoEnv { @@ -34,6 +34,6 @@ pub struct AppConfig { #[clap(long, env, default_value = "5")] pub initial_query_retry_attempts: usize, - #[command(flatten)] - pub verbosity: Verbosity, + #[clap(flatten)] + pub log: LogConfig, } diff --git a/chain/src/main.rs b/chain/src/main.rs index b55d50c4e..f3ecf6e54 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -15,7 +15,6 @@ use chain::services::{ }; use chrono::{NaiveDateTime, Utc}; use clap::Parser; -use clap_verbosity_flag::LevelFilter; use deadpool_diesel::postgres::Object; use namada_sdk::time::DateTimeUtc; use orm::migrations::run_migrations; @@ -31,8 +30,6 @@ use shared::validator::ValidatorSet; use tendermint_rpc::HttpClient; use tokio_retry::strategy::{jitter, ExponentialBackoff}; use tokio_retry::Retry; -use tracing::Level; -use tracing_subscriber::FmtSubscriber; #[tokio::main] async fn main() -> Result<(), MainError> { @@ -50,19 +47,7 @@ async fn main() -> Result<(), MainError> { checksums.add(code_path, code.to_lowercase()); } - let log_level = match config.verbosity.log_level_filter() { - LevelFilter::Off => None, - LevelFilter::Error => Some(Level::ERROR), - LevelFilter::Warn => Some(Level::WARN), - LevelFilter::Info => Some(Level::INFO), - LevelFilter::Debug => Some(Level::DEBUG), - LevelFilter::Trace => Some(Level::TRACE), - }; - if let Some(log_level) = log_level { - let subscriber = - FmtSubscriber::builder().with_max_level(log_level).finish(); - tracing::subscriber::set_global_default(subscriber).unwrap(); - } + config.log.init(); let client = Arc::new(client); diff --git a/governance/Cargo.toml b/governance/Cargo.toml index dbaad7495..8a369efe1 100644 --- a/governance/Cargo.toml +++ b/governance/Cargo.toml @@ -15,7 +15,6 @@ path = "src/main.rs" [dependencies] tokio.workspace = true tracing.workspace = true -tracing-subscriber.workspace = true chrono.workspace = true clap.workspace = true anyhow.workspace = true @@ -27,7 +26,6 @@ futures.workspace = true deadpool-diesel.workspace = true diesel.workspace = true orm.workspace = true -clap-verbosity-flag.workspace = true tokio-retry.workspace = true [build-dependencies] diff --git a/governance/src/config.rs b/governance/src/config.rs index fe10c015d..cf5b68397 100644 --- a/governance/src/config.rs +++ b/governance/src/config.rs @@ -1,7 +1,7 @@ use core::fmt; use std::fmt::Display; -use clap_verbosity_flag::{InfoLevel, Verbosity}; +use shared::log_config::LogConfig; #[derive(clap::ValueEnum, Clone, Debug, Copy)] pub enum CargoEnv { @@ -26,6 +26,6 @@ pub struct AppConfig { #[clap(long, env)] pub database_url: String, - #[command(flatten)] - pub verbosity: Verbosity, + #[clap(flatten)] + pub log: LogConfig, } diff --git a/governance/src/main.rs b/governance/src/main.rs index 3f1d72bec..4b61d07f1 100644 --- a/governance/src/main.rs +++ b/governance/src/main.rs @@ -4,7 +4,6 @@ use std::time::Duration; use chrono::{NaiveDateTime, Utc}; use clap::Parser; -use clap_verbosity_flag::LevelFilter; use deadpool_diesel::postgres::Object; use governance::config::AppConfig; use governance::repository; @@ -18,27 +17,12 @@ use shared::error::{AsDbError, AsRpcError, ContextDbInteractError, MainError}; use tendermint_rpc::HttpClient; use tokio::sync::{Mutex, MutexGuard}; use tokio::time::Instant; -use tracing::Level; -use tracing_subscriber::FmtSubscriber; #[tokio::main] async fn main() -> Result<(), MainError> { let config = AppConfig::parse(); - let log_level = match config.verbosity.log_level_filter() { - LevelFilter::Off => None, - LevelFilter::Error => Some(Level::ERROR), - LevelFilter::Warn => Some(Level::WARN), - LevelFilter::Info => Some(Level::INFO), - LevelFilter::Debug => Some(Level::DEBUG), - LevelFilter::Trace => Some(Level::TRACE), - }; - - if let Some(log_level) = log_level { - let subscriber = - FmtSubscriber::builder().with_max_level(log_level).finish(); - tracing::subscriber::set_global_default(subscriber).unwrap(); - } + config.log.init(); tracing::info!("version: {}", env!("VERGEN_GIT_SHA").to_string()); diff --git a/parameters/Cargo.toml b/parameters/Cargo.toml index 879e2dc72..c75b2f943 100644 --- a/parameters/Cargo.toml +++ b/parameters/Cargo.toml @@ -15,7 +15,6 @@ path = "src/main.rs" [dependencies] tokio.workspace = true tracing.workspace = true -tracing-subscriber.workspace = true chrono.workspace = true clap.workspace = true anyhow.workspace = true @@ -30,7 +29,6 @@ deadpool-diesel.workspace = true diesel.workspace = true diesel_migrations.workspace = true orm.workspace = true -clap-verbosity-flag.workspace = true tokio-retry.workspace = true smooth-operator.workspace = true diff --git a/parameters/src/config.rs b/parameters/src/config.rs index c18044ca2..d938213b5 100644 --- a/parameters/src/config.rs +++ b/parameters/src/config.rs @@ -1,7 +1,7 @@ use core::fmt; use std::fmt::Display; -use clap_verbosity_flag::{InfoLevel, Verbosity}; +use shared::log_config::LogConfig; #[derive(clap::ValueEnum, Clone, Debug, Copy)] pub enum CargoEnv { @@ -23,9 +23,9 @@ pub struct AppConfig { #[clap(long, env)] pub database_url: String, - #[command(flatten)] - pub verbosity: Verbosity, - #[clap(long, env, default_value_t = 30)] pub sleep_for: u64, + + #[clap(flatten)] + pub log: LogConfig, } diff --git a/parameters/src/main.rs b/parameters/src/main.rs index f5245e44d..ca197d7e0 100644 --- a/parameters/src/main.rs +++ b/parameters/src/main.rs @@ -4,7 +4,6 @@ use std::time::Duration; use chrono::NaiveDateTime; use clap::Parser; -use clap_verbosity_flag::LevelFilter; use deadpool_diesel::postgres::Object; use namada_sdk::state::EPOCH_SWITCH_BLOCKS_DELAY; use namada_sdk::time::{DateTimeUtc, Utc}; @@ -23,26 +22,12 @@ use shared::error::{AsDbError, AsRpcError, ContextDbInteractError, MainError}; use tendermint_rpc::HttpClient; use tokio::sync::{Mutex, MutexGuard}; use tokio::time::Instant; -use tracing::Level; -use tracing_subscriber::FmtSubscriber; #[tokio::main] async fn main() -> Result<(), MainError> { let config = AppConfig::parse(); - let log_level = match config.verbosity.log_level_filter() { - LevelFilter::Off => None, - LevelFilter::Error => Some(Level::ERROR), - LevelFilter::Warn => Some(Level::WARN), - LevelFilter::Info => Some(Level::INFO), - LevelFilter::Debug => Some(Level::DEBUG), - LevelFilter::Trace => Some(Level::TRACE), - }; - if let Some(log_level) = log_level { - let subscriber = - FmtSubscriber::builder().with_max_level(log_level).finish(); - tracing::subscriber::set_global_default(subscriber).unwrap(); - } + config.log.init(); let client = Arc::new(HttpClient::new(config.tendermint_url.as_str()).unwrap()); diff --git a/pos/Cargo.toml b/pos/Cargo.toml index cc032cea3..ec972c52d 100644 --- a/pos/Cargo.toml +++ b/pos/Cargo.toml @@ -15,7 +15,6 @@ path = "src/main.rs" [dependencies] tokio.workspace = true tracing.workspace = true -tracing-subscriber.workspace = true chrono.workspace = true clap.workspace = true anyhow.workspace = true @@ -28,7 +27,6 @@ deadpool-diesel.workspace = true diesel.workspace = true diesel_migrations.workspace = true orm.workspace = true -clap-verbosity-flag.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/pos/src/config.rs b/pos/src/config.rs index 6e14e0018..723e3e86e 100644 --- a/pos/src/config.rs +++ b/pos/src/config.rs @@ -1,7 +1,7 @@ use core::fmt; use std::fmt::Display; -use clap_verbosity_flag::{InfoLevel, Verbosity}; +use shared::log_config::LogConfig; #[derive(clap::ValueEnum, Clone, Debug, Copy)] pub enum CargoEnv { @@ -23,6 +23,6 @@ pub struct AppConfig { #[clap(long, env)] pub database_url: String, - #[command(flatten)] - pub verbosity: Verbosity, + #[clap(flatten)] + pub log: LogConfig, } diff --git a/pos/src/main.rs b/pos/src/main.rs index 807003a41..76f7842c1 100644 --- a/pos/src/main.rs +++ b/pos/src/main.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use chrono::{NaiveDateTime, Utc}; use clap::Parser; -use clap_verbosity_flag::LevelFilter; use deadpool_diesel::postgres::Object; use namada_sdk::time::DateTimeUtc; use orm::crawler_state::EpochStateInsertDb; @@ -17,26 +16,12 @@ use shared::crawler; use shared::crawler_state::{CrawlerName, EpochCrawlerState}; use shared::error::{AsDbError, AsRpcError, ContextDbInteractError, MainError}; use tendermint_rpc::HttpClient; -use tracing::Level; -use tracing_subscriber::FmtSubscriber; #[tokio::main] async fn main() -> Result<(), MainError> { let config = AppConfig::parse(); - let log_level = match config.verbosity.log_level_filter() { - LevelFilter::Off => None, - LevelFilter::Error => Some(Level::ERROR), - LevelFilter::Warn => Some(Level::WARN), - LevelFilter::Info => Some(Level::INFO), - LevelFilter::Debug => Some(Level::DEBUG), - LevelFilter::Trace => Some(Level::TRACE), - }; - if let Some(log_level) = log_level { - let subscriber = - FmtSubscriber::builder().with_max_level(log_level).finish(); - tracing::subscriber::set_global_default(subscriber).unwrap(); - } + config.log.init(); let client = Arc::new(HttpClient::new(config.tendermint_url.as_str()).unwrap()); diff --git a/rewards/src/config.rs b/rewards/src/config.rs index fe10c015d..cf5b68397 100644 --- a/rewards/src/config.rs +++ b/rewards/src/config.rs @@ -1,7 +1,7 @@ use core::fmt; use std::fmt::Display; -use clap_verbosity_flag::{InfoLevel, Verbosity}; +use shared::log_config::LogConfig; #[derive(clap::ValueEnum, Clone, Debug, Copy)] pub enum CargoEnv { @@ -26,6 +26,6 @@ pub struct AppConfig { #[clap(long, env)] pub database_url: String, - #[command(flatten)] - pub verbosity: Verbosity, + #[clap(flatten)] + pub log: LogConfig, } diff --git a/rewards/src/main.rs b/rewards/src/main.rs index 40bf70769..a28e09f54 100644 --- a/rewards/src/main.rs +++ b/rewards/src/main.rs @@ -4,7 +4,6 @@ use std::time::Duration; use chrono::NaiveDateTime; use clap::Parser; -use clap_verbosity_flag::LevelFilter; use deadpool_diesel::postgres::Object; use namada_sdk::time::{DateTimeUtc, Utc}; use orm::migrations::run_migrations; @@ -17,27 +16,12 @@ use shared::crawler_state::{CrawlerName, IntervalCrawlerState}; use shared::error::{AsDbError, AsRpcError, ContextDbInteractError, MainError}; use tendermint_rpc::HttpClient; use tokio::time::sleep; -use tracing::Level; -use tracing_subscriber::FmtSubscriber; #[tokio::main] async fn main() -> Result<(), MainError> { let config = AppConfig::parse(); - let log_level = match config.verbosity.log_level_filter() { - LevelFilter::Off => None, - LevelFilter::Error => Some(Level::ERROR), - LevelFilter::Warn => Some(Level::WARN), - LevelFilter::Info => Some(Level::INFO), - LevelFilter::Debug => Some(Level::DEBUG), - LevelFilter::Trace => Some(Level::TRACE), - }; - - if let Some(log_level) = log_level { - let subscriber = - FmtSubscriber::builder().with_max_level(log_level).finish(); - tracing::subscriber::set_global_default(subscriber).unwrap(); - } + config.log.init(); tracing::info!("version: {}", env!("VERGEN_GIT_SHA").to_string()); diff --git a/seeder/Cargo.toml b/seeder/Cargo.toml index 68a5fb6ea..3c3c42f5b 100644 --- a/seeder/Cargo.toml +++ b/seeder/Cargo.toml @@ -15,14 +15,12 @@ path = "src/main.rs" [dependencies] tokio.workspace = true tracing.workspace = true -tracing-subscriber.workspace = true clap.workspace = true anyhow.workspace = true shared.workspace = true deadpool-diesel.workspace = true diesel.workspace = true orm.workspace = true -clap-verbosity-flag.workspace = true rand.workspace = true [build-dependencies] diff --git a/seeder/src/config.rs b/seeder/src/config.rs index 7ef53f2b0..8918c2dce 100644 --- a/seeder/src/config.rs +++ b/seeder/src/config.rs @@ -1,7 +1,7 @@ use core::fmt; use std::fmt::Display; -use clap_verbosity_flag::{InfoLevel, Verbosity}; +use shared::log_config::LogConfig; #[derive(clap::ValueEnum, Clone, Debug, Copy)] pub enum CargoEnv { @@ -41,6 +41,6 @@ pub struct AppConfig { #[clap(long, env, default_value_t = 10)] pub total_balances: u64, - #[command(flatten)] - pub verbosity: Verbosity, + #[clap(flatten)] + pub log: LogConfig, } diff --git a/seeder/src/main.rs b/seeder/src/main.rs index d763cd144..4ef7df909 100644 --- a/seeder/src/main.rs +++ b/seeder/src/main.rs @@ -1,6 +1,5 @@ use anyhow::Context; use clap::Parser; -use clap_verbosity_flag::LevelFilter; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper}; use orm::balances::BalanceChangesInsertDb; use orm::bond::BondInsertDb; @@ -28,27 +27,12 @@ use shared::rewards::Reward; use shared::unbond::Unbond; use shared::validator::Validator; use shared::vote::GovernanceVote; -use tracing::Level; -use tracing_subscriber::FmtSubscriber; #[tokio::main] async fn main() -> anyhow::Result<(), MainError> { let config = AppConfig::parse(); - let log_level = match config.verbosity.log_level_filter() { - LevelFilter::Off => None, - LevelFilter::Error => Some(Level::ERROR), - LevelFilter::Warn => Some(Level::WARN), - LevelFilter::Info => Some(Level::INFO), - LevelFilter::Debug => Some(Level::DEBUG), - LevelFilter::Trace => Some(Level::TRACE), - }; - - if let Some(log_level) = log_level { - let subscriber = - FmtSubscriber::builder().with_max_level(log_level).finish(); - tracing::subscriber::set_global_default(subscriber).unwrap(); - } + config.log.init(); tracing::info!("version: {}", env!("VERGEN_GIT_SHA").to_string()); diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 96e457161..c8a8ae21e 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -17,6 +17,8 @@ anyhow.workspace = true async-stream.workspace = true bimap.workspace = true bigdecimal.workspace = true +clap.workspace = true +clap-verbosity-flag.workspace = true futures-core.workspace = true futures-util.workspace = true futures.workspace = true @@ -35,5 +37,6 @@ thiserror.workspace = true tokio-retry.workspace = true tokio.workspace = true tracing.workspace = true +tracing-subscriber.workspace = true fake.workspace = true rand.workspace = true \ No newline at end of file diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 989a454c0..cc23a7825 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -10,6 +10,7 @@ pub mod gas; pub mod genesis; pub mod header; pub mod id; +pub mod log_config; pub mod parameters; pub mod proposal; pub mod public_key; diff --git a/shared/src/log_config.rs b/shared/src/log_config.rs new file mode 100644 index 000000000..4999e5056 --- /dev/null +++ b/shared/src/log_config.rs @@ -0,0 +1,48 @@ +use core::fmt; +use std::fmt::Display; + +use clap_verbosity_flag::{InfoLevel, LevelFilter, Verbosity}; +use tracing::Level; +use tracing_subscriber::FmtSubscriber; + +#[derive(clap::ValueEnum, Clone, Debug, Copy)] +pub enum LogFormat { + Json, + Text, +} + +impl Display for LogFormat { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", format!("{:?}", self).to_lowercase()) + } +} + +#[derive(clap::Parser)] +pub struct LogConfig { + #[command(flatten)] + pub verbosity: Verbosity, + + #[clap(long, env, default_value_t = LogFormat::Text, help = "Logging format")] + pub log_format: LogFormat, +} + +impl LogConfig { + pub fn init(&self) { + let log_level = match self.verbosity.log_level_filter() { + LevelFilter::Off => None, + LevelFilter::Error => Some(Level::ERROR), + LevelFilter::Warn => Some(Level::WARN), + LevelFilter::Info => Some(Level::INFO), + LevelFilter::Debug => Some(Level::DEBUG), + LevelFilter::Trace => Some(Level::TRACE), + }; + if let Some(log_level) = log_level { + let subscriber = FmtSubscriber::builder().with_max_level(log_level); + + match self.log_format { + LogFormat::Text => subscriber.init(), + LogFormat::Json => subscriber.json().flatten_event(true).init(), + }; + } + } +} diff --git a/transactions/Cargo.toml b/transactions/Cargo.toml index 3ab4192b5..57ff1f28d 100644 --- a/transactions/Cargo.toml +++ b/transactions/Cargo.toml @@ -15,7 +15,6 @@ path = "src/main.rs" [dependencies] tokio.workspace = true tracing.workspace = true -tracing-subscriber.workspace = true chrono.workspace = true clap.workspace = true anyhow.workspace = true @@ -28,7 +27,6 @@ deadpool-diesel.workspace = true diesel.workspace = true diesel_migrations.workspace = true orm.workspace = true -clap-verbosity-flag.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/transactions/src/config.rs b/transactions/src/config.rs index 22e793640..16b32ac28 100644 --- a/transactions/src/config.rs +++ b/transactions/src/config.rs @@ -1,7 +1,7 @@ use core::fmt; use std::fmt::Display; -use clap_verbosity_flag::{InfoLevel, Verbosity}; +use shared::log_config::LogConfig; #[derive(clap::ValueEnum, Clone, Debug, Copy)] pub enum CargoEnv { @@ -26,6 +26,6 @@ pub struct AppConfig { #[clap(long, env)] pub database_url: String, - #[command(flatten)] - pub verbosity: Verbosity, + #[clap(flatten)] + pub log: LogConfig, } diff --git a/transactions/src/main.rs b/transactions/src/main.rs index 961e6c348..f6bbfc65e 100644 --- a/transactions/src/main.rs +++ b/transactions/src/main.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use anyhow::Context; use chrono::{NaiveDateTime, Utc}; use clap::Parser; -use clap_verbosity_flag::LevelFilter; use deadpool_diesel::postgres::Object; use orm::migrations::run_migrations; use shared::block::Block; @@ -14,8 +13,6 @@ use shared::crawler::crawl; use shared::crawler_state::BlockCrawlerState; use shared::error::{AsDbError, AsRpcError, ContextDbInteractError, MainError}; use tendermint_rpc::HttpClient; -use tracing::Level; -use tracing_subscriber::FmtSubscriber; use transactions::app_state::AppState; use transactions::config::AppConfig; use transactions::repository::transactions as transaction_repo; @@ -28,19 +25,7 @@ use transactions::services::{ async fn main() -> Result<(), MainError> { let config = AppConfig::parse(); - let log_level = match config.verbosity.log_level_filter() { - LevelFilter::Off => None, - LevelFilter::Error => Some(Level::ERROR), - LevelFilter::Warn => Some(Level::WARN), - LevelFilter::Info => Some(Level::INFO), - LevelFilter::Debug => Some(Level::DEBUG), - LevelFilter::Trace => Some(Level::TRACE), - }; - if let Some(log_level) = log_level { - let subscriber = - FmtSubscriber::builder().with_max_level(log_level).finish(); - tracing::subscriber::set_global_default(subscriber).unwrap(); - } + config.log.init(); let client = Arc::new(HttpClient::new(config.tendermint_url.as_str()).unwrap()); From f7061168b56fbe0f92ca33f06cdb29d601976229 Mon Sep 17 00:00:00 2001 From: Mateusz Jasiuk Date: Wed, 4 Dec 2024 14:31:15 +0100 Subject: [PATCH 04/14] fix: insert bonds and unbonds in chunks (#178) --- chain/src/main.rs | 15 +++++--- chain/src/repository/pos.rs | 68 +++++++++++++++++++++++++++++++++++-- chain/src/services/utils.rs | 3 +- orm/src/schema.rs | 2 +- 4 files changed, 79 insertions(+), 9 deletions(-) diff --git a/chain/src/main.rs b/chain/src/main.rs index f3ecf6e54..b5a8d9516 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -380,8 +380,9 @@ async fn try_initial_query( let tokens = query_tokens(client).await.into_rpc_error()?; - // This can sometimes fail if the last block height in the node has moved forward after we queried - // for it. In that case, query_all_balances returns an Err indicating that it can only be used for + // This can sometimes fail if the last block height in the node has moved + // forward after we queried for it. In that case, query_all_balances + // returns an Err indicating that it can only be used for // the last block. This function will be retried in that case. let balances = query_all_balances(client, block_height) .await @@ -461,8 +462,14 @@ async fn try_initial_query( validator_set, )?; - repository::pos::insert_bonds(transaction_conn, bonds)?; - repository::pos::insert_unbonds(transaction_conn, unbonds)?; + repository::pos::insert_bonds_in_chunks( + transaction_conn, + bonds, + )?; + repository::pos::insert_unbonds_in_chunks( + transaction_conn, + unbonds, + )?; repository::crawler_state::upsert_crawler_state( transaction_conn, diff --git a/chain/src/repository/pos.rs b/chain/src/repository/pos.rs index a03180584..83322f6b9 100644 --- a/chain/src/repository/pos.rs +++ b/chain/src/repository/pos.rs @@ -1,10 +1,12 @@ use std::collections::HashSet; use anyhow::Context; +use diesel::sql_types::BigInt; use diesel::upsert::excluded; use diesel::{ - BoolExpressionMethods, ExpressionMethods, OptionalEmptyChangesetExtension, - PgConnection, QueryDsl, RunQueryDsl, SelectableHelper, + sql_query, BoolExpressionMethods, ExpressionMethods, + OptionalEmptyChangesetExtension, PgConnection, QueryDsl, QueryableByName, + RunQueryDsl, SelectableHelper, }; use orm::bond::BondInsertDb; use orm::schema::{bonds, pos_rewards, unbonds, validators}; @@ -18,6 +20,20 @@ use shared::id::Id; use shared::unbond::{UnbondAddresses, Unbonds}; use shared::validator::{ValidatorMetadataChange, ValidatorSet}; +pub const MAX_PARAM_SIZE: u16 = u16::MAX; + +#[derive(QueryableByName)] +struct UnbondsColCount { + #[diesel(sql_type = BigInt)] + count: i64, +} + +#[derive(QueryableByName)] +struct BondsColCount { + #[diesel(sql_type = BigInt)] + count: i64, +} + pub fn clear_bonds( transaction_conn: &mut PgConnection, addresses: Vec<(Id, Id)>, @@ -50,6 +66,30 @@ pub fn clear_bonds( anyhow::Ok(()) } +pub fn insert_bonds_in_chunks( + transaction_conn: &mut PgConnection, + bonds: Bonds, +) -> anyhow::Result<()> { + let bonds_col_count = sql_query( + "SELECT COUNT(*) + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'bonds';", + ) + .get_result::(transaction_conn)?; + + for chunk in bonds + // We have to divide MAX_PARAM_SIZE by the number of columns in the + // balances table to get the correct number of rows in the + // chunk. + .chunks((MAX_PARAM_SIZE as i64 / bonds_col_count.count) as usize) + { + insert_bonds(transaction_conn, chunk.to_vec())? + } + + anyhow::Ok(()) +} + pub fn insert_bonds( transaction_conn: &mut PgConnection, bonds: Bonds, @@ -87,6 +127,30 @@ pub fn insert_bonds( anyhow::Ok(()) } +pub fn insert_unbonds_in_chunks( + transaction_conn: &mut PgConnection, + unbonds: Unbonds, +) -> anyhow::Result<()> { + let unbonds_col_count = sql_query( + "SELECT COUNT(*) + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'unbonds';", + ) + .get_result::(transaction_conn)?; + + for chunk in unbonds + // We have to divide MAX_PARAM_SIZE by the number of columns in the + // balances table to get the correct number of rows in the + // chunk. + .chunks((MAX_PARAM_SIZE as i64 / unbonds_col_count.count) as usize) + { + insert_unbonds(transaction_conn, chunk.to_vec())? + } + + anyhow::Ok(()) +} + pub fn insert_unbonds( transaction_conn: &mut PgConnection, unbonds: Unbonds, diff --git a/chain/src/services/utils.rs b/chain/src/services/utils.rs index 64dcfc57e..1e430f86c 100644 --- a/chain/src/services/utils.rs +++ b/chain/src/services/utils.rs @@ -1,9 +1,8 @@ use namada_sdk::borsh::BorshDeserialize; use namada_sdk::queries::RPC; use namada_sdk::storage::{self, PrefixValue}; -use tendermint_rpc::HttpClient; - use shared::block::BlockHeight; +use tendermint_rpc::HttpClient; /// Query a range of storage values with a matching prefix and decode them with /// [`BorshDeserialize`]. Returns an iterator of the storage keys paired with diff --git a/orm/src/schema.rs b/orm/src/schema.rs index 854563909..cf18897c1 100644 --- a/orm/src/schema.rs +++ b/orm/src/schema.rs @@ -77,11 +77,11 @@ pub mod sql_types { diesel::table! { balance_changes (id) { id -> Int4, + height -> Int4, owner -> Varchar, #[max_length = 64] token -> Varchar, raw_amount -> Numeric, - height -> Int4, } } From e45baf93dc2daecabc48138a29826fe514ac3d90 Mon Sep 17 00:00:00 2001 From: Fraccaroli Gianmarco Date: Wed, 4 Dec 2024 14:39:22 +0100 Subject: [PATCH 05/14] v1.0.0 (#176) * v1.0.0 * ci: minor fix --- .github/workflows/deploy.yml | 5 +++-- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8b344b15c..d4b3211ac 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,9 +10,10 @@ on: - "v?[0-9]+.[0-9]+.[0-9]+-[a-z]+-rc" permissions: - contents: read + packages: write pages: write id-token: write + contents: read jobs: swagger-ui: @@ -84,7 +85,7 @@ jobs: context: . file: Dockerfile build-args: PACKAGE=${{ matrix.docker.package }} - push: startsWith(github.ref, 'refs/tags/v') + push: ${{ startsWith(github.ref, 'refs/tags/v') }} tags: ${{ env.REGISTRY_URL }}/anoma/namada-indexer:${{ matrix.docker.image }}-${{ steps.get_version.outputs.version-without-v }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha diff --git a/Cargo.toml b/Cargo.toml index 0d4e558d9..098fe6ad6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ authors = ["Heliax "] edition = "2021" license = "GPL-3.0" readme = "README.md" -version = "0.1.0" +version = "1.0.0" [workspace.dependencies] clokwerk = "0.4.0" From 64471a05e5fddd62dbb80d38546388a4d134c92c Mon Sep 17 00:00:00 2001 From: Joel Nordell <94570446+joel-u410@users.noreply.github.com> Date: Thu, 5 Dec 2024 02:49:55 -0600 Subject: [PATCH 06/14] refactor: improve insert_balance_in_chunks -> insert_balances (#172) * [chain] use compile-time column count for insert_balance_in_chunks * [chain] Rename insert_balance_in_chunks -> insert_balances; remove non-chunked fn --- chain/src/main.rs | 4 +- chain/src/repository/balance.rs | 86 +++++++++++++-------------------- shared/src/lib.rs | 1 + shared/src/tuple_len.rs | 54 +++++++++++++++++++++ 4 files changed, 90 insertions(+), 55 deletions(-) create mode 100644 shared/src/tuple_len.rs diff --git a/chain/src/main.rs b/chain/src/main.rs index b5a8d9516..cd81f2fd0 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -285,7 +285,7 @@ async fn crawling_fn( ibc_tokens, )?; - repository::balance::insert_balance_in_chunks( + repository::balance::insert_balances( transaction_conn, balances, )?; @@ -442,7 +442,7 @@ async fn try_initial_query( "Inserting {} balances...", balances.len() ); - repository::balance::insert_balance_in_chunks( + repository::balance::insert_balances( transaction_conn, balances, )?; diff --git a/chain/src/repository/balance.rs b/chain/src/repository/balance.rs index cfd3ee59f..3269765d0 100644 --- a/chain/src/repository/balance.rs +++ b/chain/src/repository/balance.rs @@ -1,61 +1,41 @@ use anyhow::Context; -use diesel::sql_types::BigInt; -use diesel::{sql_query, PgConnection, QueryableByName, RunQueryDsl}; +use diesel::{PgConnection, RunQueryDsl}; use orm::balances::BalanceChangesInsertDb; use orm::schema::{balance_changes, ibc_token, token}; use orm::token::{IbcTokenInsertDb, TokenInsertDb}; use shared::balance::Balances; use shared::token::Token; +use shared::tuple_len::TupleLen; pub const MAX_PARAM_SIZE: u16 = u16::MAX; -#[derive(QueryableByName)] -struct BalanceColCount { - #[diesel(sql_type = BigInt)] - count: i64, -} - -pub fn insert_balance( +pub fn insert_balances( transaction_conn: &mut PgConnection, balances: Balances, ) -> anyhow::Result<()> { - diesel::insert_into(balance_changes::table) - .values::<&Vec>( - &balances - .into_iter() - .map(BalanceChangesInsertDb::from_balance) - .collect::>(), - ) - .on_conflict(( - balance_changes::columns::owner, - balance_changes::columns::token, - balance_changes::columns::height, - )) - .do_nothing() - .execute(transaction_conn) - .context("Failed to update balances in db")?; - - anyhow::Ok(()) -} - -pub fn insert_balance_in_chunks( - transaction_conn: &mut PgConnection, - balances: Balances, -) -> anyhow::Result<()> { - let balances_col_count = sql_query( - "SELECT COUNT(*) - FROM information_schema.columns - WHERE table_schema = 'public' - AND table_name = 'balances';", - ) - .get_result::(transaction_conn)?; + let balances_col_count = balance_changes::all_columns.len() as i64; for chunk in balances // We have to divide MAX_PARAM_SIZE by the number of columns in the // balances table to get the correct number of rows in the // chunk. - .chunks((MAX_PARAM_SIZE as i64 / balances_col_count.count) as usize) + .chunks((MAX_PARAM_SIZE as i64 / balances_col_count) as usize) { - insert_balance(transaction_conn, chunk.to_vec())? + diesel::insert_into(balance_changes::table) + .values::<&Vec>( + &chunk + .iter() + .cloned() + .map(BalanceChangesInsertDb::from_balance) + .collect::>(), + ) + .on_conflict(( + balance_changes::columns::owner, + balance_changes::columns::token, + balance_changes::columns::height, + )) + .do_nothing() + .execute(transaction_conn) + .context("Failed to update balances in db")?; } anyhow::Ok(()) @@ -116,7 +96,7 @@ mod tests { let db = TestDb::new(); db.run_test(|conn| { - insert_balance(conn, vec![])?; + insert_balances(conn, vec![])?; let queried_balance = query_all_balances(conn)?; @@ -152,7 +132,7 @@ mod tests { insert_tokens(conn, vec![token.clone()])?; - insert_balance(conn, vec![balance.clone()])?; + insert_balances(conn, vec![balance.clone()])?; let queried_balance = query_balance_by_address(conn, owner, token)?; @@ -197,7 +177,7 @@ mod tests { ..(balance.clone()) }; - insert_balance(conn, vec![new_balance])?; + insert_balances(conn, vec![new_balance])?; let queried_balance = query_balance_by_address(conn, owner.clone(), token.clone())?; @@ -255,7 +235,7 @@ mod tests { seed_tokens_from_balance(conn, vec![new_balance.clone()])?; - insert_balance(conn, vec![new_balance])?; + insert_balances(conn, vec![new_balance])?; let queried_balance = query_balance_by_address(conn, owner.clone(), token.clone())?; @@ -312,7 +292,7 @@ mod tests { ..(balance.clone()) }; - insert_balance(conn, vec![new_balance])?; + insert_balances(conn, vec![new_balance])?; let queried_balance = query_balance_by_address(conn, owner.clone(), token.clone())?; @@ -366,7 +346,7 @@ mod tests { ..(balance.clone()) }; - let res = insert_balance(conn, vec![new_balance]); + let res = insert_balances(conn, vec![new_balance]); // Conflicting insert succeeds, but is ignored assert!(res.is_ok()); @@ -398,7 +378,7 @@ mod tests { seed_tokens_from_balance(conn, fake_balances.clone())?; - insert_balance(conn, fake_balances.clone())?; + insert_balances(conn, fake_balances.clone())?; assert_eq!(query_all_balances(conn)?.len(), fake_balances.len()); @@ -432,7 +412,7 @@ mod tests { insert_tokens(conn, vec![token.clone()])?; - insert_balance(conn, vec![balance.clone()])?; + insert_balances(conn, vec![balance.clone()])?; let queried_balance = query_balance_by_address(conn, owner, token)?; @@ -446,7 +426,7 @@ mod tests { /// Test that we can insert more than u16::MAX balances #[tokio::test] - async fn test_insert_balance_in_chunks_with_max_param_size_plus_one() { + async fn test_insert_balances_with_max_param_size_plus_one() { let db = TestDb::new(); db.run_test(|conn| { @@ -464,7 +444,7 @@ mod tests { insert_tokens(conn, vec![token])?; - let res = insert_balance_in_chunks(conn, balances); + let res = insert_balances(conn, balances); assert!(res.is_ok()); @@ -476,7 +456,7 @@ mod tests { /// Test that we can insert less than u16::MAX balances using chunks #[tokio::test] - async fn test_insert_balance_in_chunks_with_1000_params() { + async fn test_insert_balances_with_1000_params() { let db = TestDb::new(); db.run_test(|conn| { @@ -497,7 +477,7 @@ mod tests { seed_tokens_from_balance(conn, balances.clone())?; - let res = insert_balance_in_chunks(conn, balances); + let res = insert_balances(conn, balances); assert!(res.is_ok()); diff --git a/shared/src/lib.rs b/shared/src/lib.rs index cc23a7825..9d0f5ad46 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -18,6 +18,7 @@ pub mod rewards; pub mod ser; pub mod token; pub mod transaction; +pub mod tuple_len; pub mod unbond; pub mod utils; pub mod validator; diff --git a/shared/src/tuple_len.rs b/shared/src/tuple_len.rs new file mode 100644 index 000000000..2410ac952 --- /dev/null +++ b/shared/src/tuple_len.rs @@ -0,0 +1,54 @@ +// +// The TupleLen trait allows compile-time checking of the length of a tuple. This is useful for +// statically determining the number of columns in a diesel schema table. +// +// Use it like this: +// +// let num_columns = orm::schema::(table_name)::all_columns.len(); +// +// If you need to support tuples with more than 12 elements, you can add more type parameters to +// the tuple! macro invocation at the bottom of this file. +// + +pub trait TupleLen { + fn len(&self) -> usize; + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +// Base case for empty tuple +impl TupleLen for () { + fn len(&self) -> usize { + 0 + } +} + +macro_rules! peel { + ($name:ident,) => {}; + ($first:ident, $($rest:ident,)+) => { + tuple! { $($rest,)+ } + }; +} + +macro_rules! tuple { + () => {}; + ( $($name:ident,)+ ) => { + impl<$($name),+> TupleLen for ($($name,)+) { + #[inline] + fn len(&self) -> usize { + count_idents!($($name),+) + } + } + peel! { $($name,)+ } + } +} + +macro_rules! count_idents { + () => { 0 }; + ($name:ident) => { 1 }; + ($first:ident, $($rest:ident),+) => { 1 + count_idents!($($rest),+) }; +} + +// Initial invocation with maximum number of type parameters +tuple! { L, K, J, I, H, G, F, E, D, C, B, A, } From cdec92dcea5b61fa2a34cf5e2e1a226246bb1b6b Mon Sep 17 00:00:00 2001 From: Fraccaroli Gianmarco Date: Mon, 9 Dec 2024 11:40:08 +0100 Subject: [PATCH 07/14] update readme (#186) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index daf8980c3..903504821 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ A set of microservices that crawler data from a namada node, store them in a pos > 🔧 This is currently being worked on. Don't expect things to work! 🔧 +# Namadillo integration + +When using this project as a backend for [Namadillo](https://github.com/anoma/namada-interface), always checkout the latest tag, as the `main` branch could have an incompatible set of APIs. + ## Architecture The indexer is composed of a set microservices and a webserver, each one of these lives in his own crate. Each microservice is responsible of indexing some data from the chain and store them in the postgres database. Right now, there are 4 microservices: From cc30098ef7ab454089c65510fefc4a5a2d2cca9a Mon Sep 17 00:00:00 2001 From: Mateusz Jasiuk Date: Mon, 9 Dec 2024 14:17:42 +0100 Subject: [PATCH 08/14] feat: insert votes in chunks (#180) * refactor: insert bonds to use col count macro * feat: insert votes in chunks --- chain/src/main.rs | 10 ++---- chain/src/repository/balance.rs | 10 +++--- chain/src/repository/gov.rs | 20 +++++++++++ chain/src/repository/mod.rs | 1 + chain/src/repository/pos.rs | 63 +++++++++------------------------ chain/src/repository/utils.rs | 4 +++ shared/src/tuple_len.rs | 10 +++--- 7 files changed, 52 insertions(+), 66 deletions(-) create mode 100644 chain/src/repository/utils.rs diff --git a/chain/src/main.rs b/chain/src/main.rs index cd81f2fd0..5ef4d183e 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -462,14 +462,8 @@ async fn try_initial_query( validator_set, )?; - repository::pos::insert_bonds_in_chunks( - transaction_conn, - bonds, - )?; - repository::pos::insert_unbonds_in_chunks( - transaction_conn, - unbonds, - )?; + repository::pos::insert_bonds(transaction_conn, bonds)?; + repository::pos::insert_unbonds(transaction_conn, unbonds)?; repository::crawler_state::upsert_crawler_state( transaction_conn, diff --git a/chain/src/repository/balance.rs b/chain/src/repository/balance.rs index 3269765d0..101d08722 100644 --- a/chain/src/repository/balance.rs +++ b/chain/src/repository/balance.rs @@ -6,7 +6,8 @@ use orm::token::{IbcTokenInsertDb, TokenInsertDb}; use shared::balance::Balances; use shared::token::Token; use shared::tuple_len::TupleLen; -pub const MAX_PARAM_SIZE: u16 = u16::MAX; + +use super::utils::MAX_PARAM_SIZE; pub fn insert_balances( transaction_conn: &mut PgConnection, @@ -14,11 +15,8 @@ pub fn insert_balances( ) -> anyhow::Result<()> { let balances_col_count = balance_changes::all_columns.len() as i64; - for chunk in balances - // We have to divide MAX_PARAM_SIZE by the number of columns in the - // balances table to get the correct number of rows in the - // chunk. - .chunks((MAX_PARAM_SIZE as i64 / balances_col_count) as usize) + for chunk in + balances.chunks((MAX_PARAM_SIZE as i64 / balances_col_count) as usize) { diesel::insert_into(balance_changes::table) .values::<&Vec>( diff --git a/chain/src/repository/gov.rs b/chain/src/repository/gov.rs index ac62cfcde..7882ed6cb 100644 --- a/chain/src/repository/gov.rs +++ b/chain/src/repository/gov.rs @@ -7,8 +7,11 @@ use orm::governance_proposal::GovernanceProposalInsertDb; use orm::governance_votes::GovernanceProposalVoteInsertDb; use orm::schema::{governance_proposals, governance_votes}; use shared::proposal::{GovernanceProposal, TallyType}; +use shared::tuple_len::TupleLen; use shared::vote::GovernanceVote; +use super::utils::MAX_PARAM_SIZE; + pub fn insert_proposals( transaction_conn: &mut PgConnection, proposals: Vec<(GovernanceProposal, TallyType)>, @@ -45,6 +48,23 @@ pub fn insert_proposals( pub fn insert_votes( transaction_conn: &mut PgConnection, proposals_votes: HashSet, +) -> anyhow::Result<()> { + let votes_col_count = governance_votes::all_columns.len() as i64; + + for chunk in proposals_votes + .into_iter() + .collect::>() + .chunks((MAX_PARAM_SIZE as i64 / votes_col_count) as usize) + { + insert_votes_chunk(transaction_conn, chunk.to_vec())? + } + + anyhow::Ok(()) +} + +fn insert_votes_chunk( + transaction_conn: &mut PgConnection, + proposals_votes: Vec, ) -> anyhow::Result<()> { diesel::insert_into(governance_votes::table) .values::<&Vec>( diff --git a/chain/src/repository/mod.rs b/chain/src/repository/mod.rs index 611cf592a..efdb8fcdc 100644 --- a/chain/src/repository/mod.rs +++ b/chain/src/repository/mod.rs @@ -3,3 +3,4 @@ pub mod crawler_state; pub mod gov; pub mod pos; pub mod revealed_pk; +mod utils; diff --git a/chain/src/repository/pos.rs b/chain/src/repository/pos.rs index 83322f6b9..0960c229e 100644 --- a/chain/src/repository/pos.rs +++ b/chain/src/repository/pos.rs @@ -1,12 +1,10 @@ use std::collections::HashSet; use anyhow::Context; -use diesel::sql_types::BigInt; use diesel::upsert::excluded; use diesel::{ - sql_query, BoolExpressionMethods, ExpressionMethods, - OptionalEmptyChangesetExtension, PgConnection, QueryDsl, QueryableByName, - RunQueryDsl, SelectableHelper, + BoolExpressionMethods, ExpressionMethods, OptionalEmptyChangesetExtension, + PgConnection, QueryDsl, RunQueryDsl, SelectableHelper, }; use orm::bond::BondInsertDb; use orm::schema::{bonds, pos_rewards, unbonds, validators}; @@ -17,22 +15,11 @@ use orm::validators::{ use shared::block::Epoch; use shared::bond::Bonds; use shared::id::Id; +use shared::tuple_len::TupleLen; use shared::unbond::{UnbondAddresses, Unbonds}; use shared::validator::{ValidatorMetadataChange, ValidatorSet}; -pub const MAX_PARAM_SIZE: u16 = u16::MAX; - -#[derive(QueryableByName)] -struct UnbondsColCount { - #[diesel(sql_type = BigInt)] - count: i64, -} - -#[derive(QueryableByName)] -struct BondsColCount { - #[diesel(sql_type = BigInt)] - count: i64, -} +use super::utils::MAX_PARAM_SIZE; pub fn clear_bonds( transaction_conn: &mut PgConnection, @@ -66,31 +53,22 @@ pub fn clear_bonds( anyhow::Ok(()) } -pub fn insert_bonds_in_chunks( +pub fn insert_bonds( transaction_conn: &mut PgConnection, bonds: Bonds, ) -> anyhow::Result<()> { - let bonds_col_count = sql_query( - "SELECT COUNT(*) - FROM information_schema.columns - WHERE table_schema = 'public' - AND table_name = 'bonds';", - ) - .get_result::(transaction_conn)?; + let bonds_col_count = bonds::all_columns.len() as i64; - for chunk in bonds - // We have to divide MAX_PARAM_SIZE by the number of columns in the - // balances table to get the correct number of rows in the - // chunk. - .chunks((MAX_PARAM_SIZE as i64 / bonds_col_count.count) as usize) + for chunk in + bonds.chunks((MAX_PARAM_SIZE as i64 / bonds_col_count) as usize) { - insert_bonds(transaction_conn, chunk.to_vec())? + insert_bonds_chunk(transaction_conn, chunk.to_vec())? } anyhow::Ok(()) } -pub fn insert_bonds( +fn insert_bonds_chunk( transaction_conn: &mut PgConnection, bonds: Bonds, ) -> anyhow::Result<()> { @@ -127,31 +105,22 @@ pub fn insert_bonds( anyhow::Ok(()) } -pub fn insert_unbonds_in_chunks( +pub fn insert_unbonds( transaction_conn: &mut PgConnection, unbonds: Unbonds, ) -> anyhow::Result<()> { - let unbonds_col_count = sql_query( - "SELECT COUNT(*) - FROM information_schema.columns - WHERE table_schema = 'public' - AND table_name = 'unbonds';", - ) - .get_result::(transaction_conn)?; + let unbonds_col_count = unbonds::all_columns.len() as i64; - for chunk in unbonds - // We have to divide MAX_PARAM_SIZE by the number of columns in the - // balances table to get the correct number of rows in the - // chunk. - .chunks((MAX_PARAM_SIZE as i64 / unbonds_col_count.count) as usize) + for chunk in + unbonds.chunks((MAX_PARAM_SIZE as i64 / unbonds_col_count) as usize) { - insert_unbonds(transaction_conn, chunk.to_vec())? + insert_unbonds_chunk(transaction_conn, chunk.to_vec())? } anyhow::Ok(()) } -pub fn insert_unbonds( +fn insert_unbonds_chunk( transaction_conn: &mut PgConnection, unbonds: Unbonds, ) -> anyhow::Result<()> { diff --git a/chain/src/repository/utils.rs b/chain/src/repository/utils.rs new file mode 100644 index 000000000..bd4b8ce6d --- /dev/null +++ b/chain/src/repository/utils.rs @@ -0,0 +1,4 @@ +// Represents maximum number of parameters that we can insert into postgres in +// one go. To get the number of rows that we can insert in one chunk, we have to +// divide MAX_PARAM_SIZE by the number of columns in the given table. +pub const MAX_PARAM_SIZE: u16 = u16::MAX; diff --git a/shared/src/tuple_len.rs b/shared/src/tuple_len.rs index 2410ac952..a4f51c73f 100644 --- a/shared/src/tuple_len.rs +++ b/shared/src/tuple_len.rs @@ -1,13 +1,13 @@ -// -// The TupleLen trait allows compile-time checking of the length of a tuple. This is useful for -// statically determining the number of columns in a diesel schema table. +// The TupleLen trait allows compile-time checking of the length of a tuple. +// This is useful for statically determining the number of columns in a diesel +// schema table. // // Use it like this: // // let num_columns = orm::schema::(table_name)::all_columns.len(); // -// If you need to support tuples with more than 12 elements, you can add more type parameters to -// the tuple! macro invocation at the bottom of this file. +// If you need to support tuples with more than 12 elements, you can add more +// type parameters to the tuple! macro invocation at the bottom of this file. // pub trait TupleLen { From a8fde04832bc71c14f17be4a6ba033238b9d5584 Mon Sep 17 00:00:00 2001 From: Fraccaroli Gianmarco Date: Mon, 9 Dec 2024 15:10:15 +0100 Subject: [PATCH 09/14] return latest processed block (#162) --- swagger.yml | 2 ++ webserver/src/handler/crawler_state.rs | 1 + webserver/src/response/crawler_state.rs | 1 + webserver/src/service/crawler_state.rs | 2 ++ 4 files changed, 6 insertions(+) diff --git a/swagger.yml b/swagger.yml index 6888fed45..1fe04b436 100644 --- a/swagger.yml +++ b/swagger.yml @@ -635,6 +635,8 @@ paths: enum: [chain, governance, parameters, pos, rewards, transactions] timestamp: type: number + last_processed_block: + type: number components: schemas: diff --git a/webserver/src/handler/crawler_state.rs b/webserver/src/handler/crawler_state.rs index eecbf5bf3..8490bc2dd 100644 --- a/webserver/src/handler/crawler_state.rs +++ b/webserver/src/handler/crawler_state.rs @@ -41,6 +41,7 @@ pub async fn get_crawlers_timestamps( || CrawlersTimestamps { name: variant.to_string(), timestamp: 0, + last_processed_block: None }, |ct| ct.clone(), ) diff --git a/webserver/src/response/crawler_state.rs b/webserver/src/response/crawler_state.rs index d12fe51dc..d7f44571e 100644 --- a/webserver/src/response/crawler_state.rs +++ b/webserver/src/response/crawler_state.rs @@ -5,4 +5,5 @@ use serde::{Deserialize, Serialize}; pub struct CrawlersTimestamps { pub name: String, pub timestamp: i64, + pub last_processed_block: Option } diff --git a/webserver/src/service/crawler_state.rs b/webserver/src/service/crawler_state.rs index 0ca9c805c..661ec4adb 100644 --- a/webserver/src/service/crawler_state.rs +++ b/webserver/src/service/crawler_state.rs @@ -49,6 +49,8 @@ impl CrawlerStateService { .map(|crawler| CrawlersTimestamps { name: crawler.name.to_string(), timestamp: crawler.timestamp.and_utc().timestamp(), + last_processed_block: crawler.last_processed_block + }) .collect::>() }) From 7868f20dbe375d4e2dc1f8a8de9f6b26302c821d Mon Sep 17 00:00:00 2001 From: Fraccaroli Gianmarco Date: Mon, 9 Dec 2024 15:10:23 +0100 Subject: [PATCH 10/14] improve governance apis (#161) * return sha256 of wasm code * added endpoint to retreive governance data * update swagger --- Cargo.toml | 1 + swagger.yml | 14 ++++++++++++++ webserver/Cargo.toml | 1 + webserver/src/app.rs | 4 ++++ webserver/src/error/governance.rs | 3 +++ webserver/src/handler/governance.rs | 19 +++++++++++++++++++ webserver/src/response/governance.rs | 8 +++++++- webserver/src/service/governance.rs | 13 +++++++++++++ 8 files changed, 62 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 098fe6ad6..32f08c61d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,3 +86,4 @@ rand = "0.8.5" bigdecimal = "0.4.5" strum = "0.26.3" strum_macros = "0.26.3" +sha256 = "1.5.0" \ No newline at end of file diff --git a/swagger.yml b/swagger.yml index 1fe04b436..f56f1c739 100644 --- a/swagger.yml +++ b/swagger.yml @@ -357,6 +357,20 @@ paths: application/json: schema: $ref: '#/components/schemas/Proposal' + /api/v1/gov/proposal/{id}/data: + get: + summary: Get a governance proposal data by proposal id + parameters: + - in: path + name: id + schema: + type: integer + minimum: 0 + required: true + description: Proposal id + responses: + '200': + description: A Governance proposal data. /api/v1/gov/proposal/{id}/votes: get: summary: Get all the votes for a governance proposal diff --git a/webserver/Cargo.toml b/webserver/Cargo.toml index 39fcb69fa..586a7888c 100644 --- a/webserver/Cargo.toml +++ b/webserver/Cargo.toml @@ -50,6 +50,7 @@ shared.workspace = true strum.workspace = true strum_macros.workspace = true axum-prometheus = "0.7.0" +sha256.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/webserver/src/app.rs b/webserver/src/app.rs index 5e5ae6d42..fe1496ee0 100644 --- a/webserver/src/app.rs +++ b/webserver/src/app.rs @@ -84,6 +84,10 @@ impl ApplicationServer { "/gov/proposal/:id", get(gov_handlers::get_governance_proposal_by_id), ) + .route( + "/gov/proposal/:id/data", + get(gov_handlers::get_proposal_data_by_proposal_id), + ) .route( "/gov/proposal/:id/votes", get(gov_handlers::get_governance_proposal_votes), diff --git a/webserver/src/error/governance.rs b/webserver/src/error/governance.rs index 5d6252477..50ed5b5b8 100644 --- a/webserver/src/error/governance.rs +++ b/webserver/src/error/governance.rs @@ -10,6 +10,8 @@ pub enum GovernanceError { TooShortPattern(usize), #[error("Proposal {0} not found")] NotFound(u64), + #[error("Proposal {0} has no associated data")] + DataNotFound(u64), #[error("Database error: {0}")] Database(String), #[error("Unknown error: {0}")] @@ -21,6 +23,7 @@ impl IntoResponse for GovernanceError { let status_code = match self { GovernanceError::TooShortPattern(_) => StatusCode::BAD_REQUEST, GovernanceError::NotFound(_) => StatusCode::NOT_FOUND, + GovernanceError::DataNotFound(_) => StatusCode::NOT_FOUND, GovernanceError::Unknown(_) | GovernanceError::Database(_) => { StatusCode::INTERNAL_SERVER_ERROR } diff --git a/webserver/src/handler/governance.rs b/webserver/src/handler/governance.rs index 8430099b1..f49dfdccb 100644 --- a/webserver/src/handler/governance.rs +++ b/webserver/src/handler/governance.rs @@ -64,6 +64,25 @@ pub async fn get_governance_proposal_by_id( } } +#[debug_handler] +pub async fn get_proposal_data_by_proposal_id( + _headers: HeaderMap, + Path(proposal_id): Path, + State(state): State, +) -> Result { + let proposal = state.gov_service.find_proposal_data(proposal_id).await?; + + if let Some(data) = proposal { + if let Some(data) = data { + Ok(data) + } else { + Err(GovernanceError::DataNotFound(proposal_id).into()) + } + } else { + Err(GovernanceError::NotFound(proposal_id).into()) + } +} + #[debug_handler] pub async fn get_governance_proposal_votes( _headers: HeaderMap, diff --git a/webserver/src/response/governance.rs b/webserver/src/response/governance.rs index 9a75061d4..60fa32128 100644 --- a/webserver/src/response/governance.rs +++ b/webserver/src/response/governance.rs @@ -7,6 +7,7 @@ use orm::governance_proposal::{ }; use orm::governance_votes::{GovernanceProposalVoteDb, GovernanceVoteKindDb}; use serde::{Deserialize, Serialize}; +use sha256::digest; use super::utils::{epoch_progress, time_between_epochs}; @@ -182,7 +183,12 @@ impl Proposal { TallyType::LessOneHalfOverOneThirdNay } }, - data: value.data, + data: match value.kind { + GovernanceProposalKindDb::DefaultWithWasm => { + value.data.map(digest) + } + _ => value.data, + }, author: value.author, start_epoch: value.start_epoch.to_string(), end_epoch: value.end_epoch.to_string(), diff --git a/webserver/src/service/governance.rs b/webserver/src/service/governance.rs index fb12653ed..02dcf626f 100644 --- a/webserver/src/service/governance.rs +++ b/webserver/src/service/governance.rs @@ -68,6 +68,19 @@ impl GovernanceService { )) } + pub async fn find_proposal_data( + &self, + proposal_id: u64, + ) -> Result>, GovernanceError> { + let db_proposal = self + .governance_repo + .find_governance_proposals_by_id(proposal_id as i32) + .await + .map_err(GovernanceError::Database)?; + + Ok(db_proposal.map(|proposal| proposal.data)) + } + pub async fn find_all_governance_proposals( &self, status: Option, From 32b14eb286ede364021e6582f6af932f7ad83f90 Mon Sep 17 00:00:00 2001 From: Joel Nordell <94570446+joel-u410@users.noreply.github.com> Date: Mon, 9 Dec 2024 08:41:18 -0600 Subject: [PATCH 11/14] enhancement: start chain crawler from crawler_state when possible, only doing initial_query when necessary (#181) * enhancement: start chain crawler from crawler_state when possible, only doing initial_query when necessary * refactor: handle 3 possible results from initial crawl, to avoid re-crawling the same block in the success case --- chain/src/main.rs | 87 +++++++++++++++++++++++++++++++++++----- chain/src/services/db.rs | 37 ++++++++++++----- 2 files changed, 104 insertions(+), 20 deletions(-) diff --git a/chain/src/main.rs b/chain/src/main.rs index 5ef4d183e..9ae1e74ac 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -60,17 +60,84 @@ async fn main() -> Result<(), MainError> { .context_db_interact_error() .into_db_error()?; - initial_query( - &client, - &conn, - config.initial_query_retry_time, - config.initial_query_retry_attempts, - ) - .await?; - - let crawler_state = db_service::get_chain_crawler_state(&conn) + // See if we can start from existing crawler_state + let crawler_state = match db_service::try_get_chain_crawler_state(&conn) .await - .into_db_error()?; + .into_db_error()? + { + Some(crawler_state) => { + tracing::info!( + "Found chain crawler state, attempting initial crawl at block {}...", + crawler_state.last_processed_block + ); + + // Try to run crawler_fn with the last processed block + let crawl_result = crawling_fn( + crawler_state.last_processed_block, + client.clone(), + conn.clone(), + checksums.clone(), + ) + .await; + + match crawl_result { + Err(MainError::RpcError) => { + // If there was an RpcError, it likely means the block was pruned from the node. + // We need to do an initial_query in that case. + tracing::error!( + "Failed to query block {}, starting from initial_query ...", + crawler_state.last_processed_block, + ); + None + } + Err(_) => { + // If any other type of error occurred, we should not increment + // last_processed_block but crawl from there without initial_query + tracing::info!( + "Initial crawl had an error (not RpcError), continuing from block {}...", + crawler_state.last_processed_block + ); + Some(crawler_state) + } + Ok(_) => { + // If the crawl was successful, increment last_processed block and continue from there. + let next_block = crawler_state.last_processed_block + 1; + tracing::info!( + "Initial crawl was successful, continuing from block {}...", + next_block + ); + Some(ChainCrawlerState { + last_processed_block: next_block, + ..crawler_state + }) + } + } + } + None => { + tracing::info!( + "No chain crawler state found, starting from initial_query..." + ); + None + } + }; + + // Handle cases where we need to perform initial query + let crawler_state = match crawler_state { + Some(state) => state, + None => { + initial_query( + &client, + &conn, + config.initial_query_retry_time, + config.initial_query_retry_attempts, + ) + .await?; + + db_service::get_chain_crawler_state(&conn) + .await + .into_db_error()? + } + }; crawl( move |block_height| { diff --git a/chain/src/services/db.rs b/chain/src/services/db.rs index 4793c1bb4..6e07e25ee 100644 --- a/chain/src/services/db.rs +++ b/chain/src/services/db.rs @@ -1,6 +1,6 @@ use anyhow::Context; use deadpool_diesel::postgres::Object; -use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; +use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; use orm::crawler_state::{ ChainCrawlerStateDb, CrawlerNameDb, EpochCrawlerStateDb, }; @@ -9,10 +9,10 @@ use shared::block::{BlockHeight, Epoch}; use shared::crawler_state::{ChainCrawlerState, EpochCrawlerState}; use shared::error::ContextDbInteractError; -pub async fn get_chain_crawler_state( +pub async fn try_get_chain_crawler_state( conn: &Object, -) -> anyhow::Result { - let crawler_state: ChainCrawlerStateDb = conn +) -> anyhow::Result> { + let crawler_state: Option = conn .interact(move |conn| { crawler_state::table .filter(crawler_state::name.eq(CrawlerNameDb::Chain)) @@ -23,17 +23,34 @@ pub async fn get_chain_crawler_state( crawler_state::dsl::timestamp, )) .first(conn) + .optional() }) .await .context_db_interact_error()? .context("Failed to read chain crawler state from the db")?; - Ok(ChainCrawlerState { - last_processed_block: crawler_state.last_processed_block as BlockHeight, - last_processed_epoch: crawler_state.last_processed_epoch as Epoch, - first_block_in_epoch: crawler_state.first_block_in_epoch as BlockHeight, - timestamp: crawler_state.timestamp.and_utc().timestamp(), - }) + match crawler_state { + Some(crawler_state) => Ok(Some(ChainCrawlerState { + last_processed_block: crawler_state.last_processed_block + as BlockHeight, + last_processed_epoch: crawler_state.last_processed_epoch as Epoch, + first_block_in_epoch: crawler_state.first_block_in_epoch + as BlockHeight, + timestamp: crawler_state.timestamp.and_utc().timestamp(), + })), + None => Ok(None), + } +} + +pub async fn get_chain_crawler_state( + conn: &Object, +) -> anyhow::Result { + if let Some(crawler_state) = try_get_chain_crawler_state(conn).await? { + Ok(crawler_state) + } else { + Err(anyhow::format_err!("Chain crawler state not found")) + .context_db_interact_error() + } } pub async fn get_pos_crawler_state( From bd3a2e1468dec1a56c23e89b46ba254b129e6b64 Mon Sep 17 00:00:00 2001 From: Joel Nordell <94570446+joel-u410@users.noreply.github.com> Date: Tue, 10 Dec 2024 03:09:43 -0600 Subject: [PATCH 12/14] enhancement: webserver optional redis (#188) * [webserver] Add common log config like other services * [webserver] Make redis cache optional --- shared/src/log_config.rs | 2 +- webserver/src/appstate.rs | 38 ++++++++++++++++++++++---------------- webserver/src/config.rs | 7 ++++++- webserver/src/main.rs | 4 +--- 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/shared/src/log_config.rs b/shared/src/log_config.rs index 4999e5056..7ad06e846 100644 --- a/shared/src/log_config.rs +++ b/shared/src/log_config.rs @@ -17,7 +17,7 @@ impl Display for LogFormat { } } -#[derive(clap::Parser)] +#[derive(clap::Parser, Clone)] pub struct LogConfig { #[command(flatten)] pub verbosity: Verbosity, diff --git a/webserver/src/appstate.rs b/webserver/src/appstate.rs index 66515e213..874f15796 100644 --- a/webserver/src/appstate.rs +++ b/webserver/src/appstate.rs @@ -7,11 +7,11 @@ use deadpool_redis::{Config, Connection, Pool as CachePool}; #[derive(Clone)] pub struct AppState { db: DbPool, - cache: CachePool, + cache: Option, } impl AppState { - pub fn new(db_url: String, cache_url: String) -> Self { + pub fn new(db_url: String, cache_url: Option) -> Self { let max_pool_size = env::var("DATABASE_POOL_SIZE") .unwrap_or_else(|_| 16.to_string()) .parse::() @@ -35,27 +35,33 @@ impl AppState { } }; - let cache_pool = Config::from_url(cache_url) - .create_pool(Some(deadpool_redis::Runtime::Tokio1)); - let cache_pool = match cache_pool { - Ok(pool) => pool, - Err(e) => { - tracing::info!("Error building redis pool: {}", e.to_string()); - exit(1); + let cache = cache_url.map(|url| { + let cache_pool = Config::from_url(url) + .create_pool(Some(deadpool_redis::Runtime::Tokio1)); + + match cache_pool { + Ok(pool) => pool, + Err(e) => { + tracing::info!( + "Error building redis pool: {}", + e.to_string() + ); + exit(1); + } } - }; + }); - Self { - db: pool, - cache: cache_pool, - } + Self { db: pool, cache } } pub async fn get_db_connection(&self) -> Object { self.db.get().await.unwrap() } - pub async fn get_cache_connection(&self) -> Connection { - self.cache.get().await.unwrap() + pub async fn get_cache_connection(&self) -> Option { + match &self.cache { + None => None, + Some(cache) => Some(cache.get().await.unwrap()), + } } } diff --git a/webserver/src/config.rs b/webserver/src/config.rs index 171efd986..475695faf 100644 --- a/webserver/src/config.rs +++ b/webserver/src/config.rs @@ -1,3 +1,5 @@ +use shared::log_config::LogConfig; + #[derive(clap::ValueEnum, Clone, Debug, Copy)] pub enum CargoEnv { Development, @@ -10,7 +12,7 @@ pub struct AppConfig { pub port: u16, #[clap(long, env)] - pub cache_url: String, + pub cache_url: Option, #[clap(long, env)] pub database_url: String, @@ -20,4 +22,7 @@ pub struct AppConfig { #[clap(long, env)] pub tendermint_url: String, + + #[clap(flatten)] + pub log: LogConfig, } diff --git a/webserver/src/main.rs b/webserver/src/main.rs index 19dad7ffb..5047deea2 100644 --- a/webserver/src/main.rs +++ b/webserver/src/main.rs @@ -7,9 +7,7 @@ use webserver::config::AppConfig; async fn main() -> anyhow::Result<()> { let config = AppConfig::parse(); - tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .init(); + config.log.init(); ApplicationServer::serve(config) .await From 718e836bb0a7e91c62731cc42bda84958ff91253 Mon Sep 17 00:00:00 2001 From: Fraccaroli Gianmarco Date: Tue, 10 Dec 2024 11:09:32 +0100 Subject: [PATCH 13/14] pgf: store pgf receipients and update balances (#167) * added pgf migrations * parse pgf balances * added webserver endpoints * change update balance logic * feat: remove extra path --------- Co-authored-by: Mateusz Jasiuk --- Cargo.toml | 16 +-- chain/src/main.rs | 41 ++++++-- chain/src/repository/mod.rs | 1 + chain/src/repository/pgf.rs | 27 +++++ chain/src/services/namada.rs | 15 +++ governance/Cargo.toml | 1 + governance/run.sh | 3 +- governance/src/main.rs | 99 ++++++++++++++++++- governance/src/repository/governance.rs | 34 ++++++- governance/src/repository/mod.rs | 1 + governance/src/repository/pgf.rs | 54 ++++++++++ governance/src/services/namada.rs | 22 ++++- .../2024-04-18-102935_init_balances/up.sql | 4 +- .../down.sql | 5 + .../up.sql | 22 +++++ orm/src/balances.rs | 17 ++++ orm/src/lib.rs | 1 + orm/src/pgf.rs | 68 +++++++++++++ orm/src/schema.rs | 33 +++++++ pos/run.sh | 4 + rustfmt.toml | 1 - shared/src/block.rs | 4 +- shared/src/lib.rs | 1 + shared/src/pgf.rs | 35 +++++++ shared/src/transaction.rs | 6 -- webserver/src/app.rs | 12 ++- webserver/src/dto/mod.rs | 1 + webserver/src/dto/pgf.rs | 8 ++ webserver/src/error/api.rs | 4 + webserver/src/error/mod.rs | 1 + webserver/src/error/pgf.rs | 25 +++++ webserver/src/handler/mod.rs | 1 + webserver/src/handler/pgf.rs | 42 ++++++++ webserver/src/repository/mod.rs | 1 + webserver/src/repository/pgf.rs | 69 +++++++++++++ webserver/src/response/mod.rs | 1 + webserver/src/response/pgf.rs | 44 +++++++++ webserver/src/service/mod.rs | 1 + webserver/src/service/pgf.rs | 65 ++++++++++++ webserver/src/state/common.rs | 3 + 40 files changed, 757 insertions(+), 36 deletions(-) create mode 100644 chain/src/repository/pgf.rs create mode 100644 governance/src/repository/pgf.rs create mode 100644 orm/migrations/2024-11-29-091248_public_good_funding/down.sql create mode 100644 orm/migrations/2024-11-29-091248_public_good_funding/up.sql create mode 100644 orm/src/pgf.rs create mode 100644 shared/src/pgf.rs create mode 100644 webserver/src/dto/pgf.rs create mode 100644 webserver/src/error/pgf.rs create mode 100644 webserver/src/handler/pgf.rs create mode 100644 webserver/src/repository/pgf.rs create mode 100644 webserver/src/response/pgf.rs create mode 100644 webserver/src/service/pgf.rs diff --git a/Cargo.toml b/Cargo.toml index 32f08c61d..7ecc508c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,14 +40,14 @@ axum-extra = { version = "0.9.3", features = ["query"] } chrono = { version = "0.4.30", features = ["serde"] } async-trait = "0.1.73" anyhow = "1.0.75" -namada_core = { git = "https://github.com/anoma/namada", tag = "v0.46.0" } -namada_sdk = { git = "https://github.com/anoma/namada", tag = "v0.46.0", default-features = false, features = ["std", "async-send", "download-params"] } -namada_tx = { git = "https://github.com/anoma/namada", tag = "v0.46.0" } -namada_governance = { git = "https://github.com/anoma/namada", tag = "v0.46.0" } -namada_ibc = { git = "https://github.com/anoma/namada", tag = "v0.46.0" } -namada_token = { git = "https://github.com/anoma/namada", tag = "v0.46.0" } -namada_parameters = { git = "https://github.com/anoma/namada", tag = "v0.46.0" } -namada_proof_of_stake = { git = "https://github.com/anoma/namada", tag = "v0.46.0" } +namada_core = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" } +namada_sdk = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments", default-features = false, features = ["std", "async-send", "download-params"] } +namada_tx = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" } +namada_governance = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" } +namada_ibc = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" } +namada_token = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" } +namada_parameters = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" } +namada_proof_of_stake = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" } tendermint = "0.38.0" tendermint-config = "0.38.0" tendermint-rpc = { version = "0.38.0", features = ["http-client"] } diff --git a/chain/src/main.rs b/chain/src/main.rs index 9ae1e74ac..03aab9774 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::convert::identity; use std::sync::Arc; @@ -18,6 +19,7 @@ use clap::Parser; use deadpool_diesel::postgres::Object; use namada_sdk::time::DateTimeUtc; use orm::migrations::run_migrations; +use repository::pgf as namada_pgf_repository; use shared::block::Block; use shared::block_result::BlockResult; use shared::checksums::Checksums; @@ -231,17 +233,36 @@ async fn crawling_fn( .map(Token::Ibc) .collect::>(); - let addresses = block.addresses_with_balance_change(native_token); + let addresses = block.addresses_with_balance_change(&native_token); - let balances = - namada_service::query_balance(&client, &addresses, block_height) - .await - .into_rpc_error()?; - tracing::debug!( - block = block_height, - "Updating balance for {} addresses...", - addresses.len() - ); + let pgf_receipient_addresses = if first_block_in_epoch.eq(&block_height) { + conn.interact(move |conn| { + namada_pgf_repository::get_pgf_receipients_balance_changes( + conn, + &native_token, + ) + }) + .await + .context_db_interact_error() + .and_then(identity) + .into_db_error()? + } else { + HashSet::default() + }; + + let all_balance_changed_addresses = pgf_receipient_addresses + .union(&addresses) + .cloned() + .collect::>(); + + let balances = namada_service::query_balance( + &client, + &all_balance_changed_addresses, + block_height, + ) + .await + .into_rpc_error()?; + tracing::info!("Updating balance for {} addresses...", addresses.len()); let next_governance_proposal_id = namada_service::query_next_governance_id(&client, block_height) diff --git a/chain/src/repository/mod.rs b/chain/src/repository/mod.rs index efdb8fcdc..296150acf 100644 --- a/chain/src/repository/mod.rs +++ b/chain/src/repository/mod.rs @@ -1,6 +1,7 @@ pub mod balance; pub mod crawler_state; pub mod gov; +pub mod pgf; pub mod pos; pub mod revealed_pk; mod utils; diff --git a/chain/src/repository/pgf.rs b/chain/src/repository/pgf.rs new file mode 100644 index 000000000..ff9844070 --- /dev/null +++ b/chain/src/repository/pgf.rs @@ -0,0 +1,27 @@ +use std::collections::HashSet; + +use anyhow::Context; +use diesel::{PgConnection, QueryDsl, RunQueryDsl, SelectableHelper}; +use orm::pgf::PublicGoodFundingPaymentDb; +use orm::schema::public_good_funding; +use shared::id::Id; +use shared::token::Token; +use shared::utils::BalanceChange; + +pub fn get_pgf_receipients_balance_changes( + transaction_conn: &mut PgConnection, + native_token: &Id, +) -> anyhow::Result> { + public_good_funding::table + .select(PublicGoodFundingPaymentDb::as_select()) + .load(transaction_conn) + .map(|data| { + data.into_iter() + .map(|payment| BalanceChange { + address: Id::Account(payment.receipient), + token: Token::Native(native_token.clone()), + }) + .collect::>() + }) + .context("Failed to update governance votes in db") +} diff --git a/chain/src/services/namada.rs b/chain/src/services/namada.rs index 1ef4774c8..abc6a36d1 100644 --- a/chain/src/services/namada.rs +++ b/chain/src/services/namada.rs @@ -706,3 +706,18 @@ pub(super) fn to_block_height( ) -> NamadaSdkBlockHeight { NamadaSdkBlockHeight::from(block_height as u64) } + +pub async fn get_pgf_receipients( + client: &HttpClient, + native_token: Id, +) -> HashSet { + let payments = rpc::query_pgf_fundings(client).await.unwrap_or_default(); + + payments + .into_iter() + .map(|payment| BalanceChange { + address: Id::Account(payment.detail.target()), + token: Token::Native(native_token.clone()), + }) + .collect::>() +} diff --git a/governance/Cargo.toml b/governance/Cargo.toml index 8a369efe1..f73e7cb2c 100644 --- a/governance/Cargo.toml +++ b/governance/Cargo.toml @@ -27,6 +27,7 @@ deadpool-diesel.workspace = true diesel.workspace = true orm.workspace = true tokio-retry.workspace = true +serde_json.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/governance/run.sh b/governance/run.sh index c966e9f4b..f65885b61 100755 --- a/governance/run.sh +++ b/governance/run.sh @@ -1,4 +1,5 @@ . ../.env export TENDERMINT_URL export DATABASE_URL -cargo run -- --sleep-for 15 + +SLEEP_FOR=1 cargo run -- --sleep-for 15 diff --git a/governance/src/main.rs b/governance/src/main.rs index 4b61d07f1..3c82bb398 100644 --- a/governance/src/main.rs +++ b/governance/src/main.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::convert::identity; use std::sync::Arc; use std::time::Duration; @@ -9,11 +10,15 @@ use governance::config::AppConfig; use governance::repository; use governance::services::namada as namada_service; use governance::state::AppState; +use namada_governance::storage::proposal::{AddRemove, PGFAction, PGFTarget}; use namada_sdk::time::DateTimeUtc; use orm::migrations::run_migrations; +use shared::balance::Amount as NamadaAmount; use shared::crawler; use shared::crawler_state::{CrawlerName, IntervalCrawlerState}; use shared::error::{AsDbError, AsRpcError, ContextDbInteractError, MainError}; +use shared::id::Id; +use shared::pgf::{PaymentKind, PaymentRecurrence, PgfAction, PgfPayment}; use tendermint_rpc::HttpClient; use tokio::sync::{Mutex, MutexGuard}; use tokio::time::Instant; @@ -85,11 +90,12 @@ async fn crawling_fn( tracing::info!("Starting to update proposals..."); - tracing::info!("Query epoch..."); let epoch = namada_service::query_last_epoch(&client) .await .into_rpc_error()?; + tracing::info!("Fetched epoch is {} ...", epoch); + let running_governance_proposals = conn .interact(move |conn| { repository::governance::get_all_running_proposals(conn) @@ -116,6 +122,95 @@ async fn crawling_fn( proposals_statuses.len() ); + let pgf_payments = conn + .interact(move |conn| { + repository::governance::get_all_pgf_executed_proposals_data( + conn, epoch, + ) + }) + .await + .context_db_interact_error() + .and_then(identity) + .into_db_error()? + .into_iter() + .filter_map(|(id, data)| { + if let Some(data) = data { + if let Ok(fundings) = + serde_json::from_str::>(&data) + { + Some((id, fundings)) + } else { + None + } + } else { + None + } + }) + .flat_map(|(id, data)| { + data.into_iter() + .map(|action| match action { + PGFAction::Retro(target) => match target { + PGFTarget::Internal(inner) => PgfPayment { + proposal_id: id, + recurrence: PaymentRecurrence::Retro, + kind: PaymentKind::Native, + receipient: Id::from(inner.target), + amount: NamadaAmount::from(inner.amount), + action: None, + }, + PGFTarget::Ibc(inner) => PgfPayment { + proposal_id: id, + recurrence: PaymentRecurrence::Retro, + kind: PaymentKind::Ibc, + receipient: Id::Account(inner.target), + amount: NamadaAmount::from(inner.amount), + action: None, + }, + }, + PGFAction::Continuous(add_remove) => match add_remove { + AddRemove::Add(target) => match target { + PGFTarget::Internal(inner) => PgfPayment { + proposal_id: id, + recurrence: PaymentRecurrence::Continuous, + kind: PaymentKind::Native, + receipient: Id::from(inner.target), + amount: NamadaAmount::from(inner.amount), + action: Some(PgfAction::Add), + }, + PGFTarget::Ibc(inner) => PgfPayment { + proposal_id: id, + recurrence: PaymentRecurrence::Continuous, + kind: PaymentKind::Ibc, + receipient: Id::Account(inner.target), + amount: NamadaAmount::from(inner.amount), + action: Some(PgfAction::Add), + }, + }, + AddRemove::Remove(target) => match target { + PGFTarget::Internal(inner) => PgfPayment { + proposal_id: id, + recurrence: PaymentRecurrence::Continuous, + kind: PaymentKind::Native, + receipient: Id::from(inner.target), + amount: NamadaAmount::from(inner.amount), + action: Some(PgfAction::Remove), + }, + PGFTarget::Ibc(inner) => PgfPayment { + proposal_id: id, + recurrence: PaymentRecurrence::Continuous, + kind: PaymentKind::Ibc, + receipient: Id::Account(inner.target), + amount: NamadaAmount::from(inner.amount), + action: Some(PgfAction::Remove), + }, + }, + }, + }) + .collect::>() + }) + .collect::>(); + tracing::info!("Got {} pgf payments...", pgf_payments.len()); + let timestamp = DateTimeUtc::now().0.timestamp(); let crawler_state = IntervalCrawlerState { timestamp }; @@ -130,6 +225,8 @@ async fn crawling_fn( )?; } + repository::pgf::update_pgf(transaction_conn, pgf_payments)?; + repository::crawler_state::upsert_crawler_state( transaction_conn, (CrawlerName::Governance, crawler_state).into(), diff --git a/governance/src/repository/governance.rs b/governance/src/repository/governance.rs index 9ed813703..dab607e69 100644 --- a/governance/src/repository/governance.rs +++ b/governance/src/repository/governance.rs @@ -5,7 +5,8 @@ use diesel::{ RunQueryDsl, }; use orm::governance_proposal::{ - GovernanceProposalResultDb, GovernanceProposalUpdateStatusDb, + GovernanceProposalKindDb, GovernanceProposalResultDb, + GovernanceProposalUpdateStatusDb, }; use orm::schema::governance_proposals; use shared::utils::GovernanceProposalShort; @@ -41,6 +42,37 @@ pub fn get_all_running_proposals( .collect::, _>>() } +pub fn get_all_pgf_executed_proposals_data( + conn: &mut PgConnection, + current_epoch: u32, +) -> anyhow::Result)>> { + governance_proposals::table + .filter( + governance_proposals::dsl::result + .eq(GovernanceProposalResultDb::Passed) + .and( + governance_proposals::dsl::activation_epoch + .eq(current_epoch as i32), + ) + .and( + governance_proposals::dsl::kind + .eq(GovernanceProposalKindDb::PgfFunding), + ), + ) + .select(( + governance_proposals::dsl::id, + governance_proposals::dsl::data, + )) + .load_iter::<(i32, Option), DefaultLoadingMode>(conn) + .context("Failed to get governance proposal ids from db")? + .map(|result| { + let (id, data) = + result.context("Failed to deserialize proposal from db")?; + anyhow::Ok((id as u64, data)) + }) + .collect::)>, _>>() +} + pub fn update_proposal_status( transaction_conn: &mut PgConnection, proposal_id: u64, diff --git a/governance/src/repository/mod.rs b/governance/src/repository/mod.rs index 48712ecaf..83438f839 100644 --- a/governance/src/repository/mod.rs +++ b/governance/src/repository/mod.rs @@ -1,2 +1,3 @@ pub mod crawler_state; pub mod governance; +pub mod pgf; diff --git a/governance/src/repository/pgf.rs b/governance/src/repository/pgf.rs new file mode 100644 index 000000000..4d4216062 --- /dev/null +++ b/governance/src/repository/pgf.rs @@ -0,0 +1,54 @@ +use anyhow::Context; +use diesel::query_dsl::methods::FilterDsl; +use diesel::{ + BoolExpressionMethods, ExpressionMethods, PgConnection, RunQueryDsl, +}; +use orm::pgf::{PaymentRecurrenceDb, PublicGoodFundingPaymentInsertDb}; +use orm::schema::public_good_funding; +use shared::pgf::{PaymentRecurrence, PgfPayment}; + +pub fn update_pgf( + transaction_conn: &mut PgConnection, + pgf_payments: Vec, +) -> anyhow::Result<()> { + diesel::insert_into(public_good_funding::table) + .values::>( + pgf_payments + .clone() + .into_iter() + .filter(|payment| { + matches!(payment.recurrence, PaymentRecurrence::Retro) + || (matches!( + payment.recurrence, + PaymentRecurrence::Continuous + ) && matches!( + payment.action, + Some(shared::pgf::PgfAction::Add) + )) + }) + .map(PublicGoodFundingPaymentInsertDb::from_pgf_payment) + .collect::>(), + ) + .on_conflict_do_nothing() + .execute(transaction_conn) + .context("Failed to update balance_changes in db")?; + + for payment in pgf_payments.into_iter().filter(|payment| { + matches!(payment.recurrence, PaymentRecurrence::Continuous) + && matches!(payment.action, Some(shared::pgf::PgfAction::Remove)) + }) { + diesel::delete( + public_good_funding::table.filter( + public_good_funding::dsl::receipient + .eq(payment.receipient.to_string()) + .and( + public_good_funding::dsl::payment_recurrence + .eq(PaymentRecurrenceDb::Continuous), + ), + ), + ) + .execute(transaction_conn)?; + } + + anyhow::Ok(()) +} diff --git a/governance/src/services/namada.rs b/governance/src/services/namada.rs index d3849b214..8d1ccd281 100644 --- a/governance/src/services/namada.rs +++ b/governance/src/services/namada.rs @@ -1,11 +1,22 @@ use anyhow::Context; use futures::StreamExt; +use namada_sdk::queries::RPC; use namada_sdk::rpc; -use shared::block::Epoch; +use shared::block::{BlockHeight, Epoch}; +use shared::id::Id; use shared::proposal::{GovernanceProposalResult, GovernanceProposalStatus}; use shared::utils::GovernanceProposalShort; use tendermint_rpc::HttpClient; +pub async fn query_latest_block_height( + client: &HttpClient, +) -> anyhow::Result { + let block = rpc::query_block(client) + .await + .with_context(|| "Failed to query Namada's epoch epoch".to_string())?; + Ok(block.map(|block| block.height.0 as u32).unwrap_or(0_u32)) +} + pub async fn query_last_epoch(client: &HttpClient) -> anyhow::Result { let epoch = rpc::query_epoch(client) .await @@ -13,6 +24,15 @@ pub async fn query_last_epoch(client: &HttpClient) -> anyhow::Result { Ok(epoch.0 as Epoch) } +pub async fn get_native_token(client: &HttpClient) -> anyhow::Result { + let native_token = RPC + .shell() + .native_token(client) + .await + .context("Failed to query native token")?; + Ok(Id::from(native_token)) +} + pub async fn get_governance_proposals_updates( client: &HttpClient, proposal_data: Vec, diff --git a/orm/migrations/2024-04-18-102935_init_balances/up.sql b/orm/migrations/2024-04-18-102935_init_balances/up.sql index ad2b14c1c..e7a85f861 100644 --- a/orm/migrations/2024-04-18-102935_init_balances/up.sql +++ b/orm/migrations/2024-04-18-102935_init_balances/up.sql @@ -9,9 +9,7 @@ CREATE TABLE balance_changes ( CONSTRAINT fk_balances_token FOREIGN KEY(token) REFERENCES token(address) ON DELETE CASCADE ); -ALTER TABLE balance_changes ADD UNIQUE (owner, token, height); - -CREATE INDEX index_balance_changes_owner_token_height ON balance_changes (owner, token, height); +CREATE UNIQUE INDEX index_balance_changes_owner_token_height ON balance_changes (owner, token, height); CREATE VIEW balances AS SELECT diff --git a/orm/migrations/2024-11-29-091248_public_good_funding/down.sql b/orm/migrations/2024-11-29-091248_public_good_funding/down.sql new file mode 100644 index 000000000..03471130a --- /dev/null +++ b/orm/migrations/2024-11-29-091248_public_good_funding/down.sql @@ -0,0 +1,5 @@ +-- This file should undo anything in `up.sql` +DROP TABLE IF EXISTS public_good_funding; + +DROP TYPE PAYMENT_KIND; +DROP TYPE PAYMENT_RECURRENCE; \ No newline at end of file diff --git a/orm/migrations/2024-11-29-091248_public_good_funding/up.sql b/orm/migrations/2024-11-29-091248_public_good_funding/up.sql new file mode 100644 index 000000000..887a72569 --- /dev/null +++ b/orm/migrations/2024-11-29-091248_public_good_funding/up.sql @@ -0,0 +1,22 @@ +CREATE TYPE PAYMENT_RECURRENCE AS ENUM ( + 'continuous', + 'retro' +); + +CREATE TYPE PAYMENT_KIND AS ENUM ( + 'ibc', + 'native' +); + +CREATE TABLE public_good_funding ( + id SERIAL PRIMARY KEY, + proposal_id INT NOT NULL, + payment_recurrence PAYMENT_RECURRENCE NOT NULL, + payment_kind PAYMENT_KIND NOT NULL, + receipient VARCHAR NOT NULL, + amount NUMERIC(78, 0) NOT NULL, + CONSTRAINT fk_proposal_id FOREIGN KEY(proposal_id) REFERENCES governance_proposals(id) ON DELETE CASCADE +); + +CREATE INDEX index_public_good_funding_receipient ON public_good_funding (receipient); +CREATE UNIQUE INDEX index_public_good_funding_receipient_proposal_id ON public_good_funding (receipient, proposal_id); \ No newline at end of file diff --git a/orm/src/balances.rs b/orm/src/balances.rs index 046f69dd8..d0730f624 100644 --- a/orm/src/balances.rs +++ b/orm/src/balances.rs @@ -3,6 +3,9 @@ use std::str::FromStr; use bigdecimal::BigDecimal; use diesel::{Insertable, Queryable, Selectable}; use shared::balance::Balance; +use shared::block::BlockHeight; +use shared::id::Id; +use shared::pgf::PgfPayment; use shared::token::Token; use crate::schema::balance_changes; @@ -44,4 +47,18 @@ impl BalanceChangesInsertDb { height: balance.height as i32, } } + + pub fn from_pgf_retro( + payment: PgfPayment, + token: Id, + block_height: BlockHeight, + ) -> Self { + Self { + owner: payment.receipient.to_string(), + height: block_height as i32, + token: token.to_string(), + raw_amount: BigDecimal::from_str(&payment.amount.to_string()) + .expect("Invalid amount"), + } + } } diff --git a/orm/src/lib.rs b/orm/src/lib.rs index 24d01b10b..82aa2cdd3 100644 --- a/orm/src/lib.rs +++ b/orm/src/lib.rs @@ -8,6 +8,7 @@ pub mod group_by_macros; pub mod helpers; pub mod migrations; pub mod parameters; +pub mod pgf; pub mod pos_rewards; pub mod revealed_pk; pub mod schema; diff --git a/orm/src/pgf.rs b/orm/src/pgf.rs new file mode 100644 index 000000000..9f3cf19de --- /dev/null +++ b/orm/src/pgf.rs @@ -0,0 +1,68 @@ +use std::str::FromStr; + +use bigdecimal::BigDecimal; +use diesel::{Insertable, Queryable}; +use serde::{Deserialize, Serialize}; +use shared::pgf::{PaymentKind, PaymentRecurrence, PgfPayment}; + +use crate::schema::public_good_funding; + +#[derive(Debug, Clone, Serialize, Deserialize, diesel_derive_enum::DbEnum)] +#[ExistingTypePath = "crate::schema::sql_types::PaymentRecurrence"] +pub enum PaymentRecurrenceDb { + Continuous, + Retro, +} + +#[derive(Debug, Clone, Serialize, Deserialize, diesel_derive_enum::DbEnum)] +#[ExistingTypePath = "crate::schema::sql_types::PaymentKind"] +pub enum PaymentKindDb { + Ibc, + Native, +} + +impl From for PaymentRecurrenceDb { + fn from(value: PaymentRecurrence) -> Self { + match value { + PaymentRecurrence::Continuous => Self::Continuous, + PaymentRecurrence::Retro => Self::Retro, + } + } +} + +impl From for PaymentKindDb { + fn from(value: PaymentKind) -> Self { + match value { + PaymentKind::Native => Self::Native, + PaymentKind::Ibc => Self::Ibc, + } + } +} + +#[derive(Insertable, Clone, Queryable, diesel::Selectable, Debug)] +#[diesel(table_name = public_good_funding)] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct PublicGoodFundingPaymentDb { + pub payment_recurrence: PaymentRecurrenceDb, + pub proposal_id: i32, + pub payment_kind: PaymentKindDb, + pub receipient: String, + pub amount: BigDecimal, +} + +pub type PublicGoodFundingPaymentInsertDb = PublicGoodFundingPaymentDb; + +impl PublicGoodFundingPaymentInsertDb { + pub fn from_pgf_payment(pgf_payment: PgfPayment) -> Self { + Self { + proposal_id: pgf_payment.proposal_id as i32, + payment_recurrence: PaymentRecurrenceDb::from( + pgf_payment.recurrence, + ), + payment_kind: PaymentKindDb::from(pgf_payment.kind), + receipient: pgf_payment.receipient.to_string(), + amount: BigDecimal::from_str(&pgf_payment.amount.to_string()) + .expect("Invalid amount"), + } + } +} diff --git a/orm/src/schema.rs b/orm/src/schema.rs index cf18897c1..0314a7053 100644 --- a/orm/src/schema.rs +++ b/orm/src/schema.rs @@ -33,6 +33,22 @@ pub mod sql_types { #[diesel(postgres_type(name = "governance_tally_type"))] pub struct GovernanceTallyType; + #[derive( + diesel::query_builder::QueryId, + std::fmt::Debug, + diesel::sql_types::SqlType, + )] + #[diesel(postgres_type(name = "payment_kind"))] + pub struct PaymentKind; + + #[derive( + diesel::query_builder::QueryId, + std::fmt::Debug, + diesel::sql_types::SqlType, + )] + #[diesel(postgres_type(name = "payment_recurrence"))] + pub struct PaymentRecurrence; + #[derive( diesel::query_builder::QueryId, std::fmt::Debug, @@ -214,6 +230,21 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + use super::sql_types::PaymentRecurrence; + use super::sql_types::PaymentKind; + + public_good_funding (id) { + id -> Int4, + proposal_id -> Int4, + payment_recurrence -> PaymentRecurrence, + payment_kind -> PaymentKind, + receipient -> Varchar, + amount -> Numeric, + } +} + diesel::table! { revealed_pk (id) { id -> Int4, @@ -285,6 +316,7 @@ diesel::joinable!(governance_votes -> governance_proposals (proposal_id)); diesel::joinable!(ibc_token -> token (address)); diesel::joinable!(inner_transactions -> wrapper_transactions (wrapper_id)); diesel::joinable!(pos_rewards -> validators (validator_id)); +diesel::joinable!(public_good_funding -> governance_proposals (proposal_id)); diesel::joinable!(unbonds -> validators (validator_id)); diesel::allow_tables_to_appear_in_same_query!( @@ -299,6 +331,7 @@ diesel::allow_tables_to_appear_in_same_query!( ibc_token, inner_transactions, pos_rewards, + public_good_funding, revealed_pk, token, unbonds, diff --git a/pos/run.sh b/pos/run.sh index df0434ec7..4bf5e11d9 100755 --- a/pos/run.sh +++ b/pos/run.sh @@ -1,4 +1,8 @@ . ../.env export TENDERMINT_URL export DATABASE_URL + +echo $TENDERMINT_URL +echo $DATABASE_URL + cargo run diff --git a/rustfmt.toml b/rustfmt.toml index f7a0911e7..6bfc04d34 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -24,7 +24,6 @@ format_macro_matchers = true format_strings = true group_imports = "StdExternalCrate" hard_tabs = false -show_parse_errors = true ignore = [] imports_granularity = "Module" imports_indent = "Block" diff --git a/shared/src/block.rs b/shared/src/block.rs index 3097c4348..7827455e2 100644 --- a/shared/src/block.rs +++ b/shared/src/block.rs @@ -339,7 +339,7 @@ impl Block { // TODO: move this and process_inner_tx_for_balance to a separate module pub fn addresses_with_balance_change( &self, - native_token: Id, + native_token: &Id, ) -> HashSet { self.transactions .iter() @@ -347,7 +347,7 @@ impl Block { let mut balance_changes: Vec = inners_txs .iter() .filter_map(|tx| { - self.process_inner_tx_for_balance(tx, &native_token) + self.process_inner_tx_for_balance(tx, native_token) }) .flatten() .collect(); diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 9d0f5ad46..e495659e8 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -12,6 +12,7 @@ pub mod header; pub mod id; pub mod log_config; pub mod parameters; +pub mod pgf; pub mod proposal; pub mod public_key; pub mod rewards; diff --git a/shared/src/pgf.rs b/shared/src/pgf.rs new file mode 100644 index 000000000..c5e9698b8 --- /dev/null +++ b/shared/src/pgf.rs @@ -0,0 +1,35 @@ +use serde::Serialize; + +use crate::balance::Amount; +use crate::id::Id; + +#[derive(Serialize, Debug, Clone)] +#[serde(untagged)] +pub enum PaymentRecurrence { + Continuous, + Retro, +} + +#[derive(Serialize, Debug, Clone)] +#[serde(untagged)] +pub enum PaymentKind { + Ibc, + Native, +} + +#[derive(Serialize, Debug, Clone)] +#[serde(untagged)] +pub enum PgfAction { + Add, + Remove, +} + +#[derive(Debug, Clone)] +pub struct PgfPayment { + pub proposal_id: u64, + pub recurrence: PaymentRecurrence, + pub kind: PaymentKind, + pub receipient: Id, + pub amount: Amount, + pub action: Option, +} diff --git a/shared/src/transaction.rs b/shared/src/transaction.rs index f8d4200c8..60de6ce02 100644 --- a/shared/src/transaction.rs +++ b/shared/src/transaction.rs @@ -215,12 +215,6 @@ pub struct Transaction { pub fee: Fee, } -#[derive(Debug, Clone)] -pub struct Transaction2 { - pub wrapper: WrapperTransaction, - pub inners: InnerTransaction, -} - #[derive(Debug, Clone)] pub struct WrapperTransaction { pub tx_id: Id, diff --git a/webserver/src/app.rs b/webserver/src/app.rs index fe1496ee0..fcb26cd3d 100644 --- a/webserver/src/app.rs +++ b/webserver/src/app.rs @@ -21,8 +21,8 @@ use crate::config::AppConfig; use crate::handler::{ balance as balance_handlers, chain as chain_handlers, crawler_state as crawler_state_handlers, gas as gas_handlers, - governance as gov_handlers, pk as pk_handlers, pos as pos_handlers, - transaction as transaction_handlers, + governance as gov_handlers, pgf as pgf_service, pk as pk_handlers, + pos as pos_handlers, transaction as transaction_handlers, }; use crate::state::common::CommonState; @@ -133,6 +133,14 @@ impl ApplicationServer { "/chain/epoch/latest", get(chain_handlers::get_last_processed_epoch), ) + .route( + "/pgf/payments", + get(pgf_service::get_pgf_continuous_payments), + ) + .route( + "/pgf/paymenents/:proposal_id", + get(pgf_service::get_pgf_payment_by_proposal_id), + ) .route( "/crawlers/timestamps", get(crawler_state_handlers::get_crawlers_timestamps), diff --git a/webserver/src/dto/mod.rs b/webserver/src/dto/mod.rs index 0c60f9ceb..23ea9f818 100644 --- a/webserver/src/dto/mod.rs +++ b/webserver/src/dto/mod.rs @@ -1,3 +1,4 @@ pub mod crawler_state; pub mod governance; +pub mod pgf; pub mod pos; diff --git a/webserver/src/dto/pgf.rs b/webserver/src/dto/pgf.rs new file mode 100644 index 000000000..9c87a87a4 --- /dev/null +++ b/webserver/src/dto/pgf.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; +use validator::Validate; + +#[derive(Clone, Serialize, Deserialize, Validate)] +pub struct PgfQueryParams { + #[validate(range(min = 1, max = 10000))] + pub page: Option, +} diff --git a/webserver/src/error/api.rs b/webserver/src/error/api.rs index 1db8ed5f3..86335a524 100644 --- a/webserver/src/error/api.rs +++ b/webserver/src/error/api.rs @@ -6,6 +6,7 @@ use super::chain::ChainError; use super::crawler_state::CrawlerStateError; use super::gas::GasError; use super::governance::GovernanceError; +use super::pgf::PgfError; use super::pos::PoSError; use super::revealed_pk::RevealedPkError; use super::transaction::TransactionError; @@ -27,6 +28,8 @@ pub enum ApiError { #[error(transparent)] GasError(#[from] GasError), #[error(transparent)] + PgfError(#[from] PgfError), + #[error(transparent)] CrawlerStateError(#[from] CrawlerStateError), } @@ -40,6 +43,7 @@ impl IntoResponse for ApiError { ApiError::GovernanceError(error) => error.into_response(), ApiError::RevealedPkError(error) => error.into_response(), ApiError::GasError(error) => error.into_response(), + ApiError::PgfError(error) => error.into_response(), ApiError::CrawlerStateError(error) => error.into_response(), } } diff --git a/webserver/src/error/mod.rs b/webserver/src/error/mod.rs index 94f2c2724..092b8e379 100644 --- a/webserver/src/error/mod.rs +++ b/webserver/src/error/mod.rs @@ -4,6 +4,7 @@ pub mod chain; pub mod crawler_state; pub mod gas; pub mod governance; +pub mod pgf; pub mod pos; pub mod revealed_pk; pub mod transaction; diff --git a/webserver/src/error/pgf.rs b/webserver/src/error/pgf.rs new file mode 100644 index 000000000..c85f19a2f --- /dev/null +++ b/webserver/src/error/pgf.rs @@ -0,0 +1,25 @@ +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use thiserror::Error; + +use crate::response::api::ApiErrorResponse; + +#[derive(Error, Debug)] +pub enum PgfError { + #[error("Database error: {0}")] + Database(String), + #[error("Unknown error: {0}")] + Unknown(String), +} + +impl IntoResponse for PgfError { + fn into_response(self) -> Response { + let status_code = match self { + PgfError::Unknown(_) | PgfError::Database(_) => { + StatusCode::INTERNAL_SERVER_ERROR + } + }; + + ApiErrorResponse::send(status_code.as_u16(), Some(self.to_string())) + } +} diff --git a/webserver/src/handler/mod.rs b/webserver/src/handler/mod.rs index be4fc91e8..9b0dd7931 100644 --- a/webserver/src/handler/mod.rs +++ b/webserver/src/handler/mod.rs @@ -3,6 +3,7 @@ pub mod chain; pub mod crawler_state; pub mod gas; pub mod governance; +pub mod pgf; pub mod pk; pub mod pos; pub mod transaction; diff --git a/webserver/src/handler/pgf.rs b/webserver/src/handler/pgf.rs new file mode 100644 index 000000000..6811e672f --- /dev/null +++ b/webserver/src/handler/pgf.rs @@ -0,0 +1,42 @@ +use axum::extract::{Path, State}; +use axum::http::HeaderMap; +use axum::Json; +use axum_extra::extract::Query; +use axum_macros::debug_handler; + +use crate::dto::pgf::PgfQueryParams; +use crate::error::api::ApiError; +use crate::response::pgf::PgfPayment; +use crate::response::utils::PaginatedResponse; +use crate::state::common::CommonState; + +#[debug_handler] +pub async fn get_pgf_continuous_payments( + _headers: HeaderMap, + Query(query): Query, + State(state): State, +) -> Result>>, ApiError> { + let page = query.page.unwrap_or(1); + + let (pgf_payments, total_pages, total_items) = + state.pgf_service.get_all_pgf_payments(page).await?; + + let response = + PaginatedResponse::new(pgf_payments, page, total_pages, total_items); + + Ok(Json(response)) +} + +#[debug_handler] +pub async fn get_pgf_payment_by_proposal_id( + _headers: HeaderMap, + Path(proposal_id): Path, + State(state): State, +) -> Result>, ApiError> { + let pgf_payment = state + .pgf_service + .find_pfg_payment_by_proposal_id(proposal_id) + .await?; + + Ok(Json(pgf_payment)) +} diff --git a/webserver/src/repository/mod.rs b/webserver/src/repository/mod.rs index 0ffcdf615..8153b5ddd 100644 --- a/webserver/src/repository/mod.rs +++ b/webserver/src/repository/mod.rs @@ -2,6 +2,7 @@ pub mod balance; pub mod chain; pub mod gas; pub mod governance; +pub mod pgf; pub mod pos; pub mod revealed_pk; pub mod tranasaction; diff --git a/webserver/src/repository/pgf.rs b/webserver/src/repository/pgf.rs new file mode 100644 index 000000000..e88ad323f --- /dev/null +++ b/webserver/src/repository/pgf.rs @@ -0,0 +1,69 @@ +use axum::async_trait; +use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl, SelectableHelper}; +use orm::pgf::PublicGoodFundingPaymentDb; +use orm::schema::public_good_funding; + +use super::utils::{Paginate, PaginatedResponseDb}; +use crate::appstate::AppState; + +#[derive(Clone)] +pub struct PgfRepo { + pub(crate) app_state: AppState, +} + +#[async_trait] +pub trait PgfRepoTrait { + fn new(app_state: AppState) -> Self; + + async fn get_pgf_continuous_payments( + &self, + page: i64, + ) -> Result, String>; + + async fn find_pgf_payment_by_proposal_id( + &self, + proposal_id: i32, + ) -> Result, String>; +} + +#[async_trait] +impl PgfRepoTrait for PgfRepo { + fn new(app_state: AppState) -> Self { + Self { app_state } + } + + async fn get_pgf_continuous_payments( + &self, + page: i64, + ) -> Result, String> { + let conn = self.app_state.get_db_connection().await; + + conn.interact(move |conn| { + public_good_funding::table + .select(PublicGoodFundingPaymentDb::as_select()) + .order(public_good_funding::columns::proposal_id.desc()) + .paginate(page) + .load_and_count_pages(conn) + }) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string()) + } + + async fn find_pgf_payment_by_proposal_id( + &self, + proposal_id: i32, + ) -> Result, String> { + let conn = self.app_state.get_db_connection().await; + + conn.interact(move |conn| { + public_good_funding::table + .find(proposal_id) + .select(PublicGoodFundingPaymentDb::as_select()) + .first(conn) + .ok() + }) + .await + .map_err(|e| e.to_string()) + } +} diff --git a/webserver/src/response/mod.rs b/webserver/src/response/mod.rs index d5201da87..e00cfb59d 100644 --- a/webserver/src/response/mod.rs +++ b/webserver/src/response/mod.rs @@ -4,6 +4,7 @@ pub mod chain; pub mod crawler_state; pub mod gas; pub mod governance; +pub mod pgf; pub mod pos; pub mod revealed_pk; pub mod transaction; diff --git a/webserver/src/response/pgf.rs b/webserver/src/response/pgf.rs new file mode 100644 index 000000000..9d2f2ee07 --- /dev/null +++ b/webserver/src/response/pgf.rs @@ -0,0 +1,44 @@ +use orm::pgf::{PaymentKindDb, PaymentRecurrenceDb}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PaymentRecurrence { + Retro, + Continuous, +} + +impl From for PaymentRecurrence { + fn from(value: PaymentRecurrenceDb) -> Self { + match value { + PaymentRecurrenceDb::Continuous => Self::Continuous, + PaymentRecurrenceDb::Retro => Self::Retro, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PaymentKind { + Native, + Ibc, +} + +impl From for PaymentKind { + fn from(value: PaymentKindDb) -> Self { + match value { + PaymentKindDb::Ibc => Self::Ibc, + PaymentKindDb::Native => Self::Native, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PgfPayment { + pub payment_recurrence: PaymentRecurrence, + pub proposal_id: i32, + pub payment_kind: PaymentKind, + pub receipient: String, + pub amount: String, +} diff --git a/webserver/src/service/mod.rs b/webserver/src/service/mod.rs index 3dda95f0c..bc433827f 100644 --- a/webserver/src/service/mod.rs +++ b/webserver/src/service/mod.rs @@ -3,6 +3,7 @@ pub mod chain; pub mod crawler_state; pub mod gas; pub mod governance; +pub mod pgf; pub mod pos; pub mod revealed_pk; pub mod transaction; diff --git a/webserver/src/service/pgf.rs b/webserver/src/service/pgf.rs new file mode 100644 index 000000000..984489b19 --- /dev/null +++ b/webserver/src/service/pgf.rs @@ -0,0 +1,65 @@ +use crate::appstate::AppState; +use crate::error::pgf::PgfError; +use crate::repository::pgf::{PgfRepo, PgfRepoTrait}; +use crate::response::pgf::{PaymentKind, PaymentRecurrence, PgfPayment}; + +#[derive(Clone)] +pub struct PgfService { + pgf_repo: PgfRepo, +} + +impl PgfService { + pub fn new(app_state: AppState) -> Self { + Self { + pgf_repo: PgfRepo::new(app_state.clone()), + } + } + + pub async fn get_all_pgf_payments( + &self, + page: u64, + ) -> Result<(Vec, u64, u64), PgfError> { + let (payments, total_pages, total_items) = self + .pgf_repo + .get_pgf_continuous_payments(page as i64) + .await + .map_err(PgfError::Database)?; + + let payments = payments + .into_iter() + .map(|payment| PgfPayment { + payment_recurrence: PaymentRecurrence::from( + payment.payment_recurrence, + ), + proposal_id: payment.proposal_id, + payment_kind: PaymentKind::from(payment.payment_kind), + receipient: payment.receipient, + amount: payment.amount.to_string(), + }) + .collect(); + + Ok((payments, total_pages as u64, total_items as u64)) + } + + pub async fn find_pfg_payment_by_proposal_id( + &self, + proposal_id: u64, + ) -> Result, PgfError> { + let payment = self + .pgf_repo + .find_pgf_payment_by_proposal_id(proposal_id as i32) + .await + .map_err(PgfError::Database)? + .map(|payment| PgfPayment { + payment_recurrence: PaymentRecurrence::from( + payment.payment_recurrence, + ), + proposal_id: payment.proposal_id, + payment_kind: PaymentKind::from(payment.payment_kind), + receipient: payment.receipient, + amount: payment.amount.to_string(), + }); + + Ok(payment) + } +} diff --git a/webserver/src/state/common.rs b/webserver/src/state/common.rs index f50fafb06..7580d706b 100644 --- a/webserver/src/state/common.rs +++ b/webserver/src/state/common.rs @@ -7,6 +7,7 @@ use crate::service::chain::ChainService; use crate::service::crawler_state::CrawlerStateService; use crate::service::gas::GasService; use crate::service::governance::GovernanceService; +use crate::service::pgf::PgfService; use crate::service::pos::PosService; use crate::service::revealed_pk::RevealedPkService; use crate::service::transaction::TransactionService; @@ -20,6 +21,7 @@ pub struct CommonState { pub revealed_pk_service: RevealedPkService, pub gas_service: GasService, pub transaction_service: TransactionService, + pub pgf_service: PgfService, pub crawler_state_service: CrawlerStateService, pub client: HttpClient, pub config: AppConfig, @@ -34,6 +36,7 @@ impl CommonState { chain_service: ChainService::new(data.clone()), revealed_pk_service: RevealedPkService::new(data.clone()), gas_service: GasService::new(data.clone()), + pgf_service: PgfService::new(data.clone()), transaction_service: TransactionService::new(data.clone()), crawler_state_service: CrawlerStateService::new(data.clone()), client, From b763c291a372bfe9c651cce0c1c6172e99718a26 Mon Sep 17 00:00:00 2001 From: Joel Nordell <94570446+joel-u410@users.noreply.github.com> Date: Tue, 10 Dec 2024 04:18:35 -0600 Subject: [PATCH 14/14] enhancement: store raw data for unknown transaction kinds in order to capture them so they can be parsed later (#183) * enhancement: store raw data for unknown transaction kinds in order to capture them so they can be parsed later * [checksums] Add all tx_* wasms from namada-sdk * [transactions] Add on_conflict handling so that old blocks can be re-crawled for transactions without failing --- orm/src/transactions.rs | 2 +- parameters/src/repository/parameters.rs | 2 + shared/src/checksums.rs | 21 ++++++-- shared/src/transaction.rs | 57 ++++++++++++++++++--- transactions/src/repository/transactions.rs | 10 ++++ 5 files changed, 81 insertions(+), 11 deletions(-) diff --git a/orm/src/transactions.rs b/orm/src/transactions.rs index 29a06182c..b4f820fb3 100644 --- a/orm/src/transactions.rs +++ b/orm/src/transactions.rs @@ -58,7 +58,7 @@ impl From for TransactionKindDb { TransactionKind::BecomeValidator(_) => { TransactionKindDb::BecomeValidator } - TransactionKind::Unknown => TransactionKindDb::Unknown, + TransactionKind::Unknown(_) => TransactionKindDb::Unknown, } } } diff --git a/parameters/src/repository/parameters.rs b/parameters/src/repository/parameters.rs index 1c769ef9f..301409e7b 100644 --- a/parameters/src/repository/parameters.rs +++ b/parameters/src/repository/parameters.rs @@ -19,6 +19,8 @@ pub fn upsert_chain_parameters( .eq(excluded(chain_parameters::max_block_time)), chain_parameters::cubic_slashing_window_length .eq(excluded(chain_parameters::cubic_slashing_window_length)), + chain_parameters::checksums + .eq(excluded(chain_parameters::checksums)), )) .execute(transaction_conn) .context("Failed to update chain_parameters state in db")?; diff --git a/shared/src/checksums.rs b/shared/src/checksums.rs index 27389744c..e1130c870 100644 --- a/shared/src/checksums.rs +++ b/shared/src/checksums.rs @@ -1,9 +1,13 @@ use bimap::BiMap; use namada_sdk::tx::{ - TX_BECOME_VALIDATOR_WASM, TX_BOND_WASM, TX_CHANGE_COMMISSION_WASM, - TX_CHANGE_METADATA_WASM, TX_CLAIM_REWARDS_WASM, TX_IBC_WASM, - TX_INIT_PROPOSAL, TX_REDELEGATE_WASM, TX_REVEAL_PK, TX_TRANSFER_WASM, - TX_UNBOND_WASM, TX_VOTE_PROPOSAL, TX_WITHDRAW_WASM, + TX_BECOME_VALIDATOR_WASM, TX_BOND_WASM, TX_BRIDGE_POOL_WASM, + TX_CHANGE_COMMISSION_WASM, TX_CHANGE_CONSENSUS_KEY_WASM, + TX_CHANGE_METADATA_WASM, TX_CLAIM_REWARDS_WASM, + TX_DEACTIVATE_VALIDATOR_WASM, TX_IBC_WASM, TX_INIT_ACCOUNT_WASM, + TX_INIT_PROPOSAL, TX_REACTIVATE_VALIDATOR_WASM, TX_REDELEGATE_WASM, + TX_RESIGN_STEWARD, TX_REVEAL_PK, TX_TRANSFER_WASM, TX_UNBOND_WASM, + TX_UNJAIL_VALIDATOR_WASM, TX_UPDATE_ACCOUNT_WASM, + TX_UPDATE_STEWARD_COMMISSION, TX_VOTE_PROPOSAL, TX_WITHDRAW_WASM, }; use serde::{Deserialize, Serialize}; @@ -44,6 +48,15 @@ impl Checksums { TX_CHANGE_COMMISSION_WASM.to_string(), TX_IBC_WASM.to_string(), TX_BECOME_VALIDATOR_WASM.to_string(), + TX_INIT_ACCOUNT_WASM.to_string(), + TX_UNJAIL_VALIDATOR_WASM.to_string(), + TX_DEACTIVATE_VALIDATOR_WASM.to_string(), + TX_REACTIVATE_VALIDATOR_WASM.to_string(), + TX_UPDATE_ACCOUNT_WASM.to_string(), + TX_BRIDGE_POOL_WASM.to_string(), + TX_CHANGE_CONSENSUS_KEY_WASM.to_string(), + TX_RESIGN_STEWARD.to_string(), + TX_UPDATE_STEWARD_COMMISSION.to_string(), ] } } diff --git a/shared/src/transaction.rs b/shared/src/transaction.rs index 60de6ce02..79cabbcb0 100644 --- a/shared/src/transaction.rs +++ b/shared/src/transaction.rs @@ -29,6 +29,39 @@ pub struct RevealPkData { pub public_key: PublicKey, } +// Capture details for unknown transactions so we can store them in the db +#[derive(Serialize, Debug, Clone)] +pub struct UnknownTransaction { + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "serialize_optional_bytes_to_hex")] + pub data: Option>, +} + +fn serialize_optional_bytes_to_hex( + bytes: &Option>, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + bytes + .as_ref() + .map(|b| { + let mut s = String::with_capacity(2 + (b.len() * 2)); // "0x" + 2 chars per byte + s.push_str("0x"); + for byte in b { + use std::fmt::Write; + write!(s, "{:02x}", byte).unwrap(); + } + s + }) + .serialize(serializer) +} + #[derive(Serialize, Debug, Clone)] #[serde(untagged)] pub enum TransactionKind { @@ -48,7 +81,7 @@ pub enum TransactionKind { CommissionChange(Option), RevealPk(Option), BecomeValidator(Option>), - Unknown, + Unknown(Option), } impl TransactionKind { @@ -56,7 +89,7 @@ impl TransactionKind { serde_json::to_string(&self).ok() } - pub fn from(tx_kind_name: &str, data: &[u8]) -> Self { + pub fn from(id: &str, tx_kind_name: &str, data: &[u8]) -> Self { match tx_kind_name { "tx_transfer" => { let data = if let Ok(data) = Transfer::try_from_slice(data) { @@ -174,7 +207,11 @@ impl TransactionKind { } _ => { tracing::warn!("Unknown transaction kind: {}", tx_kind_name); - TransactionKind::Unknown + TransactionKind::Unknown(Some(UnknownTransaction { + id: Some(id.to_string()), + name: Some(tx_kind_name.to_string()), + data: Some(data.to_vec()), + })) } } } @@ -325,12 +362,20 @@ impl Transaction { if let Some(tx_kind_name) = checksums.get_name_by_id(&id) { - TransactionKind::from(&tx_kind_name, &tx_data) + TransactionKind::from(&id, &tx_kind_name, &tx_data) } else { - TransactionKind::Unknown + TransactionKind::Unknown(Some(UnknownTransaction { + id: Some(id), + name: None, + data: Some(tx_data.clone()), + })) } } else { - TransactionKind::Unknown + TransactionKind::Unknown(Some(UnknownTransaction { + id: None, + name: None, + data: Some(tx_data.clone()), + })) }; let encoded_tx_data = if !tx_data.is_empty() { diff --git a/transactions/src/repository/transactions.rs b/transactions/src/repository/transactions.rs index 8fb982179..b5f1fb47d 100644 --- a/transactions/src/repository/transactions.rs +++ b/transactions/src/repository/transactions.rs @@ -18,6 +18,15 @@ pub fn insert_inner_transactions( .map(InnerTransactionInsertDb::from) .collect::>(), ) + .on_conflict(inner_transactions::id) + .do_update() + .set(( + // Allow updating transactions kind + data so that if the indexer is updated with + // new transaction type support, we can easily go back & reindex any old transactions + // that were previously marked as "unknown". + inner_transactions::kind.eq(excluded(inner_transactions::kind)), + inner_transactions::data.eq(excluded(inner_transactions::data)), + )) .execute(transaction_conn) .context("Failed to insert inner transactions in db")?; @@ -34,6 +43,7 @@ pub fn insert_wrapper_transactions( .map(WrapperTransactionInsertDb::from) .collect::>(), ) + .on_conflict_do_nothing() .execute(transaction_conn) .context("Failed to insert wrapper transactions in db")?;