From 4686144c64bf4ebc791bf593c8ea837c594f2841 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 28 Feb 2023 12:35:05 -0500 Subject: [PATCH 01/26] :wrench: fix: config will rewrite itself on read error --- config/src/lib.rs | 38 +++++++- heimdall/src/cfg/mod.rs | 2 +- heimdall/src/dump/mod.rs | 183 +++++++++++++++++++++++++++++++++++++ heimdall/src/dump/tests.rs | 11 +++ heimdall/src/heimdall.rs | 21 ++++- 5 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 heimdall/src/dump/mod.rs create mode 100644 heimdall/src/dump/tests.rs diff --git a/config/src/lib.rs b/config/src/lib.rs index 9cf8eee0..4556c6a5 100644 --- a/config/src/lib.rs +++ b/config/src/lib.rs @@ -5,7 +5,7 @@ use clap::{AppSettings, Parser}; use serde::{Deserialize, Serialize}; use heimdall_common::{ io::{ - file::{read_file, write_file}, + file::{read_file, write_file, delete_path}, logging::*, } }; @@ -14,6 +14,7 @@ use heimdall_common::{ pub static DEFAULT_CONFIG: &str = "rpc_url = \"\" local_rpc_url = \"http://localhost:8545\" etherscan_api_key = \"\" +transpose_api_key = \"\" "; @@ -39,7 +40,8 @@ pub struct ConfigArgs { pub struct Configuration { pub rpc_url: String, pub local_rpc_url: String, - pub etherscan_api_key: String + pub etherscan_api_key: String, + pub transpose_api_key: String } @@ -61,6 +63,24 @@ pub fn write_config(contents: String) { } +#[allow(deprecated)] +pub fn delete_config() { + match home_dir() { + Some(mut home) => { + home.push(".bifrost"); + home.push("config.toml"); + + let _ = delete_path(&home.into_os_string().to_str().unwrap().to_string()); + } + None => { + let (logger, _) = Logger::new(""); + logger.error("couldn't resolve the bifrost directory. Is your $HOME variable set correctly?"); + std::process::exit(1) + } + } +} + + #[allow(deprecated)] pub fn read_config() -> String { match home_dir() { @@ -93,7 +113,16 @@ pub fn get_config() -> Configuration { let contents = read_config(); // toml parse from contents into Configuration - let config: Configuration = toml::from_str(&contents).unwrap(); + let config: Configuration = match toml::from_str(&contents) { + Ok(config) => config, + Err(e) => { + let (logger, _) = Logger::new(""); + logger.error(&format!("failed to parse config file: {}", e)); + logger.info("regenerating config file..."); + delete_config(); + return get_config() + } + }; config } @@ -112,6 +141,9 @@ pub fn update_config(key: &String, value: &String) { "etherscan_api_key" => { contents.etherscan_api_key = value.to_string(); } + "transpose_api_key" => { + contents.transpose_api_key = value.to_string(); + } _ => { let (logger, _) = Logger::new(""); logger.error(&format!("unknown configuration key \'{key}\' .")); diff --git a/heimdall/src/cfg/mod.rs b/heimdall/src/cfg/mod.rs index b1bb9795..1810ce37 100644 --- a/heimdall/src/cfg/mod.rs +++ b/heimdall/src/cfg/mod.rs @@ -142,7 +142,7 @@ pub fn cfg(args: CFGArgs) { std::process::exit(1); } - // create new provider + // create new provider#[warn(unused_imports)] let provider = match Provider::::try_from(&args.rpc_url) { Ok(provider) => provider, Err(_) => { diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs new file mode 100644 index 00000000..e4d8e77e --- /dev/null +++ b/heimdall/src/dump/mod.rs @@ -0,0 +1,183 @@ +mod tests; + +use std::env; +use std::fs; +use heimdall_cache::read_cache; +use heimdall_cache::store_cache; + +use clap::{AppSettings, Parser}; +use ethers::{ + core::types::{Address}, + providers::{Middleware, Provider, Http}, +}; +use heimdall_common::{ + ether::evm::{ + vm::VM + }, + constants::{ ADDRESS_REGEX, BYTECODE_REGEX }, + io::{ logging::* }, +}; + + +#[derive(Debug, Clone, Parser)] +#[clap(about = "Dump the value of all storage slots accessed by a contract", + after_help = "For more information, read the wiki: https://jbecker.dev/r/heimdall-rs/wiki", + global_setting = AppSettings::DeriveDisplayOrder, + override_usage = "heimdall dump [OPTIONS]")] +pub struct DumpArgs { + + /// The target to find and dump the storage slots of. + #[clap(required=true)] + pub target: String, + + /// Set the output verbosity level, 1 - 5. + #[clap(flatten)] + pub verbose: clap_verbosity_flag::Verbosity, + + /// The output directory to write the output to + #[clap(long="output", short, default_value = "", hide_default_value = true)] + pub output: String, + + /// The RPC provider to use for fetching on-chain data. + #[clap(long="rpc-url", short, default_value = "", hide_default_value = true)] + pub rpc_url: String, + + /// When prompted, always select the default value. + #[clap(long, short)] + pub default: bool, +} + +pub fn dump(args: DumpArgs) { + use std::time::Instant; + let now = Instant::now(); + + let (logger, mut trace)= Logger::new(args.verbose.log_level().unwrap().as_str()); + + // truncate target for prettier display + let mut shortened_target = args.target.clone(); + if shortened_target.len() > 66 { + shortened_target = shortened_target.chars().take(66).collect::() + "..." + &shortened_target.chars().skip(shortened_target.len() - 16).collect::(); + } + + // add the call to the trace + let dump_call = trace.add_call( + 0, line!(), + "heimdall".to_string(), + "dump".to_string(), + vec![shortened_target], + "()".to_string() + ); + + // parse the output directory + let mut output_dir: String; + if &args.output.len() <= &0 { + output_dir = match env::current_dir() { + Ok(dir) => dir.into_os_string().into_string().unwrap(), + Err(_) => { + logger.error("failed to get current directory."); + std::process::exit(1); + } + }; + output_dir.push_str("/output"); + } + else { + output_dir = args.output.clone(); + } + + // fetch bytecode + let contract_bytecode: String; + if ADDRESS_REGEX.is_match(&args.target).unwrap() { + + // push the address to the output directory + if &output_dir != &args.output { + output_dir.push_str(&format!("/{}", &args.target)); + } + + // create new runtime block + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + // We are working with a contract address, so we need to fetch the bytecode from the RPC provider. + contract_bytecode = rt.block_on(async { + + // check the cache for a matching address + match read_cache(&format!("contract.{}", &args.target)) { + Some(bytecode) => { + logger.debug(&format!("found cached bytecode for '{}' .", &args.target)); + return bytecode; + }, + None => {} + } + + // make sure the RPC provider isn't empty + if &args.rpc_url.len() <= &0 { + logger.error("fetching an on-chain contract requires an RPC provider. Use `heimdall dump --help` for more information."); + std::process::exit(1); + } + + // create new provider + let provider = match Provider::::try_from(&args.rpc_url) { + Ok(provider) => provider, + Err(_) => { + logger.error(&format!("failed to connect to RPC provider '{}' .", &args.rpc_url)); + std::process::exit(1) + } + }; + + // safely unwrap the address + let address = match args.target.parse::
() { + Ok(address) => address, + Err(_) => { + logger.error(&format!("failed to parse address '{}' .", &args.target)); + std::process::exit(1) + } + }; + + // fetch the bytecode at the address + let bytecode_as_bytes = match provider.get_code(address, None).await { + Ok(bytecode) => bytecode, + Err(_) => { + logger.error(&format!("failed to fetch bytecode from '{}' .", &args.target)); + std::process::exit(1) + } + }; + + // cache the results + store_cache(&format!("contract.{}", &args.target), bytecode_as_bytes.to_string().replacen("0x", "", 1), None); + + bytecode_as_bytes.to_string().replacen("0x", "", 1) + }); + + } + else if BYTECODE_REGEX.is_match(&args.target).unwrap() { + contract_bytecode = args.target.clone(); + } + else { + + // push the address to the output directory + if &output_dir != &args.output { + output_dir.push_str("/local"); + } + + // We are analyzing a file, so we need to read the bytecode from the file. + contract_bytecode = match fs::read_to_string(&args.target) { + Ok(contents) => { + if BYTECODE_REGEX.is_match(&contents).unwrap() && contents.len() % 2 == 0 { + contents.replacen("0x", "", 1) + } + else { + logger.error(&format!("file '{}' doesn't contain valid bytecode.", &args.target)); + std::process::exit(1) + } + }, + Err(_) => { + logger.error(&format!("failed to open file '{}' .", &args.target)); + std::process::exit(1) + } + }; + } + + logger.debug(&format!("Dumped storage slots in {:?}.", now.elapsed())); +} \ No newline at end of file diff --git a/heimdall/src/dump/tests.rs b/heimdall/src/dump/tests.rs new file mode 100644 index 00000000..e11c4f51 --- /dev/null +++ b/heimdall/src/dump/tests.rs @@ -0,0 +1,11 @@ +#[cfg(test)] +mod benchmark { + + +} + + +#[cfg(test)] +mod test { + +} \ No newline at end of file diff --git a/heimdall/src/heimdall.rs b/heimdall/src/heimdall.rs index 167c1ebf..01ad2772 100644 --- a/heimdall/src/heimdall.rs +++ b/heimdall/src/heimdall.rs @@ -2,6 +2,7 @@ use std::{panic}; use backtrace::Backtrace; mod cfg; +mod dump; mod decode; mod decompile; @@ -13,6 +14,7 @@ use heimdall_cache::{cache, CacheArgs}; use heimdall_common::{ether::evm::disassemble::*, io::{logging::Logger}}; use decompile::{decompile, DecompilerArgs}; use decode::{decode, DecodeArgs}; +use dump::{dump, DumpArgs}; use cfg::{cfg, CFGArgs}; #[derive(Debug, Parser)] @@ -51,6 +53,9 @@ pub enum Subcommands { #[clap(name = "cache", about = "Manage heimdall-rs' cached files")] Cache(CacheArgs), + + #[clap(name = "dump", about = "Dump the value of all storage slots accessed by a contract")] + Dump(DumpArgs), } fn main() { @@ -130,6 +135,20 @@ fn main() { cfg(cmd); } + + Subcommands::Dump(mut cmd) => { + + // if the user has not specified a rpc url, use the default + match cmd.rpc_url.as_str() { + "" => { + cmd.rpc_url = configuration.rpc_url; + } + _ => {} + }; + + dump(cmd); + } + Subcommands::Config(cmd) => { config(cmd); @@ -139,7 +158,5 @@ fn main() { Subcommands::Cache(cmd) => { _ = cache(cmd); } - - } } From db51d3dfae2967cc1fcea1d8d4c79a2b9d512e96 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Wed, 1 Mar 2023 13:25:18 -0500 Subject: [PATCH 02/26] :sparkles: feat: tui changes --- Cargo.lock | 67 +++++++ common/src/lib.rs | 3 +- common/src/resources/mod.rs | 1 + common/src/resources/transpose.rs | 122 +++++++++++++ heimdall/Cargo.toml | 2 + heimdall/src/dump/mod.rs | 274 +++++++++++++++------------- heimdall/src/dump/tui_views/main.rs | 58 ++++++ heimdall/src/dump/tui_views/mod.rs | 1 + heimdall/src/heimdall.rs | 8 + 9 files changed, 406 insertions(+), 130 deletions(-) create mode 100644 common/src/resources/mod.rs create mode 100644 common/src/resources/transpose.rs create mode 100644 heimdall/src/dump/tui_views/main.rs create mode 100644 heimdall/src/dump/tui_views/mod.rs diff --git a/Cargo.lock b/Cargo.lock index d3336fa2..b991a791 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,6 +345,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + [[package]] name = "cc" version = "1.0.79" @@ -547,6 +553,31 @@ dependencies = [ "libc", ] +[[package]] +name = "crossterm" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" +dependencies = [ + "bitflags", + "crossterm_winapi", + "libc", + "mio", + "parking_lot 0.12.1", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -1283,6 +1314,7 @@ dependencies = [ "clap", "clap-verbosity-flag", "colored", + "crossterm", "dot", "ethers", "fancy-regex", @@ -1296,6 +1328,7 @@ dependencies = [ "serde_json", "strsim", "tokio", + "tui", ] [[package]] @@ -2513,6 +2546,27 @@ dependencies = [ "keccak", ] +[[package]] +name = "signal-hook" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -2849,6 +2903,19 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "tui" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1" +dependencies = [ + "bitflags", + "cassowary", + "crossterm", + "unicode-segmentation", + "unicode-width", +] + [[package]] name = "typenum" version = "1.16.0" diff --git a/common/src/lib.rs b/common/src/lib.rs index 412f9a91..98cce3b1 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -4,4 +4,5 @@ pub mod ether; pub mod constants; pub mod io; pub mod utils; -pub mod testing; \ No newline at end of file +pub mod testing; +pub mod resources; \ No newline at end of file diff --git a/common/src/resources/mod.rs b/common/src/resources/mod.rs new file mode 100644 index 00000000..8fb402d0 --- /dev/null +++ b/common/src/resources/mod.rs @@ -0,0 +1 @@ +pub mod transpose; \ No newline at end of file diff --git a/common/src/resources/transpose.rs b/common/src/resources/transpose.rs new file mode 100644 index 00000000..282ae01c --- /dev/null +++ b/common/src/resources/transpose.rs @@ -0,0 +1,122 @@ +use std::{io::Read, time::{Instant, Duration}}; + +use indicatif::ProgressBar; +use reqwest::header::{HeaderMap}; +use serde_json::Value; + +use crate::{io::logging::Logger}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct TransposeStats { + count: u128, + size: u128, + time: u128, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct TransposeResponse { + status: String, + stats: TransposeStats, + results: Vec +} + +fn _call_transpose(query: String, api_key: &String) -> Option { + let mut headers = HeaderMap::new(); + headers.insert("Content-Type", "application/json".parse().unwrap()); + headers.insert("X-API-KEY", api_key.parse().unwrap()); + + // make the request + let client = reqwest::blocking::Client::builder().redirect(reqwest::redirect::Policy::none()).build() .unwrap(); + let mut response = match client.post("https://api.transpose.io/sql").headers(headers).body(query).send() { + Ok(res) => res, + Err(e) => { + let (logger, _) = Logger::new("TRACE"); + logger.error(&format!("failed to get transaction list from Transpose: {}", e)); + std::process::exit(1) + } + }; + + // parse body + let mut body = String::new(); + match response.read_to_string(&mut body) { + Ok(_) => { + Some(match serde_json::from_str(&body) { + Ok(json) => json, + Err(e) => { + let (logger, _) = Logger::new("TRACE"); + logger.error(&format!("failed to parse transaction list from Transpose: {}", e)); + std::process::exit(1) + } + }) + }, + Err(e) => { + let (logger, _) = Logger::new("TRACE"); + logger.error(&format!("failed to get transaction list from Transpose: {}", e)); + std::process::exit(1) + } + } +} + +pub fn get_transaction_list(address: &String, api_key: &String, logger: &Logger) -> Vec<(u128, String)> { + + // get a new progress bar + let transaction_list_progress = ProgressBar::new_spinner(); + transaction_list_progress.enable_steady_tick(Duration::from_millis(100)); + transaction_list_progress.set_style(logger.info_spinner()); + transaction_list_progress.set_message(format!("fetching transactions from '{}' .", address)); + let start_time = Instant::now(); + + // build the SQL query + let query = format!("{{\"sql\":\"SELECT block_number, transaction_hash FROM ethereum.transactions WHERE to_address = '{}' ORDER BY block_number ASC\",\"parameters\":{{}},\"options\":{{}}}}", address); + + let response = match _call_transpose(query, api_key) { + Some(response) => response, + None => { + logger.error(&format!("failed to get transaction list from Transpose")); + std::process::exit(1) + } + }; + + transaction_list_progress.finish_and_clear(); + logger.debug(&format!("fetching transactions took {:?}", start_time.elapsed())); + + let mut transactions = Vec::new(); + + // parse the results + for result in response.results { + let block_number: u128 = match result.get("block_number") { + Some(block_number) => match block_number.as_u64() { + Some(block_number) => block_number as u128, + None => { + logger.error(&format!("failed to parse block_number from Transpose")); + std::process::exit(1) + } + }, + None => { + logger.error(&format!("failed to fetch block_number from Transpose response")); + std::process::exit(1) + } + }; + let transaction_hash: String = match result.get("transaction_hash") { + Some(transaction_hash) => match transaction_hash.as_str() { + Some(transaction_hash) => transaction_hash.to_string(), + None => { + logger.error(&format!("failed to parse transaction_hash from Transpose")); + std::process::exit(1) + } + }, + None => { + logger.error(&format!("failed to fetch transaction_hash from Transpose response")); + std::process::exit(1) + } + }; + + transactions.push((block_number, transaction_hash)); + } + + // sort the transactions by block number + transactions.sort_by(|a, b| a.0.cmp(&b.0)); + + transactions +} \ No newline at end of file diff --git a/heimdall/Cargo.toml b/heimdall/Cargo.toml index e7386de0..0fd26490 100644 --- a/heimdall/Cargo.toml +++ b/heimdall/Cargo.toml @@ -25,6 +25,8 @@ fancy-regex = "0.10.0" lazy_static = "1.4.0" petgraph = "0.6.2" dot = "0.1.4" +tui = "0.19" +crossterm = "0.25" [[bin]] name = "heimdall" diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index e4d8e77e..74c69c24 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -1,22 +1,22 @@ mod tests; +mod tui_views; -use std::env; -use std::fs; -use heimdall_cache::read_cache; -use heimdall_cache::store_cache; - +use std::sync::{Arc, Mutex}; +use std::time::Instant; +use std::{io}; use clap::{AppSettings, Parser}; -use ethers::{ - core::types::{Address}, - providers::{Middleware, Provider, Http}, -}; +use crossterm::event::{EnableMouseCapture, DisableMouseCapture}; +use crossterm::execute; +use crossterm::terminal::{enable_raw_mode, EnterAlternateScreen, disable_raw_mode, LeaveAlternateScreen}; +use ethers::types::U256; +use heimdall_common::resources::transpose::get_transaction_list; use heimdall_common::{ - ether::evm::{ - vm::VM - }, - constants::{ ADDRESS_REGEX, BYTECODE_REGEX }, io::{ logging::* }, }; +use tui::backend::Backend; +use tui::{Frame, backend::CrosstermBackend, Terminal}; + +use tui_views::main::render_tui_view_main; #[derive(Debug, Clone, Parser)] @@ -42,142 +42,158 @@ pub struct DumpArgs { #[clap(long="rpc-url", short, default_value = "", hide_default_value = true)] pub rpc_url: String, + /// Your Transpose.io API Key. + #[clap(long="transpose-api-key", short, default_value = "", hide_default_value = true)] + pub transpose_api_key: String, + /// When prompted, always select the default value. #[clap(long, short)] pub default: bool, } -pub fn dump(args: DumpArgs) { - use std::time::Instant; - let now = Instant::now(); - - let (logger, mut trace)= Logger::new(args.verbose.log_level().unwrap().as_str()); - - // truncate target for prettier display - let mut shortened_target = args.target.clone(); - if shortened_target.len() > 66 { - shortened_target = shortened_target.chars().take(66).collect::() + "..." + &shortened_target.chars().skip(shortened_target.len() - 16).collect::(); - } +#[derive(Debug, Clone)] +pub struct StorageSlot { + pub slot: U256, + pub alias: Option, + pub value: Option +} - // add the call to the trace - let dump_call = trace.add_call( - 0, line!(), - "heimdall".to_string(), - "dump".to_string(), - vec![shortened_target], - "()".to_string() - ); +#[derive(Debug, Clone)] +pub struct Transaction { + pub indexed: bool, + pub hash: String, + pub block: u128, +} - // parse the output directory - let mut output_dir: String; - if &args.output.len() <= &0 { - output_dir = match env::current_dir() { - Ok(dir) => dir.into_os_string().into_string().unwrap(), - Err(_) => { - logger.error("failed to get current directory."); - std::process::exit(1); - } - }; - output_dir.push_str("/output"); - } - else { - output_dir = args.output.clone(); - } +#[derive(Debug, Clone)] +pub struct DumpState { + pub args: DumpArgs, + pub transactions: Vec, + pub storage: Vec, + pub view: TUIView, + pub start_time: Instant, +} - // fetch bytecode - let contract_bytecode: String; - if ADDRESS_REGEX.is_match(&args.target).unwrap() { +#[derive(Debug, Clone)] +#[allow(dead_code)] +pub enum TUIView { + Main, + CommandPalette, +} - // push the address to the output directory - if &output_dir != &args.output { - output_dir.push_str(&format!("/{}", &args.target)); - } - // create new runtime block - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - // We are working with a contract address, so we need to fetch the bytecode from the RPC provider. - contract_bytecode = rt.block_on(async { - - // check the cache for a matching address - match read_cache(&format!("contract.{}", &args.target)) { - Some(bytecode) => { - logger.debug(&format!("found cached bytecode for '{}' .", &args.target)); - return bytecode; - }, - None => {} - } +fn render_ui( + f: &mut Frame, + state: &mut DumpState +) { + match state.view { + TUIView::Main => { render_tui_view_main(f, state) }, + _ => {} + } + } - // make sure the RPC provider isn't empty - if &args.rpc_url.len() <= &0 { - logger.error("fetching an on-chain contract requires an RPC provider. Use `heimdall dump --help` for more information."); - std::process::exit(1); - } +pub fn dump(args: DumpArgs) { + use std::time::Instant; + let now = Instant::now(); - // create new provider - let provider = match Provider::::try_from(&args.rpc_url) { - Ok(provider) => provider, - Err(_) => { - logger.error(&format!("failed to connect to RPC provider '{}' .", &args.rpc_url)); - std::process::exit(1) - } - }; - - // safely unwrap the address - let address = match args.target.parse::
() { - Ok(address) => address, - Err(_) => { - logger.error(&format!("failed to parse address '{}' .", &args.target)); - std::process::exit(1) - } - }; - - // fetch the bytecode at the address - let bytecode_as_bytes = match provider.get_code(address, None).await { - Ok(bytecode) => bytecode, - Err(_) => { - logger.error(&format!("failed to fetch bytecode from '{}' .", &args.target)); - std::process::exit(1) - } - }; + let (logger, _)= Logger::new(args.verbose.log_level().unwrap().as_str()); - // cache the results - store_cache(&format!("contract.{}", &args.target), bytecode_as_bytes.to_string().replacen("0x", "", 1), None); + // check if transpose api key is set + if &args.transpose_api_key.len() <= &0 { + logger.error("you must provide a Transpose API key."); + logger.info("you can get a free API key at https://app.transpose.io"); + std::process::exit(1); + } - bytecode_as_bytes.to_string().replacen("0x", "", 1) + // parse the output directory + // let mut output_dir: String; + // if &args.output.len() <= &0 { + // output_dir = match env::current_dir() { + // Ok(dir) => dir.into_os_string().into_string().unwrap(), + // Err(_) => { + // logger.error("failed to get current directory."); + // std::process::exit(1); + // } + // }; + // output_dir.push_str("/output"); + // } + // else { + // output_dir = args.output.clone(); + // } + + // fetch transactions + let transaction_list = get_transaction_list(&args.target, &args.transpose_api_key, &logger); + + // convert to vec of Transaction + let mut transactions: Vec = Vec::new(); + for transaction in transaction_list { + transactions.push(Transaction { + indexed: false, + hash: transaction.1, + block: transaction.0 }); - - } - else if BYTECODE_REGEX.is_match(&args.target).unwrap() { - contract_bytecode = args.target.clone(); } - else { - - // push the address to the output directory - if &output_dir != &args.output { - output_dir.push_str("/local"); - } - // We are analyzing a file, so we need to read the bytecode from the file. - contract_bytecode = match fs::read_to_string(&args.target) { - Ok(contents) => { - if BYTECODE_REGEX.is_match(&contents).unwrap() && contents.len() % 2 == 0 { - contents.replacen("0x", "", 1) + // create new state + let mut state = DumpState { + args: args.clone(), + transactions: transactions, + storage: Vec::new(), + view: TUIView::Main, + start_time: Instant::now(), + }; + + // in a new thread, start the TUI + let tui_thread = std::thread::spawn(move || { + + // create new TUI terminal + enable_raw_mode().unwrap(); + let mut stdout = io::stdout(); + execute!( + stdout, + EnterAlternateScreen, + EnableMouseCapture + ).unwrap(); + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend).unwrap(); + + // while user does not click CTRL+C + loop { + terminal.draw(|f| { render_ui(f, &mut state); }).unwrap(); + + // check for user input + if let Ok(event) = crossterm::event::read() { + match event { + crossterm::event::Event::Key(key) => { + match key.code { + crossterm::event::KeyCode::Char('q') => { + break; + }, + _ => {} + } + }, + _ => {} } - else { - logger.error(&format!("file '{}' doesn't contain valid bytecode.", &args.target)); - std::process::exit(1) - } - }, - Err(_) => { - logger.error(&format!("failed to open file '{}' .", &args.target)); - std::process::exit(1) } - }; + } + + // cleanup + disable_raw_mode().unwrap(); + execute!( + terminal.backend_mut(), + LeaveAlternateScreen, + DisableMouseCapture + ).unwrap(); + terminal.show_cursor().unwrap(); + }); + + for tx in state.transactions.iter_mut() { + tx.indexed = true; + std::thread::sleep(std::time::Duration::from_millis(100)); } + // wait for the TUI thread to finish + tui_thread.join().unwrap(); + logger.debug(&format!("Dumped storage slots in {:?}.", now.elapsed())); } \ No newline at end of file diff --git a/heimdall/src/dump/tui_views/main.rs b/heimdall/src/dump/tui_views/main.rs new file mode 100644 index 00000000..3bbcc35c --- /dev/null +++ b/heimdall/src/dump/tui_views/main.rs @@ -0,0 +1,58 @@ +use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction}, widgets::{Gauge, ListItem, List, Block, Borders}, style::{Style, Color, Modifier}}; + +use crate::dump::{DumpState}; + +pub fn render_tui_view_main( + f: &mut Frame, + state: &mut DumpState +) { + + // build main layout + let main_layout = Layout::default() + .direction(Direction::Vertical) + .margin(1) + .constraints( + [ + Constraint::Length(3), + Constraint::Percentage(100), + ].as_ref() + ).split(f.size()); + + let min_block_number = state.transactions.iter().min_by_key(|t| t.block).unwrap().block; + let max_block_number = state.transactions.iter().max_by_key(|t| t.block).unwrap().block; + let max_indexed_block_number = match state.transactions.iter().filter(|t| t.indexed).max_by_key(|t| t.block) { + Some(t) => t.block, + None => min_block_number + }; + + let blocks_indexed = max_indexed_block_number - min_block_number; + let percent_indexed = (max_indexed_block_number - min_block_number) as f64 / (max_block_number - min_block_number) as f64 * 100.0; + let elapsed_seconds = state.start_time.elapsed().as_secs(); + let blocks_per_second = blocks_indexed as f64 / elapsed_seconds as f64; + + + + // render progress bar + let progress = Gauge::default() + .block(Block::default().title(" Dump Progress ").borders(Borders::ALL)) + .gauge_style(Style::default().fg(Color::White).bg(Color::DarkGray)) + .percent(percent_indexed as u16) + .label(format!( + "Block {}/{} ({:.1}%). {} Blocks Per Second. ETA: {}", + max_indexed_block_number, + max_block_number, + percent_indexed, + blocks_per_second, + 0 + )); + + let items = [ListItem::new("Item 1"), ListItem::new("Item 2"), ListItem::new("Item 3")]; + let list = List::new(items) + .block(Block::default().title("List").borders(Borders::ALL)) + .style(Style::default().fg(Color::White)) + .highlight_style(Style::default().add_modifier(Modifier::ITALIC)) + .highlight_symbol(">>"); + + f.render_widget(progress, main_layout[0]); + f.render_widget(list, main_layout[1]); +} \ No newline at end of file diff --git a/heimdall/src/dump/tui_views/mod.rs b/heimdall/src/dump/tui_views/mod.rs new file mode 100644 index 00000000..5a8f6491 --- /dev/null +++ b/heimdall/src/dump/tui_views/mod.rs @@ -0,0 +1 @@ +pub mod main; \ No newline at end of file diff --git a/heimdall/src/heimdall.rs b/heimdall/src/heimdall.rs index 01ad2772..4d5cc28f 100644 --- a/heimdall/src/heimdall.rs +++ b/heimdall/src/heimdall.rs @@ -146,6 +146,14 @@ fn main() { _ => {} }; + // if the user has not specified a transpose api key, use the default + match cmd.transpose_api_key.as_str() { + "" => { + cmd.transpose_api_key = configuration.transpose_api_key; + } + _ => {} + }; + dump(cmd); } From d7e688199d07fe12365a90540177a5bf137c6db9 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Thu, 2 Mar 2023 15:03:11 -0500 Subject: [PATCH 03/26] :sparkles: feat: `heimdall dump` mvp v1 --- Cargo.lock | 20 +++ common/Cargo.toml | 3 +- common/src/io/mod.rs | 1 - common/src/utils/mod.rs | 2 + common/src/utils/threading.rs | 45 ++++++ common/src/utils/time.rs | 19 +++ heimdall/src/dump/mod.rs | 239 +++++++++++++++++++++++----- heimdall/src/dump/tui_views/main.rs | 114 ++++++++++--- heimdall/src/dump/util.rs | 87 ++++++++++ heimdall/src/heimdall.rs | 14 +- test.txt | 19 +++ 11 files changed, 491 insertions(+), 72 deletions(-) create mode 100644 common/src/utils/threading.rs create mode 100644 common/src/utils/time.rs create mode 100644 heimdall/src/dump/util.rs create mode 100644 test.txt diff --git a/Cargo.lock b/Cargo.lock index b991a791..1e919238 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -553,6 +553,25 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossterm" version = "0.25.0" @@ -1349,6 +1368,7 @@ dependencies = [ "clap", "clap-verbosity-flag", "colored", + "crossbeam-channel", "ethers", "fancy-regex", "heimdall-cache", diff --git a/common/Cargo.toml b/common/Cargo.toml index 06793809..1441c8a8 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -19,4 +19,5 @@ ethers = "1.0.2" reqwest = { version = "0.11.11", features = ["blocking"] } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } -heimdall-cache = { path = "./../cache" } \ No newline at end of file +heimdall-cache = { path = "./../cache" } +crossbeam-channel = "0.5.7" \ No newline at end of file diff --git a/common/src/io/mod.rs b/common/src/io/mod.rs index 678c6d0b..1d901943 100644 --- a/common/src/io/mod.rs +++ b/common/src/io/mod.rs @@ -1,4 +1,3 @@ pub mod logging; pub mod file; - mod tests; \ No newline at end of file diff --git a/common/src/utils/mod.rs b/common/src/utils/mod.rs index 8ffeba79..d6193b18 100644 --- a/common/src/utils/mod.rs +++ b/common/src/utils/mod.rs @@ -1,2 +1,4 @@ pub mod strings; pub mod http; +pub mod time; +pub mod threading; \ No newline at end of file diff --git a/common/src/utils/threading.rs b/common/src/utils/threading.rs new file mode 100644 index 00000000..bf5ccca9 --- /dev/null +++ b/common/src/utils/threading.rs @@ -0,0 +1,45 @@ +use std::thread; +use std::sync::{Arc}; +use crossbeam_channel::{unbounded}; + +pub fn task_pool R + Send + Sync + 'static>( + items: Vec, + num_threads: usize, + f: F, +) -> Vec { + let (tx, rx) = unbounded(); + let mut handles = Vec::new(); + + // Split items into chunks for each thread to process + let chunk_size = (items.len() + num_threads - 1) / num_threads; + let chunks = items.chunks(chunk_size); + + // Share ownership of f across threads with Arc + let shared_f = Arc::new(f); + + for chunk in chunks { + let chunk = chunk.to_owned(); + let tx = tx.clone(); + // Share ownership of shared_f with each thread with Arc + let shared_f = Arc::clone(&shared_f); + let handle = thread::spawn(move || { + let chunk_results: Vec = chunk.into_iter().map(|item| shared_f(item)).collect(); + tx.send(chunk_results).unwrap(); + }); + handles.push(handle); + } + + // Wait for all threads to finish and collect the results + let mut results = Vec::new(); + for _ in 0..num_threads { + let chunk_results = rx.recv().unwrap(); + results.extend(chunk_results); + } + + // Wait for all threads to finish + for handle in handles { + handle.join().unwrap(); + } + + results +} diff --git a/common/src/utils/time.rs b/common/src/utils/time.rs new file mode 100644 index 00000000..15780188 --- /dev/null +++ b/common/src/utils/time.rs @@ -0,0 +1,19 @@ +pub fn calculate_eta(items_per_second: f64, items_remaining: usize) -> u128 { + + (items_remaining as f64 / items_per_second) as u128 +} + +pub fn format_eta(seconds_remaining: u128) -> String { + let days = seconds_remaining / 86400; + let hours = (seconds_remaining % 86400) / 3600; + let minutes = (seconds_remaining % 3600) / 60; + let seconds = seconds_remaining % 60; + + format!( + "{}{}{}{}", + if days > 0 { format!("{}d ", days) } else { String::new() }, + if hours > 0 { format!("{}h ", hours) } else { String::new() }, + if minutes > 0 { format!("{}m ", minutes) } else { String::new() }, + if seconds > 0 { format!("{}s ", seconds) } else { String::from("0s") }, + ) +} \ No newline at end of file diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 74c69c24..c69c272a 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -1,23 +1,29 @@ mod tests; +mod util; mod tui_views; -use std::sync::{Arc, Mutex}; -use std::time::Instant; +use std::collections::HashMap; +use std::str::FromStr; +use std::sync::{Mutex}; +use std::time::{Instant, Duration}; use std::{io}; use clap::{AppSettings, Parser}; -use crossterm::event::{EnableMouseCapture, DisableMouseCapture}; +use crossterm::event::{EnableMouseCapture}; use crossterm::execute; -use crossterm::terminal::{enable_raw_mode, EnterAlternateScreen, disable_raw_mode, LeaveAlternateScreen}; -use ethers::types::U256; +use crossterm::terminal::{enable_raw_mode, EnterAlternateScreen}; +use ethers::types::{H256, H160, Diff}; use heimdall_common::resources::transpose::get_transaction_list; use heimdall_common::{ io::{ logging::* }, + utils::{ threading::task_pool } }; use tui::backend::Backend; use tui::{Frame, backend::CrosstermBackend, Terminal}; use tui_views::main::render_tui_view_main; +use lazy_static::lazy_static; +use self::util::{get_storage_diff, cleanup_terminal}; #[derive(Debug, Clone, Parser)] #[clap(about = "Dump the value of all storage slots accessed by a contract", @@ -49,31 +55,57 @@ pub struct DumpArgs { /// When prompted, always select the default value. #[clap(long, short)] pub default: bool, + + /// The number of threads to use + #[clap(long, short, default_value = "4", hide_default_value = true)] + pub threads: usize, } #[derive(Debug, Clone)] pub struct StorageSlot { - pub slot: U256, pub alias: Option, - pub value: Option + pub modified_at: u128, + pub value: H256, } #[derive(Debug, Clone)] pub struct Transaction { pub indexed: bool, pub hash: String, - pub block: u128, + pub block_number: u128, } #[derive(Debug, Clone)] pub struct DumpState { pub args: DumpArgs, + pub scroll_index: usize, pub transactions: Vec, - pub storage: Vec, + pub storage: HashMap, pub view: TUIView, pub start_time: Instant, } +impl DumpState { + pub fn new() -> Self { + Self { + args: DumpArgs { + target: String::new(), + verbose: clap_verbosity_flag::Verbosity::new(1, 0), + output: String::new(), + rpc_url: String::new(), + transpose_api_key: String::new(), + default: false, + threads: 4, + }, + scroll_index: 0, + transactions: Vec::new(), + storage: HashMap::new(), + view: TUIView::Main, + start_time: Instant::now(), + } + } +} + #[derive(Debug, Clone)] #[allow(dead_code)] pub enum TUIView { @@ -81,6 +113,9 @@ pub enum TUIView { CommandPalette, } +lazy_static! { + static ref DUMP_STATE: Mutex = Mutex::new(DumpState::new()); +} fn render_ui( f: &mut Frame, @@ -93,7 +128,6 @@ fn render_ui( } pub fn dump(args: DumpArgs) { - use std::time::Instant; let now = Instant::now(); let (logger, _)= Logger::new(args.verbose.log_level().unwrap().as_str()); @@ -121,6 +155,15 @@ pub fn dump(args: DumpArgs) { // output_dir = args.output.clone(); // } + // convert the target to an H160 + let addr_hash = match H160::from_str(&args.target) { + Ok(addr) => addr, + Err(_) => { + logger.error(&format!("failed to parse target '{}' .", &args.target)); + std::process::exit(1); + } + }; + // fetch transactions let transaction_list = get_transaction_list(&args.target, &args.transpose_api_key, &logger); @@ -130,18 +173,22 @@ pub fn dump(args: DumpArgs) { transactions.push(Transaction { indexed: false, hash: transaction.1, - block: transaction.0 + block_number: transaction.0 }); } - // create new state - let mut state = DumpState { + // update state + let mut state = DUMP_STATE.lock().unwrap(); + *state = DumpState { args: args.clone(), transactions: transactions, - storage: Vec::new(), + scroll_index: 0, + storage: HashMap::new(), view: TUIView::Main, start_time: Instant::now(), }; + drop(state); + // in a new thread, start the TUI let tui_thread = std::thread::spawn(move || { @@ -149,48 +196,154 @@ pub fn dump(args: DumpArgs) { // create new TUI terminal enable_raw_mode().unwrap(); let mut stdout = io::stdout(); - execute!( - stdout, - EnterAlternateScreen, - EnableMouseCapture - ).unwrap(); + execute!(stdout, EnterAlternateScreen, EnableMouseCapture).unwrap(); let backend = CrosstermBackend::new(stdout); let mut terminal = Terminal::new(backend).unwrap(); - // while user does not click CTRL+C loop { + let mut state = DUMP_STATE.lock().unwrap(); terminal.draw(|f| { render_ui(f, &mut state); }).unwrap(); + drop(state); // check for user input - if let Ok(event) = crossterm::event::read() { - match event { - crossterm::event::Event::Key(key) => { - match key.code { - crossterm::event::KeyCode::Char('q') => { - break; - }, - _ => {} - } - }, - _ => {} + if crossterm::event::poll(Duration::from_millis(100)).unwrap() { + if let Ok(event) = crossterm::event::read() { + match event { + crossterm::event::Event::Key(key) => { + match key.code { + + // quit + crossterm::event::KeyCode::Char('q') => { break; }, + + // scroll down + crossterm::event::KeyCode::Down => { + let mut state = DUMP_STATE.lock().unwrap(); + state.scroll_index += 1; + drop(state); + }, + + // scroll up + crossterm::event::KeyCode::Up => { + let mut state = DUMP_STATE.lock().unwrap(); + if state.scroll_index > 0 { + state.scroll_index -= 1; + } + drop(state); + }, + + _ => {} + } + }, + crossterm::event::Event::Mouse(mouse) => { + match mouse.kind { + + // scroll down + crossterm::event::MouseEventKind::ScrollDown => { + let mut state = DUMP_STATE.lock().unwrap(); + state.scroll_index += 1; + drop(state); + }, + + // scroll up + crossterm::event::MouseEventKind::ScrollUp => { + let mut state = DUMP_STATE.lock().unwrap(); + if state.scroll_index > 0 { + state.scroll_index -= 1; + } + drop(state); + }, + _ => {} + } + }, + _ => {} + } } } } - // cleanup - disable_raw_mode().unwrap(); - execute!( - terminal.backend_mut(), - LeaveAlternateScreen, - DisableMouseCapture - ).unwrap(); - terminal.show_cursor().unwrap(); + cleanup_terminal(); }); - for tx in state.transactions.iter_mut() { - tx.indexed = true; - std::thread::sleep(std::time::Duration::from_millis(100)); - } + // index transactions in a new thread + std::thread::spawn(move || { + let state = DUMP_STATE.lock().unwrap(); + let transactions = state.transactions.clone(); + drop(state); + + task_pool(transactions, args.threads, move |tx| { + + // get the storage diff for this transaction + let state_diff = get_storage_diff(&tx, &args); + + // unlock state + let mut state = DUMP_STATE.lock().unwrap(); + + // find the transaction in the state + let tx = state.transactions.iter_mut().find(|t| t.hash == tx.hash).unwrap(); + let block_number = tx.block_number.clone(); + tx.indexed = true; + + + // unwrap the state diff + match state_diff { + Some(state_diff) => { + + // get diff for this address + match state_diff.0.get(&addr_hash) { + Some(diff) => { + + // build diff of StorageSlots and append to state + for (slot, diff_type) in &diff.storage { + + // parse value from diff type + let value = match diff_type { + Diff::Born(value) => value, + Diff::Changed(changed) => &changed.to, + Diff::Died(_) => { + state.storage.remove(slot); + continue; + } + _ => continue, + }; + + // get the slot from the state + match state.storage.get_mut(slot) { + Some(slot) => { + + // update slot if it's newer + if slot.modified_at > block_number { + continue; + } + + slot.value = *value; + slot.modified_at = block_number; + }, + None => { + + // insert into state + state.storage.insert( + *slot, + StorageSlot { + value: *value, + modified_at: block_number, + alias: None, + } + ); + } + } + } + + }, + None => {} + } + }, + None => {} + } + + // drop state + drop(state); + }); + }); // wait for the TUI thread to finish tui_thread.join().unwrap(); diff --git a/heimdall/src/dump/tui_views/main.rs b/heimdall/src/dump/tui_views/main.rs index 3bbcc35c..7a9120de 100644 --- a/heimdall/src/dump/tui_views/main.rs +++ b/heimdall/src/dump/tui_views/main.rs @@ -1,4 +1,5 @@ -use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction}, widgets::{Gauge, ListItem, List, Block, Borders}, style::{Style, Color, Modifier}}; +use heimdall_common::utils::{time::{calculate_eta, format_eta}, strings::encode_hex}; +use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction}, widgets::{Gauge, Block, Borders, Cell, Row, Table}, style::{Style, Color, Modifier}}; use crate::dump::{DumpState}; @@ -18,41 +19,102 @@ pub fn render_tui_view_main( ].as_ref() ).split(f.size()); - let min_block_number = state.transactions.iter().min_by_key(|t| t.block).unwrap().block; - let max_block_number = state.transactions.iter().max_by_key(|t| t.block).unwrap().block; - let max_indexed_block_number = match state.transactions.iter().filter(|t| t.indexed).max_by_key(|t| t.block) { - Some(t) => t.block, + let min_block_number = state.transactions.iter().min_by_key(|t| t.block_number).unwrap().block_number; + let max_block_number = state.transactions.iter().max_by_key(|t| t.block_number).unwrap().block_number; + let max_indexed_block_number = match state.transactions.iter().filter(|t| t.indexed).max_by_key(|t| t.block_number) { + Some(t) => t.block_number, None => min_block_number }; - let blocks_indexed = max_indexed_block_number - min_block_number; - let percent_indexed = (max_indexed_block_number - min_block_number) as f64 / (max_block_number - min_block_number) as f64 * 100.0; + // calculate progress and stats + let transactions_indexed = state.transactions.iter().filter(|t| t.indexed).count(); + let transactions_total = state.transactions.len(); + let transactions_remaining = transactions_total - transactions_indexed; + let percent_indexed = (transactions_indexed as f64 / transactions_total as f64) * 100.0; let elapsed_seconds = state.start_time.elapsed().as_secs(); - let blocks_per_second = blocks_indexed as f64 / elapsed_seconds as f64; - - + let transactions_per_second = transactions_indexed as f64 / elapsed_seconds as f64; // render progress bar let progress = Gauge::default() .block(Block::default().title(" Dump Progress ").borders(Borders::ALL)) .gauge_style(Style::default().fg(Color::White).bg(Color::DarkGray)) .percent(percent_indexed as u16) - .label(format!( - "Block {}/{} ({:.1}%). {} Blocks Per Second. ETA: {}", - max_indexed_block_number, - max_block_number, - percent_indexed, - blocks_per_second, - 0 - )); - - let items = [ListItem::new("Item 1"), ListItem::new("Item 2"), ListItem::new("Item 3")]; - let list = List::new(items) - .block(Block::default().title("List").borders(Borders::ALL)) - .style(Style::default().fg(Color::White)) - .highlight_style(Style::default().add_modifier(Modifier::ITALIC)) - .highlight_symbol(">>"); + .label( + if transactions_indexed != transactions_total { + format!( + "Block {}/{} ({:.2}%). {:.2} TPS. ETA: {}", + max_indexed_block_number, + max_block_number, + percent_indexed, + transactions_per_second, + format_eta(calculate_eta(transactions_per_second, transactions_remaining)) + ) + } + else { + String::from("Storage Slot Dump Complete") + } + ); + + // build header cells + let header_cells = ["Slot", "Block Number", "Value"] + .iter() + .map(|h| Cell::from(*h).style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD))); + + // build header row + let header = Row::new(header_cells) + .style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD)) + .height(1) + .bottom_margin(1); + + // ensure scroll index is within bounds + if state.scroll_index >= state.storage.len() { + state.scroll_index = state.storage.len() - 1; + } + + // render storage slot list + let mut all_rows = Vec::new(); + let mut storage_iter = state.storage.iter().collect::>(); + + // sort by slot + storage_iter.sort_by_key(|(slot, _)| *slot); + + for (i, (slot, value)) in storage_iter.iter().enumerate() { + all_rows.push( + Row::new(vec![ + Cell::from(format!("0x{}", encode_hex(slot.to_fixed_bytes().into()))), + Cell::from(value.modified_at.to_string()), + Cell::from(format!("0x{}", encode_hex(value.value.to_fixed_bytes().into()))), + ]) + .style( + if i == state.scroll_index { + Style::default().fg(Color::White).bg(Color::DarkGray) + } + else { + Style::default().fg(Color::White) + } + ) + .height(1) + .bottom_margin(0) + ); + } + + // build rows of items to display + let num_items = std::cmp::min(main_layout[1].height as usize - 4, all_rows.len()); + let visible_rows = match state.scroll_index + num_items <= all_rows.len() { + true => all_rows[state.scroll_index..state.scroll_index + num_items].to_vec(), + false => all_rows[all_rows.len() - num_items..all_rows.len()].to_vec(), + }; + + // render table + let table = Table::new(visible_rows) + .header(header) + .block(Block::default().borders(Borders::ALL).title("Table")) + .widths(&[ + Constraint::Length(68), + Constraint::Length(14), + Constraint::Percentage(100), + ]); f.render_widget(progress, main_layout[0]); - f.render_widget(list, main_layout[1]); + f.render_widget(table, main_layout[1]); } \ No newline at end of file diff --git a/heimdall/src/dump/util.rs b/heimdall/src/dump/util.rs new file mode 100644 index 00000000..eea70f3e --- /dev/null +++ b/heimdall/src/dump/util.rs @@ -0,0 +1,87 @@ +use std::{str::FromStr, io}; + +use crossterm::{terminal::{disable_raw_mode, LeaveAlternateScreen}, execute, event::DisableMouseCapture}; +use ethers::{types::{StateDiff, H256, TraceType}, providers::{Provider, Http, Middleware}}; +use heimdall_cache::{read_cache, store_cache}; +use heimdall_common::io::logging::Logger; +use tui::{backend::CrosstermBackend, Terminal}; + +use super::{Transaction, DumpArgs}; + +pub fn cleanup_terminal() { + let stdout = io::stdout(); + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend).unwrap(); + disable_raw_mode().unwrap(); + execute!(terminal.backend_mut(), LeaveAlternateScreen, DisableMouseCapture).unwrap(); + terminal.show_cursor().unwrap(); +} + +pub fn get_storage_diff(tx: &Transaction, args: &DumpArgs) -> Option { + + // create new logger + let (logger, _)= Logger::new(args.verbose.log_level().unwrap().as_str()); + + // create new runtime block + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let state_diff = rt.block_on(async { + + // check the cache for a matching address + match read_cache(&format!("diff.{}", &tx.hash)) { + Some(state_diff) => { + logger.debug(&format!("found cached storage diff for '{}' .", &tx.hash)); + return state_diff; + }, + None => {} + } + + // make sure the RPC provider isn't empty + if &args.rpc_url.len() <= &0 { + cleanup_terminal(); + logger.error("fetching an on-chain transaction requires an RPC provider. Use `heimdall dump --help` for more information."); + std::process::exit(1); + } + + // create new provider + let provider = match Provider::::try_from(&args.rpc_url) { + Ok(provider) => provider, + Err(_) => { + cleanup_terminal(); + logger.error(&format!("failed to connect to RPC provider '{}' .", &args.rpc_url)); + std::process::exit(1) + } + }; + + // safely unwrap the transaction hash + let transaction_hash = match H256::from_str(&tx.hash) { + Ok(transaction_hash) => transaction_hash, + Err(_) => { + cleanup_terminal(); + logger.error(&format!("failed to parse transaction hash '{}' .", &tx.hash)); + std::process::exit(1) + } + }; + + // fetch the state diff for the transaction + let state_diff = match provider.trace_replay_transaction(transaction_hash, vec![TraceType::StateDiff]).await { + Ok(traces) => traces.state_diff, + Err(e) => { + cleanup_terminal(); + logger.error(&format!("failed to replay and trace transaction '{}' . does your RPC provider support it?", &tx.hash)); + logger.error(&format!("error: '{}' .", e)); + std::process::exit(1) + } + }; + + // write the state diff to the cache + store_cache(&format!("diff.{}", &tx.hash), &state_diff, None); + + state_diff + }); + + state_diff +} \ No newline at end of file diff --git a/heimdall/src/heimdall.rs b/heimdall/src/heimdall.rs index 4d5cc28f..46090c2e 100644 --- a/heimdall/src/heimdall.rs +++ b/heimdall/src/heimdall.rs @@ -1,4 +1,4 @@ -use std::{panic}; +use std::{panic, io}; use backtrace::Backtrace; mod cfg; @@ -9,6 +9,7 @@ mod decompile; use clap::{Parser, Subcommand}; use colored::Colorize; +use crossterm::{terminal::{disable_raw_mode, LeaveAlternateScreen}, execute, event::DisableMouseCapture}; use heimdall_config::{config, get_config, ConfigArgs}; use heimdall_cache::{cache, CacheArgs}; use heimdall_common::{ether::evm::disassemble::*, io::{logging::Logger}}; @@ -16,6 +17,7 @@ use decompile::{decompile, DecompilerArgs}; use decode::{decode, DecodeArgs}; use dump::{dump, DumpArgs}; use cfg::{cfg, CFGArgs}; +use tui::{backend::CrosstermBackend, Terminal}; #[derive(Debug, Parser)] #[clap( @@ -64,6 +66,16 @@ fn main() { // handle catching panics with panic::set_hook( Box::new(|panic_info| { + + // cleanup the terminal + let stdout = io::stdout(); + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend).unwrap(); + disable_raw_mode().unwrap(); + execute!(terminal.backend_mut(), LeaveAlternateScreen, DisableMouseCapture).unwrap(); + terminal.show_cursor().unwrap(); + + // print the panic message let backtrace = Backtrace::new(); let (logger, _)= Logger::new("TRACE"); logger.fatal( diff --git a/test.txt b/test.txt new file mode 100644 index 00000000..9e5bfde1 --- /dev/null +++ b/test.txt @@ -0,0 +1,19 @@ +AccountDiff { + balance: Same, + nonce: Same, + code: Same, + storage: { + 0x0000000000000000000000000000000000000000000000000000000000000068: Changed( + ChangedType { + from: 0x000000000000000000000000000000000000000000000000000000000000000f, + to: 0x0000000000000000000000000000000000000000000000000000000000000010, + }, + ), + 0xde793a4a6d5d86e1bbe481113da56fdf7ce0aa54ed0e14b7beec6fde6a7fdff9: Changed( + ChangedType { + from: 0x0000000000000000000000000000000000000000000000000000000000000000, + to: 0x0000000000000000000000000000000000000000000000000000000000000001, + }, + ), + }, +} \ No newline at end of file From ba1eb8573d0de07d2ddbcbe532d62636b01a2779 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Mon, 6 Mar 2023 13:32:09 -0500 Subject: [PATCH 04/26] :zap: perf: include diff from traces, csv dumping --- common/src/resources/transpose.rs | 2 +- heimdall/src/dump/mod.rs | 195 ++++++++++++++---- .../src/dump/tui_views/command_palette.rs | 61 ++++++ heimdall/src/dump/tui_views/decode_slot.rs | 97 +++++++++ heimdall/src/dump/tui_views/main.rs | 53 +---- heimdall/src/dump/tui_views/mod.rs | 4 +- heimdall/src/dump/util/csv.rs | 33 +++ heimdall/src/dump/{util.rs => util/mod.rs} | 7 +- heimdall/src/dump/util/table.rs | 57 +++++ test.txt | 19 -- 10 files changed, 420 insertions(+), 108 deletions(-) create mode 100644 heimdall/src/dump/tui_views/command_palette.rs create mode 100644 heimdall/src/dump/tui_views/decode_slot.rs create mode 100644 heimdall/src/dump/util/csv.rs rename heimdall/src/dump/{util.rs => util/mod.rs} (94%) create mode 100644 heimdall/src/dump/util/table.rs delete mode 100644 test.txt diff --git a/common/src/resources/transpose.rs b/common/src/resources/transpose.rs index 282ae01c..17b994b3 100644 --- a/common/src/resources/transpose.rs +++ b/common/src/resources/transpose.rs @@ -68,7 +68,7 @@ pub fn get_transaction_list(address: &String, api_key: &String, logger: &Logger) let start_time = Instant::now(); // build the SQL query - let query = format!("{{\"sql\":\"SELECT block_number, transaction_hash FROM ethereum.transactions WHERE to_address = '{}' ORDER BY block_number ASC\",\"parameters\":{{}},\"options\":{{}}}}", address); + let query = format!("{{\"sql\":\"SELECT block_number, transaction_hash FROM (SELECT transaction_hash, block_number FROM ethereum.transactions WHERE to_address = '{address}' UNION SELECT transaction_hash, block_number FROM ethereum.traces WHERE to_address = '{address}') x\",\"parameters\":{{}},\"options\":{{}}}}"); let response = match _call_transpose(query, api_key) { Some(response) => response, diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index c69c272a..9d3db119 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; use std::str::FromStr; use std::sync::{Mutex}; use std::time::{Instant, Duration}; -use std::{io}; +use std::{io, env}; use clap::{AppSettings, Parser}; use crossterm::event::{EnableMouseCapture}; use crossterm::execute; @@ -23,6 +23,9 @@ use tui::{Frame, backend::CrosstermBackend, Terminal}; use tui_views::main::render_tui_view_main; use lazy_static::lazy_static; +use self::tui_views::command_palette::render_tui_command_palette; +use self::tui_views::decode_slot::render_tui_decode_slot; +use self::util::csv::write_storage_to_csv; use self::util::{get_storage_diff, cleanup_terminal}; #[derive(Debug, Clone, Parser)] @@ -64,8 +67,8 @@ pub struct DumpArgs { #[derive(Debug, Clone)] pub struct StorageSlot { pub alias: Option, - pub modified_at: u128, pub value: H256, + pub modifiers: Vec<(u128, String)>, } #[derive(Debug, Clone)] @@ -79,10 +82,12 @@ pub struct Transaction { pub struct DumpState { pub args: DumpArgs, pub scroll_index: usize, + pub selection_size: usize, pub transactions: Vec, pub storage: HashMap, pub view: TUIView, pub start_time: Instant, + pub input_buffer: String, } impl DumpState { @@ -98,31 +103,38 @@ impl DumpState { threads: 4, }, scroll_index: 0, + selection_size: 1, transactions: Vec::new(), storage: HashMap::new(), view: TUIView::Main, start_time: Instant::now(), + input_buffer: String::new(), } } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] #[allow(dead_code)] pub enum TUIView { + Killed, Main, CommandPalette, + DecodeSelected } lazy_static! { static ref DUMP_STATE: Mutex = Mutex::new(DumpState::new()); } +#[allow(unreachable_patterns)] fn render_ui( f: &mut Frame, state: &mut DumpState ) { match state.view { TUIView::Main => { render_tui_view_main(f, state) }, + TUIView::CommandPalette => { render_tui_command_palette(f, state) }, + TUIView::DecodeSelected => { render_tui_decode_slot(f, state) }, _ => {} } } @@ -140,20 +152,17 @@ pub fn dump(args: DumpArgs) { } // parse the output directory - // let mut output_dir: String; - // if &args.output.len() <= &0 { - // output_dir = match env::current_dir() { - // Ok(dir) => dir.into_os_string().into_string().unwrap(), - // Err(_) => { - // logger.error("failed to get current directory."); - // std::process::exit(1); - // } - // }; - // output_dir.push_str("/output"); - // } - // else { - // output_dir = args.output.clone(); - // } + let mut output_dir = args.output.clone(); + if &args.output.len() <= &0 { + output_dir = match env::current_dir() { + Ok(dir) => dir.into_os_string().into_string().unwrap(), + Err(_) => { + logger.error("failed to get current directory."); + std::process::exit(1); + } + }; + output_dir.push_str("/output"); + } // convert the target to an H160 let addr_hash = match H160::from_str(&args.target) { @@ -164,6 +173,11 @@ pub fn dump(args: DumpArgs) { } }; + // push the address to the output directory + if &output_dir != &args.output { + output_dir.push_str(&format!("/{}", &args.target)); + } + // fetch transactions let transaction_list = get_transaction_list(&args.target, &args.transpose_api_key, &logger); @@ -183,9 +197,11 @@ pub fn dump(args: DumpArgs) { args: args.clone(), transactions: transactions, scroll_index: 0, + selection_size: 1, storage: HashMap::new(), view: TUIView::Main, start_time: Instant::now(), + input_buffer: String::new(), }; drop(state); @@ -206,54 +222,140 @@ pub fn dump(args: DumpArgs) { drop(state); // check for user input - if crossterm::event::poll(Duration::from_millis(100)).unwrap() { + if crossterm::event::poll(Duration::from_millis(10)).unwrap() { if let Ok(event) = crossterm::event::read() { match event { crossterm::event::Event::Key(key) => { + let mut state = DUMP_STATE.lock().unwrap(); + + // ignore key events if command palette is open + if state.view == TUIView::CommandPalette { + match key.code { + + // handle keys in command palette + crossterm::event::KeyCode::Char(c) => { + state.input_buffer.push(c); + }, + + // handle backspace + crossterm::event::KeyCode::Backspace => { + state.input_buffer.pop(); + }, + + // enter command + crossterm::event::KeyCode::Enter => { + let mut split = state.input_buffer.split(" "); + let command = split.next().unwrap(); + let _args = split.collect::>(); + + match command { + ":q" => { + state.view = TUIView::Killed; + break; + } + ":quit" => { + state.view = TUIView::Killed; + break; + } + _ => { + state.view = TUIView::Main; + } + } + }, + + // close command palette + crossterm::event::KeyCode::Esc => { + state.view = TUIView::Main; + } + _ => {} + } + + drop(state); + continue; + } + match key.code { - // quit - crossterm::event::KeyCode::Char('q') => { break; }, + // main on escape + crossterm::event::KeyCode::Esc => { + state.view = TUIView::Main; + }, + + // select transaction + crossterm::event::KeyCode::Right => { + state.view = TUIView::DecodeSelected; + }, + + // deselect transaction + crossterm::event::KeyCode::Left => { + state.view = TUIView::Main; + }, // scroll down crossterm::event::KeyCode::Down => { - let mut state = DUMP_STATE.lock().unwrap(); + state.selection_size = 1; state.scroll_index += 1; - drop(state); }, // scroll up crossterm::event::KeyCode::Up => { - let mut state = DUMP_STATE.lock().unwrap(); + state.selection_size = 1; if state.scroll_index > 0 { state.scroll_index -= 1; } - drop(state); + }, + + // toggle command palette on ":" + crossterm::event::KeyCode::Char(':') => { + match state.view { + TUIView::CommandPalette => { + state.view = TUIView::Main; + } + _ => { + state.input_buffer = String::from(":"); + state.view = TUIView::CommandPalette; + } + } }, _ => {} } + drop(state) }, crossterm::event::Event::Mouse(mouse) => { + let mut state = DUMP_STATE.lock().unwrap(); match mouse.kind { // scroll down crossterm::event::MouseEventKind::ScrollDown => { - let mut state = DUMP_STATE.lock().unwrap(); - state.scroll_index += 1; - drop(state); + + // if shift is held, increase selection size + if mouse.modifiers.contains(crossterm::event::KeyModifiers::SHIFT) { + state.selection_size += 1; + } + else { + state.selection_size = 1; + state.scroll_index += 1; + } }, // scroll up crossterm::event::MouseEventKind::ScrollUp => { - let mut state = DUMP_STATE.lock().unwrap(); - if state.scroll_index > 0 { - state.scroll_index -= 1; + + // if shift is held, increase selection size + if mouse.modifiers.contains(crossterm::event::KeyModifiers::SHIFT) { + state.selection_size -= 1; + } + else { + state.selection_size = 1; + if state.scroll_index > 0 { + state.scroll_index -= 1; + } } - drop(state); }, _ => {} } + drop(state); }, _ => {} } @@ -279,10 +381,9 @@ pub fn dump(args: DumpArgs) { let mut state = DUMP_STATE.lock().unwrap(); // find the transaction in the state - let tx = state.transactions.iter_mut().find(|t| t.hash == tx.hash).unwrap(); + let txs = state.transactions.iter_mut().find(|t| t.hash == tx.hash).unwrap(); let block_number = tx.block_number.clone(); - tx.indexed = true; - + txs.indexed = true; // unwrap the state diff match state_diff { @@ -310,13 +411,12 @@ pub fn dump(args: DumpArgs) { match state.storage.get_mut(slot) { Some(slot) => { - // update slot if it's newer - if slot.modified_at > block_number { - continue; + // update value if newest modifier + if slot.modifiers.iter().all(|m| m.0 < block_number) { + slot.value = *value; } - - slot.value = *value; - slot.modified_at = block_number; + + slot.modifiers.push((block_number, tx.hash.clone().to_owned())); }, None => { @@ -325,7 +425,7 @@ pub fn dump(args: DumpArgs) { *slot, StorageSlot { value: *value, - modified_at: block_number, + modifiers: vec![(block_number, tx.hash.clone().to_owned())], alias: None, } ); @@ -346,7 +446,18 @@ pub fn dump(args: DumpArgs) { }); // wait for the TUI thread to finish - tui_thread.join().unwrap(); + match tui_thread.join() { + Ok(_) => {}, + Err(e) => { + logger.error("failed to join TUI thread."); + logger.error(&format!("{:?}", e)); + std::process::exit(1); + } + } + + // write storage slots to csv + let state = DUMP_STATE.lock().unwrap(); + write_storage_to_csv(&output_dir.clone(), &state, &logger); logger.debug(&format!("Dumped storage slots in {:?}.", now.elapsed())); } \ No newline at end of file diff --git a/heimdall/src/dump/tui_views/command_palette.rs b/heimdall/src/dump/tui_views/command_palette.rs new file mode 100644 index 00000000..af761b67 --- /dev/null +++ b/heimdall/src/dump/tui_views/command_palette.rs @@ -0,0 +1,61 @@ +use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction}, widgets::{Block, Borders, Cell, Row, Table, Paragraph}, style::{Style, Color, Modifier}}; + +use crate::dump::{DumpState, util::table::build_rows}; + +pub fn render_tui_command_palette( + f: &mut Frame, + state: &mut DumpState +) { + + // build main layout + let main_layout = Layout::default() + .direction(Direction::Vertical) + .margin(1) + .constraints( + [ + Constraint::Length(3), + Constraint::Percentage(100), + ].as_ref() + ).split(f.size()); + + // add command paragraph input + let input_buffer = state.input_buffer.clone(); + let command_input = Paragraph::new(input_buffer) + .style(Style::default().fg(Color::White)) + .block( + Block::default() + .title(" Command ") + .borders(Borders::ALL) + ); + + // build header cells + let header_cells = ["Slot", "Block Number", "Value"] + .iter() + .map(|h| Cell::from(*h).style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD))); + + // build header row + let header = Row::new(header_cells) + .style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD)) + .height(1) + .bottom_margin(1); + + let rows = build_rows( + state, + main_layout[1].height as usize - 4 + ); + + // render table + let table = Table::new(rows) + .header(header) + .block(Block::default().borders(Borders::ALL) + .title(format!(" Storage for Contract {} ", &state.args.target))) + .widths(&[ + Constraint::Length(68), + Constraint::Length(14), + Constraint::Percentage(68), + Constraint::Percentage(68), + ]); + + f.render_widget(command_input, main_layout[0]); + f.render_widget(table, main_layout[1]); +} \ No newline at end of file diff --git a/heimdall/src/dump/tui_views/decode_slot.rs b/heimdall/src/dump/tui_views/decode_slot.rs new file mode 100644 index 00000000..2bbb9469 --- /dev/null +++ b/heimdall/src/dump/tui_views/decode_slot.rs @@ -0,0 +1,97 @@ +use heimdall_common::utils::{strings::encode_hex}; +use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction}, widgets::{Block, Borders, Cell, Row, Table}, style::{Style, Color, Modifier}}; + +use crate::dump::{DumpState}; + +pub fn render_tui_decode_slot( + f: &mut Frame, + state: &mut DumpState +) { + + // build main layout + let main_layout = Layout::default() + .direction(Direction::Vertical) + .margin(1) + .constraints( + [ + Constraint::Length(std::cmp::min(4+state.selection_size, 15) as u16), + Constraint::Percentage(100), + ].as_ref() + ).split(f.size()); + + // build header cells + let header_cells = ["Slot", "Block Number", "Value", "Modifiers"] + .iter() + .map(|h| Cell::from(*h).style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD))); + + // build header row + let header = Row::new(header_cells) + .style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD)) + .height(1) + .bottom_margin(1); + + // ensure scroll index is within bounds + if state.scroll_index >= state.storage.len() { + state.scroll_index = state.storage.len() - 1; + } + + // render storage slot list + let mut all_rows = Vec::new(); + let mut storage_iter = state.storage.iter().collect::>(); + + // sort by slot + storage_iter.sort_by_key(|(slot, _)| *slot); + for (i, (slot, value)) in storage_iter.iter().enumerate() { + if i >= state.scroll_index && i < state.scroll_index + state.selection_size { + all_rows.push( + Row::new(vec![ + Cell::from(format!("0x{}", encode_hex(slot.to_fixed_bytes().into()))), + Cell::from(value.modifiers.iter().max_by_key(|m| m.0).unwrap().0.to_string()), + Cell::from(format!("0x{}", encode_hex(value.value.to_fixed_bytes().into()))) + ]) + .style(Style::default().fg(Color::White)) + .height(1) + .bottom_margin(0) + ); + } + } + + // if all_rows > 10, add ... + if all_rows.len() > 10 { + + // save the last row + let last_row = all_rows.pop().unwrap(); + + // slice to 10 + all_rows = all_rows[0..9].to_vec(); + + // add ellipsis + all_rows.push( + Row::new(vec![ + Cell::from("..."), + Cell::from("..."), + Cell::from("..."), + ]) + .style(Style::default().fg(Color::White)) + .height(1) + .bottom_margin(0) + ); + + // add last row + all_rows.push(last_row); + } + + // render table + let table = Table::new(all_rows) + .header(header) + .block(Block::default().borders(Borders::ALL) + .title(" Decoding Storage Values ")) + .widths(&[ + Constraint::Length(68), + Constraint::Length(14), + Constraint::Percentage(68), + Constraint::Percentage(68), + ]); + + f.render_widget(table, main_layout[0]); +} \ No newline at end of file diff --git a/heimdall/src/dump/tui_views/main.rs b/heimdall/src/dump/tui_views/main.rs index 7a9120de..419b6c4f 100644 --- a/heimdall/src/dump/tui_views/main.rs +++ b/heimdall/src/dump/tui_views/main.rs @@ -1,7 +1,7 @@ -use heimdall_common::utils::{time::{calculate_eta, format_eta}, strings::encode_hex}; +use heimdall_common::utils::{time::{calculate_eta, format_eta}}; use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction}, widgets::{Gauge, Block, Borders, Cell, Row, Table}, style::{Style, Color, Modifier}}; -use crate::dump::{DumpState}; +use crate::dump::{DumpState, util::table::build_rows}; pub fn render_tui_view_main( f: &mut Frame, @@ -66,49 +66,16 @@ pub fn render_tui_view_main( .height(1) .bottom_margin(1); - // ensure scroll index is within bounds - if state.scroll_index >= state.storage.len() { - state.scroll_index = state.storage.len() - 1; - } - - // render storage slot list - let mut all_rows = Vec::new(); - let mut storage_iter = state.storage.iter().collect::>(); - - // sort by slot - storage_iter.sort_by_key(|(slot, _)| *slot); - - for (i, (slot, value)) in storage_iter.iter().enumerate() { - all_rows.push( - Row::new(vec![ - Cell::from(format!("0x{}", encode_hex(slot.to_fixed_bytes().into()))), - Cell::from(value.modified_at.to_string()), - Cell::from(format!("0x{}", encode_hex(value.value.to_fixed_bytes().into()))), - ]) - .style( - if i == state.scroll_index { - Style::default().fg(Color::White).bg(Color::DarkGray) - } - else { - Style::default().fg(Color::White) - } - ) - .height(1) - .bottom_margin(0) - ); - } - - // build rows of items to display - let num_items = std::cmp::min(main_layout[1].height as usize - 4, all_rows.len()); - let visible_rows = match state.scroll_index + num_items <= all_rows.len() { - true => all_rows[state.scroll_index..state.scroll_index + num_items].to_vec(), - false => all_rows[all_rows.len() - num_items..all_rows.len()].to_vec(), - }; - + let rows = build_rows( + state, + main_layout[1].height as usize - 4 + ); + // render table - let table = Table::new(visible_rows) + let table = Table::new(rows) .header(header) - .block(Block::default().borders(Borders::ALL).title("Table")) + .block(Block::default().borders(Borders::ALL) + .title(format!(" Storage for Contract {} ", &state.args.target))) .widths(&[ Constraint::Length(68), Constraint::Length(14), diff --git a/heimdall/src/dump/tui_views/mod.rs b/heimdall/src/dump/tui_views/mod.rs index 5a8f6491..c2ce6bb5 100644 --- a/heimdall/src/dump/tui_views/mod.rs +++ b/heimdall/src/dump/tui_views/mod.rs @@ -1 +1,3 @@ -pub mod main; \ No newline at end of file +pub mod main; +pub mod command_palette; +pub mod decode_slot; \ No newline at end of file diff --git a/heimdall/src/dump/util/csv.rs b/heimdall/src/dump/util/csv.rs new file mode 100644 index 00000000..7fbb075c --- /dev/null +++ b/heimdall/src/dump/util/csv.rs @@ -0,0 +1,33 @@ +use heimdall_common::{utils::strings::encode_hex, io::{file::write_lines_to_file, logging::Logger}}; + +use crate::dump::DumpState; + +pub fn write_storage_to_csv(output_dir: &String, state: &DumpState, logger: &Logger) { + let mut lines = { + let mut lines = Vec::new(); + + // sort by key ascending + let mut storage_iter = state.storage.iter().collect::>(); + storage_iter.sort_by_key(|(slot, _)| *slot); + + for (slot, slot_data) in storage_iter { + lines.push( + format!( + "{},{},{},{}", + encode_hex(slot.to_fixed_bytes().into()), + encode_hex(slot_data.value.to_fixed_bytes().into()), + slot_data.modifiers.iter().max_by_key(|m| m.0).unwrap().0.to_string(), + slot_data.alias.as_ref().unwrap_or(&String::from("None")) + ) + ); + } + lines + }; + + // add header + lines.insert(0, String::from("slot,value,last_modified,alias")); + + // save to file + write_lines_to_file(&format!("{output_dir}/storage_dump.csv"), lines); + logger.success(&format!("wrote storage dump to to '{output_dir}/storage_dump.csv' .")); +} \ No newline at end of file diff --git a/heimdall/src/dump/util.rs b/heimdall/src/dump/util/mod.rs similarity index 94% rename from heimdall/src/dump/util.rs rename to heimdall/src/dump/util/mod.rs index eea70f3e..a026e62d 100644 --- a/heimdall/src/dump/util.rs +++ b/heimdall/src/dump/util/mod.rs @@ -1,3 +1,6 @@ +pub mod csv; +pub mod table; + use std::{str::FromStr, io}; use crossterm::{terminal::{disable_raw_mode, LeaveAlternateScreen}, execute, event::DisableMouseCapture}; @@ -33,7 +36,6 @@ pub fn get_storage_diff(tx: &Transaction, args: &DumpArgs) -> Option // check the cache for a matching address match read_cache(&format!("diff.{}", &tx.hash)) { Some(state_diff) => { - logger.debug(&format!("found cached storage diff for '{}' .", &tx.hash)); return state_diff; }, None => {} @@ -78,7 +80,8 @@ pub fn get_storage_diff(tx: &Transaction, args: &DumpArgs) -> Option }; // write the state diff to the cache - store_cache(&format!("diff.{}", &tx.hash), &state_diff, None); + let expiry = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() + 60 * 60 * 24 * 7; + store_cache(&format!("diff.{}", &tx.hash), &state_diff, Some(expiry)); state_diff }); diff --git a/heimdall/src/dump/util/table.rs b/heimdall/src/dump/util/table.rs new file mode 100644 index 00000000..1ea5af2c --- /dev/null +++ b/heimdall/src/dump/util/table.rs @@ -0,0 +1,57 @@ +use heimdall_common::utils::strings::encode_hex; +use tui::{widgets::{Row, Cell}, style::{Style, Color}}; + +use crate::dump::DumpState; + +pub fn build_rows(mut state: &mut DumpState, max_row_height: usize) -> Vec> { + + // ensure scroll index is within bounds + if state.scroll_index >= state.storage.len() && state.scroll_index != 0 { + state.scroll_index = state.storage.len() - 1; + } + + // render storage slot list + let mut rows = Vec::new(); + let mut storage_iter = state.storage.iter().collect::>(); + + // sort storage slots by slot + storage_iter.sort_by_key(|(slot, _)| *slot); + let num_items = std::cmp::min(max_row_height, storage_iter.len()); + + let indices = match state.scroll_index + num_items <= storage_iter.len() { + true => state.scroll_index..state.scroll_index + num_items, + false => storage_iter.len() - num_items..storage_iter.len(), + }; + + // slice storage_iter + for (i, (slot, value)) in storage_iter[indices.clone()].iter().enumerate() { + rows.push( + Row::new(vec![ + Cell::from(format!("0x{}", encode_hex(slot.to_fixed_bytes().into()))), + Cell::from(value.modifiers.iter().max_by_key(|m| m.0).unwrap().0.to_string()), + Cell::from(format!("0x{}", encode_hex(value.value.to_fixed_bytes().into()))), + Cell::from(value.modifiers.len().to_string()) + ]) + .style( + if storage_iter.len() - state.scroll_index < num_items { + if (num_items - i <= storage_iter.len() - state.scroll_index) && (num_items - i > storage_iter.len() - state.scroll_index - state.selection_size){ + Style::default().fg(Color::White).bg(Color::DarkGray) + } + else { + Style::default().fg(Color::White) + } + } + else if i == 0 || i < state.selection_size { + Style::default().fg(Color::White).bg(Color::DarkGray) + } + else { + Style::default().fg(Color::White) + } + ) + .height(1) + .bottom_margin(0) + ); + } + + rows +} \ No newline at end of file diff --git a/test.txt b/test.txt deleted file mode 100644 index 9e5bfde1..00000000 --- a/test.txt +++ /dev/null @@ -1,19 +0,0 @@ -AccountDiff { - balance: Same, - nonce: Same, - code: Same, - storage: { - 0x0000000000000000000000000000000000000000000000000000000000000068: Changed( - ChangedType { - from: 0x000000000000000000000000000000000000000000000000000000000000000f, - to: 0x0000000000000000000000000000000000000000000000000000000000000010, - }, - ), - 0xde793a4a6d5d86e1bbe481113da56fdf7ce0aa54ed0e14b7beec6fde6a7fdff9: Changed( - ChangedType { - from: 0x0000000000000000000000000000000000000000000000000000000000000000, - to: 0x0000000000000000000000000000000000000000000000000000000000000001, - }, - ), - }, -} \ No newline at end of file From 7629870c20d7b5824fa34e8ccfeaa98c7f2cdab0 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 7 Mar 2023 12:45:52 -0500 Subject: [PATCH 05/26] :wrench: fix: remove `-t` as alias for `--threads` --- heimdall/src/dump/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 9d3db119..896b1eda 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -60,7 +60,7 @@ pub struct DumpArgs { pub default: bool, /// The number of threads to use - #[clap(long, short, default_value = "4", hide_default_value = true)] + #[clap(long, default_value = "4", hide_default_value = true)] pub threads: usize, } @@ -370,6 +370,7 @@ pub fn dump(args: DumpArgs) { std::thread::spawn(move || { let state = DUMP_STATE.lock().unwrap(); let transactions = state.transactions.clone(); + let args = state.args.clone(); drop(state); task_pool(transactions, args.threads, move |tx| { @@ -459,5 +460,6 @@ pub fn dump(args: DumpArgs) { let state = DUMP_STATE.lock().unwrap(); write_storage_to_csv(&output_dir.clone(), &state, &logger); + logger.info(&format!("Dumped {} storage values from '{}' .", state.storage.len(), &args.target)); logger.debug(&format!("Dumped storage slots in {:?}.", now.elapsed())); } \ No newline at end of file From 3ea5b4ede7bce3e6f77804718360fb59ede2d630 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 7 Mar 2023 13:32:55 -0500 Subject: [PATCH 06/26] :sparkles: feat: `heimdall cache size` --- cache/src/lib.rs | 21 +++++++++++++++++++-- cache/src/util.rs | 15 +++++++++++++++ heimdall/src/dump/mod.rs | 3 --- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/cache/src/lib.rs b/cache/src/lib.rs index d29c4bf7..53bcd100 100644 --- a/cache/src/lib.rs +++ b/cache/src/lib.rs @@ -35,6 +35,9 @@ pub enum Subcommands { #[clap(name = "ls", about = "Lists all cached objects in ~/.bifrost/cache")] Ls(NoArguments), + + #[clap(name = "size", about = "Prints the size of the cache in ~/.bifrost/cache")] + Size(NoArguments), } @@ -188,7 +191,7 @@ pub fn store_cache(key: &str, value: T, expiry: Option) where T: Seriali write_file(&cache_file.to_str().unwrap().to_string(), &binary_string); } - +#[allow(deprecated)] pub fn cache(args: CacheArgs) -> Result<(), Box> { match args.sub { Subcommands::Clean(_) => { @@ -202,8 +205,22 @@ pub fn cache(args: CacheArgs) -> Result<(), Box> { for (i, key) in keys.iter().enumerate() { println!("{i:>5} : {}", key); } - }, + Subcommands::Size(_) => { + let home = home_dir().unwrap(); + let cache_dir = home.join(".bifrost").join("cache"); + let mut size = 0; + + for entry in cache_dir.read_dir().unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + let metadata = std::fs::metadata(path).unwrap(); + size += metadata.len(); + } + + println!("Cached objects: {}", keys("*").len()); + println!("Cache size: {}", prettify_bytes(size)); + } } Ok(()) diff --git a/cache/src/util.rs b/cache/src/util.rs index e0129f70..d0c20a39 100644 --- a/cache/src/util.rs +++ b/cache/src/util.rs @@ -19,6 +19,21 @@ use std::{ io::{Write, Read}, process::Command, num::ParseIntError }; +pub fn prettify_bytes(bytes: u64) -> String { + if bytes < 1024 { + return format!("{} B", bytes); + } else if bytes < 1024 * 1024 { + let kb = bytes / 1024; + return format!("{} KB", kb); + } else if bytes < 1024 * 1024 * 1024 { + let mb = bytes / (1024 * 1024); + return format!("{} MB", mb); + } else { + let gb = bytes / (1024 * 1024 * 1024); + return format!("{} GB", gb); + } +} + pub fn write_file(_path: &String, contents: &String) -> Option { let path = std::path::Path::new(_path); diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 896b1eda..26c076c9 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -140,8 +140,6 @@ fn render_ui( } pub fn dump(args: DumpArgs) { - let now = Instant::now(); - let (logger, _)= Logger::new(args.verbose.log_level().unwrap().as_str()); // check if transpose api key is set @@ -461,5 +459,4 @@ pub fn dump(args: DumpArgs) { write_storage_to_csv(&output_dir.clone(), &state, &logger); logger.info(&format!("Dumped {} storage values from '{}' .", state.storage.len(), &args.target)); - logger.debug(&format!("Dumped storage slots in {:?}.", now.elapsed())); } \ No newline at end of file From 41e6b0486cce5ee6b871dfae8a248541a1d2da16 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Thu, 9 Mar 2023 13:16:16 -0500 Subject: [PATCH 07/26] :zap: perf: allow specifying block bounds --- common/src/resources/transpose.rs | 17 +++- common/src/utils/strings.rs | 10 ++ heimdall/src/dump/constants.rs | 17 ++++ heimdall/src/dump/mod.rs | 71 +++++++++++--- .../src/dump/tui_views/command_palette.rs | 10 +- heimdall/src/dump/tui_views/decode_slot.rs | 97 ------------------- heimdall/src/dump/tui_views/main.rs | 5 +- heimdall/src/dump/tui_views/mod.rs | 3 +- heimdall/src/dump/util/table.rs | 26 ++++- 9 files changed, 131 insertions(+), 125 deletions(-) create mode 100644 heimdall/src/dump/constants.rs delete mode 100644 heimdall/src/dump/tui_views/decode_slot.rs diff --git a/common/src/resources/transpose.rs b/common/src/resources/transpose.rs index 17b994b3..2a07c5a6 100644 --- a/common/src/resources/transpose.rs +++ b/common/src/resources/transpose.rs @@ -58,7 +58,12 @@ fn _call_transpose(query: String, api_key: &String) -> Option } } -pub fn get_transaction_list(address: &String, api_key: &String, logger: &Logger) -> Vec<(u128, String)> { +pub fn get_transaction_list( + address: &String, + api_key: &String, + bounds: (&u128, &u128), + logger: &Logger +) -> Vec<(u128, String)> { // get a new progress bar let transaction_list_progress = ProgressBar::new_spinner(); @@ -68,7 +73,15 @@ pub fn get_transaction_list(address: &String, api_key: &String, logger: &Logger) let start_time = Instant::now(); // build the SQL query - let query = format!("{{\"sql\":\"SELECT block_number, transaction_hash FROM (SELECT transaction_hash, block_number FROM ethereum.transactions WHERE to_address = '{address}' UNION SELECT transaction_hash, block_number FROM ethereum.traces WHERE to_address = '{address}') x\",\"parameters\":{{}},\"options\":{{}}}}"); + let query = format!( + "{{\"sql\":\"SELECT block_number, transaction_hash FROM (SELECT transaction_hash, block_number FROM ethereum.transactions WHERE to_address = '{}' AND block_number BETWEEN {} AND {} UNION SELECT transaction_hash, block_number FROM ethereum.traces WHERE to_address = '{}' AND block_number BETWEEN {} AND {}) x\",\"parameters\":{{}},\"options\":{{}}}}", + address, + bounds.0, + bounds.1, + address, + bounds.0, + bounds.1 + ); let response = match _call_transpose(query, api_key) { Some(response) => response, diff --git a/common/src/utils/strings.rs b/common/src/utils/strings.rs index be56da1e..25e54d8f 100644 --- a/common/src/utils/strings.rs +++ b/common/src/utils/strings.rs @@ -40,6 +40,16 @@ pub fn encode_hex_reduced(s: U256) -> String { } } +// convert a hex string to ascii +pub fn hex_to_ascii(s: &str) -> String { + let mut result = String::new(); + for i in 0..s.len() / 2 { + let byte = u8::from_str_radix(&s[2 * i..2 * i + 2], 16).unwrap(); + result.push(byte as char); + } + result +} + // replace the last occurrence of a string with a new string pub fn replace_last(s: String, old: &str, new: &str) -> String { diff --git a/heimdall/src/dump/constants.rs b/heimdall/src/dump/constants.rs new file mode 100644 index 00000000..1fe0dd62 --- /dev/null +++ b/heimdall/src/dump/constants.rs @@ -0,0 +1,17 @@ +use std::sync::Mutex; + +use lazy_static::lazy_static; + +use crate::dump::DumpState; + + +lazy_static! { + pub static ref DUMP_STATE: Mutex = Mutex::new(DumpState::new()); + pub static ref DECODE_AS_TYPES: Vec = vec![ + "bytes32".to_string(), + "bool".to_string(), + "address".to_string(), + "string".to_string(), + "uint256".to_string() + ]; +} \ No newline at end of file diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 26c076c9..ab991cdc 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -1,10 +1,10 @@ mod tests; mod util; +mod constants; mod tui_views; use std::collections::HashMap; use std::str::FromStr; -use std::sync::{Mutex}; use std::time::{Instant, Duration}; use std::{io, env}; use clap::{AppSettings, Parser}; @@ -21,10 +21,9 @@ use tui::backend::Backend; use tui::{Frame, backend::CrosstermBackend, Terminal}; use tui_views::main::render_tui_view_main; -use lazy_static::lazy_static; +use self::constants::{DUMP_STATE, DECODE_AS_TYPES}; use self::tui_views::command_palette::render_tui_command_palette; -use self::tui_views::decode_slot::render_tui_decode_slot; use self::util::csv::write_storage_to_csv; use self::util::{get_storage_diff, cleanup_terminal}; @@ -62,6 +61,14 @@ pub struct DumpArgs { /// The number of threads to use #[clap(long, default_value = "4", hide_default_value = true)] pub threads: usize, + + /// The block number to start dumping from. + #[clap(long, short, default_value = "0", hide_default_value = true)] + pub from_block: u128, + + /// The block number to stop dumping at. + #[clap(long, short, default_value = "9999999999", hide_default_value = true)] + pub to_block: u128, } #[derive(Debug, Clone)] @@ -69,6 +76,7 @@ pub struct StorageSlot { pub alias: Option, pub value: H256, pub modifiers: Vec<(u128, String)>, + pub decode_as_type_index: usize, } #[derive(Debug, Clone)] @@ -101,6 +109,8 @@ impl DumpState { transpose_api_key: String::new(), default: false, threads: 4, + from_block: 0, + to_block: 9999999999, }, scroll_index: 0, selection_size: 1, @@ -119,11 +129,6 @@ pub enum TUIView { Killed, Main, CommandPalette, - DecodeSelected -} - -lazy_static! { - static ref DUMP_STATE: Mutex = Mutex::new(DumpState::new()); } #[allow(unreachable_patterns)] @@ -134,7 +139,6 @@ fn render_ui( match state.view { TUIView::Main => { render_tui_view_main(f, state) }, TUIView::CommandPalette => { render_tui_command_palette(f, state) }, - TUIView::DecodeSelected => { render_tui_decode_slot(f, state) }, _ => {} } } @@ -177,7 +181,7 @@ pub fn dump(args: DumpArgs) { } // fetch transactions - let transaction_list = get_transaction_list(&args.target, &args.transpose_api_key, &logger); + let transaction_list = get_transaction_list(&args.target, &args.transpose_api_key, (&args.from_block, &args.to_block), &logger); // convert to vec of Transaction let mut transactions: Vec = Vec::new(); @@ -281,12 +285,54 @@ pub fn dump(args: DumpArgs) { // select transaction crossterm::event::KeyCode::Right => { - state.view = TUIView::DecodeSelected; + + // increment decode_as_type_index on all selected transactions + let scroll_index = state.scroll_index.clone(); + let selection_size = state.selection_size.clone(); + let mut storage_iter = state.storage.iter_mut().collect::>(); + storage_iter.sort_by_key(|(slot, _)| *slot); + + for (i, (_, value)) in storage_iter.iter_mut().enumerate() { + if i >= scroll_index && i < scroll_index + selection_size { + + // saturating increment + if value.decode_as_type_index + 1 >= DECODE_AS_TYPES.len() { + value.decode_as_type_index = 0; + } else { + value.decode_as_type_index += 1; + } + + } + else if i >= scroll_index + selection_size { + break; + } + } }, // deselect transaction crossterm::event::KeyCode::Left => { - state.view = TUIView::Main; + + // decrement decode_as_type_index on all selected transactions + let scroll_index = state.scroll_index.clone(); + let selection_size = state.selection_size.clone(); + let mut storage_iter = state.storage.iter_mut().collect::>(); + storage_iter.sort_by_key(|(slot, _)| *slot); + + for (i, (_, value)) in storage_iter.iter_mut().enumerate() { + if i >= scroll_index && i < scroll_index + selection_size { + + // saturating decrement + if value.decode_as_type_index == 0 { + value.decode_as_type_index = DECODE_AS_TYPES.len() - 1; + } else { + value.decode_as_type_index -= 1; + } + + } + else if i >= scroll_index + selection_size { + break; + } + } }, // scroll down @@ -426,6 +472,7 @@ pub fn dump(args: DumpArgs) { value: *value, modifiers: vec![(block_number, tx.hash.clone().to_owned())], alias: None, + decode_as_type_index: 0 } ); } diff --git a/heimdall/src/dump/tui_views/command_palette.rs b/heimdall/src/dump/tui_views/command_palette.rs index af761b67..8bdfcdfc 100644 --- a/heimdall/src/dump/tui_views/command_palette.rs +++ b/heimdall/src/dump/tui_views/command_palette.rs @@ -29,7 +29,7 @@ pub fn render_tui_command_palette( ); // build header cells - let header_cells = ["Slot", "Block Number", "Value"] + let header_cells = ["Last Modified", "Slot", "As Type", "Value"] .iter() .map(|h| Cell::from(*h).style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD))); @@ -43,17 +43,17 @@ pub fn render_tui_command_palette( state, main_layout[1].height as usize - 4 ); - + // render table let table = Table::new(rows) .header(header) .block(Block::default().borders(Borders::ALL) .title(format!(" Storage for Contract {} ", &state.args.target))) .widths(&[ - Constraint::Length(68), Constraint::Length(14), - Constraint::Percentage(68), - Constraint::Percentage(68), + Constraint::Length(68), + Constraint::Length(9), + Constraint::Percentage(100), ]); f.render_widget(command_input, main_layout[0]); diff --git a/heimdall/src/dump/tui_views/decode_slot.rs b/heimdall/src/dump/tui_views/decode_slot.rs deleted file mode 100644 index 2bbb9469..00000000 --- a/heimdall/src/dump/tui_views/decode_slot.rs +++ /dev/null @@ -1,97 +0,0 @@ -use heimdall_common::utils::{strings::encode_hex}; -use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction}, widgets::{Block, Borders, Cell, Row, Table}, style::{Style, Color, Modifier}}; - -use crate::dump::{DumpState}; - -pub fn render_tui_decode_slot( - f: &mut Frame, - state: &mut DumpState -) { - - // build main layout - let main_layout = Layout::default() - .direction(Direction::Vertical) - .margin(1) - .constraints( - [ - Constraint::Length(std::cmp::min(4+state.selection_size, 15) as u16), - Constraint::Percentage(100), - ].as_ref() - ).split(f.size()); - - // build header cells - let header_cells = ["Slot", "Block Number", "Value", "Modifiers"] - .iter() - .map(|h| Cell::from(*h).style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD))); - - // build header row - let header = Row::new(header_cells) - .style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD)) - .height(1) - .bottom_margin(1); - - // ensure scroll index is within bounds - if state.scroll_index >= state.storage.len() { - state.scroll_index = state.storage.len() - 1; - } - - // render storage slot list - let mut all_rows = Vec::new(); - let mut storage_iter = state.storage.iter().collect::>(); - - // sort by slot - storage_iter.sort_by_key(|(slot, _)| *slot); - for (i, (slot, value)) in storage_iter.iter().enumerate() { - if i >= state.scroll_index && i < state.scroll_index + state.selection_size { - all_rows.push( - Row::new(vec![ - Cell::from(format!("0x{}", encode_hex(slot.to_fixed_bytes().into()))), - Cell::from(value.modifiers.iter().max_by_key(|m| m.0).unwrap().0.to_string()), - Cell::from(format!("0x{}", encode_hex(value.value.to_fixed_bytes().into()))) - ]) - .style(Style::default().fg(Color::White)) - .height(1) - .bottom_margin(0) - ); - } - } - - // if all_rows > 10, add ... - if all_rows.len() > 10 { - - // save the last row - let last_row = all_rows.pop().unwrap(); - - // slice to 10 - all_rows = all_rows[0..9].to_vec(); - - // add ellipsis - all_rows.push( - Row::new(vec![ - Cell::from("..."), - Cell::from("..."), - Cell::from("..."), - ]) - .style(Style::default().fg(Color::White)) - .height(1) - .bottom_margin(0) - ); - - // add last row - all_rows.push(last_row); - } - - // render table - let table = Table::new(all_rows) - .header(header) - .block(Block::default().borders(Borders::ALL) - .title(" Decoding Storage Values ")) - .widths(&[ - Constraint::Length(68), - Constraint::Length(14), - Constraint::Percentage(68), - Constraint::Percentage(68), - ]); - - f.render_widget(table, main_layout[0]); -} \ No newline at end of file diff --git a/heimdall/src/dump/tui_views/main.rs b/heimdall/src/dump/tui_views/main.rs index 419b6c4f..232f3f9a 100644 --- a/heimdall/src/dump/tui_views/main.rs +++ b/heimdall/src/dump/tui_views/main.rs @@ -56,7 +56,7 @@ pub fn render_tui_view_main( ); // build header cells - let header_cells = ["Slot", "Block Number", "Value"] + let header_cells = ["Last Modified", "Slot", "As Type", "Value"] .iter() .map(|h| Cell::from(*h).style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD))); @@ -77,8 +77,9 @@ pub fn render_tui_view_main( .block(Block::default().borders(Borders::ALL) .title(format!(" Storage for Contract {} ", &state.args.target))) .widths(&[ - Constraint::Length(68), Constraint::Length(14), + Constraint::Length(68), + Constraint::Length(9), Constraint::Percentage(100), ]); diff --git a/heimdall/src/dump/tui_views/mod.rs b/heimdall/src/dump/tui_views/mod.rs index c2ce6bb5..9ecca6ed 100644 --- a/heimdall/src/dump/tui_views/mod.rs +++ b/heimdall/src/dump/tui_views/mod.rs @@ -1,3 +1,2 @@ pub mod main; -pub mod command_palette; -pub mod decode_slot; \ No newline at end of file +pub mod command_palette; \ No newline at end of file diff --git a/heimdall/src/dump/util/table.rs b/heimdall/src/dump/util/table.rs index 1ea5af2c..2767bc98 100644 --- a/heimdall/src/dump/util/table.rs +++ b/heimdall/src/dump/util/table.rs @@ -1,7 +1,8 @@ -use heimdall_common::utils::strings::encode_hex; +use ethers::{abi::{decode, ParamType}, types::U256}; +use heimdall_common::utils::strings::{encode_hex, hex_to_ascii}; use tui::{widgets::{Row, Cell}, style::{Style, Color}}; -use crate::dump::DumpState; +use crate::dump::{DumpState, constants::DECODE_AS_TYPES}; pub fn build_rows(mut state: &mut DumpState, max_row_height: usize) -> Vec> { @@ -25,12 +26,27 @@ pub fn build_rows(mut state: &mut DumpState, max_row_height: usize) -> Vec format!("0x{}", encode_hex(value.value.to_fixed_bytes().into())), + 1 => format!("{}", !value.value.is_zero()), + 2 => format!("0x{}", encode_hex(value.value.to_fixed_bytes().into()).get(24..).unwrap_or("")), + 3 => match decode(&[ParamType::String], value.value.as_bytes()) { + Ok(decoded) => decoded[0].to_string(), + Err(_) => hex_to_ascii(&encode_hex(value.value.to_fixed_bytes().into())) + }, + 4 => { + let decoded = U256::from_big_endian(&value.value.to_fixed_bytes()); + format!("{}", decoded) + }, + _ => format!("decoding error") + }; + rows.push( Row::new(vec![ - Cell::from(format!("0x{}", encode_hex(slot.to_fixed_bytes().into()))), Cell::from(value.modifiers.iter().max_by_key(|m| m.0).unwrap().0.to_string()), - Cell::from(format!("0x{}", encode_hex(value.value.to_fixed_bytes().into()))), - Cell::from(value.modifiers.len().to_string()) + Cell::from(format!("0x{}", encode_hex(slot.to_fixed_bytes().into()))), + Cell::from(DECODE_AS_TYPES[value.decode_as_type_index].clone()), + Cell::from(decoded_value), ]) .style( if storage_iter.len() - state.scroll_index < num_items { From a109b6e652a3eb47bb3b53c9d216680513ee80ca Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Sat, 11 Mar 2023 09:52:29 -0600 Subject: [PATCH 08/26] :wrench: fix: improved errors, clap qol --- common/src/resources/transpose.rs | 36 ++++++++++++++++--------------- heimdall/src/dump/mod.rs | 27 ++++++++++++----------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/common/src/resources/transpose.rs b/common/src/resources/transpose.rs index 2a07c5a6..305dcc31 100644 --- a/common/src/resources/transpose.rs +++ b/common/src/resources/transpose.rs @@ -21,18 +21,24 @@ struct TransposeResponse { results: Vec } -fn _call_transpose(query: String, api_key: &String) -> Option { +fn _call_transpose(endpoint: String, api_key: &String) -> Option { let mut headers = HeaderMap::new(); headers.insert("Content-Type", "application/json".parse().unwrap()); headers.insert("X-API-KEY", api_key.parse().unwrap()); // make the request let client = reqwest::blocking::Client::builder().redirect(reqwest::redirect::Policy::none()).build() .unwrap(); - let mut response = match client.post("https://api.transpose.io/sql").headers(headers).body(query).send() { + let mut response = match client + .get(format!("https://api.transpose.io/endpoint/{endpoint}")) + .body("{\"options\":{\"timeout\": 999999999}}") + .headers(headers) + .send() + { Ok(res) => res, Err(e) => { let (logger, _) = Logger::new("TRACE"); - logger.error(&format!("failed to get transaction list from Transpose: {}", e)); + logger.error(&format!("failed to call Transpose endpoint '{endpoint}' .")); + logger.error(&format!("error: {}", e)); std::process::exit(1) } }; @@ -45,14 +51,18 @@ fn _call_transpose(query: String, api_key: &String) -> Option Ok(json) => json, Err(e) => { let (logger, _) = Logger::new("TRACE"); - logger.error(&format!("failed to parse transaction list from Transpose: {}", e)); + logger.error(&format!("Transpose request unsucessful.")); + logger.error(&format!("error: {}", e)); + logger.debug(&format!("response body: {:?}", body)); std::process::exit(1) } }) }, Err(e) => { let (logger, _) = Logger::new("TRACE"); - logger.error(&format!("failed to get transaction list from Transpose: {}", e)); + logger.error(&format!("failed to parse Transpose response body.")); + logger.error(&format!("error: {}", e)); + logger.debug(&format!("response body: {:?}", body)); std::process::exit(1) } } @@ -72,18 +82,10 @@ pub fn get_transaction_list( transaction_list_progress.set_message(format!("fetching transactions from '{}' .", address)); let start_time = Instant::now(); - // build the SQL query - let query = format!( - "{{\"sql\":\"SELECT block_number, transaction_hash FROM (SELECT transaction_hash, block_number FROM ethereum.transactions WHERE to_address = '{}' AND block_number BETWEEN {} AND {} UNION SELECT transaction_hash, block_number FROM ethereum.traces WHERE to_address = '{}' AND block_number BETWEEN {} AND {}) x\",\"parameters\":{{}},\"options\":{{}}}}", - address, - bounds.0, - bounds.1, - address, - bounds.0, - bounds.1 - ); - - let response = match _call_transpose(query, api_key) { + let response = match _call_transpose( + format!("get-all-transactions?address={}&from_block={}&to_block={}", address, bounds.0, bounds.1), + api_key + ) { Some(response) => response, None => { logger.error(&format!("failed to get transaction list from Transpose")); diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index ab991cdc..9718796e 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -31,6 +31,7 @@ use self::util::{get_storage_diff, cleanup_terminal}; #[clap(about = "Dump the value of all storage slots accessed by a contract", after_help = "For more information, read the wiki: https://jbecker.dev/r/heimdall-rs/wiki", global_setting = AppSettings::DeriveDisplayOrder, + global_setting = AppSettings::ColoredHelp, override_usage = "heimdall dump [OPTIONS]")] pub struct DumpArgs { @@ -46,28 +47,24 @@ pub struct DumpArgs { #[clap(long="output", short, default_value = "", hide_default_value = true)] pub output: String, - /// The RPC provider to use for fetching on-chain data. + /// The RPC URL to use for fetching data. #[clap(long="rpc-url", short, default_value = "", hide_default_value = true)] pub rpc_url: String, - /// Your Transpose.io API Key. + /// Your Transpose.io API Key #[clap(long="transpose-api-key", short, default_value = "", hide_default_value = true)] pub transpose_api_key: String, - /// When prompted, always select the default value. - #[clap(long, short)] - pub default: bool, - - /// The number of threads to use + /// The number of threads to use when fetching data. #[clap(long, default_value = "4", hide_default_value = true)] pub threads: usize, /// The block number to start dumping from. - #[clap(long, short, default_value = "0", hide_default_value = true)] + #[clap(long, default_value = "0", hide_default_value = true)] pub from_block: u128, /// The block number to stop dumping at. - #[clap(long, short, default_value = "9999999999", hide_default_value = true)] + #[clap(long, default_value = "9999999999", hide_default_value = true)] pub to_block: u128, } @@ -107,7 +104,6 @@ impl DumpState { output: String::new(), rpc_url: String::new(), transpose_api_key: String::new(), - default: false, threads: 4, from_block: 0, to_block: 9999999999, @@ -148,11 +144,18 @@ pub fn dump(args: DumpArgs) { // check if transpose api key is set if &args.transpose_api_key.len() <= &0 { - logger.error("you must provide a Transpose API key."); - logger.info("you can get a free API key at https://app.transpose.io"); + logger.error("you must provide a Transpose API key, which is used to fetch all normal and internal transactions for your target."); + logger.info("you can get a free API key at https://app.transpose.io/?utm_medium=organic&utm_source=heimdall-rs"); std::process::exit(1); } + // check if the RPC url is set and supports trace_replayTransaction + get_storage_diff(&Transaction { + indexed: false, + hash: String::from("0xb95343413e459a0f97461812111254163ae53467855c0d73e0f1e7c5b8442fa3"), + block_number: 471968 + }, &args); + // parse the output directory let mut output_dir = args.output.clone(); if &args.output.len() <= &0 { From 11d22355e84b6b24bb35fa5c2d79bcca2695c688 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Mon, 13 Mar 2023 16:51:46 -0500 Subject: [PATCH 09/26] :sparkles: feat: help TUI --- heimdall/src/dump/constants.rs | 22 +++++++ heimdall/src/dump/mod.rs | 10 ++-- .../src/dump/tui_views/command_palette.rs | 10 ++-- heimdall/src/dump/tui_views/help.rs | 57 +++++++++++++++++++ heimdall/src/dump/tui_views/mod.rs | 1 + 5 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 heimdall/src/dump/tui_views/help.rs diff --git a/heimdall/src/dump/constants.rs b/heimdall/src/dump/constants.rs index 1fe0dd62..a248eb77 100644 --- a/heimdall/src/dump/constants.rs +++ b/heimdall/src/dump/constants.rs @@ -14,4 +14,26 @@ lazy_static! { "string".to_string(), "uint256".to_string() ]; + + pub static ref ABOUT_TEXT: Vec = vec![ + format!("heimdall-rs v{}", env!("CARGO_PKG_VERSION")), + "By Jonathan Becker ".to_string(), + "The storage dump module will fetch all storage slots and values accessed by any EVM contract.".to_string(), + ]; + + pub static ref HELP_MENU_COMMANDS: Vec = vec![ + ":q, :quit exit the program".to_string(), + ":h, :help display this help menu".to_string(), + ":f, :find search for a storage slot by slot or value".to_string(), + ":e, :export export the current storage dump to a file, preserving decoded values".to_string(), + ":s, :seek move the cusor up or down by a specified amount".to_string(), + ]; + + pub static ref HELP_MENU_CONTROLS: Vec = vec![ + "↑, Scroll Up move the cursor up one slot".to_string(), + "↓, Scroll Down move the cursor down one slot".to_string(), + "←, → change the decoding type of the selected slot".to_string(), + "CTRL + ↑, CTRL + ↓ move the cursor up or down by 10 slots".to_string(), + "CTRL + C, CMD + C copy the decoded value of the current slot to the clipboard".to_string(), + ]; } \ No newline at end of file diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 9718796e..0174406a 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -24,6 +24,7 @@ use tui_views::main::render_tui_view_main; use self::constants::{DUMP_STATE, DECODE_AS_TYPES}; use self::tui_views::command_palette::render_tui_command_palette; +use self::tui_views::help::render_tui_help; use self::util::csv::write_storage_to_csv; use self::util::{get_storage_diff, cleanup_terminal}; @@ -125,6 +126,7 @@ pub enum TUIView { Killed, Main, CommandPalette, + Help, } #[allow(unreachable_patterns)] @@ -135,6 +137,7 @@ fn render_ui( match state.view { TUIView::Main => { render_tui_view_main(f, state) }, TUIView::CommandPalette => { render_tui_command_palette(f, state) }, + TUIView::Help => { render_tui_help(f, state) }, _ => {} } } @@ -254,13 +257,12 @@ pub fn dump(args: DumpArgs) { let _args = split.collect::>(); match command { - ":q" => { + ":q" | ":quit" => { state.view = TUIView::Killed; break; } - ":quit" => { - state.view = TUIView::Killed; - break; + ":h" | ":help" => { + state.view = TUIView::Help; } _ => { state.view = TUIView::Main; diff --git a/heimdall/src/dump/tui_views/command_palette.rs b/heimdall/src/dump/tui_views/command_palette.rs index 8bdfcdfc..af761b67 100644 --- a/heimdall/src/dump/tui_views/command_palette.rs +++ b/heimdall/src/dump/tui_views/command_palette.rs @@ -29,7 +29,7 @@ pub fn render_tui_command_palette( ); // build header cells - let header_cells = ["Last Modified", "Slot", "As Type", "Value"] + let header_cells = ["Slot", "Block Number", "Value"] .iter() .map(|h| Cell::from(*h).style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD))); @@ -43,17 +43,17 @@ pub fn render_tui_command_palette( state, main_layout[1].height as usize - 4 ); - + // render table let table = Table::new(rows) .header(header) .block(Block::default().borders(Borders::ALL) .title(format!(" Storage for Contract {} ", &state.args.target))) .widths(&[ - Constraint::Length(14), Constraint::Length(68), - Constraint::Length(9), - Constraint::Percentage(100), + Constraint::Length(14), + Constraint::Percentage(68), + Constraint::Percentage(68), ]); f.render_widget(command_input, main_layout[0]); diff --git a/heimdall/src/dump/tui_views/help.rs b/heimdall/src/dump/tui_views/help.rs new file mode 100644 index 00000000..cb803a95 --- /dev/null +++ b/heimdall/src/dump/tui_views/help.rs @@ -0,0 +1,57 @@ +use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction, Alignment}, style::{Style, Color, Modifier}, widgets::{Paragraph, Block, Borders, Wrap}, text::Span}; + +use crate::dump::{DumpState, constants::{HELP_MENU_COMMANDS, HELP_MENU_CONTROLS, ABOUT_TEXT}}; + +pub fn render_tui_help( + f: &mut Frame, + _: &mut DumpState +) { + + // build main layout + let main_layout = Layout::default() + .direction(Direction::Vertical) + .margin(1) + .constraints( + [ + Constraint::Length(6), + Constraint::Length((HELP_MENU_COMMANDS.len() + 2).try_into().unwrap()), + Constraint::Percentage(100) + ].as_ref() + ).split(f.size()); + + // creates a new block with the given title + // https://github.com/fdehau/tui-rs/blob/master/examples/paragraph.rs + let create_block = |title| { + Block::default() + .borders(Borders::NONE) + .style(Style::default().fg(Color::White)) + .title(Span::styled( + format!("{title}"), + Style::default().add_modifier(Modifier::BOLD), + )) + }; + + // about text + let paragraph = Paragraph::new(ABOUT_TEXT.join("\n")) + .style(Style::default().fg(Color::White)) + .block(create_block("About")) + .alignment(Alignment::Left) + .wrap(Wrap { trim: true }); + f.render_widget(paragraph, main_layout[0]); + + // commands paragraph + let paragraph = Paragraph::new(HELP_MENU_COMMANDS.join("\n")) + .style(Style::default().fg(Color::White)) + .block(create_block("Commands")) + .alignment(Alignment::Left) + .wrap(Wrap { trim: true }); + f.render_widget(paragraph, main_layout[1]); + + // controls paragraph + let paragraph = Paragraph::new(HELP_MENU_CONTROLS.join("\n")) + .style(Style::default().fg(Color::White)) + .block(create_block("Controls")) + .alignment(Alignment::Left) + .wrap(Wrap { trim: true }); + f.render_widget(paragraph, main_layout[2]); +} \ No newline at end of file diff --git a/heimdall/src/dump/tui_views/mod.rs b/heimdall/src/dump/tui_views/mod.rs index 9ecca6ed..c69d1329 100644 --- a/heimdall/src/dump/tui_views/mod.rs +++ b/heimdall/src/dump/tui_views/mod.rs @@ -1,2 +1,3 @@ pub mod main; +pub mod help; pub mod command_palette; \ No newline at end of file From 0e2c54961cc61da4dbc3e19517721158e07f2bee Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 14 Mar 2023 15:34:29 -0500 Subject: [PATCH 10/26] :sparkles: feat: `:f` & `:find` to filter --- heimdall/src/dump/constants.rs | 3 +- heimdall/src/dump/mod.rs | 17 ++++++++-- .../src/dump/tui_views/command_palette.rs | 8 ++--- heimdall/src/dump/util/table.rs | 33 +++++++++++++++++-- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/heimdall/src/dump/constants.rs b/heimdall/src/dump/constants.rs index a248eb77..4545fc24 100644 --- a/heimdall/src/dump/constants.rs +++ b/heimdall/src/dump/constants.rs @@ -24,7 +24,7 @@ lazy_static! { pub static ref HELP_MENU_COMMANDS: Vec = vec![ ":q, :quit exit the program".to_string(), ":h, :help display this help menu".to_string(), - ":f, :find search for a storage slot by slot or value".to_string(), + ":f, :find search for a storage slot by slot or value".to_string(), ":e, :export export the current storage dump to a file, preserving decoded values".to_string(), ":s, :seek move the cusor up or down by a specified amount".to_string(), ]; @@ -35,5 +35,6 @@ lazy_static! { "←, → change the decoding type of the selected slot".to_string(), "CTRL + ↑, CTRL + ↓ move the cursor up or down by 10 slots".to_string(), "CTRL + C, CMD + C copy the decoded value of the current slot to the clipboard".to_string(), + "ESC clear the search filter".to_string(), ]; } \ No newline at end of file diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 0174406a..0c3bfa65 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -94,6 +94,7 @@ pub struct DumpState { pub view: TUIView, pub start_time: Instant, pub input_buffer: String, + pub filter: String, } impl DumpState { @@ -116,6 +117,7 @@ impl DumpState { view: TUIView::Main, start_time: Instant::now(), input_buffer: String::new(), + filter: String::new(), } } } @@ -210,6 +212,7 @@ pub fn dump(args: DumpArgs) { view: TUIView::Main, start_time: Instant::now(), input_buffer: String::new(), + filter: String::new(), }; drop(state); @@ -252,9 +255,10 @@ pub fn dump(args: DumpArgs) { // enter command crossterm::event::KeyCode::Enter => { + state.filter = String::new(); let mut split = state.input_buffer.split(" "); let command = split.next().unwrap(); - let _args = split.collect::>(); + let args = split.collect::>(); match command { ":q" | ":quit" => { @@ -264,16 +268,24 @@ pub fn dump(args: DumpArgs) { ":h" | ":help" => { state.view = TUIView::Help; } + ":f" | ":find" => { + if args.len() > 0 { + state.filter = args[0].to_string(); + } + state.view = TUIView::Main; + } _ => { state.view = TUIView::Main; } } }, - // close command palette + // handle escape crossterm::event::KeyCode::Esc => { + state.filter = String::new(); state.view = TUIView::Main; } + _ => {} } @@ -285,6 +297,7 @@ pub fn dump(args: DumpArgs) { // main on escape crossterm::event::KeyCode::Esc => { + state.filter = String::new(); state.view = TUIView::Main; }, diff --git a/heimdall/src/dump/tui_views/command_palette.rs b/heimdall/src/dump/tui_views/command_palette.rs index af761b67..cfc7f645 100644 --- a/heimdall/src/dump/tui_views/command_palette.rs +++ b/heimdall/src/dump/tui_views/command_palette.rs @@ -29,7 +29,7 @@ pub fn render_tui_command_palette( ); // build header cells - let header_cells = ["Slot", "Block Number", "Value"] + let header_cells = ["Last Modified", "Slot", "As Type", "Value"] .iter() .map(|h| Cell::from(*h).style(Style::default().fg(Color::White).add_modifier(Modifier::BOLD))); @@ -50,10 +50,10 @@ pub fn render_tui_command_palette( .block(Block::default().borders(Borders::ALL) .title(format!(" Storage for Contract {} ", &state.args.target))) .widths(&[ - Constraint::Length(68), Constraint::Length(14), - Constraint::Percentage(68), - Constraint::Percentage(68), + Constraint::Length(68), + Constraint::Length(9), + Constraint::Percentage(100), ]); f.render_widget(command_input, main_layout[0]); diff --git a/heimdall/src/dump/util/table.rs b/heimdall/src/dump/util/table.rs index 2767bc98..a9d1e98f 100644 --- a/heimdall/src/dump/util/table.rs +++ b/heimdall/src/dump/util/table.rs @@ -1,4 +1,4 @@ -use ethers::{abi::{decode, ParamType}, types::U256}; +use ethers::{abi::{decode, ParamType}, types::{U256}}; use heimdall_common::utils::strings::{encode_hex, hex_to_ascii}; use tui::{widgets::{Row, Cell}, style::{Style, Color}}; @@ -13,7 +13,22 @@ pub fn build_rows(mut state: &mut DumpState, max_row_height: usize) -> Vec>(); + + + // filter storage_iter by state.filter + let mut storage_iter = match state.filter.len() > 0 { + true => { + state.storage + .iter() + .filter(|(slot, value)| { + let slot = format!("0x{}", encode_hex(slot.to_fixed_bytes().into())); + let value = format!("0x{}", encode_hex(value.value.to_fixed_bytes().into())); + slot.contains(&state.filter) || value.contains(&state.filter) + }) + .collect::>() + } + false => state.storage.iter().collect::>() + }; // sort storage slots by slot storage_iter.sort_by_key(|(slot, _)| *slot); @@ -69,5 +84,19 @@ pub fn build_rows(mut state: &mut DumpState, max_row_height: usize) -> Vec Date: Tue, 14 Mar 2023 17:11:08 -0500 Subject: [PATCH 11/26] :sparkles: feat: allow copying values with ctrl c --- Cargo.lock | 86 +++++++++++++++++++++++++++++++++ common/Cargo.toml | 3 +- common/src/io/clipboard.rs | 7 +++ common/src/io/mod.rs | 3 +- heimdall/src/dump/constants.rs | 4 +- heimdall/src/dump/mod.rs | 34 ++++++++++++- heimdall/src/dump/util/csv.rs | 38 ++++++++++----- heimdall/src/dump/util/table.rs | 42 +++++++++++++++- 8 files changed, 199 insertions(+), 18 deletions(-) create mode 100644 common/src/io/clipboard.rs diff --git a/Cargo.lock b/Cargo.lock index 05c6f29b..df78938b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,6 +232,12 @@ dependencies = [ "digest 0.10.6", ] +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + [[package]] name = "block-buffer" version = "0.7.3" @@ -420,6 +426,28 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "clipboard" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a904646c0340239dcf7c51677b33928bf24fdf424b79a57909c0109075b2e7" +dependencies = [ + "clipboard-win", + "objc", + "objc-foundation", + "objc_id", + "x11-clipboard", +] + +[[package]] +name = "clipboard-win" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a093d6fed558e5fe24c3dfc85a68bb68f1c824f440d3ba5aca189e2998786b" +dependencies = [ + "winapi", +] + [[package]] name = "coins-bip32" version = "0.7.0" @@ -1404,6 +1432,7 @@ version = "0.3.4" dependencies = [ "clap", "clap-verbosity-flag", + "clipboard", "colored", "crossbeam-channel", "ethers", @@ -1732,6 +1761,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.5.0" @@ -1839,6 +1877,35 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", +] + +[[package]] +name = "objc-foundation" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" +dependencies = [ + "block", + "objc", + "objc_id", +] + +[[package]] +name = "objc_id" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92d4ddb4bd7b50d730c215ff871754d0da6b2178849f8a2a2ab69712d0c073b" +dependencies = [ + "objc", +] + [[package]] name = "object" version = "0.30.3" @@ -3406,6 +3473,25 @@ dependencies = [ "tap", ] +[[package]] +name = "x11-clipboard" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89bd49c06c9eb5d98e6ba6536cf64ac9f7ee3a009b2f53996d405b3944f6bcea" +dependencies = [ + "xcb", +] + +[[package]] +name = "xcb" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e917a3f24142e9ff8be2414e36c649d47d6cc2ba81f16201cdef96e533e02de" +dependencies = [ + "libc", + "log", +] + [[package]] name = "zeroize" version = "1.5.7" diff --git a/common/Cargo.toml b/common/Cargo.toml index d381edc7..a3e0b9c1 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -20,4 +20,5 @@ reqwest = { version = "0.11.11", features = ["blocking"] } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } heimdall-cache = { path = "./../cache" } -crossbeam-channel = "0.5.7" \ No newline at end of file +crossbeam-channel = "0.5.7" +clipboard = "0.5.0" diff --git a/common/src/io/clipboard.rs b/common/src/io/clipboard.rs new file mode 100644 index 00000000..4a892fb8 --- /dev/null +++ b/common/src/io/clipboard.rs @@ -0,0 +1,7 @@ +extern crate clipboard; +use clipboard::{ClipboardProvider, ClipboardContext}; + +pub fn copy_to_clipboard(text: &str) { + let mut ctx: ClipboardContext = ClipboardProvider::new().unwrap(); + ctx.set_contents(text.to_string()).unwrap(); +} \ No newline at end of file diff --git a/common/src/io/mod.rs b/common/src/io/mod.rs index 1d901943..0ecbe30f 100644 --- a/common/src/io/mod.rs +++ b/common/src/io/mod.rs @@ -1,3 +1,4 @@ pub mod logging; pub mod file; -mod tests; \ No newline at end of file +mod tests; +pub mod clipboard; \ No newline at end of file diff --git a/heimdall/src/dump/constants.rs b/heimdall/src/dump/constants.rs index 4545fc24..496beda1 100644 --- a/heimdall/src/dump/constants.rs +++ b/heimdall/src/dump/constants.rs @@ -25,7 +25,7 @@ lazy_static! { ":q, :quit exit the program".to_string(), ":h, :help display this help menu".to_string(), ":f, :find search for a storage slot by slot or value".to_string(), - ":e, :export export the current storage dump to a file, preserving decoded values".to_string(), + ":e, :export export the current storage dump to a file, preserving decoded values".to_string(), ":s, :seek move the cusor up or down by a specified amount".to_string(), ]; @@ -34,7 +34,7 @@ lazy_static! { "↓, Scroll Down move the cursor down one slot".to_string(), "←, → change the decoding type of the selected slot".to_string(), "CTRL + ↑, CTRL + ↓ move the cursor up or down by 10 slots".to_string(), - "CTRL + C, CMD + C copy the decoded value of the current slot to the clipboard".to_string(), + "CTRL + C, copy the decoded value of the current slot to the clipboard".to_string(), "ESC clear the search filter".to_string(), ]; } \ No newline at end of file diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 0c3bfa65..3217f1e9 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -26,6 +26,7 @@ use self::constants::{DUMP_STATE, DECODE_AS_TYPES}; use self::tui_views::command_palette::render_tui_command_palette; use self::tui_views::help::render_tui_help; use self::util::csv::write_storage_to_csv; +use self::util::table::copy_selected; use self::util::{get_storage_diff, cleanup_terminal}; #[derive(Debug, Clone, Parser)] @@ -274,6 +275,30 @@ pub fn dump(args: DumpArgs) { } state.view = TUIView::Main; } + ":s" | ":seek" => { + if args.len() > 1 { + let direction = args[0].to_lowercase(); + let amount = args[1].parse::().unwrap_or(0); + match direction.as_str() { + "up" => { + if state.scroll_index >= amount { + state.scroll_index -= amount; + } else { + state.scroll_index = 0; + } + } + "down" => { + if state.scroll_index + amount < state.storage.len() { + state.scroll_index += amount; + } else { + state.scroll_index = state.storage.len() - 1; + } + } + _ => {} + } + } + state.view = TUIView::Main; + } _ => { state.view = TUIView::Main; } @@ -295,6 +320,13 @@ pub fn dump(args: DumpArgs) { match key.code { + // copy value on MODIFIER + C + crossterm::event::KeyCode::Char('c') => { + if crossterm::event::KeyModifiers::NONE != key.modifiers { + copy_selected(&mut state) + } + }, + // main on escape crossterm::event::KeyCode::Esc => { state.filter = String::new(); @@ -521,7 +553,7 @@ pub fn dump(args: DumpArgs) { // write storage slots to csv let state = DUMP_STATE.lock().unwrap(); - write_storage_to_csv(&output_dir.clone(), &state, &logger); + write_storage_to_csv(&output_dir.clone(), &"storage_dump.csv".to_string(), &state, &logger); logger.info(&format!("Dumped {} storage values from '{}' .", state.storage.len(), &args.target)); } \ No newline at end of file diff --git a/heimdall/src/dump/util/csv.rs b/heimdall/src/dump/util/csv.rs index 7fbb075c..458f38f6 100644 --- a/heimdall/src/dump/util/csv.rs +++ b/heimdall/src/dump/util/csv.rs @@ -1,8 +1,9 @@ -use heimdall_common::{utils::strings::encode_hex, io::{file::write_lines_to_file, logging::Logger}}; +use ethers::{abi::{decode, ParamType}, types::U256}; +use heimdall_common::{utils::strings::{encode_hex, hex_to_ascii}, io::{file::write_lines_to_file, logging::Logger}}; -use crate::dump::DumpState; +use crate::dump::{DumpState, constants::DECODE_AS_TYPES}; -pub fn write_storage_to_csv(output_dir: &String, state: &DumpState, logger: &Logger) { +pub fn write_storage_to_csv(output_dir: &String, file_name: &String, state: &DumpState, logger: &Logger) { let mut lines = { let mut lines = Vec::new(); @@ -10,14 +11,29 @@ pub fn write_storage_to_csv(output_dir: &String, state: &DumpState, logger: &Log let mut storage_iter = state.storage.iter().collect::>(); storage_iter.sort_by_key(|(slot, _)| *slot); - for (slot, slot_data) in storage_iter { + for (slot, value) in storage_iter { + let decoded_value = match value.decode_as_type_index { + 0 => format!("0x{}", encode_hex(value.value.to_fixed_bytes().into())), + 1 => format!("{}", !value.value.is_zero()), + 2 => format!("0x{}", encode_hex(value.value.to_fixed_bytes().into()).get(24..).unwrap_or("")), + 3 => match decode(&[ParamType::String], value.value.as_bytes()) { + Ok(decoded) => decoded[0].to_string(), + Err(_) => hex_to_ascii(&encode_hex(value.value.to_fixed_bytes().into())) + }, + 4 => { + let decoded = U256::from_big_endian(&value.value.to_fixed_bytes()); + format!("{}", decoded) + }, + _ => format!("decoding error") + }; lines.push( format!( - "{},{},{},{}", + "{},{},{},{},{}", + value.modifiers.iter().max_by_key(|m| m.0).unwrap().0.to_string(), + value.alias.as_ref().unwrap_or(&String::from("None")), encode_hex(slot.to_fixed_bytes().into()), - encode_hex(slot_data.value.to_fixed_bytes().into()), - slot_data.modifiers.iter().max_by_key(|m| m.0).unwrap().0.to_string(), - slot_data.alias.as_ref().unwrap_or(&String::from("None")) + DECODE_AS_TYPES[value.decode_as_type_index], + decoded_value, ) ); } @@ -25,9 +41,9 @@ pub fn write_storage_to_csv(output_dir: &String, state: &DumpState, logger: &Log }; // add header - lines.insert(0, String::from("slot,value,last_modified,alias")); + lines.insert(0, String::from("last_modified,alias,slot,decoded_type,value")); // save to file - write_lines_to_file(&format!("{output_dir}/storage_dump.csv"), lines); - logger.success(&format!("wrote storage dump to to '{output_dir}/storage_dump.csv' .")); + write_lines_to_file(&format!("{output_dir}/{file_name}"), lines); + logger.success(&format!("wrote storage dump to to '{output_dir}/{file_name}' .")); } \ No newline at end of file diff --git a/heimdall/src/dump/util/table.rs b/heimdall/src/dump/util/table.rs index a9d1e98f..b0a22bef 100644 --- a/heimdall/src/dump/util/table.rs +++ b/heimdall/src/dump/util/table.rs @@ -1,5 +1,5 @@ use ethers::{abi::{decode, ParamType}, types::{U256}}; -use heimdall_common::utils::strings::{encode_hex, hex_to_ascii}; +use heimdall_common::{utils::strings::{encode_hex, hex_to_ascii}, io::clipboard::copy_to_clipboard}; use tui::{widgets::{Row, Cell}, style::{Style, Color}}; use crate::dump::{DumpState, constants::DECODE_AS_TYPES}; @@ -14,7 +14,6 @@ pub fn build_rows(mut state: &mut DumpState, max_row_height: usize) -> Vec 0 { true => { @@ -99,4 +98,43 @@ pub fn build_rows(mut state: &mut DumpState, max_row_height: usize) -> Vec 0 { + true => { + state.storage + .iter() + .filter(|(slot, value)| { + let slot = format!("0x{}", encode_hex(slot.to_fixed_bytes().into())); + let value = format!("0x{}", encode_hex(value.value.to_fixed_bytes().into())); + slot.contains(&state.filter) || value.contains(&state.filter) + }) + .collect::>() + } + false => state.storage.iter().collect::>() + }; + + // sort storage slots by slot + storage_iter.sort_by_key(|(slot, _)| *slot); + + let (_, value) = storage_iter.get_mut(state.scroll_index).unwrap(); + let decoded_value = match value.decode_as_type_index { + 0 => format!("0x{}", encode_hex(value.value.to_fixed_bytes().into())), + 1 => format!("{}", !value.value.is_zero()), + 2 => format!("0x{}", encode_hex(value.value.to_fixed_bytes().into()).get(24..).unwrap_or("")), + 3 => match decode(&[ParamType::String], value.value.as_bytes()) { + Ok(decoded) => decoded[0].to_string(), + Err(_) => hex_to_ascii(&encode_hex(value.value.to_fixed_bytes().into())) + }, + 4 => { + let decoded = U256::from_big_endian(&value.value.to_fixed_bytes()); + format!("{}", decoded) + }, + _ => format!("decoding error") + }; + + copy_to_clipboard(&decoded_value); } \ No newline at end of file From 6a3e8219c2e7162085ce14fa4cbc48440ad8cba9 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:15:25 -0500 Subject: [PATCH 12/26] :sparkles: feat: `:e` & `:export` --- heimdall/src/dump/mod.rs | 8 +++++++- heimdall/src/dump/util/csv.rs | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 3217f1e9..7889c2e3 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -217,6 +217,7 @@ pub fn dump(args: DumpArgs) { }; drop(state); + let _output_dir = output_dir.clone(); // in a new thread, start the TUI let tui_thread = std::thread::spawn(move || { @@ -275,6 +276,11 @@ pub fn dump(args: DumpArgs) { } state.view = TUIView::Main; } + ":e" | ":export" => { + if args.len() > 0 { + write_storage_to_csv(&output_dir.clone(), &args[0].to_string(), &state); + } + } ":s" | ":seek" => { if args.len() > 1 { let direction = args[0].to_lowercase(); @@ -553,7 +559,7 @@ pub fn dump(args: DumpArgs) { // write storage slots to csv let state = DUMP_STATE.lock().unwrap(); - write_storage_to_csv(&output_dir.clone(), &"storage_dump.csv".to_string(), &state, &logger); + write_storage_to_csv(&_output_dir, &"storage_dump.csv".to_string(), &state); logger.info(&format!("Dumped {} storage values from '{}' .", state.storage.len(), &args.target)); } \ No newline at end of file diff --git a/heimdall/src/dump/util/csv.rs b/heimdall/src/dump/util/csv.rs index 458f38f6..7d132739 100644 --- a/heimdall/src/dump/util/csv.rs +++ b/heimdall/src/dump/util/csv.rs @@ -3,7 +3,7 @@ use heimdall_common::{utils::strings::{encode_hex, hex_to_ascii}, io::{file::wri use crate::dump::{DumpState, constants::DECODE_AS_TYPES}; -pub fn write_storage_to_csv(output_dir: &String, file_name: &String, state: &DumpState, logger: &Logger) { +pub fn write_storage_to_csv(output_dir: &String, file_name: &String, state: &DumpState) { let mut lines = { let mut lines = Vec::new(); @@ -45,5 +45,4 @@ pub fn write_storage_to_csv(output_dir: &String, file_name: &String, state: &Dum // save to file write_lines_to_file(&format!("{output_dir}/{file_name}"), lines); - logger.success(&format!("wrote storage dump to to '{output_dir}/{file_name}' .")); } \ No newline at end of file From dde3af558e1ef085bb8bf983d2fa48e417973bf0 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:24:06 -0500 Subject: [PATCH 13/26] =?UTF-8?q?=F0=9F=94=A8=20build:=20fix=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pull_request_checks.yml | 4 +++- .github/workflows/release.yml | 4 +++- heimdall/src/dump/util/csv.rs | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request_checks.yml b/.github/workflows/pull_request_checks.yml index 0e6a1388..20ab49b8 100644 --- a/.github/workflows/pull_request_checks.yml +++ b/.github/workflows/pull_request_checks.yml @@ -16,7 +16,9 @@ jobs: - uses: actions/checkout@v3 - name: Compile working-directory: ./heimdall - run: cargo build --verbose + run: | + cargo clean + cargo build - name: Run Tests working-directory: ./heimdall diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b058239a..221fa2c0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,7 +25,9 @@ jobs: steps: - uses: actions/checkout@v2 - name: Build Binaries - run: cargo build --release + run: | + cargo clean + cargo build --release - name: Upload Binaries uses: svenstaro/upload-release-action@v2 with: diff --git a/heimdall/src/dump/util/csv.rs b/heimdall/src/dump/util/csv.rs index 7d132739..112b9fd4 100644 --- a/heimdall/src/dump/util/csv.rs +++ b/heimdall/src/dump/util/csv.rs @@ -1,5 +1,5 @@ use ethers::{abi::{decode, ParamType}, types::U256}; -use heimdall_common::{utils::strings::{encode_hex, hex_to_ascii}, io::{file::write_lines_to_file, logging::Logger}}; +use heimdall_common::{utils::strings::{encode_hex, hex_to_ascii}, io::{file::write_lines_to_file}}; use crate::dump::{DumpState, constants::DECODE_AS_TYPES}; From 61ef9b354c2d27adae5cea099dba15a6c0f0b3f6 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:46:04 -0500 Subject: [PATCH 14/26] =?UTF-8?q?=F0=9F=94=A8=20build:=20fix=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pull_request_checks.yml | 9 ++++++++- .github/workflows/release.yml | 8 ++++++++ .github/workflows/rust_build.yml | 9 +++++++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request_checks.yml b/.github/workflows/pull_request_checks.yml index 20ab49b8..ea8136be 100644 --- a/.github/workflows/pull_request_checks.yml +++ b/.github/workflows/pull_request_checks.yml @@ -14,7 +14,14 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Compile + + - name: Install Prerequisites + run: | + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then + sudo apt-get install xorg-dev + fi + + - name: Build Binaries working-directory: ./heimdall run: | cargo clean diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 221fa2c0..f9fb36d4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,10 +24,18 @@ jobs: steps: - uses: actions/checkout@v2 + + - name: Install Prerequisites + run: | + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then + sudo apt-get install xorg-dev + fi + - name: Build Binaries run: | cargo clean cargo build --release + - name: Upload Binaries uses: svenstaro/upload-release-action@v2 with: diff --git a/.github/workflows/rust_build.yml b/.github/workflows/rust_build.yml index 331a5011..da1194b9 100644 --- a/.github/workflows/rust_build.yml +++ b/.github/workflows/rust_build.yml @@ -7,11 +7,16 @@ env: jobs: build: - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 + + - name: Install Prerequisites + run: | + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then + sudo apt-get install xorg-dev + fi + - name: Compile working-directory: ./heimdall run: cargo build --verbose From a57e68ae5cafed3be97c93913f0de572cddfc3bd Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:49:28 -0500 Subject: [PATCH 15/26] =?UTF-8?q?=F0=9F=94=A8=20build:=20fix=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pull_request_checks.yml | 5 +---- .github/workflows/rust_build.yml | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pull_request_checks.yml b/.github/workflows/pull_request_checks.yml index ea8136be..ba0f895a 100644 --- a/.github/workflows/pull_request_checks.yml +++ b/.github/workflows/pull_request_checks.yml @@ -16,10 +16,7 @@ jobs: - uses: actions/checkout@v3 - name: Install Prerequisites - run: | - if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then - sudo apt-get install xorg-dev - fi + run: sudo apt-get install xorg-dev - name: Build Binaries working-directory: ./heimdall diff --git a/.github/workflows/rust_build.yml b/.github/workflows/rust_build.yml index da1194b9..e07c5292 100644 --- a/.github/workflows/rust_build.yml +++ b/.github/workflows/rust_build.yml @@ -12,10 +12,7 @@ jobs: - uses: actions/checkout@v3 - name: Install Prerequisites - run: | - if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then - sudo apt-get install xorg-dev - fi + run: sudo apt-get install xorg-dev - name: Compile working-directory: ./heimdall From afe4f9dd4c711d065b83796d8ad088236d8cbf2d Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:56:36 -0500 Subject: [PATCH 16/26] :wrench: fix: remove `clipboard` --- .github/workflows/pull_request_checks.yml | 3 - .github/workflows/release.yml | 6 -- .github/workflows/rust_build.yml | 3 - Cargo.lock | 86 ----------------------- common/Cargo.toml | 3 +- common/src/io/clipboard.rs | 5 +- 6 files changed, 2 insertions(+), 104 deletions(-) diff --git a/.github/workflows/pull_request_checks.yml b/.github/workflows/pull_request_checks.yml index ba0f895a..96775e57 100644 --- a/.github/workflows/pull_request_checks.yml +++ b/.github/workflows/pull_request_checks.yml @@ -15,9 +15,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install Prerequisites - run: sudo apt-get install xorg-dev - - name: Build Binaries working-directory: ./heimdall run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f9fb36d4..c7203f14 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,12 +24,6 @@ jobs: steps: - uses: actions/checkout@v2 - - - name: Install Prerequisites - run: | - if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then - sudo apt-get install xorg-dev - fi - name: Build Binaries run: | diff --git a/.github/workflows/rust_build.yml b/.github/workflows/rust_build.yml index e07c5292..becbfa15 100644 --- a/.github/workflows/rust_build.yml +++ b/.github/workflows/rust_build.yml @@ -11,9 +11,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install Prerequisites - run: sudo apt-get install xorg-dev - - name: Compile working-directory: ./heimdall run: cargo build --verbose diff --git a/Cargo.lock b/Cargo.lock index df78938b..05c6f29b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,12 +232,6 @@ dependencies = [ "digest 0.10.6", ] -[[package]] -name = "block" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" - [[package]] name = "block-buffer" version = "0.7.3" @@ -426,28 +420,6 @@ dependencies = [ "os_str_bytes", ] -[[package]] -name = "clipboard" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a904646c0340239dcf7c51677b33928bf24fdf424b79a57909c0109075b2e7" -dependencies = [ - "clipboard-win", - "objc", - "objc-foundation", - "objc_id", - "x11-clipboard", -] - -[[package]] -name = "clipboard-win" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a093d6fed558e5fe24c3dfc85a68bb68f1c824f440d3ba5aca189e2998786b" -dependencies = [ - "winapi", -] - [[package]] name = "coins-bip32" version = "0.7.0" @@ -1432,7 +1404,6 @@ version = "0.3.4" dependencies = [ "clap", "clap-verbosity-flag", - "clipboard", "colored", "crossbeam-channel", "ethers", @@ -1761,15 +1732,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "malloc_buf" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" -dependencies = [ - "libc", -] - [[package]] name = "memchr" version = "2.5.0" @@ -1877,35 +1839,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" -[[package]] -name = "objc" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" -dependencies = [ - "malloc_buf", -] - -[[package]] -name = "objc-foundation" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" -dependencies = [ - "block", - "objc", - "objc_id", -] - -[[package]] -name = "objc_id" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92d4ddb4bd7b50d730c215ff871754d0da6b2178849f8a2a2ab69712d0c073b" -dependencies = [ - "objc", -] - [[package]] name = "object" version = "0.30.3" @@ -3473,25 +3406,6 @@ dependencies = [ "tap", ] -[[package]] -name = "x11-clipboard" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89bd49c06c9eb5d98e6ba6536cf64ac9f7ee3a009b2f53996d405b3944f6bcea" -dependencies = [ - "xcb", -] - -[[package]] -name = "xcb" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e917a3f24142e9ff8be2414e36c649d47d6cc2ba81f16201cdef96e533e02de" -dependencies = [ - "libc", - "log", -] - [[package]] name = "zeroize" version = "1.5.7" diff --git a/common/Cargo.toml b/common/Cargo.toml index a3e0b9c1..d381edc7 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -20,5 +20,4 @@ reqwest = { version = "0.11.11", features = ["blocking"] } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } heimdall-cache = { path = "./../cache" } -crossbeam-channel = "0.5.7" -clipboard = "0.5.0" +crossbeam-channel = "0.5.7" \ No newline at end of file diff --git a/common/src/io/clipboard.rs b/common/src/io/clipboard.rs index 4a892fb8..42a76c9c 100644 --- a/common/src/io/clipboard.rs +++ b/common/src/io/clipboard.rs @@ -1,7 +1,4 @@ -extern crate clipboard; -use clipboard::{ClipboardProvider, ClipboardContext}; pub fn copy_to_clipboard(text: &str) { - let mut ctx: ClipboardContext = ClipboardProvider::new().unwrap(); - ctx.set_contents(text.to_string()).unwrap(); + } \ No newline at end of file From 4db7249c7529642f0c60dd9d71f2dfa092d8a39c Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 14 Mar 2023 20:08:57 -0500 Subject: [PATCH 17/26] :sparkles: feat: add `--no-tui` flag --- common/src/io/clipboard.rs | 2 +- heimdall/src/dump/constants.rs | 1 - heimdall/src/dump/mod.rs | 26 ++++++++++++++++++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/common/src/io/clipboard.rs b/common/src/io/clipboard.rs index 42a76c9c..3dd5e589 100644 --- a/common/src/io/clipboard.rs +++ b/common/src/io/clipboard.rs @@ -1,4 +1,4 @@ -pub fn copy_to_clipboard(text: &str) { +pub fn copy_to_clipboard(_text: &str) { } \ No newline at end of file diff --git a/heimdall/src/dump/constants.rs b/heimdall/src/dump/constants.rs index 496beda1..633b3953 100644 --- a/heimdall/src/dump/constants.rs +++ b/heimdall/src/dump/constants.rs @@ -34,7 +34,6 @@ lazy_static! { "↓, Scroll Down move the cursor down one slot".to_string(), "←, → change the decoding type of the selected slot".to_string(), "CTRL + ↑, CTRL + ↓ move the cursor up or down by 10 slots".to_string(), - "CTRL + C, copy the decoded value of the current slot to the clipboard".to_string(), "ESC clear the search filter".to_string(), ]; } \ No newline at end of file diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 7889c2e3..5c19bf73 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -68,6 +68,10 @@ pub struct DumpArgs { /// The block number to stop dumping at. #[clap(long, default_value = "9999999999", hide_default_value = true)] pub to_block: u128, + + /// Whether to skip opening the TUI. + #[clap(long)] + pub no_tui: bool, } #[derive(Debug, Clone)] @@ -110,6 +114,7 @@ impl DumpState { threads: 4, from_block: 0, to_block: 9999999999, + no_tui: false, }, scroll_index: 0, selection_size: 1, @@ -222,6 +227,11 @@ pub fn dump(args: DumpArgs) { // in a new thread, start the TUI let tui_thread = std::thread::spawn(move || { + // if no TUI is requested, just run the dump + if args.no_tui { + return; + } + // create new TUI terminal enable_raw_mode().unwrap(); let mut stdout = io::stdout(); @@ -467,7 +477,7 @@ pub fn dump(args: DumpArgs) { }); // index transactions in a new thread - std::thread::spawn(move || { + let dump_thread = std::thread::spawn(move || { let state = DUMP_STATE.lock().unwrap(); let transactions = state.transactions.clone(); let args = state.args.clone(); @@ -547,6 +557,18 @@ pub fn dump(args: DumpArgs) { }); }); + // if no-tui flag is set, wait for the indexing thread to finish + if args.no_tui { + match dump_thread.join() { + Ok(_) => {}, + Err(e) => { + logger.error("failed to join indexer thread."); + logger.error(&format!("{:?}", e)); + std::process::exit(1); + } + } + } + // wait for the TUI thread to finish match tui_thread.join() { Ok(_) => {}, @@ -560,6 +582,6 @@ pub fn dump(args: DumpArgs) { // write storage slots to csv let state = DUMP_STATE.lock().unwrap(); write_storage_to_csv(&_output_dir, &"storage_dump.csv".to_string(), &state); - + logger.success(&format!("Wrote storage dump to '{}/storage_dump.csv'.", _output_dir)); logger.info(&format!("Dumped {} storage values from '{}' .", state.storage.len(), &args.target)); } \ No newline at end of file From 562cc4af9f8bfc4d8a0c6a9fe0e7f6b80b5e9ea9 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 14 Mar 2023 20:11:51 -0500 Subject: [PATCH 18/26] :wrench: fix: main after export --- heimdall/src/dump/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 5c19bf73..26ae784c 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -290,6 +290,7 @@ pub fn dump(args: DumpArgs) { if args.len() > 0 { write_storage_to_csv(&output_dir.clone(), &args[0].to_string(), &state); } + state.view = TUIView::Main; } ":s" | ":seek" => { if args.len() > 1 { From 52db3dc59b07a0d0a52fbdf1578b6f15e68c16b9 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Tue, 14 Mar 2023 22:12:27 -0500 Subject: [PATCH 19/26] :wrench: fix: csv output fix --- common/src/utils/strings.rs | 5 +++++ heimdall/src/dump/util/csv.rs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/common/src/utils/strings.rs b/common/src/utils/strings.rs index 25e54d8f..30313694 100644 --- a/common/src/utils/strings.rs +++ b/common/src/utils/strings.rs @@ -47,6 +47,11 @@ pub fn hex_to_ascii(s: &str) -> String { let byte = u8::from_str_radix(&s[2 * i..2 * i + 2], 16).unwrap(); result.push(byte as char); } + + // remove newlines + result = result.replace("\r", ""); + result = result.replace("\n", ""); + result } diff --git a/heimdall/src/dump/util/csv.rs b/heimdall/src/dump/util/csv.rs index 112b9fd4..e5c8a8a8 100644 --- a/heimdall/src/dump/util/csv.rs +++ b/heimdall/src/dump/util/csv.rs @@ -28,7 +28,7 @@ pub fn write_storage_to_csv(output_dir: &String, file_name: &String, state: &Dum }; lines.push( format!( - "{},{},{},{},{}", + "\"{}\",\"{}\",\"{}\",\"{}\",\"{}\"", value.modifiers.iter().max_by_key(|m| m.0).unwrap().0.to_string(), value.alias.as_ref().unwrap_or(&String::from("None")), encode_hex(slot.to_fixed_bytes().into()), From 49c524abeaec5c7309948280996c1ba61a09c366 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Wed, 15 Mar 2023 11:44:50 -0500 Subject: [PATCH 20/26] :wrench: fix: include contract creation tx --- common/src/resources/transpose.rs | 74 ++++++++++++++++++++++++++++--- common/src/utils/strings.rs | 1 - heimdall/src/dump/mod.rs | 45 ++++++++++++------- 3 files changed, 97 insertions(+), 23 deletions(-) diff --git a/common/src/resources/transpose.rs b/common/src/resources/transpose.rs index 305dcc31..0880db7c 100644 --- a/common/src/resources/transpose.rs +++ b/common/src/resources/transpose.rs @@ -21,7 +21,7 @@ struct TransposeResponse { results: Vec } -fn _call_transpose(endpoint: String, api_key: &String) -> Option { +fn _call_transpose(endpoint: String, api_key: &String, logger: &Logger) -> Option { let mut headers = HeaderMap::new(); headers.insert("Content-Type", "application/json".parse().unwrap()); headers.insert("X-API-KEY", api_key.parse().unwrap()); @@ -36,7 +36,6 @@ fn _call_transpose(endpoint: String, api_key: &String) -> Option res, Err(e) => { - let (logger, _) = Logger::new("TRACE"); logger.error(&format!("failed to call Transpose endpoint '{endpoint}' .")); logger.error(&format!("error: {}", e)); std::process::exit(1) @@ -50,8 +49,8 @@ fn _call_transpose(endpoint: String, api_key: &String) -> Option json, Err(e) => { - let (logger, _) = Logger::new("TRACE"); logger.error(&format!("Transpose request unsucessful.")); + logger.debug(&format!("curl: curl -X GET \"https://api.transpose.io/endpoint/{endpoint}\" -H \"accept: application/json\" -H \"Content-Type: application/json\" -H \"X-API-KEY: {api_key}\" -d \"{{\\\"options\\\":{{\\\"timeout\\\": 999999999}}}}\"")); logger.error(&format!("error: {}", e)); logger.debug(&format!("response body: {:?}", body)); std::process::exit(1) @@ -59,7 +58,6 @@ fn _call_transpose(endpoint: String, api_key: &String) -> Option { - let (logger, _) = Logger::new("TRACE"); logger.error(&format!("failed to parse Transpose response body.")); logger.error(&format!("error: {}", e)); logger.debug(&format!("response body: {:?}", body)); @@ -83,8 +81,9 @@ pub fn get_transaction_list( let start_time = Instant::now(); let response = match _call_transpose( - format!("get-all-transactions?address={}&from_block={}&to_block={}", address, bounds.0, bounds.1), - api_key + format!("get-all-transactions/1?address={}&from_block={}&to_block={}", address, bounds.0, bounds.1), + api_key, + logger ) { Some(response) => response, None => { @@ -134,4 +133,67 @@ pub fn get_transaction_list( transactions.sort_by(|a, b| a.0.cmp(&b.0)); transactions +} + +pub fn get_contract_creation( + address: &String, + api_key: &String, + logger: &Logger +) -> Option<(u128, String)> { + + // get a new progress bar + let transaction_list_progress = ProgressBar::new_spinner(); + transaction_list_progress.enable_steady_tick(Duration::from_millis(100)); + transaction_list_progress.set_style(logger.info_spinner()); + transaction_list_progress.set_message(format!("fetching '{address}''s creation tx .")); + let start_time = Instant::now(); + + let response = match _call_transpose( + format!("fast-contract-address-creation-lookup/1?contract_address={address}"), + api_key, + logger + ) { + Some(response) => response, + None => { + logger.error(&format!("failed to get creation tx from Transpose")); + std::process::exit(1) + } + }; + + transaction_list_progress.finish_and_clear(); + logger.debug(&format!("fetching contract creation took {:?}", start_time.elapsed())); + + // parse the results + for result in response.results { + let block_number: u128 = match result.get("block_number") { + Some(block_number) => match block_number.as_u64() { + Some(block_number) => block_number as u128, + None => { + logger.error(&format!("failed to parse block_number from Transpose")); + std::process::exit(1) + } + }, + None => { + logger.error(&format!("failed to fetch block_number from Transpose response")); + std::process::exit(1) + } + }; + let transaction_hash: String = match result.get("transaction_hash") { + Some(transaction_hash) => match transaction_hash.as_str() { + Some(transaction_hash) => transaction_hash.to_string(), + None => { + logger.error(&format!("failed to parse transaction_hash from Transpose")); + std::process::exit(1) + } + }, + None => { + logger.error(&format!("failed to fetch transaction_hash from Transpose response")); + std::process::exit(1) + } + }; + + return Some((block_number, transaction_hash)); + }; + + None } \ No newline at end of file diff --git a/common/src/utils/strings.rs b/common/src/utils/strings.rs index 30313694..891e78a7 100644 --- a/common/src/utils/strings.rs +++ b/common/src/utils/strings.rs @@ -55,7 +55,6 @@ pub fn hex_to_ascii(s: &str) -> String { result } - // replace the last occurrence of a string with a new string pub fn replace_last(s: String, old: &str, new: &str) -> String { let new = new.chars().rev().collect::(); diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 26ae784c..46a2dd09 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -12,7 +12,7 @@ use crossterm::event::{EnableMouseCapture}; use crossterm::execute; use crossterm::terminal::{enable_raw_mode, EnterAlternateScreen}; use ethers::types::{H256, H160, Diff}; -use heimdall_common::resources::transpose::get_transaction_list; +use heimdall_common::resources::transpose::{get_transaction_list, get_contract_creation}; use heimdall_common::{ io::{ logging::* }, utils::{ threading::task_pool } @@ -160,12 +160,22 @@ pub fn dump(args: DumpArgs) { std::process::exit(1); } - // check if the RPC url is set and supports trace_replayTransaction - get_storage_diff(&Transaction { + // get the contract creation tx + let contract_creation_tx = match get_contract_creation(&args.target, &args.transpose_api_key, &logger) { + Some(tx) => tx, + None => { + logger.error("failed to get contract creation transaction. Is the target a contract address?"); + std::process::exit(1); + } + }; + + // add the contract creation tx to the transactions list to be indexed + let mut transactions: Vec = Vec::new(); + transactions.push(Transaction { indexed: false, - hash: String::from("0xb95343413e459a0f97461812111254163ae53467855c0d73e0f1e7c5b8442fa3"), - block_number: 471968 - }, &args); + hash: contract_creation_tx.1, + block_number: contract_creation_tx.0 + }); // parse the output directory let mut output_dir = args.output.clone(); @@ -198,7 +208,6 @@ pub fn dump(args: DumpArgs) { let transaction_list = get_transaction_list(&args.target, &args.transpose_api_key, (&args.from_block, &args.to_block), &logger); // convert to vec of Transaction - let mut transactions: Vec = Vec::new(); for transaction in transaction_list { transactions.push(Transaction { indexed: false, @@ -484,7 +493,10 @@ pub fn dump(args: DumpArgs) { let args = state.args.clone(); drop(state); - task_pool(transactions, args.threads, move |tx| { + // the number of threads cannot exceed the number of transactions + let num_indexing_threads = std::cmp::min(transactions.len(), args.threads); + + task_pool(transactions, num_indexing_threads, move |tx| { // get the storage diff for this transaction let state_diff = get_storage_diff(&tx, &args); @@ -569,14 +581,15 @@ pub fn dump(args: DumpArgs) { } } } - - // wait for the TUI thread to finish - match tui_thread.join() { - Ok(_) => {}, - Err(e) => { - logger.error("failed to join TUI thread."); - logger.error(&format!("{:?}", e)); - std::process::exit(1); + else { + // wait for the TUI thread to finish + match tui_thread.join() { + Ok(_) => {}, + Err(e) => { + logger.error("failed to join TUI thread."); + logger.error(&format!("{:?}", e)); + std::process::exit(1); + } } } From d53c6e8259ef6a9e4e522cd99447b8fb18712667 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Wed, 15 Mar 2023 12:59:40 -0500 Subject: [PATCH 21/26] :wrench: fix: transpose sql, no endpoints can be called ext team --- common/src/resources/transpose.rs | 39 +- heimdall/src/dump/constants.rs | 3 +- heimdall/src/dump/mod.rs | 363 +----------------- heimdall/src/dump/structures/dump_state.rs | 46 +++ heimdall/src/dump/structures/mod.rs | 3 + heimdall/src/dump/structures/storage_slot.rs | 9 + heimdall/src/dump/structures/transaction.rs | 6 + .../src/dump/tui_views/command_palette.rs | 2 +- heimdall/src/dump/tui_views/help.rs | 2 +- heimdall/src/dump/tui_views/main.rs | 2 +- heimdall/src/dump/tui_views/mod.rs | 28 +- heimdall/src/dump/util/csv.rs | 2 +- heimdall/src/dump/util/mod.rs | 3 +- heimdall/src/dump/util/table.rs | 2 +- heimdall/src/dump/util/threads/mod.rs | 1 + heimdall/src/dump/util/threads/tui.rs | 261 +++++++++++++ 16 files changed, 408 insertions(+), 364 deletions(-) create mode 100644 heimdall/src/dump/structures/dump_state.rs create mode 100644 heimdall/src/dump/structures/mod.rs create mode 100644 heimdall/src/dump/structures/storage_slot.rs create mode 100644 heimdall/src/dump/structures/transaction.rs create mode 100644 heimdall/src/dump/util/threads/mod.rs create mode 100644 heimdall/src/dump/util/threads/tui.rs diff --git a/common/src/resources/transpose.rs b/common/src/resources/transpose.rs index 0880db7c..55f1d484 100644 --- a/common/src/resources/transpose.rs +++ b/common/src/resources/transpose.rs @@ -21,22 +21,27 @@ struct TransposeResponse { results: Vec } -fn _call_transpose(endpoint: String, api_key: &String, logger: &Logger) -> Option { +fn _call_transpose(query: String, api_key: &String, logger: &Logger) -> Option { let mut headers = HeaderMap::new(); headers.insert("Content-Type", "application/json".parse().unwrap()); headers.insert("X-API-KEY", api_key.parse().unwrap()); // make the request - let client = reqwest::blocking::Client::builder().redirect(reqwest::redirect::Policy::none()).build() .unwrap(); + let client = reqwest::blocking::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(999999999)) + .build() + .unwrap(); + let mut response = match client - .get(format!("https://api.transpose.io/endpoint/{endpoint}")) - .body("{\"options\":{\"timeout\": 999999999}}") + .post(format!("https://api.transpose.io/sql")) + .body(query.clone()) .headers(headers) .send() { Ok(res) => res, Err(e) => { - logger.error(&format!("failed to call Transpose endpoint '{endpoint}' .")); + logger.error(&format!("failed to call Transpose .")); logger.error(&format!("error: {}", e)); std::process::exit(1) } @@ -50,7 +55,7 @@ fn _call_transpose(endpoint: String, api_key: &String, logger: &Logger) -> Optio Ok(json) => json, Err(e) => { logger.error(&format!("Transpose request unsucessful.")); - logger.debug(&format!("curl: curl -X GET \"https://api.transpose.io/endpoint/{endpoint}\" -H \"accept: application/json\" -H \"Content-Type: application/json\" -H \"X-API-KEY: {api_key}\" -d \"{{\\\"options\\\":{{\\\"timeout\\\": 999999999}}}}\"")); + logger.debug(&format!("curl: curl -X GET \"https://api.transpose.io/sql\" -H \"accept: application/json\" -H \"Content-Type: application/json\" -H \"X-API-KEY: {api_key}\" -d {query}")); logger.error(&format!("error: {}", e)); logger.debug(&format!("response body: {:?}", body)); std::process::exit(1) @@ -80,8 +85,19 @@ pub fn get_transaction_list( transaction_list_progress.set_message(format!("fetching transactions from '{}' .", address)); let start_time = Instant::now(); + // build the SQL query + let query = format!( + "{{\"sql\":\"SELECT block_number, transaction_hash FROM (SELECT transaction_hash, block_number FROM ethereum.transactions WHERE to_address = '{}' AND block_number BETWEEN {} AND {} UNION SELECT transaction_hash, block_number FROM ethereum.traces WHERE to_address = '{}' AND block_number BETWEEN {} AND {}) x\",\"parameters\":{{}},\"options\":{{\"timeout\": 999999999}}}}", + address, + bounds.0, + bounds.1, + address, + bounds.0, + bounds.1 + ); + let response = match _call_transpose( - format!("get-all-transactions/1?address={}&from_block={}&to_block={}", address, bounds.0, bounds.1), + query, api_key, logger ) { @@ -148,8 +164,15 @@ pub fn get_contract_creation( transaction_list_progress.set_message(format!("fetching '{address}''s creation tx .")); let start_time = Instant::now(); + // build the SQL query + let query = format!( + "{{\"sql\":\"SELECT block_number, transaction_hash FROM ethereum.transactions WHERE TIMESTAMP = ( SELECT created_timestamp FROM ethereum.accounts WHERE address = '{}' ) AND contract_address = '{}'\",\"parameters\":{{}},\"options\":{{\"timeout\": 999999999}}}}", + address, + address, + ); + let response = match _call_transpose( - format!("fast-contract-address-creation-lookup/1?contract_address={address}"), + query, api_key, logger ) { diff --git a/heimdall/src/dump/constants.rs b/heimdall/src/dump/constants.rs index 633b3953..56bc085a 100644 --- a/heimdall/src/dump/constants.rs +++ b/heimdall/src/dump/constants.rs @@ -2,8 +2,7 @@ use std::sync::Mutex; use lazy_static::lazy_static; -use crate::dump::DumpState; - +use crate::dump::structures::dump_state::DumpState; lazy_static! { pub static ref DUMP_STATE: Mutex = Mutex::new(DumpState::new()); diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 46a2dd09..3edc8761 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -2,32 +2,24 @@ mod tests; mod util; mod constants; mod tui_views; +mod structures; use std::collections::HashMap; use std::str::FromStr; -use std::time::{Instant, Duration}; -use std::{io, env}; +use std::time::{Instant}; +use std::env; use clap::{AppSettings, Parser}; -use crossterm::event::{EnableMouseCapture}; -use crossterm::execute; -use crossterm::terminal::{enable_raw_mode, EnterAlternateScreen}; -use ethers::types::{H256, H160, Diff}; +use ethers::types::{H160, Diff}; use heimdall_common::resources::transpose::{get_transaction_list, get_contract_creation}; -use heimdall_common::{ - io::{ logging::* }, - utils::{ threading::task_pool } -}; -use tui::backend::Backend; -use tui::{Frame, backend::CrosstermBackend, Terminal}; - -use tui_views::main::render_tui_view_main; - -use self::constants::{DUMP_STATE, DECODE_AS_TYPES}; -use self::tui_views::command_palette::render_tui_command_palette; -use self::tui_views::help::render_tui_help; +use heimdall_common::{io::{ logging::* }, utils::{ threading::task_pool }}; + +use self::constants::{DUMP_STATE}; +use self::structures::dump_state::DumpState; +use self::structures::storage_slot::StorageSlot; +use self::structures::transaction::Transaction; +use self::tui_views::{TUIView}; use self::util::csv::write_storage_to_csv; -use self::util::table::copy_selected; -use self::util::{get_storage_diff, cleanup_terminal}; +use self::util::{get_storage_diff}; #[derive(Debug, Clone, Parser)] #[clap(about = "Dump the value of all storage slots accessed by a contract", @@ -74,82 +66,6 @@ pub struct DumpArgs { pub no_tui: bool, } -#[derive(Debug, Clone)] -pub struct StorageSlot { - pub alias: Option, - pub value: H256, - pub modifiers: Vec<(u128, String)>, - pub decode_as_type_index: usize, -} - -#[derive(Debug, Clone)] -pub struct Transaction { - pub indexed: bool, - pub hash: String, - pub block_number: u128, -} - -#[derive(Debug, Clone)] -pub struct DumpState { - pub args: DumpArgs, - pub scroll_index: usize, - pub selection_size: usize, - pub transactions: Vec, - pub storage: HashMap, - pub view: TUIView, - pub start_time: Instant, - pub input_buffer: String, - pub filter: String, -} - -impl DumpState { - pub fn new() -> Self { - Self { - args: DumpArgs { - target: String::new(), - verbose: clap_verbosity_flag::Verbosity::new(1, 0), - output: String::new(), - rpc_url: String::new(), - transpose_api_key: String::new(), - threads: 4, - from_block: 0, - to_block: 9999999999, - no_tui: false, - }, - scroll_index: 0, - selection_size: 1, - transactions: Vec::new(), - storage: HashMap::new(), - view: TUIView::Main, - start_time: Instant::now(), - input_buffer: String::new(), - filter: String::new(), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -#[allow(dead_code)] -pub enum TUIView { - Killed, - Main, - CommandPalette, - Help, -} - -#[allow(unreachable_patterns)] -fn render_ui( - f: &mut Frame, - state: &mut DumpState -) { - match state.view { - TUIView::Main => { render_tui_view_main(f, state) }, - TUIView::CommandPalette => { render_tui_command_palette(f, state) }, - TUIView::Help => { render_tui_help(f, state) }, - _ => {} - } - } - pub fn dump(args: DumpArgs) { let (logger, _)= Logger::new(args.verbose.log_level().unwrap().as_str()); @@ -232,258 +148,11 @@ pub fn dump(args: DumpArgs) { drop(state); let _output_dir = output_dir.clone(); + let _args = args.clone(); // in a new thread, start the TUI let tui_thread = std::thread::spawn(move || { - - // if no TUI is requested, just run the dump - if args.no_tui { - return; - } - - // create new TUI terminal - enable_raw_mode().unwrap(); - let mut stdout = io::stdout(); - execute!(stdout, EnterAlternateScreen, EnableMouseCapture).unwrap(); - let backend = CrosstermBackend::new(stdout); - let mut terminal = Terminal::new(backend).unwrap(); - - loop { - let mut state = DUMP_STATE.lock().unwrap(); - terminal.draw(|f| { render_ui(f, &mut state); }).unwrap(); - drop(state); - - // check for user input - if crossterm::event::poll(Duration::from_millis(10)).unwrap() { - if let Ok(event) = crossterm::event::read() { - match event { - crossterm::event::Event::Key(key) => { - let mut state = DUMP_STATE.lock().unwrap(); - - // ignore key events if command palette is open - if state.view == TUIView::CommandPalette { - match key.code { - - // handle keys in command palette - crossterm::event::KeyCode::Char(c) => { - state.input_buffer.push(c); - }, - - // handle backspace - crossterm::event::KeyCode::Backspace => { - state.input_buffer.pop(); - }, - - // enter command - crossterm::event::KeyCode::Enter => { - state.filter = String::new(); - let mut split = state.input_buffer.split(" "); - let command = split.next().unwrap(); - let args = split.collect::>(); - - match command { - ":q" | ":quit" => { - state.view = TUIView::Killed; - break; - } - ":h" | ":help" => { - state.view = TUIView::Help; - } - ":f" | ":find" => { - if args.len() > 0 { - state.filter = args[0].to_string(); - } - state.view = TUIView::Main; - } - ":e" | ":export" => { - if args.len() > 0 { - write_storage_to_csv(&output_dir.clone(), &args[0].to_string(), &state); - } - state.view = TUIView::Main; - } - ":s" | ":seek" => { - if args.len() > 1 { - let direction = args[0].to_lowercase(); - let amount = args[1].parse::().unwrap_or(0); - match direction.as_str() { - "up" => { - if state.scroll_index >= amount { - state.scroll_index -= amount; - } else { - state.scroll_index = 0; - } - } - "down" => { - if state.scroll_index + amount < state.storage.len() { - state.scroll_index += amount; - } else { - state.scroll_index = state.storage.len() - 1; - } - } - _ => {} - } - } - state.view = TUIView::Main; - } - _ => { - state.view = TUIView::Main; - } - } - }, - - // handle escape - crossterm::event::KeyCode::Esc => { - state.filter = String::new(); - state.view = TUIView::Main; - } - - _ => {} - } - - drop(state); - continue; - } - - match key.code { - - // copy value on MODIFIER + C - crossterm::event::KeyCode::Char('c') => { - if crossterm::event::KeyModifiers::NONE != key.modifiers { - copy_selected(&mut state) - } - }, - - // main on escape - crossterm::event::KeyCode::Esc => { - state.filter = String::new(); - state.view = TUIView::Main; - }, - - // select transaction - crossterm::event::KeyCode::Right => { - - // increment decode_as_type_index on all selected transactions - let scroll_index = state.scroll_index.clone(); - let selection_size = state.selection_size.clone(); - let mut storage_iter = state.storage.iter_mut().collect::>(); - storage_iter.sort_by_key(|(slot, _)| *slot); - - for (i, (_, value)) in storage_iter.iter_mut().enumerate() { - if i >= scroll_index && i < scroll_index + selection_size { - - // saturating increment - if value.decode_as_type_index + 1 >= DECODE_AS_TYPES.len() { - value.decode_as_type_index = 0; - } else { - value.decode_as_type_index += 1; - } - - } - else if i >= scroll_index + selection_size { - break; - } - } - }, - - // deselect transaction - crossterm::event::KeyCode::Left => { - - // decrement decode_as_type_index on all selected transactions - let scroll_index = state.scroll_index.clone(); - let selection_size = state.selection_size.clone(); - let mut storage_iter = state.storage.iter_mut().collect::>(); - storage_iter.sort_by_key(|(slot, _)| *slot); - - for (i, (_, value)) in storage_iter.iter_mut().enumerate() { - if i >= scroll_index && i < scroll_index + selection_size { - - // saturating decrement - if value.decode_as_type_index == 0 { - value.decode_as_type_index = DECODE_AS_TYPES.len() - 1; - } else { - value.decode_as_type_index -= 1; - } - - } - else if i >= scroll_index + selection_size { - break; - } - } - }, - - // scroll down - crossterm::event::KeyCode::Down => { - state.selection_size = 1; - state.scroll_index += 1; - }, - - // scroll up - crossterm::event::KeyCode::Up => { - state.selection_size = 1; - if state.scroll_index > 0 { - state.scroll_index -= 1; - } - }, - - // toggle command palette on ":" - crossterm::event::KeyCode::Char(':') => { - match state.view { - TUIView::CommandPalette => { - state.view = TUIView::Main; - } - _ => { - state.input_buffer = String::from(":"); - state.view = TUIView::CommandPalette; - } - } - }, - - _ => {} - } - drop(state) - }, - crossterm::event::Event::Mouse(mouse) => { - let mut state = DUMP_STATE.lock().unwrap(); - match mouse.kind { - - // scroll down - crossterm::event::MouseEventKind::ScrollDown => { - - // if shift is held, increase selection size - if mouse.modifiers.contains(crossterm::event::KeyModifiers::SHIFT) { - state.selection_size += 1; - } - else { - state.selection_size = 1; - state.scroll_index += 1; - } - }, - - // scroll up - crossterm::event::MouseEventKind::ScrollUp => { - - // if shift is held, increase selection size - if mouse.modifiers.contains(crossterm::event::KeyModifiers::SHIFT) { - state.selection_size -= 1; - } - else { - state.selection_size = 1; - if state.scroll_index > 0 { - state.scroll_index -= 1; - } - } - }, - _ => {} - } - drop(state); - }, - _ => {} - } - } - } - } - - cleanup_terminal(); + util::threads::tui::handle(args, output_dir); }); // index transactions in a new thread @@ -571,7 +240,7 @@ pub fn dump(args: DumpArgs) { }); // if no-tui flag is set, wait for the indexing thread to finish - if args.no_tui { + if _args.no_tui { match dump_thread.join() { Ok(_) => {}, Err(e) => { @@ -597,5 +266,5 @@ pub fn dump(args: DumpArgs) { let state = DUMP_STATE.lock().unwrap(); write_storage_to_csv(&_output_dir, &"storage_dump.csv".to_string(), &state); logger.success(&format!("Wrote storage dump to '{}/storage_dump.csv'.", _output_dir)); - logger.info(&format!("Dumped {} storage values from '{}' .", state.storage.len(), &args.target)); + logger.info(&format!("Dumped {} storage values from '{}' .", state.storage.len(), &_args.target)); } \ No newline at end of file diff --git a/heimdall/src/dump/structures/dump_state.rs b/heimdall/src/dump/structures/dump_state.rs new file mode 100644 index 00000000..8f1fefc1 --- /dev/null +++ b/heimdall/src/dump/structures/dump_state.rs @@ -0,0 +1,46 @@ +use std::{collections::HashMap, time::Instant}; + +use ethers::types::H256; + +use crate::dump::{DumpArgs, tui_views::TUIView}; + +use super::{storage_slot::StorageSlot, transaction::Transaction}; + +#[derive(Debug, Clone)] +pub struct DumpState { + pub args: DumpArgs, + pub scroll_index: usize, + pub selection_size: usize, + pub transactions: Vec, + pub storage: HashMap, + pub view: TUIView, + pub start_time: Instant, + pub input_buffer: String, + pub filter: String, +} + +impl DumpState { + pub fn new() -> Self { + Self { + args: DumpArgs { + target: String::new(), + verbose: clap_verbosity_flag::Verbosity::new(1, 0), + output: String::new(), + rpc_url: String::new(), + transpose_api_key: String::new(), + threads: 4, + from_block: 0, + to_block: 9999999999, + no_tui: false, + }, + scroll_index: 0, + selection_size: 1, + transactions: Vec::new(), + storage: HashMap::new(), + view: TUIView::Main, + start_time: Instant::now(), + input_buffer: String::new(), + filter: String::new(), + } + } +} \ No newline at end of file diff --git a/heimdall/src/dump/structures/mod.rs b/heimdall/src/dump/structures/mod.rs new file mode 100644 index 00000000..0dbad750 --- /dev/null +++ b/heimdall/src/dump/structures/mod.rs @@ -0,0 +1,3 @@ +pub mod dump_state; +pub mod transaction; +pub mod storage_slot; \ No newline at end of file diff --git a/heimdall/src/dump/structures/storage_slot.rs b/heimdall/src/dump/structures/storage_slot.rs new file mode 100644 index 00000000..b73e48e2 --- /dev/null +++ b/heimdall/src/dump/structures/storage_slot.rs @@ -0,0 +1,9 @@ +use ethers::types::H256; + +#[derive(Debug, Clone)] +pub struct StorageSlot { + pub alias: Option, + pub value: H256, + pub modifiers: Vec<(u128, String)>, + pub decode_as_type_index: usize, +} \ No newline at end of file diff --git a/heimdall/src/dump/structures/transaction.rs b/heimdall/src/dump/structures/transaction.rs new file mode 100644 index 00000000..618d44d6 --- /dev/null +++ b/heimdall/src/dump/structures/transaction.rs @@ -0,0 +1,6 @@ +#[derive(Debug, Clone)] +pub struct Transaction { + pub indexed: bool, + pub hash: String, + pub block_number: u128, +} \ No newline at end of file diff --git a/heimdall/src/dump/tui_views/command_palette.rs b/heimdall/src/dump/tui_views/command_palette.rs index cfc7f645..a100ba71 100644 --- a/heimdall/src/dump/tui_views/command_palette.rs +++ b/heimdall/src/dump/tui_views/command_palette.rs @@ -1,6 +1,6 @@ use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction}, widgets::{Block, Borders, Cell, Row, Table, Paragraph}, style::{Style, Color, Modifier}}; -use crate::dump::{DumpState, util::table::build_rows}; +use crate::dump::{util::table::build_rows, structures::dump_state::DumpState}; pub fn render_tui_command_palette( f: &mut Frame, diff --git a/heimdall/src/dump/tui_views/help.rs b/heimdall/src/dump/tui_views/help.rs index cb803a95..a5b84d76 100644 --- a/heimdall/src/dump/tui_views/help.rs +++ b/heimdall/src/dump/tui_views/help.rs @@ -1,6 +1,6 @@ use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction, Alignment}, style::{Style, Color, Modifier}, widgets::{Paragraph, Block, Borders, Wrap}, text::Span}; -use crate::dump::{DumpState, constants::{HELP_MENU_COMMANDS, HELP_MENU_CONTROLS, ABOUT_TEXT}}; +use crate::dump::{constants::{HELP_MENU_COMMANDS, HELP_MENU_CONTROLS, ABOUT_TEXT}, structures::dump_state::DumpState}; pub fn render_tui_help( f: &mut Frame, diff --git a/heimdall/src/dump/tui_views/main.rs b/heimdall/src/dump/tui_views/main.rs index 232f3f9a..80a2245b 100644 --- a/heimdall/src/dump/tui_views/main.rs +++ b/heimdall/src/dump/tui_views/main.rs @@ -1,7 +1,7 @@ use heimdall_common::utils::{time::{calculate_eta, format_eta}}; use tui::{backend::Backend, Frame, layout::{Layout, Constraint, Direction}, widgets::{Gauge, Block, Borders, Cell, Row, Table}, style::{Style, Color, Modifier}}; -use crate::dump::{DumpState, util::table::build_rows}; +use crate::dump::{util::table::build_rows, structures::dump_state::DumpState}; pub fn render_tui_view_main( f: &mut Frame, diff --git a/heimdall/src/dump/tui_views/mod.rs b/heimdall/src/dump/tui_views/mod.rs index c69d1329..c0707858 100644 --- a/heimdall/src/dump/tui_views/mod.rs +++ b/heimdall/src/dump/tui_views/mod.rs @@ -1,3 +1,29 @@ +use tui::{backend::Backend, Frame}; + +use super::structures::dump_state::DumpState; + pub mod main; pub mod help; -pub mod command_palette; \ No newline at end of file +pub mod command_palette; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code)] +pub enum TUIView { + Killed, + Main, + CommandPalette, + Help, +} + +#[allow(unreachable_patterns)] +pub fn render_ui( + f: &mut Frame, + state: &mut DumpState +) { + match state.view { + TUIView::Main => { main::render_tui_view_main(f, state) }, + TUIView::CommandPalette => { command_palette::render_tui_command_palette(f, state) }, + TUIView::Help => { help::render_tui_help(f, state) }, + _ => {} + } +} \ No newline at end of file diff --git a/heimdall/src/dump/util/csv.rs b/heimdall/src/dump/util/csv.rs index e5c8a8a8..9c38dbd7 100644 --- a/heimdall/src/dump/util/csv.rs +++ b/heimdall/src/dump/util/csv.rs @@ -1,7 +1,7 @@ use ethers::{abi::{decode, ParamType}, types::U256}; use heimdall_common::{utils::strings::{encode_hex, hex_to_ascii}, io::{file::write_lines_to_file}}; -use crate::dump::{DumpState, constants::DECODE_AS_TYPES}; +use crate::dump::{constants::DECODE_AS_TYPES, structures::dump_state::DumpState}; pub fn write_storage_to_csv(output_dir: &String, file_name: &String, state: &DumpState) { let mut lines = { diff --git a/heimdall/src/dump/util/mod.rs b/heimdall/src/dump/util/mod.rs index a026e62d..49a2342e 100644 --- a/heimdall/src/dump/util/mod.rs +++ b/heimdall/src/dump/util/mod.rs @@ -1,5 +1,6 @@ pub mod csv; pub mod table; +pub mod threads; use std::{str::FromStr, io}; @@ -9,7 +10,7 @@ use heimdall_cache::{read_cache, store_cache}; use heimdall_common::io::logging::Logger; use tui::{backend::CrosstermBackend, Terminal}; -use super::{Transaction, DumpArgs}; +use super::{DumpArgs, structures::transaction::Transaction}; pub fn cleanup_terminal() { let stdout = io::stdout(); diff --git a/heimdall/src/dump/util/table.rs b/heimdall/src/dump/util/table.rs index b0a22bef..15fcede8 100644 --- a/heimdall/src/dump/util/table.rs +++ b/heimdall/src/dump/util/table.rs @@ -2,7 +2,7 @@ use ethers::{abi::{decode, ParamType}, types::{U256}}; use heimdall_common::{utils::strings::{encode_hex, hex_to_ascii}, io::clipboard::copy_to_clipboard}; use tui::{widgets::{Row, Cell}, style::{Style, Color}}; -use crate::dump::{DumpState, constants::DECODE_AS_TYPES}; +use crate::dump::{constants::DECODE_AS_TYPES, structures::dump_state::DumpState}; pub fn build_rows(mut state: &mut DumpState, max_row_height: usize) -> Vec> { diff --git a/heimdall/src/dump/util/threads/mod.rs b/heimdall/src/dump/util/threads/mod.rs new file mode 100644 index 00000000..44e13cf2 --- /dev/null +++ b/heimdall/src/dump/util/threads/mod.rs @@ -0,0 +1 @@ +pub mod tui; \ No newline at end of file diff --git a/heimdall/src/dump/util/threads/tui.rs b/heimdall/src/dump/util/threads/tui.rs new file mode 100644 index 00000000..d83bfc8f --- /dev/null +++ b/heimdall/src/dump/util/threads/tui.rs @@ -0,0 +1,261 @@ +use std::{io, time::Duration}; + +use crossterm::{terminal::{enable_raw_mode, EnterAlternateScreen}, execute, event::EnableMouseCapture}; +use tui::{backend::CrosstermBackend, Terminal}; + +use crate::dump::{constants::{DUMP_STATE, DECODE_AS_TYPES}, tui_views::{render_ui, TUIView}, DumpArgs, util::{csv::write_storage_to_csv, table::copy_selected, cleanup_terminal}}; + +pub fn handle( + args: DumpArgs, + output_dir: String, +) { + + // if no TUI is requested, just run the dump + if args.no_tui { + return; + } + + // create new TUI terminal + enable_raw_mode().unwrap(); + let mut stdout = io::stdout(); + execute!(stdout, EnterAlternateScreen, EnableMouseCapture).unwrap(); + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend).unwrap(); + + loop { + let mut state = DUMP_STATE.lock().unwrap(); + terminal.draw(|f| { render_ui(f, &mut state); }).unwrap(); + drop(state); + + // check for user input + if crossterm::event::poll(Duration::from_millis(10)).unwrap() { + if let Ok(event) = crossterm::event::read() { + match event { + crossterm::event::Event::Key(key) => { + let mut state = DUMP_STATE.lock().unwrap(); + + // ignore key events if command palette is open + if state.view == TUIView::CommandPalette { + match key.code { + + // handle keys in command palette + crossterm::event::KeyCode::Char(c) => { + state.input_buffer.push(c); + }, + + // handle backspace + crossterm::event::KeyCode::Backspace => { + state.input_buffer.pop(); + }, + + // enter command + crossterm::event::KeyCode::Enter => { + state.filter = String::new(); + let mut split = state.input_buffer.split(" "); + let command = split.next().unwrap(); + let args = split.collect::>(); + + match command { + ":q" | ":quit" => { + state.view = TUIView::Killed; + break; + } + ":h" | ":help" => { + state.view = TUIView::Help; + } + ":f" | ":find" => { + if args.len() > 0 { + state.filter = args[0].to_string(); + } + state.view = TUIView::Main; + } + ":e" | ":export" => { + if args.len() > 0 { + write_storage_to_csv(&output_dir.clone(), &args[0].to_string(), &state); + } + state.view = TUIView::Main; + } + ":s" | ":seek" => { + if args.len() > 1 { + let direction = args[0].to_lowercase(); + let amount = args[1].parse::().unwrap_or(0); + match direction.as_str() { + "up" => { + if state.scroll_index >= amount { + state.scroll_index -= amount; + } else { + state.scroll_index = 0; + } + } + "down" => { + if state.scroll_index + amount < state.storage.len() { + state.scroll_index += amount; + } else { + state.scroll_index = state.storage.len() - 1; + } + } + _ => {} + } + } + state.view = TUIView::Main; + } + _ => { + state.view = TUIView::Main; + } + } + }, + + // handle escape + crossterm::event::KeyCode::Esc => { + state.filter = String::new(); + state.view = TUIView::Main; + } + + _ => {} + } + + drop(state); + continue; + } + + match key.code { + + // copy value on MODIFIER + C + crossterm::event::KeyCode::Char('c') => { + if crossterm::event::KeyModifiers::NONE != key.modifiers { + copy_selected(&mut state) + } + }, + + // main on escape + crossterm::event::KeyCode::Esc => { + state.filter = String::new(); + state.view = TUIView::Main; + }, + + // select transaction + crossterm::event::KeyCode::Right => { + + // increment decode_as_type_index on all selected transactions + let scroll_index = state.scroll_index.clone(); + let selection_size = state.selection_size.clone(); + let mut storage_iter = state.storage.iter_mut().collect::>(); + storage_iter.sort_by_key(|(slot, _)| *slot); + + for (i, (_, value)) in storage_iter.iter_mut().enumerate() { + if i >= scroll_index && i < scroll_index + selection_size { + + // saturating increment + if value.decode_as_type_index + 1 >= DECODE_AS_TYPES.len() { + value.decode_as_type_index = 0; + } else { + value.decode_as_type_index += 1; + } + + } + else if i >= scroll_index + selection_size { + break; + } + } + }, + + // deselect transaction + crossterm::event::KeyCode::Left => { + + // decrement decode_as_type_index on all selected transactions + let scroll_index = state.scroll_index.clone(); + let selection_size = state.selection_size.clone(); + let mut storage_iter = state.storage.iter_mut().collect::>(); + storage_iter.sort_by_key(|(slot, _)| *slot); + + for (i, (_, value)) in storage_iter.iter_mut().enumerate() { + if i >= scroll_index && i < scroll_index + selection_size { + + // saturating decrement + if value.decode_as_type_index == 0 { + value.decode_as_type_index = DECODE_AS_TYPES.len() - 1; + } else { + value.decode_as_type_index -= 1; + } + + } + else if i >= scroll_index + selection_size { + break; + } + } + }, + + // scroll down + crossterm::event::KeyCode::Down => { + state.selection_size = 1; + state.scroll_index += 1; + }, + + // scroll up + crossterm::event::KeyCode::Up => { + state.selection_size = 1; + if state.scroll_index > 0 { + state.scroll_index -= 1; + } + }, + + // toggle command palette on ":" + crossterm::event::KeyCode::Char(':') => { + match state.view { + TUIView::CommandPalette => { + state.view = TUIView::Main; + } + _ => { + state.input_buffer = String::from(":"); + state.view = TUIView::CommandPalette; + } + } + }, + + _ => {} + } + drop(state) + }, + crossterm::event::Event::Mouse(mouse) => { + let mut state = DUMP_STATE.lock().unwrap(); + match mouse.kind { + + // scroll down + crossterm::event::MouseEventKind::ScrollDown => { + + // if shift is held, increase selection size + if mouse.modifiers.contains(crossterm::event::KeyModifiers::SHIFT) { + state.selection_size += 1; + } + else { + state.selection_size = 1; + state.scroll_index += 1; + } + }, + + // scroll up + crossterm::event::MouseEventKind::ScrollUp => { + + // if shift is held, increase selection size + if mouse.modifiers.contains(crossterm::event::KeyModifiers::SHIFT) { + state.selection_size -= 1; + } + else { + state.selection_size = 1; + if state.scroll_index > 0 { + state.scroll_index -= 1; + } + } + }, + _ => {} + } + drop(state); + }, + _ => {} + } + } + } + } + + cleanup_terminal(); +} \ No newline at end of file From e8cbe5278f84e0d887d8c7deb6fb6d6cfaebf45b Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Wed, 15 Mar 2023 20:31:37 -0500 Subject: [PATCH 22/26] =?UTF-8?q?=F0=9F=A7=B9=20clean:=20code=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- heimdall/src/dump/mod.rs | 87 +--------------------- heimdall/src/dump/util/threads/indexer.rs | 90 +++++++++++++++++++++++ heimdall/src/dump/util/threads/mod.rs | 3 +- 3 files changed, 95 insertions(+), 85 deletions(-) create mode 100644 heimdall/src/dump/util/threads/indexer.rs diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 3edc8761..5095394b 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -9,17 +9,15 @@ use std::str::FromStr; use std::time::{Instant}; use std::env; use clap::{AppSettings, Parser}; -use ethers::types::{H160, Diff}; +use ethers::types::{H160}; use heimdall_common::resources::transpose::{get_transaction_list, get_contract_creation}; -use heimdall_common::{io::{ logging::* }, utils::{ threading::task_pool }}; +use heimdall_common::{io::{ logging::* }}; use self::constants::{DUMP_STATE}; use self::structures::dump_state::DumpState; -use self::structures::storage_slot::StorageSlot; use self::structures::transaction::Transaction; use self::tui_views::{TUIView}; use self::util::csv::write_storage_to_csv; -use self::util::{get_storage_diff}; #[derive(Debug, Clone, Parser)] #[clap(about = "Dump the value of all storage slots accessed by a contract", @@ -157,86 +155,7 @@ pub fn dump(args: DumpArgs) { // index transactions in a new thread let dump_thread = std::thread::spawn(move || { - let state = DUMP_STATE.lock().unwrap(); - let transactions = state.transactions.clone(); - let args = state.args.clone(); - drop(state); - - // the number of threads cannot exceed the number of transactions - let num_indexing_threads = std::cmp::min(transactions.len(), args.threads); - - task_pool(transactions, num_indexing_threads, move |tx| { - - // get the storage diff for this transaction - let state_diff = get_storage_diff(&tx, &args); - - // unlock state - let mut state = DUMP_STATE.lock().unwrap(); - - // find the transaction in the state - let txs = state.transactions.iter_mut().find(|t| t.hash == tx.hash).unwrap(); - let block_number = tx.block_number.clone(); - txs.indexed = true; - - // unwrap the state diff - match state_diff { - Some(state_diff) => { - - // get diff for this address - match state_diff.0.get(&addr_hash) { - Some(diff) => { - - // build diff of StorageSlots and append to state - for (slot, diff_type) in &diff.storage { - - // parse value from diff type - let value = match diff_type { - Diff::Born(value) => value, - Diff::Changed(changed) => &changed.to, - Diff::Died(_) => { - state.storage.remove(slot); - continue; - } - _ => continue, - }; - - // get the slot from the state - match state.storage.get_mut(slot) { - Some(slot) => { - - // update value if newest modifier - if slot.modifiers.iter().all(|m| m.0 < block_number) { - slot.value = *value; - } - - slot.modifiers.push((block_number, tx.hash.clone().to_owned())); - }, - None => { - - // insert into state - state.storage.insert( - *slot, - StorageSlot { - value: *value, - modifiers: vec![(block_number, tx.hash.clone().to_owned())], - alias: None, - decode_as_type_index: 0 - } - ); - } - } - } - - }, - None => {} - } - }, - None => {} - } - - // drop state - drop(state); - }); + util::threads::indexer::handle(addr_hash); }); // if no-tui flag is set, wait for the indexing thread to finish diff --git a/heimdall/src/dump/util/threads/indexer.rs b/heimdall/src/dump/util/threads/indexer.rs new file mode 100644 index 00000000..de6f2a88 --- /dev/null +++ b/heimdall/src/dump/util/threads/indexer.rs @@ -0,0 +1,90 @@ +use ethers::types::{H160, Diff}; +use heimdall_common::utils::threading::task_pool; + +use crate::dump::{util::get_storage_diff, constants::DUMP_STATE, structures::storage_slot::StorageSlot}; + + +pub fn handle( + addr_hash: H160, +) { + let state = DUMP_STATE.lock().unwrap(); + let transactions = state.transactions.clone(); + let args = state.args.clone(); + drop(state); + + // the number of threads cannot exceed the number of transactions + let num_indexing_threads = std::cmp::min(transactions.len(), args.threads); + + task_pool(transactions, num_indexing_threads, move |tx| { + + // get the storage diff for this transaction + let state_diff = get_storage_diff(&tx, &args); + + // unlock state + let mut state = DUMP_STATE.lock().unwrap(); + + // find the transaction in the state + let txs = state.transactions.iter_mut().find(|t| t.hash == tx.hash).unwrap(); + let block_number = tx.block_number.clone(); + txs.indexed = true; + + // unwrap the state diff + match state_diff { + Some(state_diff) => { + + // get diff for this address + match state_diff.0.get(&addr_hash) { + Some(diff) => { + + // build diff of StorageSlots and append to state + for (slot, diff_type) in &diff.storage { + + // parse value from diff type + let value = match diff_type { + Diff::Born(value) => value, + Diff::Changed(changed) => &changed.to, + Diff::Died(_) => { + state.storage.remove(slot); + continue; + } + _ => continue, + }; + + // get the slot from the state + match state.storage.get_mut(slot) { + Some(slot) => { + + // update value if newest modifier + if slot.modifiers.iter().all(|m| m.0 < block_number) { + slot.value = *value; + } + + slot.modifiers.push((block_number, tx.hash.clone().to_owned())); + }, + None => { + + // insert into state + state.storage.insert( + *slot, + StorageSlot { + value: *value, + modifiers: vec![(block_number, tx.hash.clone().to_owned())], + alias: None, + decode_as_type_index: 0 + } + ); + } + } + } + + }, + None => {} + } + }, + None => {} + } + + // drop state + drop(state); + }); +} \ No newline at end of file diff --git a/heimdall/src/dump/util/threads/mod.rs b/heimdall/src/dump/util/threads/mod.rs index 44e13cf2..103b893e 100644 --- a/heimdall/src/dump/util/threads/mod.rs +++ b/heimdall/src/dump/util/threads/mod.rs @@ -1 +1,2 @@ -pub mod tui; \ No newline at end of file +pub mod tui; +pub mod indexer; \ No newline at end of file From aa4f80fbc88323e1be2c1ef6f35e805d72ee185b Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Wed, 15 Mar 2023 20:32:42 -0500 Subject: [PATCH 23/26] =?UTF-8?q?=F0=9F=91=B7=20build:=20bump=20version=20?= =?UTF-8?q?to=200.4.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 8 ++++---- cache/Cargo.toml | 2 +- common/Cargo.toml | 2 +- config/Cargo.toml | 2 +- heimdall/Cargo.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05c6f29b..3fc0c696 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1364,7 +1364,7 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "heimdall" -version = "0.3.4" +version = "0.4.0" dependencies = [ "backtrace", "clap", @@ -1389,7 +1389,7 @@ dependencies = [ [[package]] name = "heimdall-cache" -version = "0.3.4" +version = "0.4.0" dependencies = [ "bincode", "clap", @@ -1400,7 +1400,7 @@ dependencies = [ [[package]] name = "heimdall-common" -version = "0.3.4" +version = "0.4.0" dependencies = [ "clap", "clap-verbosity-flag", @@ -1419,7 +1419,7 @@ dependencies = [ [[package]] name = "heimdall-config" -version = "0.3.4" +version = "0.4.0" dependencies = [ "clap", "clap-verbosity-flag", diff --git a/cache/Cargo.toml b/cache/Cargo.toml index bf4601f5..b4de7c0b 100644 --- a/cache/Cargo.toml +++ b/cache/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "heimdall-cache" -version = "0.3.4" +version = "0.4.0" edition = "2021" license = "MIT" readme = "README.md" diff --git a/common/Cargo.toml b/common/Cargo.toml index d381edc7..cfc1f799 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "heimdall-common" -version = "0.3.4" +version = "0.4.0" edition = "2021" license = "MIT" readme = "README.md" diff --git a/config/Cargo.toml b/config/Cargo.toml index ba8ff684..80db3b70 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "heimdall-config" -version = "0.3.4" +version = "0.4.0" edition = "2021" license = "MIT" readme = "README.md" diff --git a/heimdall/Cargo.toml b/heimdall/Cargo.toml index 69a5f9a0..fb3c12b6 100644 --- a/heimdall/Cargo.toml +++ b/heimdall/Cargo.toml @@ -5,7 +5,7 @@ keywords = ["ethereum", "web3", "decompiler", "evm", "crypto"] license = "MIT" name = "heimdall" readme = "README.md" -version = "0.3.4" +version = "0.4.0" [dependencies] backtrace = "0.3" From 9db46b1c310093bc6cfa7aa0bd118b1ef265fc44 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Wed, 15 Mar 2023 22:52:27 -0500 Subject: [PATCH 24/26] =?UTF-8?q?=F0=9F=A7=B9=20clean:=20code=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/src/ether/mod.rs | 4 ++-- heimdall/src/dump/mod.rs | 40 +++++++++++++++++++++++------------ heimdall/src/dump/util/mod.rs | 2 ++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/common/src/ether/mod.rs b/common/src/ether/mod.rs index 9bbc4f0f..915f9991 100644 --- a/common/src/ether/mod.rs +++ b/common/src/ether/mod.rs @@ -1,3 +1,3 @@ -pub mod evm; pub mod signatures; -pub mod solidity; \ No newline at end of file +pub mod solidity; +pub mod evm; \ No newline at end of file diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 5095394b..93dee6f8 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -67,6 +67,19 @@ pub struct DumpArgs { pub fn dump(args: DumpArgs) { let (logger, _)= Logger::new(args.verbose.log_level().unwrap().as_str()); + // parse the output directory + let mut output_dir = args.output.clone(); + if &args.output.len() <= &0 { + output_dir = match env::current_dir() { + Ok(dir) => dir.into_os_string().into_string().unwrap(), + Err(_) => { + logger.error("failed to get current directory."); + std::process::exit(1); + } + }; + output_dir.push_str("/output"); + } + // check if transpose api key is set if &args.transpose_api_key.len() <= &0 { logger.error("you must provide a Transpose API key, which is used to fetch all normal and internal transactions for your target."); @@ -74,6 +87,20 @@ pub fn dump(args: DumpArgs) { std::process::exit(1); } + // // disassemble the bytecode + // let disassembled_bytecode = heimdall_common::ether::evm::disassemble::disassemble(DisassemblerArgs { + // target: args.target.clone(), + // default: true, + // verbose: args.verbose.clone(), + // output: String::new(), + // rpc_url: args.rpc_url.clone(), + // }); + + // // find and all selectors in the bytecode + // let selectors = find_function_selectors(disassembled_bytecode); + + // println!("{:?}", selectors); + // get the contract creation tx let contract_creation_tx = match get_contract_creation(&args.target, &args.transpose_api_key, &logger) { Some(tx) => tx, @@ -91,19 +118,6 @@ pub fn dump(args: DumpArgs) { block_number: contract_creation_tx.0 }); - // parse the output directory - let mut output_dir = args.output.clone(); - if &args.output.len() <= &0 { - output_dir = match env::current_dir() { - Ok(dir) => dir.into_os_string().into_string().unwrap(), - Err(_) => { - logger.error("failed to get current directory."); - std::process::exit(1); - } - }; - output_dir.push_str("/output"); - } - // convert the target to an H160 let addr_hash = match H160::from_str(&args.target) { Ok(addr) => addr, diff --git a/heimdall/src/dump/util/mod.rs b/heimdall/src/dump/util/mod.rs index 49a2342e..55ecfd11 100644 --- a/heimdall/src/dump/util/mod.rs +++ b/heimdall/src/dump/util/mod.rs @@ -12,6 +12,7 @@ use tui::{backend::CrosstermBackend, Terminal}; use super::{DumpArgs, structures::transaction::Transaction}; +// cleanup the terminal, disable raw mode, and leave the alternate screen pub fn cleanup_terminal() { let stdout = io::stdout(); let backend = CrosstermBackend::new(stdout); @@ -21,6 +22,7 @@ pub fn cleanup_terminal() { terminal.show_cursor().unwrap(); } +// get the state diff for the given transaction pub fn get_storage_diff(tx: &Transaction, args: &DumpArgs) -> Option { // create new logger From 7d63b745f4febab98a7db9f6426286dba12eeaca Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Wed, 15 Mar 2023 23:00:19 -0500 Subject: [PATCH 25/26] =?UTF-8?q?=F0=9F=A7=B9=20clean:=20code=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- heimdall/src/dump/mod.rs | 1 - heimdall/src/dump/tests.rs | 11 ----------- 2 files changed, 12 deletions(-) delete mode 100644 heimdall/src/dump/tests.rs diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 93dee6f8..67189edb 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -1,4 +1,3 @@ -mod tests; mod util; mod constants; mod tui_views; diff --git a/heimdall/src/dump/tests.rs b/heimdall/src/dump/tests.rs deleted file mode 100644 index e11c4f51..00000000 --- a/heimdall/src/dump/tests.rs +++ /dev/null @@ -1,11 +0,0 @@ -#[cfg(test)] -mod benchmark { - - -} - - -#[cfg(test)] -mod test { - -} \ No newline at end of file From 1cc85236c4848f3cc33afe2006e2a5971c0028d0 Mon Sep 17 00:00:00 2001 From: Jonathan Becker <64037729+Jon-Becker@users.noreply.github.com> Date: Wed, 15 Mar 2023 23:01:39 -0500 Subject: [PATCH 26/26] :wrench: fix: indentation --- heimdall/src/dump/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heimdall/src/dump/mod.rs b/heimdall/src/dump/mod.rs index 67189edb..158e8beb 100644 --- a/heimdall/src/dump/mod.rs +++ b/heimdall/src/dump/mod.rs @@ -22,7 +22,7 @@ use self::util::csv::write_storage_to_csv; #[clap(about = "Dump the value of all storage slots accessed by a contract", after_help = "For more information, read the wiki: https://jbecker.dev/r/heimdall-rs/wiki", global_setting = AppSettings::DeriveDisplayOrder, - global_setting = AppSettings::ColoredHelp, + global_setting = AppSettings::ColoredHelp, override_usage = "heimdall dump [OPTIONS]")] pub struct DumpArgs {