Merge pull request #70 from uptest-sc/chain_info

chain_info cli n example, bump to 1.2 and migsearch

examples/Cargo.toml

@@ -7,7 +7,7 @@ license = "MIT"
 edition = "2021"
 
 [dependencies]
-libuptest = { path = "../libuptest", version = "0.1.1", features = ["metadatadecode"] } #  "subxthelper"
+libuptest = { path = "../libuptest", version = "0.1.1", features = ["metadatadecode", "migrationsearch"] } #  "subxthelper"
 anyhow =  "1.0.70"
 tokio = { version = "1.27", features = ["full"] }
 jsonrpsee = { version = "0.16.2",  features = ["server", "ws-client", "macros", "client-ws-transport"]}

examples/examples/chain_info.rs

@@ -0,0 +1,34 @@
+/// https://github.com/uptest-sc/uptest/issues/69
+/*
+cargo run -p uptest-examples --example chain_info
+----Chain-Info----
+Chain Name: "node-template"
+Runtime version: 109
+Authoring Version: 1
+State Version: 1
+--E-O-L--
+*/
+
+use libuptest::types::RuntimeVersion;
+use libuptest::ws_mod::get_runtime_version;
+use libuptest::jsonrpseeclient::JsonrpseeClient;
+
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> { 
+    // chain at 127.0.0.1:9944
+    let client = JsonrpseeClient::with_default_url().unwrap();
+    let chain_info: RuntimeVersion = get_runtime_version(client).await.unwrap();
+    println!("----Chain-Info----");
+    println!("Chain Name: {:?}
+Runtime version: {:?}
+Authoring Version: {:?}
+State Version: {:?}",
+        chain_info.spec_name,
+        chain_info.spec_version,
+        chain_info.authoring_version,
+        chain_info.state_version
+    );
+    println!("--E-O-L--");
+    Ok(())
+}

examples/examples/migration_search_test.rs

@@ -0,0 +1,34 @@
+//use std::fmt::format;
+use libuptest::migration_search::crates_io_search::{
+    download_crate_from_crates_io, search_crates_io, Crate, Crates,
+};
+use libuptest::migration_search::decompress::tar_xvf;
+use libuptest::migration_search::file_search::test_glob;
+
+pub struct mig_find {}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    println!("Starting search, looking for pallet-balances");
+
+    let resp: Crates = search_crates_io("pallet-balan").await?;
+    for results in resp.crates.iter() {
+        println!("Found crate: {:?}", results.name);
+        //   println!("Found repo: {:?}", results.repository_url);
+        println!("Crates version is: {}", results.version);
+        let _url = results.repository_url.clone().unwrap();
+        println!("docs are at: {:?}", results.documentation);
+        // download crate to parent working dir
+        let filen =
+            download_crate_from_crates_io(results.version.clone(), results.name.clone()).await?;
+        println!("unziping file");
+        let _unzip = tar_xvf(filen.clone());
+        println!("file downloaded");
+        let _test_o = test_glob(filen).await?;
+        println!("All done")
+    }
+    //   let files_found = file_test().await?;
+
+    //  println!("{:#?}", resp);
+    Ok(())
+}

examples/examples/sub_events.rs

@@ -22,7 +22,7 @@ use libuptest::ws_mod::{blocknumber_to_blockhash, get_block_events, get_raw_meta
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     let wsinstance = "wss://polkadot-rpc-tn.dwellir.com:443"; // my node endpoint
-    let client: JsonrpseeClient =  JsonrpseeClient::new(wsinstance).unwrap();
+    let client: JsonrpseeClient = JsonrpseeClient::new(wsinstance).unwrap();
 
     println!("Subscribing");
     let mut subscrib: SubscriptionWrapper<Header> = client

libuptest/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "libuptest"
-version = "0.1.1"
+version = "0.1.2"
 edition = "2021"
 authors = ["Filip Kalebo<[email protected]>"]
 license = "MIT"
@@ -17,18 +17,20 @@ hex = { version = "0.4.3", features = ["alloc"]}
 anyhow = "1.0.69"
 serde_json = { version = "1" }
 futures = "0.3.26"
-serde = { version = "1.0.154", features = ["derive"], default-features = false }
+serde = { version = "1.0.173", features = ["derive"], default-features = false }
 fixed-hash = "0.8"
 maybe-async = { version = "0.2.7" }
 async-trait = "0.1.68"
 impl-serde = { version = "0.4.0", default-features = false }
 tokio = { version = "1.27", features = ["full"] }
 
 
+
 [features]
-all = ["metadatadecode"]#, "subxthelper"
+all = ["metadatadecode", "migrationsearch"]#, "subxthelper"
 metadatadecode = ["desub-current", "frame-metadata"]
 #subxthelper = ["subxt"]
+migrationsearch = ["reqwest", "glob", "regex", "tar", "flate2"]
 
 #[dependencies.subxt]
 #version = "0.28.0"
@@ -45,4 +47,25 @@ rev = "0932164"
 [dependencies.frame-metadata]
 version = "14.2"
 features = ["v14", "std", "scale-info"]
+optional = true
+
+[dependencies.reqwest]
+version = "0.11.18"
+features = ["json"]
+optional = true
+
+[dependencies.glob]
+version = "0.3.1"
+optional = true
+
+[dependencies.regex] 
+version = "1.9.1" 
+optional = true
+
+[dependencies.flate2] 
+version = "1.0.26"
+optional = true
+
+[dependencies.tar] 
+version = "0.4.39"
 optional = true

libuptest/src/lib.rs

@@ -22,6 +22,8 @@ pub mod ws_mod;
 
 //#[cfg(feature = "subxthelper")]
 //pub mod subxt_helper;
+#[cfg(feature = "migrationsearch")]
+pub mod migration_search;
 
 #[cfg(feature = "metadatadecode")]
 pub mod decode_extrinsic;

libuptest/src/migration_search/crates_io_search.rs

@@ -0,0 +1,105 @@
+use reqwest::header::USER_AGENT;
+use serde::{Deserialize, Serialize};
+use tokio::fs::File;
+use tokio::io::AsyncWriteExt; // for write_all()
+
+/// crates.io crates.crate.links field
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Lankar {
+    pub owner_team: String,
+    pub owner_user: String,
+    pub owners: String,
+    pub reverse_dependencies: String,
+    pub version_downloads: String,
+    pub versions: String,
+}
+
+/// crates.io crates.crate json field
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Crate {
+    pub badges: Vec<String>,
+    pub categories: Option<Vec<String>>,
+    pub created_at: String,
+    pub description: Option<String>,
+    pub documentation: Option<String>,
+    pub downloads: u32,
+    pub exact_match: bool,
+    pub homepage: Option<String>,
+    pub id: String,
+    pub keywords: Option<String>,
+    pub links: Lankar,
+    pub max_stable_version: String,
+    pub max_version: String,
+    pub name: String,
+    #[serde(rename = "newest_version")]
+    pub version: String,
+    pub recent_downloads: u32,
+    #[serde(rename = "repository")]
+    pub repository_url: Option<String>,
+    pub updated_at: Option<String>,
+    pub versions: Option<String>,
+}
+
+/// crates.io meta key field
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Metastruct {
+    pub next_page: Option<String>,
+    pub prev_page: Option<String>,
+    pub total: u32,
+}
+
+/// Wrapper used by crates.io API.
+/// crates.io search returns a json blob with crates and meta field
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Crates {
+    #[serde(rename = "crates")]
+    pub crates: Vec<Crate>,
+    pub meta: Metastruct,
+}
+
+/// download tarball/gzip of crate from crates.io, unstable version atm, does not handle errors properly
+/// return file name if all is good
+pub async fn download_crate_from_crates_io(
+    crate_version: String,
+    crate_name: String,
+) -> Result<String, reqwest::Error> {
+    let file_name = format!("{crate_name:}-{crate_version:}.gzip");
+    //https://crates.io/api/v1/crates/serde/1.0.0/download
+    let url = format!(
+        "https://crates.io/api/v1/crates/{}/{}/download",
+        crate_name, crate_version
+    );
+    let client = reqwest::Client::new();
+    println!("Downloading crate: {crate_name:?} version: {crate_version:?}");
+    let tanka = client
+        .get(url)
+        .header(USER_AGENT, "Uptest cli client")
+        .send()
+        .await?;
+    println!("crate downloaded ok");
+    let mut filen = File::create(file_name.clone()).await.unwrap();
+    filen
+        .write_all(&tanka.bytes().await.unwrap())
+        .await
+        .unwrap();
+    println!("crate saved as {file_name:}");
+
+    Ok(file_name)
+}
+
+/// search crates.io
+pub async fn search_crates_io(query: &str) -> Result<Crates, reqwest::Error> {
+    let client = reqwest::Client::new();
+    let url = format!(
+        "https://crates.io/api/v1/crates?page=1&per_page=100&q={}",
+        query
+    ); // https://crates.io/api/v1/crates?page=1&per_page=100&q=pallet-balan
+    let resp = client
+        .get(url)
+        .header(USER_AGENT, "Uptest cli client")
+        .send()
+        .await?
+        .json::<Crates>()
+        .await;
+    resp
+}

libuptest/src/migration_search/decompress.rs

@@ -0,0 +1,17 @@
+use flate2::read::GzDecoder;
+use std::fs::File; // todo change to tokio fs
+use tar::Archive;
+
+/// unzip a gzip file, like tar -xvf does on posix based systems
+pub fn tar_xvf(filename: String) -> Result<(), Box<dyn std::error::Error>> {
+    // Open the gzip file
+    let file = File::open(filename)?;
+    let decoder = GzDecoder::new(file);
+
+    // Create a tar archive from the gzip file decoder
+    let mut archive = Archive::new(decoder);
+
+    // Extract the contents of the archive
+    archive.unpack(".")?;
+    Ok(())
+}

libuptest/src/migration_search/file_search.rs

@@ -0,0 +1,50 @@
+use glob::glob;
+use regex::Regex;
+use tokio::fs::File;
+use tokio::io::{AsyncBufReadExt, BufReader};
+
+pub async fn file_find(file_name: String) -> Result<(), Box<dyn std::error::Error>> {
+    //let file_path = "/tmp/search_crates/pallet-balances-21.0.0/src/migration.rs";
+
+    // Open the file asynchronously
+    let file = File::open(&file_name).await?;
+    let reader = BufReader::new(file);
+
+    let re = Regex::new(r".*(fn\smigrate.[a-z-0-9-A-Z-_]{0,})(<.*>|)\(").unwrap();
+
+    let mut lines = reader.lines();
+
+    while let Some(line) = lines.next_line().await? {
+        // println!("line is = {}", line);
+
+        let caps = re.find(&line);
+        //   println!("The name is: {:?}", &caps);
+        //  println!("line is: {line:?}");
+        match caps {
+            Some(result) => {
+                println!("found match: {:?} in file: {file_name:}", result.as_str());
+            }
+            _ => continue, //println!("no match")
+        }
+    }
+
+    Ok(())
+}
+
+pub async fn test_glob(folder_name: String) -> Result<(), Box<dyn std::error::Error>> {
+    let folder_name = folder_name.replace(".gzip", "");
+    println!("Glob search with {folder_name:}");
+    let findo = format!("{}/*/*.rs", folder_name);
+    println!("findo: {findo:}");
+    for entry in glob(&findo).expect("Failed to read glob pattern") {
+        match entry {
+            Ok(path) => {
+                let pathen = path.display();
+                //    println!("checking path: {pathen:}");
+                let _loots = file_find(pathen.to_string()).await?;
+            }
+            Err(e) => println!("{:?}", e),
+        }
+    }
+    Ok(())
+}

libuptest/src/migration_search/mod.rs

@@ -0,0 +1,3 @@
+pub mod crates_io_search;
+pub mod decompress;
+pub mod file_search;

src/cli.rs

@@ -12,16 +12,16 @@ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR I
 use clap::{arg, Command};
 
 fn get_git_hash() -> String {
-	let output = std::process::Command::new("git")
-		.args(&["rev-parse", "--verify", "HEAD"])
-		.output().unwrap();
-	let git_hash = String::from_utf8(output.stdout).unwrap_or_default();
-	git_hash.trim().to_string()
+    let output = std::process::Command::new("git")
+        .args(&["rev-parse", "--verify", "HEAD"])
+        .output()
+        .unwrap();
+    let git_hash = String::from_utf8(output.stdout).unwrap_or_default();
+    git_hash.trim().to_string()
 }
 
 pub fn gen_cli() -> Command {
-
-     Command::new("uptest")
+    Command::new("uptest")
         .about("substrate runtime UPgrade TESTing suit")
         .version(env!("CARGO_PKG_VERSION")) // read from Cargo.tomlss
         .subcommand_required(true)
@@ -90,6 +90,23 @@ pub fn gen_cli() -> Command {
             .help_template("Usage example: uptest -c -w ws://127.0.0.1:9944")
             .arg(arg!(<ws> "ws endpoint of the chain to connect")),
         )
+        .subcommand(
+            Command::new("chain-info")
+            .short_flag('i')
+            .long_flag("chain-info")
+            .about("Displays meta information about the chain")
+            .help_template("Usage example: 
+./target/release/uptest -i ws://127.0.0.1:9944
+Uptest command line tool
+----Chain-Info----
+Chain Name: node-template
+Runtime version: 109
+Authoring Version: 1
+State Version: 1
+--E-O-L--
+")
+            .arg(arg!(<ws> "ws endpoint of the chain to connect")),
+        )
         // todo filter by custom pallet
         .subcommand(
             Command::new("pallets-storage")
@@ -121,8 +138,7 @@ pub fn gen_cli() -> Command {
             .arg(arg!(<ws> "ws endpoint of the chain to connect"))
             .arg(arg!(<block_limit> "amount of blocks of latest blocks to subscribe to").required(true)),
         )
-        // subscribe to the chain, display the latest blocks and events triggered in those blocks
-
+        // subscribe to the chain, display the latest blocks and events triggered in those blocks 
 
         // TODO: read local wasm file and submit runtime upgrade
         .subcommand(