diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 04aec7e4f..1b97090fe 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -58,11 +58,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - name: Symlink lightwalletd and zcash binaries for darkside-tests - run: ln -s /usr/bin/lightwalletd ./darkside-tests/lightwalletd_bin/ + - name: create binaries dir + run: mkdir -p ./zingo-testutils/test_binaries/bins - - name: Symlink lightwalletd and zcash binaries for integration-tests - run: ln -s /usr/bin/lightwalletd /usr/bin/zcashd /usr/bin/zcash-cli ./integration-tests/regtest/bin/ + - name: Symlink lightwalletd and zcash binaries + run: ln -s /usr/bin/lightwalletd /usr/bin/zcashd /usr/bin/zcash-cli ./zingo-testutils/test_binaries/bins/ - name: Symlink zcash parameters run: ln -s /root/.zcash-params /github/home diff --git a/.gitignore b/.gitignore index 42ff45e27..b2f9c3294 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,22 @@ -zingocli/regtest/data/zcashd/regtest/ -zingocli/regtest/data/lightwalletd/ -zingocli/regtest/data/zingo/ +!zingolib/test-data/README +!zingolib/test-data/openssl_cfg +*.json.txt +.DS_Store +/procgov +/testdata +cobertura.xml +docker-ci/lightwalletd +target +wallets/* zingocli/regtest/logs/ zingocli/regtest/zingo-wallet.dat -zingocli/regtest/bin -integration-tests/regtest/bin -target -zingolib/target zingocli/target -/testdata -/procgov -.DS_Store -*.json.txt -docker-ci/lightwalletd -zingolib/test-data/* -!zingolib/test-data/README -!zingolib/test-data/openssl_cfg +zingocli/tests/data/basic_zcashd.conf zingocli/tests/data/externalwallet_coinbaseaddress.conf zingocli/tests/data/lightwalletd.yml -zingocli/tests/data/basic_zcashd.conf -cobertura.xml zingocli/tests/times/* -wallets/* zingocli/wallets/* -darkside-tests/lightwalletd_bin/ +zingolib/target +zingolib/test-data/* +zingo-testutils/test_binaries + diff --git a/darkside-tests/src/utils.rs b/darkside-tests/src/utils.rs index cc2a076cd..94327cc1d 100644 --- a/darkside-tests/src/utils.rs +++ b/darkside-tests/src/utils.rs @@ -23,7 +23,8 @@ use zcash_primitives::{merkle_tree::read_commitment_tree, transaction::Transacti use zingo_testutils::{ self, incrementalmerkletree::frontier::CommitmentTree, - regtest::{get_cargo_manifest_dir, launch_lightwalletd}, + paths::{get_bin_dir, get_cargo_manifest_dir}, + regtest::launch_lightwalletd, scenarios::setup::TestEnvironmentGenerator, }; use zingolib::wallet::traits::DomainWalletExt; @@ -251,7 +252,7 @@ impl DarksideHandler { darkside_dir.join("logs"), darkside_dir.join("conf"), darkside_dir.join("data"), - get_cargo_manifest_dir().join("lightwalletd_bin"), + get_bin_dir(), check_interval, grpc_bind_addr, ); @@ -430,7 +431,7 @@ pub async fn init_darksidewalletd( pub fn create_chainbuild_file(test_name: &str) -> File { let path = format!( "{}/tests/data/chainbuilds/{}", - zingo_testutils::regtest::get_cargo_manifest_dir().to_string_lossy(), + get_cargo_manifest_dir().to_string_lossy(), test_name ); match fs::create_dir(path.clone()) { @@ -453,7 +454,7 @@ pub fn create_chainbuild_file(test_name: &str) -> File { pub fn load_chainbuild_file(test_name: &str) -> Vec { let path = format!( "{}/tests/data/chainbuilds/{}", - zingo_testutils::regtest::get_cargo_manifest_dir().to_string_lossy(), + get_cargo_manifest_dir().to_string_lossy(), test_name ); let filename = "hex_transactions.txt"; diff --git a/darkside-tests/tests/advanced_reorg_tests.rs b/darkside-tests/tests/advanced_reorg_tests.rs index 5ce3f0fb8..773c8b8d0 100644 --- a/darkside-tests/tests/advanced_reorg_tests.rs +++ b/darkside-tests/tests/advanced_reorg_tests.rs @@ -12,7 +12,7 @@ use darkside_tests::{ use tokio::time::sleep; use zcash_primitives::consensus::BlockHeight; -use zingo_testutils::{regtest::get_cargo_manifest_dir, scenarios::setup::ClientBuilder}; +use zingo_testutils::{paths::get_cargo_manifest_dir, scenarios::setup::ClientBuilder}; use zingoconfig::RegtestNetwork; use zingolib::lightclient::PoolBalances; use zingolib::wallet::data::summaries::ValueTransferKind; diff --git a/docs/TEST-SETUP-NOTES.txt b/docs/TEST-SETUP-NOTES.txt deleted file mode 100644 index 479806a80..000000000 --- a/docs/TEST-SETUP-NOTES.txt +++ /dev/null @@ -1,21 +0,0 @@ -cargo install cargo-nextest --locked - -## zcash install instructions for debian/ubuntu -git clone https://github.com/zcash/zcash.git -sudo apt-get install \ - build-essential pkg-config libc6-dev m4 g++-multilib \ - autoconf libtool ncurses-dev unzip git python3 python3-zmq \ - zlib1g-dev curl bsdmainutils automake libtinfo5 -./zcutil/build.sh -j$(nproc) - -## some alternate packages for arch -paru -S - build-essential -> base-devel - libc6-dev -> glibc - g++-multilib (supposedly in gcc) - ncurses-dev -> ncurses - python3-zmq - zlib1g-dev - libtinfo5 - -cargo nextest run diff --git a/docs/TEST-DARKSIDE.md b/docs/testing/TEST-DARKSIDE.md similarity index 100% rename from docs/TEST-DARKSIDE.md rename to docs/testing/TEST-DARKSIDE.md diff --git a/docs/testing/ZINGOCLI-REGTEST.md b/docs/testing/ZINGOCLI-REGTEST.md new file mode 100644 index 000000000..f6186da00 --- /dev/null +++ b/docs/testing/ZINGOCLI-REGTEST.md @@ -0,0 +1,153 @@ +`G=$(git rev-parse --show-toplevel)` + +## Regtest Mode +WARNING Experimental! +Zingocli has a set of tests based on zcashd regtest mode, by locally running a `zcashd` and `lightwalletd`. +This is now working with a simple `zingo-cli` invocation flag. + +There are pre-made directories in this repo to support ready use of regtest mode. These are found in the `$G/zingocli/regtest/` subdirectory. + +There are default config files for these binaries already in place in `$G/zingocli/regtest/conf/` which can also be edited. + +Because regtest mode has no ability to cope with an initial `zcashd` state without any blocks, +we have included files to produce an initial block height of one, with no variation between runs. +These files are copied from a 'passive' directory (`$G/zingocli/regtest/data/regtestvectors/`) +into a newly cleared 'active' data directory at the beginning of each time regtest mode is run. +This means, by default, any blocks added while zcashd is running are not retained for subsequent runs. + +The default config includes all network upgrades set to block height 1, therefore all network upgrades are active by default in regtest mode. + +# Usage example: +You must copy your compiled `zcashd`, `zcash-cli` and `lightwalletd` binaries to `$G/zingocli/regtest/bin/` or set up symlinks, etc. `zcash-cli` is also needed if you wish +to interact with your `zcashd` instance while it is running. + +From your `$G/` directory, you can run: +`cargo run --release -- --regtest` + +Note: Regtest mode only works when invoked with `cargo run`. Running `cargo build` followed by an invocation of the compiled binary will fail. + +This will start an interactive session. Individual commands can be passed to zingolib (via the cli), for example: + +`cargo run --release -- --regtest help` + +This will start `zcashd` and `lightwalletd` and then connect to these tools with an interactive `zingo-cli`. +It currently takes a few seconds to do so, even on a fast machine, to give the daemons time to boot. + +These daemons will be killed when the user exits `zingo-cli` using the `quit` command. +However, if there is an issue starting or shutting down regtest mode, it's possible you will have to shut down the daemons manually. + +You should see several diagnostic messages, and then: +`regtest detected and network set correctly! +Lightclient connecting to http://127.0.0.1:9067/` +at which point the interactive cli application should work with your regtest network. + +`zcashd`'s stdout logfile should quickly have an output of several dozen lines, and show network upgrade activation parameters at `height=1`. +`lightwalletd`'s stdout log file will show something like: +`{"app":"lightwalletd","level":"info","msg":"Got sapling height 1 block height 1 chain regtest branchID ..."}` +...which you can view with `tail -f` or your favorite tool. + +Once regtest mode is running, you can manipulate the simulated chain with `zcash-cli`. + +For example, in still another terminal instance in the `$G/zingocli/regtest/bin/` directory, you can run +`./zcash-cli -regtest -rpcuser=xxxxxx -rpcpassword=xxxxxx generate 11` to generate 11 blocks. +Please note that by adding more than 100 blocks it is difficult or impossible to rewind the chain. The config means that after the first block all network upgrades should be in place. +Other `zcash-cli` commands should work similarly. + +Invocation currently only works when being launched within a `zingolib` repo's worktree +(The paths have to know where to look for the subdirectories, they start with the top level of a `zingolib` repo, or fail immediately). + +Have fun! + +# Tree Diagrams +In `$G/zingocli`, running `tree ./regtest` +after moving binaries and running: +./regtest/ +├── bin +│   ├── lightwalletd +│   ├── zcash-cli +│   └── zcashd +├── conf +│   ├── lightwalletd.yml +│   └── zcash.conf +├── data +│   ├── lightwalletd +│   │   └── db +│   │   └── regtest +│   │   ├── blocks +│   │   └── lengths +│   ├── regtestvectors +│   │   └── regtest +│   │   ├── banlist.dat +│   │   ├── blocks +│   │   │   ├── blk00000.dat +│   │   │   ├── index +│   │   │   │   ├── 000005.ldb +│   │   │   │   ├── 000008.ldb +│   │   │   │   ├── 000009.log +│   │   │   │   ├── CURRENT +│   │   │   │   ├── LOCK +│   │   │   │   ├── LOG +│   │   │   │   ├── LOG.old +│   │   │   │   └── MANIFEST-000007 +│   │   │   └── rev00000.dat +│   │   ├── chainstate +│   │   │   ├── 000005.ldb +│   │   │   ├── 000008.ldb +│   │   │   ├── 000009.log +│   │   │   ├── CURRENT +│   │   │   ├── LOCK +│   │   │   ├── LOG +│   │   │   ├── LOG.old +│   │   │   └── MANIFEST-000007 +│   │   ├── database +│   │   │   └── log.0000000001 +│   │   ├── db.log +│   │   ├── fee_estimates.dat +│   │   ├── peers.dat +│   │   └── wallet.dat +│   ├── zcashd +│   │   └── regtest +│   │   ├── banlist.dat +│   │   ├── blocks +│   │   │   ├── blk00000.dat +│   │   │   ├── index +│   │   │   │   ├── 000005.ldb +│   │   │   │   ├── 000008.ldb +│   │   │   │   ├── 000011.ldb +│   │   │   │   ├── 000012.log +│   │   │   │   ├── CURRENT +│   │   │   │   ├── LOCK +│   │   │   │   ├── LOG +│   │   │   │   ├── LOG.old +│   │   │   │   └── MANIFEST-000010 +│   │   │   └── rev00000.dat +│   │   ├── chainstate +│   │   │   ├── 000005.ldb +│   │   │   ├── 000008.ldb +│   │   │   ├── 000011.ldb +│   │   │   ├── 000012.log +│   │   │   ├── CURRENT +│   │   │   ├── LOCK +│   │   │   ├── LOG +│   │   │   ├── LOG.old +│   │   │   └── MANIFEST-000010 +│   │   ├── database +│   │   │   └── log.0000000001 +│   │   ├── db.log +│   │   ├── fee_estimates.dat +│   │   ├── peers.dat +│   │   ├── wallet.dat +│   │   └── zcashd.pid +│   └── zingo +│   ├── zingo-wallet.dat +│   └── zingo-wallet.debug.log +├── logs +│   ├── lightwalletd +│   │   ├── stderr.log +│   │   └── stdout.log +│   └── zcashd +│   └── stdout.log +└── README.md + +# Working Commits +Tested with `zcash` commit `d6d209`, `lightwalletd` commit `f53511c`, and `zingolib` commit `89fbe118f8305051d5f1d1d95903ba3ccaec586b` or better. diff --git a/integration-tests/tests/integrations.rs b/integration-tests/tests/integrations.rs index d08ff811c..6be326717 100644 --- a/integration-tests/tests/integrations.rs +++ b/integration-tests/tests/integrations.rs @@ -15,9 +15,10 @@ use zcash_primitives::{ transaction::{fees::zip317::MINIMUM_FEE, TxId}, }; use zingo_testutils::{ - build_fvk_client, check_transaction_equality, increase_height_and_wait_for_client, - regtest::get_cargo_manifest_dir, scenarios, + self, build_fvk_client, check_transaction_equality, increase_height_and_wait_for_client, + paths::get_cargo_manifest_dir, scenarios, }; + use zingoconfig::{ChainType, RegtestNetwork, ZingoConfig, MAX_REORG}; use zingolib::{ check_client_balances, get_base_address, diff --git a/zingo-testutils/src/lib.rs b/zingo-testutils/src/lib.rs index f61f278bd..a51057d55 100644 --- a/zingo-testutils/src/lib.rs +++ b/zingo-testutils/src/lib.rs @@ -1,9 +1,5 @@ use grpc_proxy::ProxyServer; pub use incrementalmerkletree; -// #[cfg(features = "grpc-proxy")] -pub mod grpc_proxy; -pub mod regtest; - use std::collections::HashMap; use std::io::Read; use std::path::{Path, PathBuf}; @@ -24,6 +20,10 @@ use zingolib::lightclient::LightClient; use crate::scenarios::setup::TestEnvironmentGenerator; +pub mod grpc_proxy; +pub mod paths; +pub mod regtest; + pub fn build_fvks_from_wallet_capability(wallet_capability: &WalletCapability) -> [Fvk; 3] { let o_fvk = Fvk::Orchard( orchard::keys::FullViewingKey::try_from(wallet_capability) @@ -174,7 +174,7 @@ pub fn get_wallet_nym(nym: &str) -> Result<(String, PathBuf, PathBuf), String> { "sap_only" | "orch_only" | "orch_and_sapl" | "tadd_only" => { let one_sapling_wallet = format!( "{}/tests/data/wallets/v26/202302_release/regtest/{nym}/zingo-wallet.dat", - regtest::get_cargo_manifest_dir().to_string_lossy() + paths::get_cargo_manifest_dir().to_string_lossy() ); let wallet_path = Path::new(&one_sapling_wallet); let wallet_dir = wallet_path.parent().unwrap(); @@ -251,7 +251,7 @@ pub mod scenarios { REG_Z_ADDR_FROM_ABANDONART, }; - use super::super::regtest::get_regtest_dir; + use super::super::paths::get_regtest_dir; use super::{testvectors, ChildProcessHandler, RegtestManager}; use std::path::PathBuf; use tokio::time::sleep; diff --git a/zingo-testutils/src/paths.rs b/zingo-testutils/src/paths.rs new file mode 100644 index 000000000..c18e0828e --- /dev/null +++ b/zingo-testutils/src/paths.rs @@ -0,0 +1,17 @@ +use std::path::PathBuf; + +pub fn get_cargo_manifest_dir() -> PathBuf { + PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("To be inside a manifested space.")) +} + +pub fn get_regtest_dir() -> PathBuf { + get_cargo_manifest_dir().join("regtest") +} + +pub fn get_bin_dir() -> PathBuf { + let mut dir = get_cargo_manifest_dir(); + dir.pop(); + dir.join("zingo-testutils") + .join("test_binaries") + .join("bins") +} diff --git a/zingo-testutils/src/regtest.rs b/zingo-testutils/src/regtest.rs index fb2082a51..6732bc1fe 100644 --- a/zingo-testutils/src/regtest.rs +++ b/zingo-testutils/src/regtest.rs @@ -4,13 +4,6 @@ use std::io::Read; /// Simple helper to succinctly reference the project root dir. use std::path::PathBuf; use std::process::Child; -pub fn get_cargo_manifest_dir() -> PathBuf { - PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("To be inside a manifested space.")) -} - -pub fn get_regtest_dir() -> PathBuf { - get_cargo_manifest_dir().join("regtest") -} /// To manage the state associated a "regtest" run this type: /// * sets up paths to config and log directories @@ -178,7 +171,7 @@ impl RegtestManager { let regtest_dir = rootpathname; let confs_dir = regtest_dir.join("conf"); std::fs::create_dir_all(&confs_dir).expect("Couldn't create dir."); - let bin_dir = get_regtest_dir().join("bin"); + let bin_dir = super::paths::get_bin_dir(); std::fs::create_dir_all(&bin_dir).expect("Couldn't create dir."); let cli_bin = bin_dir.join("zcash-cli"); let logs_dir = regtest_dir.join("logs"); diff --git a/zingocli/regtest/bin/.gitinclude b/zingocli/regtest/bin/.gitinclude deleted file mode 100644 index 6a69f9202..000000000 --- a/zingocli/regtest/bin/.gitinclude +++ /dev/null @@ -1 +0,0 @@ -f diff --git a/zingocli/src/lib.rs b/zingocli/src/lib.rs index f940aaf62..662be73eb 100644 --- a/zingocli/src/lib.rs +++ b/zingocli/src/lib.rs @@ -338,7 +338,7 @@ to scan from the start of the blockchain." let data_dir = if let Some(dir) = matches.get_one::("data-dir") { PathBuf::from(dir.clone()) } else if is_regtest { - regtest::get_regtest_dir() + zingo_testutils::paths::get_regtest_dir() } else { PathBuf::from("wallets") };