Merge pull request #784 from AloeareV/binary_location_stabalized

Binary location stabilized

.github/workflows/test.yaml

@@ -58,11 +58,11 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Symlink lightwalletd and zcash binaries for darkside-tests
-        run: ln -s /usr/bin/lightwalletd ./darkside-tests/lightwalletd_bin/
+      - name: create binaries dir
+        run: mkdir -p ./zingo-testutils/test_binaries/bins
 
-      - name: Symlink lightwalletd and zcash binaries for integration-tests
-        run: ln -s /usr/bin/lightwalletd /usr/bin/zcashd /usr/bin/zcash-cli ./integration-tests/regtest/bin/
+      - name: Symlink lightwalletd and zcash binaries
+        run: ln -s /usr/bin/lightwalletd /usr/bin/zcashd /usr/bin/zcash-cli ./zingo-testutils/test_binaries/bins/
 
       - name: Symlink zcash parameters
         run: ln -s /root/.zcash-params /github/home

.gitignore

@@ -1,26 +1,22 @@
-zingocli/regtest/data/zcashd/regtest/
-zingocli/regtest/data/lightwalletd/
-zingocli/regtest/data/zingo/
+!zingolib/test-data/README
+!zingolib/test-data/openssl_cfg
+*.json.txt
+.DS_Store
+/procgov
+/testdata
+cobertura.xml
+docker-ci/lightwalletd
+target
+wallets/*
 zingocli/regtest/logs/
 zingocli/regtest/zingo-wallet.dat
-zingocli/regtest/bin
-integration-tests/regtest/bin
-target
-zingolib/target
 zingocli/target
-/testdata
-/procgov
-.DS_Store
-*.json.txt
-docker-ci/lightwalletd
-zingolib/test-data/*
-!zingolib/test-data/README
-!zingolib/test-data/openssl_cfg
+zingocli/tests/data/basic_zcashd.conf
 zingocli/tests/data/externalwallet_coinbaseaddress.conf
 zingocli/tests/data/lightwalletd.yml
-zingocli/tests/data/basic_zcashd.conf
-cobertura.xml
 zingocli/tests/times/*
-wallets/*
 zingocli/wallets/*
-darkside-tests/lightwalletd_bin/
+zingolib/target
+zingolib/test-data/*
+zingo-testutils/test_binaries
+

darkside-tests/src/utils.rs

@@ -23,7 +23,8 @@ use zcash_primitives::{merkle_tree::read_commitment_tree, transaction::Transacti
 use zingo_testutils::{
     self,
     incrementalmerkletree::frontier::CommitmentTree,
-    regtest::{get_cargo_manifest_dir, launch_lightwalletd},
+    paths::{get_bin_dir, get_cargo_manifest_dir},
+    regtest::launch_lightwalletd,
     scenarios::setup::TestEnvironmentGenerator,
 };
 use zingolib::wallet::traits::DomainWalletExt;
@@ -251,7 +252,7 @@ impl DarksideHandler {
             darkside_dir.join("logs"),
             darkside_dir.join("conf"),
             darkside_dir.join("data"),
-            get_cargo_manifest_dir().join("lightwalletd_bin"),
+            get_bin_dir(),
             check_interval,
             grpc_bind_addr,
         );
@@ -430,7 +431,7 @@ pub async fn init_darksidewalletd(
 pub fn create_chainbuild_file(test_name: &str) -> File {
     let path = format!(
         "{}/tests/data/chainbuilds/{}",
-        zingo_testutils::regtest::get_cargo_manifest_dir().to_string_lossy(),
+        get_cargo_manifest_dir().to_string_lossy(),
         test_name
     );
     match fs::create_dir(path.clone()) {
@@ -453,7 +454,7 @@ pub fn create_chainbuild_file(test_name: &str) -> File {
 pub fn load_chainbuild_file(test_name: &str) -> Vec<String> {
     let path = format!(
         "{}/tests/data/chainbuilds/{}",
-        zingo_testutils::regtest::get_cargo_manifest_dir().to_string_lossy(),
+        get_cargo_manifest_dir().to_string_lossy(),
         test_name
     );
     let filename = "hex_transactions.txt";

darkside-tests/tests/advanced_reorg_tests.rs

@@ -12,7 +12,7 @@ use darkside_tests::{
 
 use tokio::time::sleep;
 use zcash_primitives::consensus::BlockHeight;
-use zingo_testutils::{regtest::get_cargo_manifest_dir, scenarios::setup::ClientBuilder};
+use zingo_testutils::{paths::get_cargo_manifest_dir, scenarios::setup::ClientBuilder};
 use zingoconfig::RegtestNetwork;
 use zingolib::lightclient::PoolBalances;
 use zingolib::wallet::data::summaries::ValueTransferKind;

docs/testing/ZINGOCLI-REGTEST.md

@@ -0,0 +1,153 @@
+`G=$(git rev-parse --show-toplevel)`
+
+## Regtest Mode
+WARNING Experimental!
+Zingocli has a set of tests based on zcashd regtest mode, by locally running a `zcashd` and `lightwalletd`.
+This is now working with a simple `zingo-cli` invocation flag.
+
+There are pre-made directories in this repo to support ready use of regtest mode. These are found in the `$G/zingocli/regtest/` subdirectory.
+
+There are default config files for these binaries already in place in `$G/zingocli/regtest/conf/` which can also be edited.
+
+Because regtest mode has no ability to cope with an initial `zcashd` state without any blocks,
+we have included files to produce an initial block height of one, with no variation between runs.
+These files are copied from a 'passive' directory (`$G/zingocli/regtest/data/regtestvectors/`)
+into a newly cleared 'active' data directory at the beginning of each time regtest mode is run.
+This means, by default, any blocks added while zcashd is running are not retained for subsequent runs.
+
+The default config includes all network upgrades set to block height 1, therefore all network upgrades are active by default in regtest mode.
+
+# Usage example:
+You must copy your compiled `zcashd`, `zcash-cli` and `lightwalletd` binaries to `$G/zingocli/regtest/bin/` or set up symlinks, etc. `zcash-cli` is also needed if you wish
+to interact with your `zcashd` instance while it is running.
+
+From your `$G/` directory, you can run:
+`cargo run --release -- --regtest`
+
+Note: Regtest mode only works when invoked with `cargo run`. Running `cargo build` followed by an invocation of the compiled binary will fail.
+
+This will start an interactive session. Individual commands can be passed to zingolib (via the cli), for example:
+
+`cargo run --release -- --regtest help`
+
+This will start `zcashd` and `lightwalletd` and then connect to these tools with an interactive `zingo-cli`.
+It currently takes a few seconds to do so, even on a fast machine, to give the daemons time to boot.
+
+These daemons will be killed when the user exits `zingo-cli` using the `quit` command.
+However, if there is an issue starting or shutting down regtest mode, it's possible you will have to shut down the daemons manually.
+
+You should see several diagnostic messages, and then:
+`regtest detected and network set correctly!
+Lightclient connecting to http://127.0.0.1:9067/`
+at which point the interactive cli application should work with your regtest network.
+
+`zcashd`'s stdout logfile should quickly have an output of several dozen lines, and show network upgrade activation parameters at `height=1`.
+`lightwalletd`'s stdout log file will show something like:
+`{"app":"lightwalletd","level":"info","msg":"Got sapling height 1 block height 1 chain regtest branchID ..."}`
+...which you can view with `tail -f` or your favorite tool.
+
+Once regtest mode is running, you can manipulate the simulated chain with `zcash-cli`.
+
+For example, in still another terminal instance in the `$G/zingocli/regtest/bin/` directory, you can run
+`./zcash-cli -regtest -rpcuser=xxxxxx -rpcpassword=xxxxxx generate 11` to generate 11 blocks.
+Please note that by adding more than 100 blocks it is difficult or impossible to rewind the chain. The config means that after the first block all network upgrades should be in place.
+Other `zcash-cli` commands should work similarly.
+
+Invocation currently only works when being launched within a `zingolib` repo's worktree
+(The paths have to know where to look for the subdirectories, they start with the top level of a `zingolib` repo, or fail immediately).
+
+Have fun!
+
+# Tree Diagrams
+In `$G/zingocli`, running `tree ./regtest`
+after moving binaries and running:
+./regtest/
+├── bin
+│   ├── lightwalletd
+│   ├── zcash-cli
+│   └── zcashd
+├── conf
+│   ├── lightwalletd.yml
+│   └── zcash.conf
+├── data
+│   ├── lightwalletd
+│   │   └── db
+│   │       └── regtest
+│   │           ├── blocks
+│   │           └── lengths
+│   ├── regtestvectors
+│   │   └── regtest
+│   │       ├── banlist.dat
+│   │       ├── blocks
+│   │       │   ├── blk00000.dat
+│   │       │   ├── index
+│   │       │   │   ├── 000005.ldb
+│   │       │   │   ├── 000008.ldb
+│   │       │   │   ├── 000009.log
+│   │       │   │   ├── CURRENT
+│   │       │   │   ├── LOCK
+│   │       │   │   ├── LOG
+│   │       │   │   ├── LOG.old
+│   │       │   │   └── MANIFEST-000007
+│   │       │   └── rev00000.dat
+│   │       ├── chainstate
+│   │       │   ├── 000005.ldb
+│   │       │   ├── 000008.ldb
+│   │       │   ├── 000009.log
+│   │       │   ├── CURRENT
+│   │       │   ├── LOCK
+│   │       │   ├── LOG
+│   │       │   ├── LOG.old
+│   │       │   └── MANIFEST-000007
+│   │       ├── database
+│   │       │   └── log.0000000001
+│   │       ├── db.log
+│   │       ├── fee_estimates.dat
+│   │       ├── peers.dat
+│   │       └── wallet.dat
+│   ├── zcashd
+│   │   └── regtest
+│   │       ├── banlist.dat
+│   │       ├── blocks
+│   │       │   ├── blk00000.dat
+│   │       │   ├── index
+│   │       │   │   ├── 000005.ldb
+│   │       │   │   ├── 000008.ldb
+│   │       │   │   ├── 000011.ldb
+│   │       │   │   ├── 000012.log
+│   │       │   │   ├── CURRENT
+│   │       │   │   ├── LOCK
+│   │       │   │   ├── LOG
+│   │       │   │   ├── LOG.old
+│   │       │   │   └── MANIFEST-000010
+│   │       │   └── rev00000.dat
+│   │       ├── chainstate
+│   │       │   ├── 000005.ldb
+│   │       │   ├── 000008.ldb
+│   │       │   ├── 000011.ldb
+│   │       │   ├── 000012.log
+│   │       │   ├── CURRENT
+│   │       │   ├── LOCK
+│   │       │   ├── LOG
+│   │       │   ├── LOG.old
+│   │       │   └── MANIFEST-000010
+│   │       ├── database
+│   │       │   └── log.0000000001
+│   │       ├── db.log
+│   │       ├── fee_estimates.dat
+│   │       ├── peers.dat
+│   │       ├── wallet.dat
+│   │       └── zcashd.pid
+│   └── zingo
+│       ├── zingo-wallet.dat
+│       └── zingo-wallet.debug.log
+├── logs
+│   ├── lightwalletd
+│   │   ├── stderr.log
+│   │   └── stdout.log
+│   └── zcashd
+│       └── stdout.log
+└── README.md
+
+# Working Commits
+Tested with `zcash` commit `d6d209`, `lightwalletd` commit `f53511c`, and `zingolib` commit `89fbe118f8305051d5f1d1d95903ba3ccaec586b` or better.

integration-tests/tests/integrations.rs

@@ -15,9 +15,10 @@ use zcash_primitives::{
     transaction::{fees::zip317::MINIMUM_FEE, TxId},
 };
 use zingo_testutils::{
-    build_fvk_client, check_transaction_equality, increase_height_and_wait_for_client,
-    regtest::get_cargo_manifest_dir, scenarios,
+    self, build_fvk_client, check_transaction_equality, increase_height_and_wait_for_client,
+    paths::get_cargo_manifest_dir, scenarios,
 };
+
 use zingoconfig::{ChainType, RegtestNetwork, ZingoConfig, MAX_REORG};
 use zingolib::{
     check_client_balances, get_base_address,

zingo-testutils/src/lib.rs

@@ -1,9 +1,5 @@
 use grpc_proxy::ProxyServer;
 pub use incrementalmerkletree;
-// #[cfg(features = "grpc-proxy")]
-pub mod grpc_proxy;
-pub mod regtest;
-
 use std::collections::HashMap;
 use std::io::Read;
 use std::path::{Path, PathBuf};
@@ -24,6 +20,10 @@ use zingolib::lightclient::LightClient;
 
 use crate::scenarios::setup::TestEnvironmentGenerator;
 
+pub mod grpc_proxy;
+pub mod paths;
+pub mod regtest;
+
 pub fn build_fvks_from_wallet_capability(wallet_capability: &WalletCapability) -> [Fvk; 3] {
     let o_fvk = Fvk::Orchard(
         orchard::keys::FullViewingKey::try_from(wallet_capability)
@@ -174,7 +174,7 @@ pub fn get_wallet_nym(nym: &str) -> Result<(String, PathBuf, PathBuf), String> {
         "sap_only" | "orch_only" | "orch_and_sapl" | "tadd_only" => {
             let one_sapling_wallet = format!(
                 "{}/tests/data/wallets/v26/202302_release/regtest/{nym}/zingo-wallet.dat",
-                regtest::get_cargo_manifest_dir().to_string_lossy()
+                paths::get_cargo_manifest_dir().to_string_lossy()
             );
             let wallet_path = Path::new(&one_sapling_wallet);
             let wallet_dir = wallet_path.parent().unwrap();
@@ -251,7 +251,7 @@ pub mod scenarios {
             REG_Z_ADDR_FROM_ABANDONART,
         };
 
-        use super::super::regtest::get_regtest_dir;
+        use super::super::paths::get_regtest_dir;
         use super::{testvectors, ChildProcessHandler, RegtestManager};
         use std::path::PathBuf;
         use tokio::time::sleep;

zingo-testutils/src/paths.rs

@@ -0,0 +1,17 @@
+use std::path::PathBuf;
+
+pub fn get_cargo_manifest_dir() -> PathBuf {
+    PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("To be inside a manifested space."))
+}
+
+pub fn get_regtest_dir() -> PathBuf {
+    get_cargo_manifest_dir().join("regtest")
+}
+
+pub fn get_bin_dir() -> PathBuf {
+    let mut dir = get_cargo_manifest_dir();
+    dir.pop();
+    dir.join("zingo-testutils")
+        .join("test_binaries")
+        .join("bins")
+}

zingo-testutils/src/regtest.rs

@@ -4,13 +4,6 @@ use std::io::Read;
 ///  Simple helper to succinctly reference the project root dir.
 use std::path::PathBuf;
 use std::process::Child;
-pub fn get_cargo_manifest_dir() -> PathBuf {
-    PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("To be inside a manifested space."))
-}
-
-pub fn get_regtest_dir() -> PathBuf {
-    get_cargo_manifest_dir().join("regtest")
-}
 
 ///  To manage the state associated a "regtest" run this type:
 ///   * sets up paths to config and log directories
@@ -178,7 +171,7 @@ impl RegtestManager {
         let regtest_dir = rootpathname;
         let confs_dir = regtest_dir.join("conf");
         std::fs::create_dir_all(&confs_dir).expect("Couldn't create dir.");
-        let bin_dir = get_regtest_dir().join("bin");
+        let bin_dir = super::paths::get_bin_dir();
         std::fs::create_dir_all(&bin_dir).expect("Couldn't create dir.");
         let cli_bin = bin_dir.join("zcash-cli");
         let logs_dir = regtest_dir.join("logs");

zingocli/src/lib.rs

@@ -338,7 +338,7 @@ to scan from the start of the blockchain."
         let data_dir = if let Some(dir) = matches.get_one::<String>("data-dir") {
             PathBuf::from(dir.clone())
         } else if is_regtest {
-            regtest::get_regtest_dir()
+            zingo_testutils::paths::get_regtest_dir()
         } else {
             PathBuf::from("wallets")
         };