tools: Block Generator - allow exporting to files

tools/block-generator/Makefile

@@ -1,41 +1,58 @@
-SCENARIO = scenarios/config.allmixed.small.yml
-SKIP = --skip-runner
+SCENARIO = scenarios/benchmarks/stress.50000.yml
+PG_UP = --start-postgres
 RESETDB = --reset-db
+TIMES = 1
 REPORTS = ../../tmp/RUN_RUNNER_OUTPUTS
 DURATION = 30s
 VERBOSE = --verbose
+CONDUIT = ./conduit
+TEMPLATE = # --template file-exporter (default postgres-exporter)
+PGUSER = algorand
+PGDB = generator_db
+PGCONT = "generator-test-container"
+PGCONN = "host=localhost user=$(PGUSER) password=algorand dbname=$(PGDB) port=15432 sslmode=disable"
+PGRUNNER = --postgres-connection-string $(PGCONN)
 
 block-generator: clean-generator
 	go build
 
 clean-generator:
 	rm -f block-generator
 
-debug-blockgen:
-	python scripts/run_runner.py \
-	--conduit-binary ./conduit \
-	--scenario $(SCENARIO) \
-	--report-directory $(REPORTS) \
-	--keep-alive $(SKIP) \
-	--test-duration $(DURATION) \
-	$(RESETDB)
+pg-up:
+	docker run --name $(PGCONT) -p 15432:5432 -e POSTGRES_USER=algorand -e POSTGRES_PASSWORD=algorand -d postgres
+	sleep 5
+	docker exec -it generator-test-container psql -Ualgorand -c "create database generator_db"
+
+pg-enter:
+	docker exec -it $(PGCONT) psql -U $(PGUSER) -d $(PGDB)
 
-enter-pg:
-	docker exec -it generator-test-container psql -U algorand -d generator_db
+QUERY := -c "select count(*) from txn;"
+pg-query:
+	psql $(PGCONN) $(QUERY)
 
-clean-docker:
-	docker rm -f generator-test-container
+pg-down:
+	docker rm -f $(PGCONT)
 
 run-runner: block-generator
-	./block-generator runner --conduit-binary ./conduit \
+	./block-generator runner --conduit-binary $(CONDUIT) \
 	--keep-data-dir \
 	--test-duration $(DURATION) \
 	--conduit-log-level trace \
-	--postgres-connection-string "host=localhost user=algorand password=algorand dbname=generator_db port=15432 sslmode=disable" \
+	$(TEMPLATE) \
+	$(PGRUNNER) \
 	--scenario $(SCENARIO) \
 	$(RESETDB) \
 	$(VERBOSE) \
 	--report-directory $(REPORTS)
+	--times $(TIMES)
+
+run-file-exporter:
+	make run-runner TEMPLATE="--template file-exporter" TIMES=1 RESETDB= PGRUNNER=
+
+BENCHMARK = "organic.25000"
+benchmark-blocks-export: block-generator
+	make run-file-exporter DURATION=60s SCENARIO=scenarios/benchmarks/$(BENCHMARK).yml REPORTS=$(BENCHMARK)
 
 clean-reports:
 	rm -rf $(REPORTS)

tools/block-generator/README.md

@@ -9,6 +9,7 @@ Several scenarios were designed to mimic different block traffic patterns. Scena
 ### Organic Traffic
 
 Simulate the current mainnet traffic pattern. Approximately:
+
 * 15% payment transactions
 * 10% application transactions
 * 75% asset transactions
@@ -33,7 +34,7 @@ Block generator uses a YAML config file to describe the composition of each rand
 
 The block generator supports **payment**, **asset**, and **application** transactions. The settings are hopefully, more or less, obvious. Distributions are specified as fractions of 1.0, and the sum of all options must add up to ~1.0.
 
-Here is an example which uses all of the current options. Notice that the synthetic blocks are not required to follow algod limits, in this case the block size is specified as 99,999:
+Here is an example which uses all of the current options. Notice that the synthetic blocks are not required to follow algod limits, and that in this case the block size is specified as 99,999:
 
 ```yml
 name: "Mixed (99,999)"
@@ -104,6 +105,7 @@ Flags:
   -c, --config string   Specify the block configuration yaml file.
   -h, --help            help for daemon
   -p, --port uint       Port to start the server at. (default 4010)
+  -v, --verbose         If set the daemon will print debugging information from the generator and ledger.
 ```
 
 ### runner
@@ -156,32 +158,33 @@ Flags:
   -i, --conduit-binary string               Path to conduit binary.
   -l, --conduit-log-level string            LogLevel to use when starting Conduit. [panic, fatal, error, warn, info, debug, trace] (default "error")
       --cpuprofile string                   Path where Conduit writes its CPU profile.
-  -f, --genesis-file string                 file path to the genesis associated with the db snapshot
+  -f, --genesis-file string                 The file path to the genesis associated with the db snapshot.
   -h, --help                                help for runner
   -k, --keep-data-dir                       If set the validator will not delete the data directory after tests complete.
   -p, --metrics-port uint                   Port to start the metrics server at. (default 9999)
   -c, --postgres-connection-string string   Postgres connection string.
-  -r, --report-directory string             Location to place test reports.
-      --reset-db                            If set database will be deleted before running tests.
+  -r, --report-directory string             Location to place test reports. If --times is used, this is the prefix for multiple report directories.
+      --reset-db                            If set the database will be deleted before running tests.
       --reset-report-dir                    If set any existing report directory will be deleted before running tests.
   -s, --scenario string                     Directory containing scenarios, or specific scenario file.
+      --start-delay duration                Duration to wait before starting a test scenario. This may be useful for snapshot tests where DB maintenance occurs after loading data.
+      --template string                     Specify the conduit template to use. Choices are: file-exporter or postgres-exporter. (default "postgres-exporter")
   -d, --test-duration duration              Duration to use for each scenario. (default 5m0s)
+  -t, --times uint                          The number of times to run the scenario(s). (default 1)
       --validate                            If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure.
   -v, --verbose                             If set the runner will print debugging information from the generator and ledger.
- ```
+```
 
-## Example Run using Conduit and Postgres
+## Example Runs using Conduit
 
 A typical **runner** scenario involves:
 
-* a [scenario configuration](#scenario-configuration) file, e.g. [test_config.yml](./test_config.yml)
-* access to a `conduit` binary to query the block generator's mock Algod endpoint and ingest the synthetic blocks
+* a [scenario configuration](#scenario-configuration) file, e.g. [config.asset.xfer.yml](./scenarios/config.asset.xfer.yml) or for the example below [test_scenario.yml](./generator/test_scenario.yml)
+* access to a `conduit` binary to query the block generator's mock Algod endpoint and ingest the synthetic blocks (below it's assumed to be set in the `CONDUIT_BINARY` environment variable)
 * a datastore -such as a postgres database- to collect `conduit`'s output
 * a `conduit` config file to define its import/export behavior
 
-The `block-generator runner` subcommand has a number of options to configure behavion.
-
-### Sample Run
+### Sample Run with Postgres
 
 First you'll need to get a `conduit` binary. For example you can follow the [developer portal's instructions](https://developer.algorand.org/docs/get-details/conduit/GettingStarted/#installation) or run `go build .` inside of the directory `cmd/conduit` after downloading the `conduit` repo.
 
@@ -204,5 +207,133 @@ block-generator runner \
 
 ### Scenario Report
 
-If all goes well, the run will generate a directory named reports.
+If all goes well, the run will generate a directory named `reports`
+in the same directory in which the command was run.
 In that directory you can see the statistics of the run in the file ending with `.report`.
+
+The `block-generator runner` subcommand has a number of options to configure behavior.
+
+## Sample Run with the File Exporter
+
+It's possible to save the generated blocks to the file system.
+This enables running benchmarks and stress tests at a later time and without
+needing a live block generator. The setup is very similar to the previous Postgres example. The main change compared to the previous is to _**specify a different conduit configuration**_ template.
+
+The `block-generator runner` command in this case would look like:
+
+```sh
+block-generator runner \
+  --conduit-binary "$CONDUIT_BINARY" \
+  --report-directory reports \
+  --test-duration 30s \
+  --conduit-log-level trace \
+  --template file-exporter \
+  --keep-data-dir \
+  --scenario generator/test_scenario.yml
+```
+
+### Generated Blocks
+
+If all goes well, the run will generate a directory named `reports`
+in the same directory in which the command was run.
+In addition to the statistical report and run logs,
+there will be a directory ending with `_data` - this is conduit's
+data directory (which is saved thanks to the `--keep-data-dir` flag).
+In that directory under `exporter_file_writer/`
+the generated blocks and a genesis file will be saved.
+
+## Scenario Distribution - Configuration vs. Reality
+
+This section follows up on the [Scenario Configuration](#scenario-configuration) section to detail how each kind of transaction is actually chosen.
+Note that -especially for early rounds- there is no guarantee that the
+percentages of transaction types will resemble the configured distribution.
+
+For example consider the [Organic 25,000](scenarios/benchmarks/organic.25000.yml) scenario:
+
+```yml
+name: "Organic (25000)"
+genesis_accounts: 10000
+genesis_account_balance: 1000000000000
+tx_per_block: 25000
+
+# transaction distribution
+tx_pay_fraction: 0.05
+tx_asset_fraction: 0.75
+tx_app_fraction: 0.20
+
+# payment config
+pay_acct_create_fraction: 0.10
+pay_xfer_fraction: 0.90
+
+# asset config
+asset_create_fraction: 0.001
+asset_optin_fraction: 0.1
+asset_close_fraction: 0.05
+asset_xfer_fraction: 0.849
+asset_delete_fraction: 0
+
+# app kind config
+app_boxes_fraction: 1.0
+app_swap_fraction: 0.0
+
+# app boxes config
+app_boxes_create_fraction: 0.01
+app_boxes_optin_fraction: 0.1
+app_boxes_call_fraction: 0.89
+```
+
+We are _actually_ asking the generator for the following distribution:
+
+* `pay_acct_create_fraction = 0.005 (= 0.05 * 0.10)`
+* `pay_xfer_fraction =  0.045 (= 0.05 * 0.90)`
+* `asset_create_fraction = 0.00075 (= 0.75 * 0.001)`
+* `asset_optin_fraction = 0.075 (= 0.75 * 0.1)`
+* `asset_close_fraction = 0.0375 (= 0.75 * 0.05)`
+* `asset_xfer_fraction = 0.63675 (= 0.75 * 0.849)`
+* `asset_delete_fraction = 0`
+* `app_boxes_create_fraction = 0.002 (= 0.20 * 1.0 * 0.01)`
+* `app_boxes_optin_fraction = 0.02 (= 0.20 * 1.0 * 0.1)`
+* `app_boxes_call_fraction = 0.178 (= 0.20 * 1.0 * 0.89)`
+
+The block generator randomly chooses
+
+1. the transaction type (pay, asset, or app) according to the `transaction distribution`
+2. based on the type:
+
+   a. for payments and assets, the specific type based on the `payment config` and `asset config` distributions
+
+   b. for apps, the app kind (boxes or swaps) based on the `app kind config` distribution
+
+3. For _apps only_: the specific app call based on the `app boxes config` (and perhaps in the future `app swap config`)
+
+As each of the steps above is itself random, we only expect _approximate matching_ to the configured distribution.
+
+Furthermore, for certain asset and app transactions there may be a substitution that occurs based on the type. In particular:
+
+* for **assets**:
+  * when a requested asset txn is **create**, it is never substituted
+  * when there are no assets, an **asset create** is always substituted
+  * when a requested asset txn is **delete** but the creator doesn't hold all asset funds, an **asset close** is substitued (which itself may be substituted using the **close** rule below)
+  * when a requested asset txn is **opt in** but all accounts are already opted in, an  **asset close** is substituted (which itself may be substituted using the **close** rule below)
+  * when a requested asset txn is **transfer** but there is only one account holding it,  an **asset opt in** is substituted (which itself may be substituted using the **asset opt in** rule above)
+  * when a requested asset txn is **close** but there is only one account holding it, an **asset opt in** is substituted (which itself may be substituted using the **asset opt in** rule above)
+* for **apps**:
+  * when a requested app txn is **create**, it is never substituted
+  * when a requested app txn is **opt in**:
+    * if the sender is already opted in, an **app call** is substituted
+    * otherwise, if the sender's opt-in is pending for the round, an **app create** is substituted
+  * when a requested app txn is **call** but it's not opted into, an **app opt in** is attempted to be substituted (but this may itself be substituted for given the **app opt in** rule above)
+
+Over time, we expect the state of the generator to stabilize so that very few substitutions occur. However, especially for the first few rounds, there may be drastic differences between the config distribution and observed percentages.
+
+In particular:
+
+* for Round 1, all app transactions are replaced by **app create**
+* for Round 2, all **app call** transactions are replaced by **app opt in**
+
+Therefore, for scenarios involving a variety of app transactions, only for Round 3 and higher do we expect to see distributions comparable to those configured.
+
+> NOTE: Even in the steady state, we still expect fundamental deviations 
+> from the configured distributions in the cases of apps. This is because
+> an app call may have associated group and inner transactions. For example,
+> if an app call requires 1 sibling asset call in it's group and has 2 inner payments, this single app call will generate 1 additional asset txn and 2 payment txns.

tools/block-generator/generator/daemon.go

@@ -48,7 +48,7 @@ func init() {
 
 	DaemonCmd.Flags().StringVarP(&configFile, "config", "c", "", "Specify the block configuration yaml file.")
 	DaemonCmd.Flags().Uint64VarP(&port, "port", "p", 4010, "Port to start the server at.")
-	DaemonCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "If set the runner will print debugging information from the generator and ledger.")
+	DaemonCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "If set the daemon will print debugging information from the generator and ledger.")
 
 	DaemonCmd.MarkFlagRequired("config")
 }

tools/block-generator/generator/server.go

@@ -75,28 +75,30 @@ func help(w http.ResponseWriter, r *http.Request) {
 	fmt.Fprintf(w, "Use /v2/blocks/:blocknum: to get a block.")
 }
 
-func maybeWriteError(w http.ResponseWriter, err error) {
+func maybeWriteError(handler string, w http.ResponseWriter, err error) {
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+		msg := fmt.Sprintf("%s handler: error encountered while writing response for: %v\n", handler, err)
+		fmt.Println(msg)
+		http.Error(w, msg, http.StatusInternalServerError)
 		return
 	}
 }
 
 func getReportHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
-		maybeWriteError(w, gen.WriteReport(w))
+		maybeWriteError("report", w, gen.WriteReport(w))
 	}
 }
 
 func getStatusWaitHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
-		maybeWriteError(w, gen.WriteStatus(w))
+		maybeWriteError("status wait", w, gen.WriteStatus(w))
 	}
 }
 
 func getGenesisHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
-		maybeWriteError(w, gen.WriteGenesis(w))
+		maybeWriteError("genesis", w, gen.WriteGenesis(w))
 	}
 }
 
@@ -113,7 +115,7 @@ func getBlockHandler(gen Generator) func(w http.ResponseWriter, r *http.Request)
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-		maybeWriteError(w, gen.WriteBlock(w, round))
+		maybeWriteError("block", w, gen.WriteBlock(w, round))
 	}
 }
 
@@ -125,7 +127,7 @@ func getAccountHandler(gen Generator) func(w http.ResponseWriter, r *http.Reques
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-		maybeWriteError(w, gen.WriteAccount(w, account))
+		maybeWriteError("account", w, gen.WriteAccount(w, account))
 	}
 }
 
@@ -141,7 +143,7 @@ func getDeltasHandler(gen Generator) func(w http.ResponseWriter, r *http.Request
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-		maybeWriteError(w, gen.WriteDeltas(w, round))
+		maybeWriteError("deltas", w, gen.WriteDeltas(w, round))
 	}
 }