Ab/democracy bots (#370)

* random voting in bot-community

* democracy bot fixes

* fmt

* cosmetics

* add cli export-secret and help tester with the mnemonic of an account which will skip voting. randomize turnout too

* fix ci

* fix ci

* fix ci^3

client/Cargo.toml

@@ -7,6 +7,7 @@ version = "1.12.0"
 
 [dependencies]
 # todo migrate to clap >=3 https://github.com/encointer/encointer-node/issues/107
+array-bytes = "6.2.2"
 chrono = "0.4.35"
 clap = "2.33"
 clap-nested = "0.4.0"

client/README.md

@@ -156,3 +156,11 @@ create a proposal:
 ```
 RUST_LOG=info ../target/release/encointer-client-notee -u wss://rococo.api.encointer.org -p 443 new-community test-data/leu.rococo.json
 ```
+
+## Logging
+
+A reasonably verbose log:
+
+```bash
+export RUST_LOG=debug,substrate_api_client=warn,ws=warn,mio=warn,ac_node_api=warn,sp_io=warn,tungstenite=warn,rustls=info,soketto=info
+```

client/bootstrap_demo_community.py

@@ -43,7 +43,7 @@ def check_participant_count(client, cid, type, number):
 def check_reputation(client, cid, account, cindex, reputation):
     rep = client.reputation(account)
     print(rep)
-    if (str(cindex), f" {cid}", reputation) not in rep:
+    if (str(cindex), cid, reputation) not in rep:
         print(f"🔎 Reputation for {account} in cid {cid} cindex {cindex} is not {reputation}")
         exit(1)
 
@@ -158,9 +158,9 @@ def test_reputation_caching(client, cid):
     # check if the reputation cache was updated
     rep = client.reputation(account1)
     print(rep)
-    if ('1', ' sqm1v79dF6b', 'VerifiedLinked(2)') not in rep or (
-            '2', ' sqm1v79dF6b', 'VerifiedLinked(3)') not in rep or (
-            '3', ' sqm1v79dF6b', 'VerifiedUnlinked') not in rep:
+    if ('1', 'sqm1v79dF6b', 'VerifiedLinked(2)') not in rep or (
+            '2', 'sqm1v79dF6b', 'VerifiedLinked(3)') not in rep or (
+            '3', 'sqm1v79dF6b', 'VerifiedUnlinked') not in rep:
         print("wrong reputation")
         exit(1)
 
@@ -175,7 +175,7 @@ def test_reputation_caching(client, cid):
     rep = client.reputation(account1)
     print(rep)
     # after the registration the second reputation should now be linked
-    if ('3', ' sqm1v79dF6b', 'VerifiedLinked(4)') not in rep:
+    if ('3', 'sqm1v79dF6b', 'VerifiedLinked(4)') not in rep:
         print("reputation not linked")
         exit(1)
 
@@ -404,7 +404,7 @@ def test_democracy(client, cid):
 @click.command()
 @click.option('--client', default='../target/release/encointer-client-notee',
               help='Client binary to communicate with the chain.')
-@click.option('--signer', default='//Bob', help='optional account keypair creating the community')
+@click.option('--signer', help='optional account keypair creating the community')
 @click.option('-u', '--url', default='ws://127.0.0.1', help='URL of the chain.')
 @click.option('-p', '--port', default='9944', help='ws-port of the chain.')
 @click.option('-l', '--ipfs-local', is_flag=True, help='if set, local ipfs node is used.')

client/bot-community.py

@@ -32,7 +32,7 @@
 
 import click
 import ast
-
+import random
 from math import floor
 
 from py_client.communities import random_community_spec, COMMUNITY_SPECS_PATH
@@ -70,6 +70,7 @@ def init(ctx):
     purge_keystore_prompt()
 
     root_dir = os.path.realpath(ASSETS_PATH)
+    ipfs_cid = "QmDUMMYikh7VqTu8pvzd2G2vAd4eK7EaazXTEgqGN6AWoD"
     try:
         ipfs_cid = Ipfs.add_recursive(root_dir, ctx['ipfs_local'])
     except:
@@ -98,6 +99,7 @@ def init(ctx):
 def purge_communities():
     purge_prompt(COMMUNITY_SPECS_PATH, 'communities')
 
+
 @cli.command()
 @click.pass_obj
 def execute_current_phase(ctx):
@@ -108,17 +110,19 @@ def _execute_current_phase(client: Client):
     client = client
     cid = read_cid()
     phase = client.get_phase()
-    print(f'phase is {phase}')
+    cindex = client.get_cindex()
+    print(f'🕑 phase is {phase} and ceremony index is {cindex}')
     accounts = client.list_accounts()
     print(f'number of known accounts: {len(accounts)}')
     if phase == 'Registering':
-        print("all participants claim their potential reward")
+        print("🏆 all participants claim their potential reward")
         for account in accounts:
             client.claim_reward(account, cid)
         client.await_block(3)
 
-        total_supply = write_current_stats(client, accounts, cid)
+        update_proposal_states(client, accounts[0])
 
+        total_supply = write_current_stats(client, accounts, cid)
         if total_supply > 0:
             init_new_community_members(client, cid, len(accounts))
 
@@ -131,10 +135,14 @@ def _execute_current_phase(client: Client):
     if phase == "Assigning":
         meetups = client.list_meetups(cid)
         meetup_sizes = list(map(lambda x: len(x), meetups))
-        print(f'meetups assigned for {sum(meetup_sizes)} participants with sizes: {meetup_sizes}')
+        print(f'🔎 meetups assigned for {sum(meetup_sizes)} participants with sizes: {meetup_sizes}')
+        update_proposal_states(client, accounts[0])
+        submit_democracy_proposals(client, cid, accounts[0])
     if phase == 'Attesting':
         meetups = client.list_meetups(cid)
-        print(f'****** Performing {len(meetups)} meetups')
+        update_proposal_states(client, accounts[0])
+        vote_on_proposals(client, cid, accounts)
+        print(f'🫂 Performing {len(meetups)} meetups')
         for meetup in meetups:
             perform_meetup(client, meetup, cid)
         client.await_block()
@@ -158,7 +166,7 @@ def benchmark(ctx):
 def test(ctx):
     py_client = ctx['client']
     print('will grow population for fixed number of ceremonies')
-    for i in range(3*2+1):
+    for i in range(3 * 2 + 1):
         phase = _execute_current_phase(py_client)
         while phase == py_client.get_phase():
             print("awaiting next phase...")
@@ -219,7 +227,7 @@ def endorse_new_accounts(client: Client, cid: str, bootstrappers_and_tickets, en
     start = 0
     for endorser, endorsement_count in endorsers_and_tickets:
         # execute endorsements per bootstrapper
-        end = start+endorsement_count
+        end = start + endorsement_count
 
         print(f'bootstrapper {endorser} endorses {endorsement_count} accounts.')
 
@@ -271,7 +279,6 @@ def init_new_community_members(client: Client, cid: str, current_community_size:
         client.await_block()
         print(f'Added endorsees to community: {len(endorsees)}')
 
-
     newbies = client.create_accounts(get_newbie_amount(current_community_size + len(endorsees)))
 
     print(f'Add newbies to community {len(newbies)}')
@@ -305,7 +312,10 @@ def register_participants(client: Client, accounts, cid):
         client.await_block()
 
         for p in need_refunding:
-            client.register_participant(p, cid)
+            try:
+                client.register_participant(p, cid)
+            except ExtrinsicFeePaymentImpossible:
+                print("refunding failed")
 
 
 def perform_meetup(client: Client, meetup, cid):
@@ -318,5 +328,52 @@ def perform_meetup(client: Client, meetup, cid):
         client.attest_attendees(attestor, cid, attendees)
 
 
+def submit_democracy_proposals(client: Client, cid: str, proposer: str):
+    print("submitting new democracy proposals")
+    client.submit_update_nominal_income_proposal(proposer, 1.1, cid)
+
+
+def vote_on_proposals(client: Client, cid: str, voters: list):
+    proposals = client.get_proposals()
+    for proposal in proposals:
+        print(
+            f"checking proposal {proposal.id}, state: {proposal.state}, approval: {proposal.approval} turnout: {proposal.turnout}")
+        if proposal.state == 'Ongoing' and proposal.turnout == 0:
+            choices = ['aye', 'nay']
+            target_approval = random.random()
+            target_turnout = random.random()
+            print(
+                f"🗳 voting on proposal {proposal.id} with target approval of {target_approval * 100}% and target turnout of {target_turnout * 100}%")
+            weights = [target_approval, 1 - target_approval]
+            try:
+                active_voters = voters[0:round(len(voters) * target_turnout)]
+                print(f"will attempt to vote with {len(active_voters) - 1} accounts")
+                is_first_voter_with_rep = True
+                for voter in active_voters:
+                    reputations = [[t[1], t[0]] for t in client.reputation(voter)]
+                    if len(reputations) == 0:
+                        print(f"no reputations for {voter}. can't vote")
+                        continue
+                    if is_first_voter_with_rep:
+                        print(f"👉 will not vote with {voter}: mnemonic: {client.export_secret(voter)}")
+                        is_first_voter_with_rep = False
+                    vote = random.choices(choices, weights)[0]
+                    print(f"voting {vote} on proposal {proposal.id} with {voter} and reputations {reputations}")
+                    client.vote(voter, proposal.id, vote, reputations)
+            except:
+                print(f"voting failed")
+        client.await_block()
+
+
+def update_proposal_states(client: Client, who: str):
+    proposals = client.get_proposals()
+    for proposal in proposals:
+        print(
+            f"checking proposal {proposal.id}, state: {proposal.state}, approval: {proposal.approval} turnout: {proposal.turnout}")
+        if proposal.state in ['Ongoing', 'Confirming']:
+            print(f"updating proposal {proposal.id}")
+            client.update_proposal_state(who, proposal.id)
+
+
 if __name__ == '__main__':
     cli(obj={})

client/faucet.py

@@ -14,14 +14,13 @@
 from time import sleep
 from py_client.client import Client
 
-
 app = flask.Flask(__name__)
 app.config['DEBUG'] = True
 CLIENT = Client()
 
 
 def faucet(accounts):
-    for x in range(0, 180):  # try 100 times
+    for x in range(0, 1):  # try multiple
         try:
             CLIENT.faucet(accounts, is_faucet=True)
             CLIENT.await_block()  # wait for transaction to complete
@@ -50,5 +49,5 @@ def faucet_service():
     else:
         return "no accounts provided to drip to\n"
 
-app.run()
 
+app.run()

client/py_client/client.py

@@ -3,6 +3,7 @@
 import os
 
 from py_client.scheduler import CeremonyPhase
+from py_client.democracy import parse_proposals
 
 DEFAULT_CLIENT = '../target/release/encointer-client-notee'
 
@@ -105,6 +106,10 @@ def new_account(self):
         ret = self.run_cli_command(["new-account"])
         return ret.stdout.decode("utf-8").strip()
 
+    def export_secret(self, account):
+        ret = self.run_cli_command(["export-secret", account])
+        return ret.stdout.decode("utf-8").strip()
+
     def create_accounts(self, amount):
         return [self.new_account() for _ in range(0, amount)]
 
@@ -117,7 +122,10 @@ def faucet(self, accounts, faucet_url='http://localhost:5000/api', is_faucet=Fal
             ensure_clean_exit(ret)
         else:
             payload = {'accounts': accounts}
-            requests.get(faucet_url, params=payload)
+            try:
+                requests.get(faucet_url, params=payload, timeout=20)
+            except requests.exceptions.Timeout:
+                print("faucet timeout")
 
     def balance(self, account, cid=None):
         ret = self.run_cli_command(["balance", account], cid=cid)
@@ -129,7 +137,7 @@ def reputation(self, account):
         reputation_history = []
         lines = ret.stdout.decode("utf-8").splitlines()
         while len(lines) > 0:
-            (cindex, cid, rep) = lines.pop(0).split(',')
+            (cindex, cid, rep) = [item.strip() for item in lines.pop(0).split(',')]
             reputation_history.append(
                 (cindex, cid, rep.strip().split('::')[1]))
         return reputation_history
@@ -285,6 +293,11 @@ def submit_set_inactivity_timeout_proposal(self, account, inactivity_timeout, ci
                                    pay_fees_in_cc)
         return ret.stdout.decode("utf-8").strip()
 
+    def submit_update_nominal_income_proposal(self, account, new_income, cid=None, pay_fees_in_cc=False):
+        ret = self.run_cli_command(["submit-update-nominal-income-proposal", account, str(new_income)], cid,
+                                   pay_fees_in_cc)
+        return ret.stdout.decode("utf-8").strip()
+
     def vote(self, account, proposal_id, vote, reputations, cid=None, pay_fees_in_cc=False):
         reputations = [f'{cid}_{cindex}' for [cid, cindex] in reputations]
         reputation_vec = ','.join(reputations)
@@ -298,3 +311,6 @@ def update_proposal_state(self, account, proposal_id, cid=None, pay_fees_in_cc=F
     def list_proposals(self):
         ret = self.run_cli_command(["list-proposals"])
         return ret.stdout.decode("utf-8").strip()
+
+    def get_proposals(self):
+        return parse_proposals(self.list_proposals())

client/py_client/democracy.py

@@ -0,0 +1,39 @@
+import re
+from datetime import datetime
+
+
+class Proposal:
+    def __init__(self, id, action, started, ends, start_cindex, electorate, turnout, approval, state):
+        self.id = id
+        self.action = action
+        self.started = started
+        self.ends = ends
+        self.start_cindex = start_cindex
+        self.electorate = electorate
+        self.turnout = turnout
+        self.approval = approval
+        self.state = state
+
+
+def parse_proposals(text):
+    proposals = text.split("Proposal id:")
+    proposal_objects = []
+
+    for proposal in proposals[1:]:  # Skip the first split result as it will be an empty string
+        proposal = "Proposal id:" + proposal  # Add back the identifier
+        lines = proposal.split("\n")
+        id = int(re.search(r'\d+', lines[0]).group())
+        action = re.search(r'ProposalAction::\w+\([\w, .]+\)', lines[1]).group()
+        started = datetime.strptime(re.search(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', lines[2]).group(),
+                                    '%Y-%m-%d %H:%M:%S')
+        ends = datetime.strptime(re.search(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', lines[3]).group(),
+                                 '%Y-%m-%d %H:%M:%S')
+        start_cindex = int(re.search(r'\d+', lines[4]).group())
+        electorate = int(re.search(r'\d+', lines[5]).group())
+        turnout = int(re.search(r'\d+', lines[6]).group())
+        approval = int(re.search(r'\d+', lines[7]).group())
+        state = re.search(r'ProposalState::(\w+)', lines[8]).group(1)
+
+        proposal_objects.append(Proposal(id, action, started, ends, start_cindex, electorate, turnout, approval, state))
+
+    return proposal_objects

client/src/commands/encointer_democracy.rs

@@ -80,6 +80,7 @@ pub fn submit_update_nominal_income_proposal(
 	})
 	.into()
 }
+
 pub fn list_proposals(_args: &str, matches: &ArgMatches<'_>) -> Result<(), clap::Error> {
 	let rt = tokio::runtime::Runtime::new().unwrap();
 	rt.block_on(async {
@@ -258,7 +259,7 @@ pub fn vote(_args: &str, matches: &ArgMatches<'_>) -> Result<(), clap::Error> {
 		)
 		.unwrap();
 		ensure_payment(&api, &xt.encode().into(), tx_payment_cid_arg).await;
-		let _result = api.submit_and_watch_extrinsic_until(xt, XtStatus::InBlock).await;
+		let _result = api.submit_and_watch_extrinsic_until(xt, XtStatus::Ready).await;
 		println!("Vote submitted: {vote_raw:?} for proposal {proposal_id:?}");
 		Ok(())
 	})

client/src/commands/keystore.rs

@@ -6,7 +6,7 @@ use clap::ArgMatches;
 use log::info;
 use sp_application_crypto::{ed25519, sr25519, Ss58Codec};
 use sp_keystore::Keystore;
-use std::path::PathBuf;
+use std::{env, fs, io::Read, path::PathBuf};
 use substrate_client_keystore::{KeystoreExt, LocalKeystore};
 
 pub fn new_account(_args: &str, matches: &ArgMatches<'_>) -> Result<(), clap::Error> {
@@ -38,3 +38,19 @@ pub fn list_accounts(_args: &str, _matches: &ArgMatches<'_>) -> Result<(), clap:
 	drop(store);
 	Ok(())
 }
+
+pub fn export_secret(_args: &str, matches: &ArgMatches<'_>) -> Result<(), clap::Error> {
+	let arg_account = matches.value_of("account").unwrap();
+	let mut path = env::current_dir().expect("Failed to get current directory");
+	path.push("my_keystore");
+	let pubkey = sr25519::Public::from_ss58check(arg_account)
+		.expect("arg should be ss58 encoded public key");
+	let key_type = array_bytes::bytes2hex("", SR25519.0);
+	let key = array_bytes::bytes2hex("", pubkey);
+	path.push(key_type + key.as_str());
+	let mut file = fs::File::open(&path).expect("Failed to open keystore file");
+	let mut contents = String::new();
+	file.read_to_string(&mut contents).expect("Failed to read file contents");
+	println!("{}", contents);
+	Ok(())
+}