From 4100d8bdc10cbb0bbd93bd983cdc95a97d78255b Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Wed, 8 May 2024 20:55:16 +0200 Subject: [PATCH] Add db-sync checks for conway param update proposals --- cardano_node_tests/tests/reqs_conway.py | 4 + .../tests/tests_conway/test_pparam_update.py | 75 +++++++++ cardano_node_tests/utils/dbsync_queries.py | 112 ++++++++++++- cardano_node_tests/utils/dbsync_utils.py | 156 ++++++++++++++++++ 4 files changed, 341 insertions(+), 6 deletions(-) diff --git a/cardano_node_tests/tests/reqs_conway.py b/cardano_node_tests/tests/reqs_conway.py index 0a0ab8b89..9b5069c1c 100644 --- a/cardano_node_tests/tests/reqs_conway.py +++ b/cardano_node_tests/tests/reqs_conway.py @@ -116,6 +116,10 @@ def __r(id: str) -> requirements.Req: cip077 = __r("CIP077") cip078 = __r("CIP078") cip079 = __r("CIP079") +cip080 = __r("CIP080") +cip081 = __r("CIP081") +cip082 = __r("CIP082") +cip083 = __r("CIP083") # https://github.com/IntersectMBO/cardano-test-plans/blob/main/docs/user-stories/02-cardano-cli.md cli001 = __r("CLI001") diff --git a/cardano_node_tests/tests/tests_conway/test_pparam_update.py b/cardano_node_tests/tests/tests_conway/test_pparam_update.py index e3eced768..f1d79c829 100644 --- a/cardano_node_tests/tests/tests_conway/test_pparam_update.py +++ b/cardano_node_tests/tests/tests_conway/test_pparam_update.py @@ -17,6 +17,7 @@ from cardano_node_tests.tests.tests_conway import conway_common from cardano_node_tests.utils import clusterlib_utils from cardano_node_tests.utils import configuration +from cardano_node_tests.utils import dbsync_utils from cardano_node_tests.utils import governance_setup from cardano_node_tests.utils import governance_utils from cardano_node_tests.utils import helpers @@ -215,6 +216,7 @@ class TestPParamUpdate: @allure.link(helpers.get_vcs_link()) @pytest.mark.long + @pytest.mark.dbsync def test_pparam_update( # noqa: C901 self, cluster_lock_governance: governance_setup.GovClusterT, @@ -242,6 +244,7 @@ def test_pparam_update( # noqa: C901 cluster, governance_data = cluster_lock_governance temp_template = common.get_test_id(cluster) cost_proposal_file = DATA_DIR / "cost_models_list.json" + db_errors_final = [] # Check if total delegated stake is below the threshold. This can be used to check that # undelegated stake is treated as Abstain. If undelegated stake was treated as Yes, than @@ -691,6 +694,16 @@ def _check_proposed_pparams( else True, ) + # db-sync check + try: + dbsync_utils.check_conway_gov_action_proposal_description( + net_nodrep_prop_rec.future_pparams, net_nodrep_prop_rec.action_txid + ) + dbsync_utils.check_conway_param_update_proposal(net_nodrep_prop_rec.future_pparams) + except AssertionError as exc: + str_exc = str(exc) + db_errors_final.append(f"db-sync network params update error: {str_exc}") + # Vote on update proposals from network group that will NOT get approved by CC if configuration.HAS_CC: reqc.cip062_02.start(url=helpers.get_vcs_link()) @@ -734,6 +747,16 @@ def _check_proposed_pparams( else True, ) + # db-sync check + try: + dbsync_utils.check_conway_gov_action_proposal_description( + eco_nodrep_prop_rec.future_pparams, eco_nodrep_prop_rec.action_txid + ) + dbsync_utils.check_conway_param_update_proposal(eco_nodrep_prop_rec.future_pparams) + except AssertionError as exc: + str_exc = str(exc) + db_errors_final.append(f"db-sync economic params update error: {str_exc}") + # Vote on update proposals from economic group that will NOT get approved by CC if configuration.HAS_CC: eco_nocc_update_proposals = list(helpers.flatten(economic_g_proposals)) @@ -800,6 +823,16 @@ def _check_proposed_pparams( approve_drep=None, ) + # db-sync check + try: + dbsync_utils.check_conway_gov_action_proposal_description( + tech_nodrep_prop_rec.future_pparams, tech_nodrep_prop_rec.action_txid + ) + dbsync_utils.check_conway_param_update_proposal(tech_nodrep_prop_rec.future_pparams) + except AssertionError as exc: + str_exc = str(exc) + db_errors_final.append(f"db-sync technical params update error: {str_exc}") + # Vote on update proposals from technical group that will NOT get approved by CC if configuration.HAS_CC: tech_nocc_prop_rec = _propose_pparams_update( @@ -883,6 +916,16 @@ def _check_proposed_pparams( else True, ) + # db-sync check + try: + dbsync_utils.check_conway_gov_action_proposal_description( + gov_nodrep_prop_rec.future_pparams, gov_nodrep_prop_rec.action_txid + ) + dbsync_utils.check_conway_param_update_proposal(gov_nodrep_prop_rec.future_pparams) + except AssertionError as exc: + str_exc = str(exc) + db_errors_final.append(f"db-sync governance params update error: {str_exc}") + # Vote on update proposals from governance group that will NOT get approved by CC if configuration.HAS_CC: gov_nocc_update_proposals = list(helpers.flatten(governance_g_proposals)) @@ -935,6 +978,16 @@ def _check_proposed_pparams( else True, ) + # db-sync check + try: + dbsync_utils.check_conway_gov_action_proposal_description( + mix_nodrep_prop_rec.future_pparams, mix_nodrep_prop_rec.action_txid + ) + dbsync_utils.check_conway_param_update_proposal(mix_nodrep_prop_rec.future_pparams) + except AssertionError as exc: + str_exc = str(exc) + db_errors_final.append(f"db-sync mixed group params update error: {str_exc}") + # Vote on update proposals from mix of groups that will NOT get approved by CC if configuration.HAS_CC: mix_nocc_update_proposals = list( @@ -1007,6 +1060,17 @@ def _check_proposed_pparams( ) fin_approve_epoch = cluster.g_query.get_epoch() + # db-sync check + [r.start(url=_url) for r in (reqc.cip080, reqc.cip081, reqc.cip082, reqc.cip083)] + try: + dbsync_utils.check_conway_gov_action_proposal_description( + fin_prop_rec.future_pparams, fin_prop_rec.action_txid + ) + dbsync_utils.check_conway_param_update_proposal(fin_prop_rec.future_pparams) + except AssertionError as exc: + str_exc = str(exc) + db_errors_final.append(f"db-sync 'final' params update error: {str_exc}") + # Vote on another update proposals from mix of groups. The proposal will get approved, # but not enacted, because it comes after the "final" action that was accepted to the chain # first. @@ -1121,6 +1185,13 @@ def _check_state(state: dict): if is_spo_total_below_threshold: reqc.cip064_04.success() + # db-sync check + try: + dbsync_utils.check_conway_param_update_enactment(enact_gov_state, _cur_epoch) + except AssertionError as exc: + str_exc = str(exc) + db_errors_final.append(f"db-sync params enactment error: {str_exc}") + if proposed_pparams_errors: proposed_pparams_errors_str = "\n".join(proposed_pparams_errors) raise AssertionError(proposed_pparams_errors_str) @@ -1145,6 +1216,10 @@ def _check_state(state: dict): governance_utils.check_vote_view(cluster_obj=cluster, vote_data=fin_voted_votes.cc[0]) governance_utils.check_vote_view(cluster_obj=cluster, vote_data=fin_voted_votes.drep[0]) + if db_errors_final: + raise AssertionError("\n".join(db_errors_final)) + [r.success() for r in (reqc.cip080, reqc.cip081, reqc.cip082, reqc.cip083)] + class TestPParamData: """Tests for checking protocol parameters keys and values.""" diff --git a/cardano_node_tests/utils/dbsync_queries.py b/cardano_node_tests/utils/dbsync_queries.py index 490dabf90..338365e9c 100644 --- a/cardano_node_tests/utils/dbsync_queries.py +++ b/cardano_node_tests/utils/dbsync_queries.py @@ -325,7 +325,7 @@ class ParamProposalDBRow: protocol_minor: int min_utxo_value: int min_pool_cost: int - coins_per_utxo_word: int + coins_per_utxo_size: int cost_model_id: int price_mem: float price_step: float @@ -357,6 +357,7 @@ class ParamProposalDBRow: gov_action_deposit: int drep_deposit: int drep_activity: str + pvtpp_security_group: float min_fee_ref_script_cost_per_byte: float @@ -371,6 +372,66 @@ class EpochDBRow: epoch_number: int +@dataclasses.dataclass(frozen=True) +class EpochParamDBRow: + # pylint: disable=too-many-instance-attributes disable-next=invalid-name + id: int + epoch_no: int + min_fee_a: int + min_fee_b: int + max_block_size: int + max_tx_size: int + max_bh_size: int + key_deposit: int + pool_deposit: int + max_epoch: int + optimal_pool_count: int + influence: float + monetary_expand_rate: float + treasury_growth_rate: float + decentralisation: float + protocol_major: int + protocol_minor: int + min_utxo_value: int + min_pool_cost: int + nonce: memoryview + cost_model_id: int + price_mem: float + price_step: float + max_tx_ex_mem: int + max_tx_ex_steps: int + max_block_ex_mem: int + max_block_ex_steps: int + max_val_size: int + collateral_percent: int + max_collateral_inputs: int + block_id: int + extra_entropy: memoryview + coins_per_utxo_size: int + pvt_motion_no_confidence: float + pvt_committee_normal: float + pvt_committee_no_confidence: float + pvt_hard_fork_initiation: float + dvt_motion_no_confidence: float + dvt_committee_normal: float + dvt_committee_no_confidence: float + dvt_update_to_constitution: float + dvt_hard_fork_initiation: float + dvt_p_p_network_group: float + dvt_p_p_economic_group: float + dvt_p_p_technical_group: float + dvt_p_p_gov_group: float + dvt_treasury_withdrawal: float + committee_min_size: int + committee_max_term_length: int + gov_action_lifetime: int + gov_action_deposit: int + drep_deposit: int + drep_activity: int + pvtpp_security_group: float + min_fee_ref_script_cost_per_byte: float + + @dataclasses.dataclass(frozen=True) class CommitteeRegistrationDBRow: # pylint: disable-next=invalid-name @@ -416,7 +477,7 @@ class GovActionProposalDBRow: expiration: int voting_anchor_id: int type: str - description: str + description: dict param_proposal: int ratified_epoch: int enacted_epoch: int @@ -914,6 +975,34 @@ def query_epoch_stake( yield EpochStakeDBRow(*result) +def query_epoch_param(epoch_no: int = 0) -> EpochParamDBRow: + """Query epoch param record in db-sync.""" + query_var = epoch_no + + query = ( + "SELECT " + " id, epoch_no, min_fee_a, min_fee_b, max_block_size, max_tx_size, max_bh_size, " + " key_deposit, pool_deposit, max_epoch, optimal_pool_count, influence, " + " monetary_expand_rate, treasury_growth_rate, decentralisation, protocol_major, " + " protocol_minor, min_utxo_value, min_pool_cost, nonce, cost_model_id, price_mem, " + " price_step, max_tx_ex_mem, max_tx_ex_steps, max_block_ex_mem, max_block_ex_steps, " + " max_val_size, collateral_percent, max_collateral_inputs, block_id, extra_entropy, " + " coins_per_utxo_size, pvt_motion_no_confidence, pvt_committee_normal, " + " pvt_committee_no_confidence, pvt_hard_fork_initiation, dvt_motion_no_confidence, " + " dvt_committee_normal, dvt_committee_no_confidence, dvt_update_to_constitution, " + " dvt_hard_fork_initiation, dvt_p_p_network_group, dvt_p_p_economic_group, " + " dvt_p_p_technical_group, dvt_p_p_gov_group, dvt_treasury_withdrawal, committee_min_size, " + " committee_max_term_length, gov_action_lifetime, gov_action_deposit, drep_deposit, " + " drep_activity, pvtpp_security_group, min_fee_ref_script_cost_per_byte " + " FROM epoch_param " + " WHERE epoch_no = %s " + ) + + with execute(query=query, vars=(query_var,)) as cur: + results = cur.fetchone() + return EpochParamDBRow(*results) + + def query_blocks( pool_id_bech32: str = "", epoch_from: int = 0, epoch_to: int = 99999999 ) -> tp.Generator[BlockDBRow, None, None]: @@ -966,11 +1055,21 @@ def query_datum(datum_hash: str) -> tp.Generator[DatumDBRow, None, None]: yield DatumDBRow(*result) -def query_cost_model() -> tp.Dict[str, tp.Dict[str, tp.Any]]: - """Query last cost-model record in db-sync.""" +def query_cost_model(model_id: int = -1) -> tp.Dict[str, tp.Dict[str, tp.Any]]: + """Query last cost model record (if id not specified) in db-sync.""" query = "SELECT * FROM cost_model ORDER BY ID DESC LIMIT 1" + query_var: tp.Union[int, str] - with execute(query=query) as cur: + if model_id != -1: + id_query = "WHERE id = %s " + query_var = model_id + else: + id_query = "" + query_var = "" + + query = f"SELECT * FROM cost_model {id_query} ORDER BY ID DESC LIMIT 1" + + with execute(query=query, vars=(query_var,)) as cur: results = cur.fetchone() cost_model: tp.Dict[str, tp.Dict[str, tp.Any]] = results[1] if results else {} return cost_model @@ -1003,7 +1102,8 @@ def query_param_proposal(txhash: str = "") -> ParamProposalDBRow: " p.dvt_p_p_network_group, p.dvt_p_p_economic_group, p.dvt_p_p_technical_group," " p.dvt_p_p_gov_group, p.dvt_treasury_withdrawal, p.committee_min_size," " p.committee_max_term_length, p.gov_action_lifetime, p.gov_action_deposit," - " p.drep_deposit, p.drep_activity, p.min_fee_ref_script_cost_per_byte " + " p.drep_deposit, p.drep_activity, p.pvtpp_security_group," + " p.min_fee_ref_script_cost_per_byte " "FROM param_proposal AS p " "INNER JOIN tx ON tx.id = p.registered_tx_id " f"{hash_query}" diff --git a/cardano_node_tests/utils/dbsync_utils.py b/cardano_node_tests/utils/dbsync_utils.py index 6ff766a05..60caba2f4 100644 --- a/cardano_node_tests/utils/dbsync_utils.py +++ b/cardano_node_tests/utils/dbsync_utils.py @@ -1,6 +1,7 @@ """Functionality for interacting with db-sync.""" import functools +import json import logging import time import typing as tp @@ -848,6 +849,161 @@ def check_param_proposal(protocol_params: dict) -> tp.Optional[dbsync_queries.Pa return param_proposal_db +def _evaluate_pparam(pparam: tp.Any) -> tp.Union[float, None]: + if pparam is None: + return None + if isinstance(pparam, dict): + numerator = pparam.get("numerator", 0) + denominator = pparam.get("denominator", 1) + return float(numerator / denominator) + return float(pparam) + + +def map_params_to_db_convention(pp: dict) -> tp.Dict[str, tp.Any]: + # Get the prices of memory and steps + prices = pp.get("prices", {}) + price_mem = _evaluate_pparam(prices.get("prMem")) + price_steps = _evaluate_pparam(prices.get("prSteps")) + + dvt = pp.get("dRepVotingThresholds", {}) + pvt = pp.get("poolVotingThresholds", {}) + + params_mapping = { + # Network proposals group + "max_block_size": pp.get("maxBlockBodySize"), + "max_tx_size": pp.get("maxTxSize"), + "max_bh_size": pp.get("maxBlockHeaderSize"), + "max_val_size": pp.get("maxValSize"), + "max_tx_ex_mem": pp.get("maxTxExUnits", {}).get("exUnitsMem"), + "max_tx_ex_steps": pp.get("maxTxExUnits", {}).get("exUnitsSteps"), + "max_block_ex_mem": pp.get("maxBlockExUnits", {}).get("exUnitsMem"), + "max_block_ex_steps": pp.get("maxBlockExUnits", {}).get("exUnitsSteps"), + "max_collateral_inputs": pp.get("maxCollateralInputs"), + # Economic proposals group + "min_fee_a": pp.get("minFeeA"), + "min_fee_b": pp.get("minFeeB"), + "key_deposit": pp.get("keyDeposit"), + "pool_deposit": pp.get("poolDeposit"), + "monetary_expand_rate": _evaluate_pparam(pp.get("monetaryExpansion")), + "treasury_growth_rate": _evaluate_pparam(pp.get("treasuryCut")), + "min_pool_cost": pp.get("minPoolCost"), + "coins_per_utxo_size": pp.get("coinsPerUTxOByte"), + "min_fee_ref_script_cost_per_byte": pp.get("minFeeRefScriptCostPerByte"), + "price_mem": price_mem, + "price_step": price_steps, + # Technical proposals group + "influence": _evaluate_pparam(pp.get("poolPledgeInfluence")), + "max_epoch": pp.get("poolRetireMaxEpoch"), + "optimal_pool_count": pp.get("stakePoolTargetNum"), + "collateral_percent": pp.get("collateralPercentage"), + # Governance proposal group + # - DReps + "dvt_committee_no_confidence": _evaluate_pparam(dvt.get("committeeNoConfidence")), + "dvt_committee_normal": _evaluate_pparam(dvt.get("committeeNormal")), + "dvt_hard_fork_initiation": _evaluate_pparam(dvt.get("hardForkInitiation")), + "dvt_motion_no_confidence": _evaluate_pparam(dvt.get("motionNoConfidence")), + "dvt_p_p_economic_group": _evaluate_pparam(dvt.get("ppEconomicGroup")), + "dvt_p_p_gov_group": _evaluate_pparam(dvt.get("ppGovGroup")), + "dvt_p_p_network_group": _evaluate_pparam(dvt.get("ppNetworkGroup")), + "dvt_p_p_technical_group": _evaluate_pparam(dvt.get("ppTechnicalGroup")), + "dvt_treasury_withdrawal": _evaluate_pparam(dvt.get("treasuryWithdrawal")), + "dvt_update_to_constitution": _evaluate_pparam(dvt.get("updateToConstitution")), + # - Pools + "pvt_committee_no_confidence": _evaluate_pparam(pvt.get("committeeNoConfidence")), + "pvt_committee_normal": _evaluate_pparam(pvt.get("committeeNormal")), + "pvt_hard_fork_initiation": _evaluate_pparam(pvt.get("hardForkInitiation")), + "pvt_motion_no_confidence": _evaluate_pparam(pvt.get("motionNoConfidence")), + "pvtpp_security_group": _evaluate_pparam(pvt.get("ppSecurityGroup")), + # General + "gov_action_lifetime": pp.get("govActionLifetime"), + "gov_action_deposit": pp.get("govActionDeposit"), + "drep_deposit": pp.get("dRepDeposit"), + "drep_activity": pp.get("dRepActivity"), + "committee_min_size": pp.get("committeeMinSize"), + "committee_max_term_length": pp.get("committeeMaxTermLength"), + } + + return params_mapping + + +def _check_param_proposal( + param_proposal_db: tp.Union[dbsync_queries.ParamProposalDBRow, dbsync_queries.EpochParamDBRow], + params_map: dict, +) -> list: + """Check parameter proposal against db-sync.""" + failures = [] + + for param_db, protocol_value in params_map.items(): + if protocol_value: + db_value = getattr(param_proposal_db, param_db) + if db_value and (db_value != protocol_value): + failures.append( + f"Param value for {param_db}: {db_value}. Expected: {protocol_value}" + ) + return failures + + +def check_conway_param_update_proposal( + param_proposal_ledger: dict, +) -> tp.Optional[dbsync_queries.ParamProposalDBRow]: + """Check comparison for param proposal between ledger and db-sync.""" + param_proposal_db = dbsync_queries.query_param_proposal() + params_map = map_params_to_db_convention(param_proposal_ledger) + failures = [] + + # Get cost models + if param_proposal_db.cost_model_id: + db_cost_model = dbsync_queries.query_cost_model(param_proposal_db.cost_model_id) + pp_cost_model = param_proposal_ledger.get("costModels") + if db_cost_model != pp_cost_model: + failures.append(f"Cost model mismatch for {db_cost_model}. Expected: {pp_cost_model}") + failures.extend(_check_param_proposal(param_proposal_db, params_map)) + + if failures: + failures_str = "\n".join(failures) + msg = f"Unexpected parameter proposal values in db-sync:\n{failures_str}" + raise AssertionError(msg) + return param_proposal_db + + +def check_conway_param_update_enactment(gov_state: dict, epoch_no: int) -> list: + """Check params enactment between ledger and epoch param in db-sync.""" + curr_params_db = dbsync_queries.query_epoch_param(epoch_no) + curr_params_ledger = gov_state["currentPParams"] + params_map = map_params_to_db_convention(curr_params_ledger) + + return _check_param_proposal(curr_params_db, params_map) + + +def check_conway_gov_action_proposal_description( + update_proposal: dict, txhash: str = "" +) -> tp.Optional[dbsync_queries.GovActionProposalDBRow]: + """Check expected values in the param proposal table in db-sync.""" + db_gov_action = get_gov_action_proposals(txhash=txhash).pop() + db_gov_action_description = db_gov_action.description + db_gov_params_proposal = db_gov_action_description["contents"][1] + + # Convert dictionaries to JSON strings + db_gov_params_proposal_json = json.dumps(db_gov_params_proposal, sort_keys=True) + ledger_gov_params_proposal_json = json.dumps(update_proposal, sort_keys=True) + + if db_gov_params_proposal_json != ledger_gov_params_proposal_json: + msg = ( + f"JSON comparison {db_gov_params_proposal_json} failed in db-sync:\n" + f"Expected {ledger_gov_params_proposal_json}" + ) + raise AssertionError(msg) + return db_gov_action + + +def get_gov_action_proposals( + txhash: str = "", type: str = "" +) -> tp.List[dbsync_queries.GovActionProposalDBRow]: + """Get goverment action proposal from db-sync.""" + gov_action_proposals = list(dbsync_queries.query_gov_action_proposal(txhash, type)) + return gov_action_proposals + + def get_committee_member(cold_key: str) -> tp.Optional[dbsync_types.CommitteeRegistrationRecord]: """Get committee member data from db-sync.""" cc_members = list(dbsync_queries.query_committee_registration(cold_key=cold_key))