diff --git a/prediction_market_agent_tooling/markets/omen/data_models.py b/prediction_market_agent_tooling/markets/omen/data_models.py index d5613877..1e1b5457 100644 --- a/prediction_market_agent_tooling/markets/omen/data_models.py +++ b/prediction_market_agent_tooling/markets/omen/data_models.py @@ -24,6 +24,7 @@ ResolvedBet, ) from prediction_market_agent_tooling.tools.utils import ( + BPS_CONSTANT, DatetimeUTC, check_not_none, should_not_happen, @@ -502,7 +503,7 @@ class OmenBet(BaseModel): feeAmount: Wei outcomeIndex: int outcomeTokensTraded: Wei - transactionHash: HexAddress + transactionHash: HexBytes fpmm: OmenMarket @property @@ -785,6 +786,14 @@ class ContractPrediction(BaseModel): tx_hashes: list[HexBytes] = Field(..., alias="txHashes") estimated_probability_bps: int = Field(..., alias="estimatedProbabilityBps") + @property + def estimated_probability(self) -> Probability: + return Probability(self.estimated_probability_bps / BPS_CONSTANT) + + @property + def boolean_outcome(self) -> bool: + return self.estimated_probability > 0.5 + @computed_field # type: ignore[prop-decorator] # Mypy issue: https://github.com/python/mypy/issues/14461 @property def publisher_checksummed(self) -> ChecksumAddress: diff --git a/prediction_market_agent_tooling/markets/omen/omen.py b/prediction_market_agent_tooling/markets/omen/omen.py index ff127cf3..a1d54644 100644 --- a/prediction_market_agent_tooling/markets/omen/omen.py +++ b/prediction_market_agent_tooling/markets/omen/omen.py @@ -78,6 +78,7 @@ from prediction_market_agent_tooling.tools.hexbytes_custom import HexBytes from prediction_market_agent_tooling.tools.ipfs.ipfs_handler import IPFSHandler from prediction_market_agent_tooling.tools.utils import ( + BPS_CONSTANT, DatetimeUTC, calculate_sell_amount_in_collateral, check_not_none, @@ -448,7 +449,7 @@ def store_trades( publisher=keys.public_key, ipfs_hash=ipfs_hash_decoded, tx_hashes=tx_hashes, - estimated_probability_bps=int(traded_market.answer.p_yes * 10000), + estimated_probability_bps=int(traded_market.answer.p_yes * BPS_CONSTANT), ) tx_receipt = OmenAgentResultMappingContract().add_prediction( api_keys=keys, diff --git a/prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py b/prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py index 491134fc..0e30b6b0 100644 --- a/prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py +++ b/prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py @@ -908,3 +908,17 @@ def get_agent_results_for_market( if not items: return [] return [ContractPrediction.model_validate(i) for i in items] + + def get_agent_results_for_bet(self, bet: OmenBet) -> ContractPrediction | None: + results = [ + result + for result in self.get_agent_results_for_market(bet.fpmm.id) + if bet.transactionHash in result.tx_hashes + ] + + if not results: + return None + elif len(results) > 1: + raise RuntimeError("Multiple results found for a single bet.") + + return results[0] diff --git a/prediction_market_agent_tooling/tools/utils.py b/prediction_market_agent_tooling/tools/utils.py index d6b94b18..0bbf63a0 100644 --- a/prediction_market_agent_tooling/tools/utils.py +++ b/prediction_market_agent_tooling/tools/utils.py @@ -28,6 +28,8 @@ # For consistent results, also include seed for models that supports it. LLM_SEED = 0 +BPS_CONSTANT = 10000 + def check_not_none( value: Optional[T], diff --git a/scripts/compare_trading_vs_prediction_accuracy.py b/scripts/compare_trading_vs_prediction_accuracy.py new file mode 100644 index 00000000..eced0944 --- /dev/null +++ b/scripts/compare_trading_vs_prediction_accuracy.py @@ -0,0 +1,44 @@ +import typer +from web3 import Web3 + +from prediction_market_agent_tooling.markets.omen.omen import OmenSubgraphHandler + + +def main(public_key: str) -> None: + public_key_checksummed = Web3.to_checksum_address(public_key) + + all_bets = [ + bet + for bet in OmenSubgraphHandler().get_bets( + better_address=public_key_checksummed, + filter_by_answer_finalized_not_null=True, + ) + if bet.fpmm.is_resolved_with_valid_answer + ] + correct_bets = [ + bet for bet in all_bets if bet.boolean_outcome == bet.fpmm.boolean_outcome + ] + + all_bets_with_results = [] + all_bets_without_results = [] + correct_results = [] + for bet in all_bets: + result = OmenSubgraphHandler().get_agent_results_for_bet(bet) + if result is None: + all_bets_without_results.append(bet) + continue + all_bets_with_results.append(bet) + if result.boolean_outcome == bet.fpmm.boolean_outcome: + correct_results.append(result) + + print("N bets:", len(all_bets)) + print("Bet accuracy:", len(correct_bets) / len(all_bets) if all_bets else None) + print( + "Prediction accuracy:", + len(correct_results) / len(all_bets_with_results) if all_bets else None, + ) + print("N bets without results:", len(all_bets_without_results)) + + +if __name__ == "__main__": + typer.run(main)