From 743ab37ad560087ec599dd02ea7ff924fb93f8a4 Mon Sep 17 00:00:00 2001 From: Norbert <37236152+KatunaNorbert@users.noreply.github.com> Date: Wed, 10 Jul 2024 12:07:24 +0300 Subject: [PATCH] Fix #1325: Create basic predictoor dashboard (#1326) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * created a working dash app structure * displayed feeds data inside a table * get and display predictoors data as table * added predictoors accuracy chart * added costs table * added profit chart * update layout so plots take up more space * added some typings and descriptions * handle db connection errors and display them * added tests for utils * add search inputs for tables * added tests for plots * added readmes and documentation * increase stake bar width --------- Co-authored-by: Mustafa Tuncay Co-authored-by: Călina Cenan --- READMEs/predictoor-dashboard.md | 54 ++++++ READMEs/predictoor.md | 9 + .../dash_components/callbacks.py | 180 ++++++++++++++++++ .../dash_components/plots.py | 133 +++++++++++++ .../dash_components/util.py | 88 +++++++++ .../dash_components/view_elements.py | 154 +++++++++++++++ .../predictoor_dashboard/predictoor_dash.py | 28 +++ .../predictoor_dashboard/test/conftest.py | 30 +++ .../predictoor_dashboard/test/resources.py | 35 ++++ .../test/test_callbacks.py | 106 +++++++++++ .../predictoor_dashboard/test/test_plots.py | 152 +++++++++++++++ .../predictoor_dashboard/test/test_util.py | 91 +++++++++ pdr_backend/cli/cli_arguments.py | 17 ++ pdr_backend/cli/cli_module.py | 7 + pdr_backend/lake/base_data_store.py | 8 + pdr_backend/lake/payout.py | 96 ++++++++++ 16 files changed, 1188 insertions(+) create mode 100644 READMEs/predictoor-dashboard.md create mode 100644 pdr_backend/analytics/predictoor_dashboard/dash_components/callbacks.py create mode 100644 pdr_backend/analytics/predictoor_dashboard/dash_components/plots.py create mode 100644 pdr_backend/analytics/predictoor_dashboard/dash_components/util.py create mode 100644 pdr_backend/analytics/predictoor_dashboard/dash_components/view_elements.py create mode 100644 pdr_backend/analytics/predictoor_dashboard/predictoor_dash.py create mode 100644 pdr_backend/analytics/predictoor_dashboard/test/conftest.py create mode 100644 pdr_backend/analytics/predictoor_dashboard/test/resources.py create mode 100644 pdr_backend/analytics/predictoor_dashboard/test/test_callbacks.py create mode 100644 pdr_backend/analytics/predictoor_dashboard/test/test_plots.py create mode 100644 pdr_backend/analytics/predictoor_dashboard/test/test_util.py diff --git a/READMEs/predictoor-dashboard.md b/READMEs/predictoor-dashboard.md new file mode 100644 index 000000000..16cfdfcce --- /dev/null +++ b/READMEs/predictoor-dashboard.md @@ -0,0 +1,54 @@ +# Predictoor Dashboard + +**The Predictoor dashboard provides insights into blockchain data through interactive plots and tables.** + +## **How it works** + +Live data from a specified supported blockchain is fetched and stored in a local database. By running a Dash dashboard, this data is processed and displayed in various formats to provide valuable information and control. + +## **How to setup and run** + +To set up and run the Predictoor dashboard, follow these steps: + +1. **Fetch chain data into the lake** + +The first step is to fetch the data from the blockchain using the lake's ETL, by this command: + +```console +pdr lake etl update ./my_ppss.yaml sapphire-mainnet +``` + +For more information on how the lake works and how to configure it, refer to [`this readme`](./lake-and-etl.md). + +2. **Run the dash app from command line** + +After fetching the chain data locally into the lake, the next step is to read, process, and display the data by running the dashboard with the following command: + +```console +pdr predictoor_dashboard ./my_ppss.yaml sapphire-mainnet +``` + +This command will open a browser window where you can select predictors and feeds to visualize their performance. + +## **How to use** + +By default, the plots will be empty because no predictors or feeds are selected. + +After **selecting one or more predictors and feeds combinations** where predictions has been submitted, and the payout method called, the plots will be populated with the relevant data. + +You can then observe how your predictor bots or others have been performing on different feeds and even compare them. + +**IMPORTANT: Only predictions data where the payout method has been called are visible.** + +The displayed charts are: + +1. **Accuracy** - how predictoor accuracy has evolved durring the selected period of time. +2. **Profit** - what is the profit in OCEAN over the period. Here you can clearly see if you are profitable or not. +3. **Costs** - how much predictoor has staked for each epoch. + +## **How to contribute** + +We are constantly looking for ways to improve the Predictoor dashboard to help predictors become more profitable and are open to suggestions and ideas. + +If you are a predictor and want to help us improve this tool, [join our Discord channel](https://discord.gg/Tvqd7Z648H) and drop a message, or open a GitHub issue. + diff --git a/READMEs/predictoor.md b/READMEs/predictoor.md index 1ad332814..d1ba6c3c0 100644 --- a/READMEs/predictoor.md +++ b/READMEs/predictoor.md @@ -14,6 +14,7 @@ This README shows how to earn $ by running a predictoor bot on mainnet. 1. **[Run bot on testnet](#3-run-predictoor-bot-on-sapphire-testnet)** 1. **[Run bot on mainnet](#4-run-predictoor-bot-on-sapphire-mainnet)** 1. **[Claim payout](#5-claim-payout)** +1. **[Run dashboard to monitor and analyze performance](#6-run-dashboard)** Once you're done the main flow, you can go beyond, with any of: @@ -217,6 +218,14 @@ When running predictoors on mainnet, you have the potential to earn $. Congrats! You've gone through all the essential steps to earn $ by running a predictoor bot on mainnet. +## 6. Run Dashboard + +After running your predictor bot, you may **monitor and analyze its performance**. + +To assist with this, pdr_backend provides a dashboard that utilizes **live blockchain data** to visualize key performance metrics such as **accuracy, profit, and costs over a specific period**. + +Please refer to this **[setup guide](predictoor-dashboard.md)** for detailed instructions on how to set up, configure, and **run the dashboard** to visualize real-time data from the blockchain. + The next sections describe how to go beyond, by optimizing the model and more. # Optimize model diff --git a/pdr_backend/analytics/predictoor_dashboard/dash_components/callbacks.py b/pdr_backend/analytics/predictoor_dashboard/dash_components/callbacks.py new file mode 100644 index 000000000..badfc23d0 --- /dev/null +++ b/pdr_backend/analytics/predictoor_dashboard/dash_components/callbacks.py @@ -0,0 +1,180 @@ +from dash import Input, Output, State +import dash +from pdr_backend.analytics.predictoor_dashboard.dash_components.util import ( + get_feeds_data_from_db, + get_predictoors_data_from_db, + get_payouts_from_db, + filter_objects_by_field, +) +from pdr_backend.analytics.predictoor_dashboard.dash_components.view_elements import ( + get_graph, +) +from pdr_backend.analytics.predictoor_dashboard.dash_components.plots import ( + get_figures, +) + + +# pylint: disable=too-many-statements +def get_callbacks(app): + @app.callback( + Output("feeds-data", "data"), + Output("predictoors-data", "data"), + Output("error-message", "children"), + Input("data-folder", "data"), + ) + def get_input_data_from_db(files_dir): + try: + feeds_data = get_feeds_data_from_db(files_dir) + predictoors_data = get_predictoors_data_from_db(files_dir) + return feeds_data, predictoors_data, None + except Exception as e: + return None, None, dash.html.H3(str(e)) + + @app.callback( + Output("payouts-data", "data"), + [ + Input("feeds_table", "selected_rows"), + Input("predictoors_table", "selected_rows"), + Input("search-input-Feeds", "value"), + Input("search-input-Predictoors", "value"), + ], + State("feeds-data", "data"), + State("predictoors-data", "data"), + State("data-folder", "data"), + ) + def get_display_data_from_db( + feeds_table_selected_rows, + predictoors_table_selected_rows, + search_value_feeds, + search_value_predictoors, + feeds_data, + predictoors_data, + lake_dir, + ): + feeds_addrs = [] + predictoors_addrs = [] + if ( + len(feeds_table_selected_rows) == 0 + or len(predictoors_table_selected_rows) == 0 + ): + return dash.no_update + + current_feeds_table_data = update_feeds_table_on_search( + search_value_feeds, feeds_data + ) + for i in feeds_table_selected_rows: + feeds_addrs.append(current_feeds_table_data[i]["contract"]) + + current_predictoors_table_data = update_predictoors_table_on_search( + search_value_predictoors, predictoors_data + ) + for i in predictoors_table_selected_rows: + predictoors_addrs.append(current_predictoors_table_data[i]["user"]) + + payouts = get_payouts_from_db(feeds_addrs, predictoors_addrs, lake_dir) + + return payouts + + @app.callback( + Output("feeds_table", "columns"), + Input("feeds-data", "data"), + ) + def create_feeds_table(feeds_data): + if not feeds_data: + return dash.no_update + + for feed in feeds_data: + del feed["contract"] + + columns = [{"name": col, "id": col} for col in feeds_data[0].keys()] + return columns + + @app.callback( + Output("predictoors_table", "columns"), + Input("predictoors-data", "data"), + ) + def create_predictoors_table(predictoors_data): + if not predictoors_data: + return dash.no_update + columns = [{"name": col, "id": col} for col in predictoors_data[0].keys()] + return columns + + @app.callback( + Output("accuracy_chart", "children"), + Output("profit_chart", "children"), + Output("stake_chart", "children"), + Input("payouts-data", "data"), + Input("feeds_table", "selected_rows"), + Input("predictoors_table", "selected_rows"), + Input("search-input-Feeds", "value"), + Input("search-input-Predictoors", "value"), + State("feeds-data", "data"), + State("predictoors-data", "data"), + ) + def create_charts( + payouts_data, + feeds_table_selected_rows, + predictoors_table_selected_rows, + search_value_feeds, + search_value_predictoors, + feeds_data, + predictoors_data, + ): + feeds_addrs = [] + predictoors_addrs = [] + + ## calculate selected feeds + current_feeds_table_data = update_feeds_table_on_search( + search_value_feeds, feeds_data + ) + for i in feeds_table_selected_rows: + feeds_addrs.append( + { + "contract": current_feeds_table_data[i]["contract"], + "feed_name": f"{current_feeds_table_data[i]['pair']}-{current_feeds_table_data[i]['timeframe']}", # pylint: disable=line-too-long + } + ) + + ## calculate selected predictoors addrs + current_predictoors_table_data = update_predictoors_table_on_search( + search_value_predictoors, predictoors_data + ) + for i in predictoors_table_selected_rows: + predictoors_addrs.append(current_predictoors_table_data[i]["user"]) + + # get figures + accuracy_fig, profit_fig, stakes_fig = get_figures( + payouts_data, feeds_addrs, predictoors_addrs + ) + + return get_graph(accuracy_fig), get_graph(profit_fig), get_graph(stakes_fig) + + @app.callback( + Output("predictoors_table", "data"), + [ + Input("search-input-Predictoors", "value"), + Input("predictoors-data", "data"), + ], + ) + def update_predictoors_table_on_search(search_value, predictoors_data): + if not search_value: + return predictoors_data + + # filter predictoors by user address + filtered_data = filter_objects_by_field(predictoors_data, "user", search_value) + return filtered_data + + @app.callback( + Output("feeds_table", "data"), + [ + Input("search-input-Feeds", "value"), + Input("feeds-data", "data"), + ], + ) + def update_feeds_table_on_search(search_value, feeds_data): + if not search_value: + return feeds_data + + # filter feeds by pair address + filtered_data = filter_objects_by_field(feeds_data, "pair", search_value) + return filtered_data diff --git a/pdr_backend/analytics/predictoor_dashboard/dash_components/plots.py b/pdr_backend/analytics/predictoor_dashboard/dash_components/plots.py new file mode 100644 index 000000000..798ece1dc --- /dev/null +++ b/pdr_backend/analytics/predictoor_dashboard/dash_components/plots.py @@ -0,0 +1,133 @@ +from itertools import product +from typing import Union, List, Tuple, Optional +import plotly.graph_objects as go +from enforce_typing import enforce_types +from pdr_backend.util.time_types import UnixTimeS + + +@enforce_types +def process_payouts(payouts: List[dict], predictor: str, feed: str) -> tuple: + """ + Process payouts data for a given predictor and feed. + Args: + payouts (list): List of payouts data. + predictor (str): Predictor address. + feed (str): Feed contract address. + Returns: + tuple: Tuple of slots, accuracies, profits, and stakes. + """ + slots, accuracies, profits, stakes = [], [], [], [] + profit = predictions = correct_predictions = 0 + + for p in payouts: + if not (predictor in p["ID"] and feed in p["ID"]): + continue + + predictions += 1 + profit_change = max(p["payout"], 0) - p["stake"] + profit += profit_change + correct_predictions += p["payout"] > 0 + + slots.append(p["slot"]) + accuracies.append((correct_predictions / predictions) * 100) + profits.append(profit) + stakes.append(p["stake"]) + slot_in_date_format = [ + UnixTimeS(ts).to_milliseconds().to_dt().strftime("%m-%d %H:%M") for ts in slots + ] + return slot_in_date_format, accuracies, profits, stakes + + +@enforce_types +def create_figure( + data_traces: List[Union[go.Scatter, go.Bar]], + title: str, + yaxis_title: str, + show_legend: bool = True, +): + """ + Create a figure with the given data traces. + Args: + data_traces (list): List of data traces. + title (str): Figure title. + yaxis_title (str): Y-axis title. + show_legend (bool): Show legend. Default is True. + Returns: + go.Figure: Plotly figure. + """ + fig = go.Figure(data_traces) + legend_config = ( + { + "orientation": "h", + "yanchor": "bottom", + "y": 1.02, + "xanchor": "right", + "x": 1, + } + if show_legend + else {} + ) + + fig.update_layout( + title=title, + yaxis_title=yaxis_title, + margin={"l": 20, "r": 0, "t": 50, "b": 0}, + showlegend=show_legend, + xaxis_nticks=4, + bargap=0.1, + barmode="stack", + legend=legend_config, + ) + return fig + + +@enforce_types +def get_figures( + payouts: Optional[List], feeds: List, predictoors: List[str] +) -> Tuple[go.Figure, go.Figure, go.Figure]: + """ + Get figures for accuracy, profit, and costs. + Args: + payouts (list): List of payouts data. + feeds (list): List of feeds data. + predictoors (list): List of predictoors data. + Returns: + tuple: Tuple of accuracy, profit, and costs figures. + """ + accuracy_scatters, profit_scatters, stakes_scatters = [], [], [] + + if payouts: + for predictor, feed in product(predictoors, feeds): + slots, accuracies, profits, stakes = process_payouts( + payouts, predictor, feed["contract"] + ) + if not slots: + continue + + short_name = f"{predictor[:5]} - {feed['feed_name']}" + accuracy_scatters.append( + go.Scatter(x=slots, y=accuracies, mode="lines", name=short_name) + ) + profit_scatters.append( + go.Scatter(x=slots, y=profits, mode="lines", name=short_name) + ) + stakes_scatters.append(go.Bar(x=slots, y=stakes, name=short_name, width=5)) + + if not accuracy_scatters: + accuracy_scatters.append(go.Scatter(x=[], y=[], mode="lines", name="accuracy")) + if not profit_scatters: + profit_scatters.append(go.Scatter(x=[], y=[], mode="lines", name="profit")) + if not stakes_scatters: + stakes_scatters.append(go.Bar(x=[], y=[], name="stakes", width=5)) + + fig_accuracy = create_figure( + accuracy_scatters, "Accuracy", "'%' accuracy over time" + ) + fig_profit = create_figure( + profit_scatters, "Profit", "OCEAN profit over time", show_legend=False + ) + fig_costs = create_figure( + stakes_scatters, "Costs", "Stake (OCEAN) at a time", show_legend=False + ) + + return fig_accuracy, fig_profit, fig_costs diff --git a/pdr_backend/analytics/predictoor_dashboard/dash_components/util.py b/pdr_backend/analytics/predictoor_dashboard/dash_components/util.py new file mode 100644 index 000000000..0fc0e6fb8 --- /dev/null +++ b/pdr_backend/analytics/predictoor_dashboard/dash_components/util.py @@ -0,0 +1,88 @@ +import logging +from typing import Union, List, Dict, Any +from enforce_typing import enforce_types + +from pdr_backend.lake.duckdb_data_store import DuckDBDataStore +from pdr_backend.lake.payout import Payout +from pdr_backend.lake.prediction import Prediction + +logger = logging.getLogger("predictoor_dashboard_utils") + + +@enforce_types +def _query_db(lake_dir: str, query: str) -> Union[List[dict], Exception]: + """ + Query the database with the given query. + Args: + lake_dir (str): Path to the lake directory. + query (str): SQL query. + Returns: + dict: Query result. + """ + try: + db = DuckDBDataStore(lake_dir, read_only=True) + df = db.query_data(query) + db.duckdb_conn.close() + return df.to_dicts() if len(df) else [] + except Exception as e: + logger.error("Error querying the database: %s", e) + return [] + + +@enforce_types +def get_feeds_data_from_db(lake_dir: str): + return _query_db( + lake_dir, + f""" + SELECT contract, pair, timeframe, source FROM {Prediction.get_lake_table_name()} + GROUP BY contract, pair, timeframe, source + """, + ) + + +@enforce_types +def get_predictoors_data_from_db(lake_dir: str): + return _query_db( + lake_dir, + f""" + SELECT user FROM {Prediction.get_lake_table_name()} + GROUP BY user + """, + ) + + +@enforce_types +def get_payouts_from_db( + feed_addrs: List[str], predictoor_addrs: List[str], lake_dir: str +) -> List[dict]: + """ + Get payouts data for the given feed and predictoor addresses. + Args: + feed_addrs (list): List of feed addresses. + predictoor_addrs (list): List of predictoor addresses. + lake_dir (str): Path to the lake directory. + Returns: + list: List of payouts data. + """ + + # Constructing the SQL query + query = f"SELECT * FROM {Payout.get_lake_table_name()} WHERE (" + + # Adding conditions for the first list + query += " OR ".join([f"ID LIKE '%{item}%'" for item in feed_addrs]) + query += ") AND (" + + # Adding conditions for the second list + query += " OR ".join([f"ID LIKE '%{item}%'" for item in predictoor_addrs]) + query += ");" + + return _query_db(lake_dir, query) + + +# Function to filter the list by field containing a given string +def filter_objects_by_field( + objects: List[Dict[str, Any]], field: str, search_string: str +) -> List[Dict[str, Any]]: + return list( + filter(lambda obj: search_string.lower() in obj[field].lower(), objects) + ) diff --git a/pdr_backend/analytics/predictoor_dashboard/dash_components/view_elements.py b/pdr_backend/analytics/predictoor_dashboard/dash_components/view_elements.py new file mode 100644 index 000000000..5a78c166b --- /dev/null +++ b/pdr_backend/analytics/predictoor_dashboard/dash_components/view_elements.py @@ -0,0 +1,154 @@ +import dash_bootstrap_components as dbc +from dash import dcc, html, dash_table + + +def get_input_column(): + return html.Div( + [ + html.Div( + get_table( + table_id="feeds_table", + table_name="Feeds", + searchable_field="pair", + columns=[], + data=None, + ), + id="feeds_container", + style={"height": "50%"}, + ), + html.Div( + get_table( + table_id="predictoors_table", + table_name="Predictoors", + searchable_field="user", + columns=[], + data=None, + ), + id="predictoors_container", + style={"height": "50%"}, + ), + ], + style={ + "height": "100%", + "width": "20%", + "marginTop": "8px", + "display": "flex", + "flexDirection": "column", + "justifyContent": "space-around", + }, + ) + + +def get_graphs_column(): + return html.Div( + [ + html.Div(id="accuracy_chart"), + html.Div(id="profit_chart"), + html.Div(id="stake_chart"), + ], + id="graphs_container", + style={ + "height": "100%", + "width": "80%", + "display": "flex", + "flexDirection": "column", + "justifyContent": "space-around", + "paddingLeft": "40px", + }, + ) + + +def get_layout(): + return html.Div( + [ + dcc.Store(id="data-folder"), + dcc.Store(id="feeds-data"), + dcc.Store(id="predictoors-data"), + dcc.Store(id="payouts-data"), + dcc.Store(id="predictoors-stake-data"), + html.H1( + "Predictoor dashboard", + id="page_title", + style={"width": "100%", "textAlign": "center"}, + ), + html.Div( + id="error-message", + style={ + "display": "flex", + "width": "100%", + "justifyContent": "center", + "alignItems": "center", + "textAlign": "center", + }, + ), + dcc.Loading( + id="loading", + type="default", + children=get_main_container(), + style={ + "height": "100%", + "width": "100%", + }, + custom_spinner=html.H2(dbc.Spinner(), style={"height": "100%"}), + ), + ], + ) + + +def get_main_container(): + return html.Div( + [get_input_column(), get_graphs_column()], + style={ + "height": "100%", + "width": "100%", + "display": "flex", + "justifyContent": "space-between", + }, + ) + + +def get_table(table_id, table_name, searchable_field, columns, data): + return html.Div( + [ + html.Div( + [ + html.Span(table_name, style={"fontSize": "20px"}), + dcc.Input( + id=f"search-input-{table_name}", + type="text", + placeholder=f"Search for {searchable_field}", + debounce=True, # Trigger the input event after user stops typing + style={ + "fontSize": "15px", + "height": "100%", + }, + ), + ], + style={ + "display": "flex", + "justifyContent": "space-between", + "alignItems": "center", + }, + ), + dash_table.DataTable( + id=table_id, + columns=[{"name": col, "id": col} for col in columns], + data=data, + row_selectable="multi", # Can be 'multi' for multiple rows + selected_rows=[], + style_cell={"textAlign": "left"}, + style_table={ + "height": "38vh", + "width": "100%", + "overflow": "auto", + "marginTop": "5px", + }, + fill_width=True, + ), + ], + style={"marginBottom": "40px"}, + ) + + +def get_graph(figure): + return dcc.Graph(figure=figure, style={"width": "100%", "height": "30vh"}) diff --git a/pdr_backend/analytics/predictoor_dashboard/predictoor_dash.py b/pdr_backend/analytics/predictoor_dashboard/predictoor_dash.py new file mode 100644 index 000000000..278e0aa74 --- /dev/null +++ b/pdr_backend/analytics/predictoor_dashboard/predictoor_dash.py @@ -0,0 +1,28 @@ +import webbrowser + +import dash_bootstrap_components as dbc +from dash import Dash +from enforce_typing import enforce_types + +from pdr_backend.analytics.predictoor_dashboard.dash_components.callbacks import ( + get_callbacks, +) +from pdr_backend.analytics.predictoor_dashboard.dash_components.view_elements import ( + get_layout, +) +from pdr_backend.ppss.ppss import PPSS + +app = Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) +app.config["suppress_callback_exceptions"] = True +app.layout = get_layout() +get_callbacks(app) + + +@enforce_types +def predictoor_dash(ppss: PPSS, debug_mode: bool): + port = 8050 + if not debug_mode: + webbrowser.open(f"http://127.0.0.1:{port}/") + folder = ppss.lake_ss.lake_dir + app.layout.children[0].data = folder + app.run(debug=debug_mode, port=port) diff --git a/pdr_backend/analytics/predictoor_dashboard/test/conftest.py b/pdr_backend/analytics/predictoor_dashboard/test/conftest.py new file mode 100644 index 000000000..2c7bcc293 --- /dev/null +++ b/pdr_backend/analytics/predictoor_dashboard/test/conftest.py @@ -0,0 +1,30 @@ +import pytest + +from pdr_backend.lake.prediction import ( + mock_first_predictions, + mock_daily_predictions, +) +from pdr_backend.lake.payout import ( + mock_payouts, + mock_payouts_related_with_predictions, +) + + +@pytest.fixture() +def _sample_first_predictions(): + return mock_first_predictions() + + +@pytest.fixture() +def _sample_daily_predictions(): + return mock_daily_predictions() + + +@pytest.fixture() +def _sample_payouts(): + return mock_payouts() + + +@pytest.fixture() +def _sample_payouts_related_with_predictions(): + return mock_payouts_related_with_predictions() diff --git a/pdr_backend/analytics/predictoor_dashboard/test/resources.py b/pdr_backend/analytics/predictoor_dashboard/test/resources.py new file mode 100644 index 000000000..e0ac5681b --- /dev/null +++ b/pdr_backend/analytics/predictoor_dashboard/test/resources.py @@ -0,0 +1,35 @@ +from enforce_typing import enforce_types + +from pdr_backend.lake.duckdb_data_store import DuckDBDataStore +from pdr_backend.ppss.ppss import mock_ppss +from pdr_backend.lake.plutil import _object_list_to_df +from pdr_backend.lake.prediction import Prediction +from pdr_backend.lake.payout import Payout + + +def _prepare_test_db(tmpdir, sample_data, table_name): + ppss = mock_ppss( + [{"predict": "binance BTC/USDT c 5m", "train_on": "binance BTC/USDT c 5m"}], + "sapphire-mainnet", + str(tmpdir), + ) + + db = DuckDBDataStore(str(ppss.lake_ss.lake_dir)) + + sample_data_df = _object_list_to_df(sample_data) + db.insert_from_df( + sample_data_df, + table_name, + ) + + db.duckdb_conn.close() + + return ppss, sample_data_df + + +@enforce_types +def _clear_test_db(directory: str): + db = DuckDBDataStore(directory) + db.drop_table(Payout.get_lake_table_name()) + db.drop_table(Prediction.get_lake_table_name()) + db.duckdb_conn.close() diff --git a/pdr_backend/analytics/predictoor_dashboard/test/test_callbacks.py b/pdr_backend/analytics/predictoor_dashboard/test/test_callbacks.py new file mode 100644 index 000000000..0eb2020d0 --- /dev/null +++ b/pdr_backend/analytics/predictoor_dashboard/test/test_callbacks.py @@ -0,0 +1,106 @@ +import time +import pytest + +from dash import Dash +import dash_bootstrap_components as dbc +from selenium.webdriver.common.keys import Keys + +from pdr_backend.analytics.predictoor_dashboard.dash_components.callbacks import ( + get_callbacks, +) +from pdr_backend.analytics.predictoor_dashboard.dash_components.view_elements import ( + get_layout, +) +from pdr_backend.analytics.predictoor_dashboard.test.resources import ( + _prepare_test_db, + _clear_test_db, +) +from pdr_backend.lake.payout import Payout +from pdr_backend.lake.prediction import Prediction + + +@pytest.fixture +def _test_analytics_app(): + app = Dash(__name__) + get_callbacks(app) + return app + + +def test_get_input_data_from_db( + tmpdir, + _sample_daily_predictions, + _sample_payouts_related_with_predictions, + dash_duo, +): + _clear_test_db(str(tmpdir)) + + _prepare_test_db( + tmpdir, + _sample_payouts_related_with_predictions, + table_name=Payout.get_lake_table_name(), + ) + + ppss, _ = _prepare_test_db( + tmpdir, + _sample_daily_predictions, + table_name=Prediction.get_lake_table_name(), + ) + + app = Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) + app.config["suppress_callback_exceptions"] = True + app.layout = get_layout() + get_callbacks(app) + app.layout.children[0].data = ppss.lake_ss.lake_dir + + dash_duo.start_server(app) + dash_duo.wait_for_element("#feeds_table") + dash_duo.wait_for_element("#predictoors_table") + + dash_duo.wait_for_element("#feeds_table tbody tr") + dash_duo.wait_for_element("#predictoors_table tbody tr") + + # ----TEST SEARCH INPUT WITH "OCEAN" VALUE----# + dash_duo.find_element("#search-input-Feeds").send_keys("OCEAN" + Keys.ENTER) + # wait 5 seconds + time.sleep(2) + assert len(dash_duo.find_elements("#feeds_table tbody tr")) == 1 + + # ----TEST SEARCH INPUT WITH "BTC" VALUE----# + # reset the search input + dash_duo.find_element("#search-input-Feeds").clear() + # enter "BTC" phrase to the "search-input-Feeds" input + dash_duo.find_element("#search-input-Feeds").send_keys("BTC" + Keys.ENTER) + time.sleep(2) + # check if the "Feeds" table has only two rows + assert len(dash_duo.find_elements("#feeds_table tbody tr")) == 2 + + # ----TEST SEARCH INPUT WITH "ADA" VALUE----# + # reset the search input + dash_duo.find_element("#search-input-Feeds").clear() + # enter "BTC" phrase to the "search-input-Feeds" input + dash_duo.find_element("#search-input-Feeds").send_keys("ADA" + Keys.ENTER) + time.sleep(2) + + # check if the "Feeds" table has only two rows + assert len(dash_duo.find_elements("#feeds_table tbody tr")) == 2 + + # ----TEST PREDICTOOR INPUT WITH "0xaaa" VALUE----# + dash_duo.find_element("#search-input-Predictoors").send_keys("0xaaa" + Keys.ENTER) + time.sleep(2) + assert len(dash_duo.find_elements("#predictoors_table tbody tr")) == 1 + + # ----TEST PREDICTOOR INPUT WITH "0xd2" VALUE----# + # clear input + dash_duo.find_element("#search-input-Predictoors").clear() + # enter "0xd2" phrase to the "search-input-Predictoors" input + dash_duo.find_element("#search-input-Predictoors").send_keys("0xd2" + Keys.ENTER) + time.sleep(2) + + # check if the "Predictoors" table has only one row + assert len(dash_duo.find_elements("#predictoors_table tbody tr")) == 2 + + # click on the checkbox in the second row of the "Feeds" table + dash_duo.find_element("#feeds_table tbody tr:nth-child(2) input").click() + + # click on the checkbox in the first row of the "Predictoors" table + dash_duo.find_element("#predictoors_table tbody tr input").click() diff --git a/pdr_backend/analytics/predictoor_dashboard/test/test_plots.py b/pdr_backend/analytics/predictoor_dashboard/test/test_plots.py new file mode 100644 index 000000000..f2e74dfb7 --- /dev/null +++ b/pdr_backend/analytics/predictoor_dashboard/test/test_plots.py @@ -0,0 +1,152 @@ +from unittest.mock import patch + +from enforce_typing import enforce_types + +from pdr_backend.analytics.predictoor_dashboard.dash_components.plots import ( + process_payouts, + create_figure, + get_figures, +) +from pdr_backend.util.time_types import UnixTimeS + + +@enforce_types +def test_process_payouts( + _sample_payouts, +): + ## convert List[Payout] to List[dict] + payouts = [p.__dict__ for p in _sample_payouts] + + user = "0xeb18bad7365a40e36a41fb8734eb0b855d13b74f" + feed = "0x18f54cc21b7a2fdd011bea06bba7801b280e3151" + result = process_payouts(payouts, user, feed) + + ## filter payouts by user and feed + filtered_payouts = [p for p in payouts if user in p["ID"] and feed in p["ID"]] + filtered_payouts = sorted(filtered_payouts, key=lambda x: x["slot"]) + + assert len(result) == 4 + + slots, accuracies, profits, stakes = result + + assert len(slots) == len(filtered_payouts) + assert slots[0] == UnixTimeS( + filtered_payouts[0]["slot"] + ).to_milliseconds().to_dt().strftime("%m-%d %H:%M") + + ## calculate accuracies + test_accuracies = [ + (sum(p["payout"] > 0 for p in filtered_payouts[: i + 1]) / (i + 1)) * 100 + for i in range(len(filtered_payouts)) + ] + + assert len(accuracies) == len(test_accuracies) + + for i, accuracy in enumerate(accuracies): + assert accuracy == test_accuracies[i] + + ## calculate profits + test_profits = [ + sum( + (p["payout"] - p["stake"]) if p["payout"] > 0 else -p["stake"] + for p in filtered_payouts[: i + 1] + ) + for i in range(len(filtered_payouts)) + ] + + assert len(profits) == len(test_profits) + + # check if profits are the same + for i, profit in enumerate(profits): + assert profit == test_profits[i] + + test_stakes = [p["stake"] for p in filtered_payouts] + + assert len(stakes) == len(test_stakes) + + # check if stakes are the same + for i, stake in enumerate(stakes): + assert stake == test_stakes[i] + + +class MockFigure: + def __init__(self, data_traces): + self.data_traces = data_traces + self.layout = {} + self.update_layout_called = 0 + + def update_layout(self, **kwargs): + self.layout.update(kwargs) + self.update_layout_called += 1 + return self.layout + + +@enforce_types +@patch("plotly.graph_objects.Figure", new=MockFigure) +def test_create_figure(): + result = create_figure( + data_traces=[], + title="title", + yaxis_title="yaxis_title", + show_legend=True, + ) + + assert isinstance(result, MockFigure) + assert result.data_traces == [] + assert result.layout == { + "bargap": 0.1, + "barmode": "stack", + "title": "title", + "yaxis_title": "yaxis_title", + "margin": {"l": 20, "r": 0, "t": 50, "b": 0}, + "showlegend": True, + "xaxis_nticks": 4, + "legend": { + "orientation": "h", + "yanchor": "bottom", + "y": 1.02, + "xanchor": "right", + "x": 1, + }, + } + + assert result.update_layout_called == 1 + + +@enforce_types +@patch("plotly.graph_objects.Figure", new=MockFigure) +def test_get_figures( + _sample_payouts, +): + ## convert List[Payout] to List[dict] + payouts = [p.__dict__ for p in _sample_payouts] + sample_feeds = [ + {"contract": "0x18f54cc21b7a2fdd011bea06bba7801b280e3151", "feed_name": "Feed1"} + ] + sample_predictoors = ["0xeb18bad7365a40e36a41fb8734eb0b855d13b74f"] + + fig_accuracy, fig_profit, fig_costs = get_figures( + payouts, sample_feeds, sample_predictoors + ) + + # Check if figures are instances of MockFigure + assert isinstance(fig_accuracy, MockFigure) + assert isinstance(fig_profit, MockFigure) + assert isinstance(fig_costs, MockFigure) + + # Check if the figures have the correct layout and data traces + assert len(fig_accuracy.data_traces) == 1 + assert len(fig_profit.data_traces) == 1 + assert len(fig_costs.data_traces) == 1 + + assert fig_accuracy.layout["title"] == "Accuracy" + assert fig_profit.layout["title"] == "Profit" + assert fig_costs.layout["title"] == "Costs" + + assert fig_accuracy.layout["yaxis_title"] == "'%' accuracy over time" + assert fig_profit.layout["yaxis_title"] == "OCEAN profit over time" + assert fig_costs.layout["yaxis_title"] == "Stake (OCEAN) at a time" + + assert fig_accuracy.update_layout_called == 1 + assert fig_profit.update_layout_called == 1 + assert fig_costs.update_layout_called == 1 diff --git a/pdr_backend/analytics/predictoor_dashboard/test/test_util.py b/pdr_backend/analytics/predictoor_dashboard/test/test_util.py new file mode 100644 index 000000000..34a9a00cf --- /dev/null +++ b/pdr_backend/analytics/predictoor_dashboard/test/test_util.py @@ -0,0 +1,91 @@ +from unittest.mock import patch + +from enforce_typing import enforce_types +from pdr_backend.analytics.predictoor_dashboard.dash_components.util import ( + _query_db, + get_feeds_data_from_db, + get_predictoors_data_from_db, + get_payouts_from_db, +) + +from pdr_backend.analytics.predictoor_dashboard.test.resources import ( + _prepare_test_db, + _clear_test_db, +) +from pdr_backend.lake.prediction import Prediction +from pdr_backend.lake.payout import Payout + + +@enforce_types +@patch( + "pdr_backend.analytics.predictoor_dashboard.dash_components.util.DuckDBDataStore" +) +def test_query_db( + mock_duckdb_data_store, +): + lake_dir = "lake_dir" + query = "query" + _query_db(lake_dir, query) + mock_duckdb_data_store.assert_called_once_with(lake_dir, read_only=True) + mock_duckdb_data_store.return_value.query_data.assert_called_once_with(query) + + +@enforce_types +def test_get_feeds_data_from_db( + tmpdir, + _sample_first_predictions, +): + ppss, sample_data_df = _prepare_test_db( + tmpdir, _sample_first_predictions, Prediction.get_lake_table_name() + ) + + result = get_feeds_data_from_db(ppss.lake_ss.lake_dir) + + assert isinstance(result, list) + assert len(result) == len(sample_data_df) + + _clear_test_db(ppss.lake_ss.lake_dir) + + +@enforce_types +def test_get_predictoors_data_from_db( + tmpdir, + _sample_first_predictions, +): + ppss, sample_data_df = _prepare_test_db( + tmpdir, _sample_first_predictions, Prediction.get_lake_table_name() + ) + + result = get_predictoors_data_from_db(ppss.lake_ss.lake_dir) + + grouped_sample = sample_data_df.unique("user") + + assert isinstance(result, list) + assert len(result) == len(grouped_sample) + + _clear_test_db(ppss.lake_ss.lake_dir) + + +@enforce_types +def test_get_payouts_from_db( + tmpdir, + _sample_payouts, +): + ppss, _ = _prepare_test_db( + tmpdir, _sample_payouts, table_name=Payout.get_lake_table_name() + ) + + result = get_payouts_from_db([], [], ppss.lake_ss.lake_dir) + + assert len(result) == 0 + + result = get_payouts_from_db( + ["0x18f54cc21b7a2fdd011bea06bba7801b280e3151"], + ["0xeb18bad7365a40e36a41fb8734eb0b855d13b74f"], + ppss.lake_ss.lake_dir, + ) + + assert isinstance(result, list) + assert len(result) == 2 + + _clear_test_db(ppss.lake_ss.lake_dir) diff --git a/pdr_backend/cli/cli_arguments.py b/pdr_backend/cli/cli_arguments.py index 593cc8f6b..873352e13 100644 --- a/pdr_backend/cli/cli_arguments.py +++ b/pdr_backend/cli/cli_arguments.py @@ -50,6 +50,7 @@ pdr lake raw|etl drop PPSS_FILE NETWORK ST pdr lake describe --HTML PPSS_FILE NETWORK pdr lake validate PPSS_FILE NETWORK + pdr predictoor_dashboard PPSS_FILE NETWORK pdr analytics PPSS_FILE NETWORK Utilities: @@ -630,6 +631,19 @@ def __init__(self, description: str, command_name: str): ) +class PredictoorDashboardArgParser( + CustomArgParser, PPSS_Mixin, NETWORK_Mixin, DEBUG_Mixin +): + # pylint: disable=unused-argument + def __init__(self, description: str, command_name: str): + super().__init__(description=description) + + self.add_arguments_bulk( + command_name, + ["PPSS", "NETWORK", "DEBUG"], + ) + + # below, list each entry in defined_parsers in same order as HELP_LONG defined_parsers = { # main tools @@ -676,6 +690,9 @@ def __init__(self, description: str, command_name: str): "do_topup": TopupArgParser("Topup OCEAN and ROSE in dfbuyer, trueval, ..", "topup"), "do_sim_plots": SimPlotsArgParser("Visualize simulation data", "sim_plots"), "do_arima_plots": ArimaPlotsArgParser("Visualize ARIMA data", "arima_plots"), + "do_predictoor_dashboard": PredictoorDashboardArgParser( + "Visualize Predictoor data", "predictoor_dashboard" + ), } diff --git a/pdr_backend/cli/cli_module.py b/pdr_backend/cli/cli_module.py index 2ac532085..ebb350685 100644 --- a/pdr_backend/cli/cli_module.py +++ b/pdr_backend/cli/cli_module.py @@ -33,6 +33,7 @@ from pdr_backend.sim.sim_dash import sim_dash from pdr_backend.sim.sim_engine import SimEngine from pdr_backend.statutil.arima_dash import arima_dash +from pdr_backend.analytics.predictoor_dashboard.predictoor_dash import predictoor_dash from pdr_backend.trader.approach1.trader_agent1 import TraderAgent1 from pdr_backend.trader.approach2.trader_agent2 import TraderAgent2 from pdr_backend.trueval.trueval_agent import TruevalAgent @@ -301,3 +302,9 @@ def do_arima_plots(args, nested_args=None): nested_override_args=nested_args, ) arima_dash(ppss, args.debug_mode) + + +@enforce_types +# pylint: disable=unused-argument +def do_predictoor_dashboard(args, nested_args=None): + predictoor_dash(args.PPSS, args.debug_mode) diff --git a/pdr_backend/lake/base_data_store.py b/pdr_backend/lake/base_data_store.py index 7b0ec18a0..595206ccc 100644 --- a/pdr_backend/lake/base_data_store.py +++ b/pdr_backend/lake/base_data_store.py @@ -30,3 +30,11 @@ def __init__(self, base_path: str, read_only: bool = False): @classmethod def clear_instances(cls): cls._instances = {} + + @enforce_types + def delete_own_instance(self): + read_only_key = "0" if self.read_only is False else "1" + pattern_key = f"{self.base_path}_{read_only_key}" + + if pattern_key in self._instances: + del self._instances[pattern_key] diff --git a/pdr_backend/lake/payout.py b/pdr_backend/lake/payout.py index 71c524391..383b58b7c 100644 --- a/pdr_backend/lake/payout.py +++ b/pdr_backend/lake/payout.py @@ -110,6 +110,14 @@ def mock_payouts() -> List[Payout]: return [mock_payout(payout_tuple) for payout_tuple in _PAYOUT_TUPS] +@enforce_types +def mock_payouts_related_with_predictions() -> List[Payout]: + return [ + mock_payout(payout_tuple) + for payout_tuple in _PAYOUT_TUPS_RELATED_WITH_PREDICTIONS + ] + + _PAYOUT_TUPS = [ ( # pylint: disable=line-too-long @@ -196,3 +204,91 @@ def mock_payouts() -> List[Payout]: 0.41, ), ] + + +_PAYOUT_TUPS_RELATED_WITH_PREDICTIONS = [ + ( + # pylint: disable=line-too-long + "0x18f54cc21b7a2fdd011bea06bba7801b280e3151-1704152700-0xd2a24cb4ff2584bad80ff5f109034a891c3d88dd", + "0xd2a24cb4ff2584bad80ff5f109034a891c3d88dd", # user + 1704153558, # timestamp + "ADA/USDT", # token + 1704152700, # slot + 1.4, # payout + True, # predictedValue + 0.919372744934776618, # revenue + 7.635901006590730052, # roundSumStakesUp + 17.728238320965607921, # roundSumStakes + 0.41, # stake + ), + ( + # pylint: disable=line-too-long + "0x18f54cc21b7a2fdd011bea06bba7801b280e3151-1704152700-0xfb223c3583aa934273173a55c226d598a149441c", + "0xfb223c3583aa934273173a55c226d598a149441c", + 1704153051, + "ADA/USDT", + 1704152700, + 3.786517720904995824, + False, + 0.919372744934776618, + 7.635901006590730052, + 17.728238320965607921, + 2.049314196396558, + ), + ( + # pylint: disable=line-too-long + "0x18f54cc21b7a2fdd011bea06bba7801b280e3151-1704153000-0xd2a24cb4ff2584bad80ff5f109034a891c3d88dd", + "0xd2a24cb4ff2584bad80ff5f109034a891c3d88dd", + 1704153351, + "ADA/USDT", + 1704153000, + 3.687473663992716148, + False, + 0.919372744934776618, + 11.201148268567394458, + 25.423083432944667468, + 1.9908170679122585, + ), + ( + # pylint: disable=line-too-long + "0x18f54cc21b7a2fdd011bea06bba7801b280e3151-1704153000-0x04a5e9f565dfa83c77581d1022b9ef041f55210b", + "0x04a5e9f565dfa83c77581d1022b9ef041f55210b", + 1704153504, + "ADA/USDT", + 1704153000, + 6.334665366356455078, + False, + 0.919372744934776618, + 11.201148268567394458, + 25.423083432944667468, + 3.4200000000000004, + ), + ( + # pylint: disable=line-too-long + "0x18f54cc21b7a2fdd011bea06bba7801b280e3151-1704153000-0x7149ceca72c61991018ed80788bea3f3f4540c3c", + "0x7149ceca72c61991018ed80788bea3f3f4540c3c", + 1704153534, + "ADA/USDT", + 1704153000, + 1.463270654801637113, + False, + 0.919372744934776618, + 11.201148268567394458, + 25.423083432944667468, + 0.79, + ), + ( + # pylint: disable=line-too-long + "0x18f54cc21b7a2fdd011bea06bba7801b280e3151-1704153000-0xeb18bad7365a40e36a41fb8734eb0b855d13b74f", + "0xeb18bad7365a40e36a41fb8734eb0b855d13b74f", + 1704153558, + "ADA/USDT", + 1704153000, + 0.0, + True, + 0.919372744934776618, + 11.201148268567394458, + 25.423083432944667468, + 0.41, + ), +]