From 620cc10024d879b980d024758079fbd6144db516 Mon Sep 17 00:00:00 2001 From: drupman Date: Thu, 5 Oct 2023 22:09:42 -0300 Subject: [PATCH 1/9] (feat) get bots data paths from everywhere --- pages/strategy_performance/app.py | 24 ++++++++++++++++++------ utils/os_utils.py | 20 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index eff2af63..b57b7219 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -4,6 +4,7 @@ import plotly.graph_objects as go import math import plotly.express as px +from utils.os_utils import get_bots_data_paths from utils.database_manager import DatabaseManager from utils.graphs import CandlesGraph from utils.st_utils import initialize_st_page @@ -28,10 +29,16 @@ def get_databases(): - sqlite_files = [db_name for db_name in os.listdir("data") if db_name.endswith(".sqlite")] - databases_list = [DatabaseManager(db) for db in sqlite_files] - if len(databases_list) > 0: - return {database.db_name: database for database in databases_list} + databases = {} + bots_data_paths = get_bots_data_paths() + for source_name, source_path in bots_data_paths.items(): + sqlite_files = {} + for db_name in os.listdir(source_path): + if db_name.endswith(".sqlite"): + sqlite_files[db_name] = os.path.join(source_path, db_name) + databases[source_name] = sqlite_files + if len(databases) > 0: + return {key: value for key, value in databases.items() if value} else: return None @@ -233,6 +240,8 @@ def hr_str(hr): yanchor="bottom" ), ) + return fig + return fig @@ -282,9 +291,12 @@ def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_ro selected_db = DatabaseManager(uploaded_db.name) dbs = get_databases() if dbs is not None: - db_names = [x.db_name for x in dbs.values()] + bot_source = st.selectbox("Choose your database source:", dbs.keys()) + db_names = [x for x in dbs[bot_source]] selected_db_name = st.selectbox("Select a database to start:", db_names) - selected_db = dbs[selected_db_name] + executors_path = os.path.dirname(dbs[bot_source][selected_db_name]) + selected_db = DatabaseManager(db_name=dbs[bot_source][selected_db_name], + executors_path=executors_path) else: st.warning("Ups! No databases were founded. Start uploading one") selected_db = None diff --git a/utils/os_utils.py b/utils/os_utils.py index ba1e47b4..e7b73ccb 100644 --- a/utils/os_utils.py +++ b/utils/os_utils.py @@ -104,6 +104,26 @@ def load_controllers(path): return controllers +def get_bots_data_paths(): + root_directory = os.getcwd() + bots_data_paths = {} + + # Walk through the directory tree + for dirpath, dirnames, filenames in os.walk(root_directory): + for dirname in dirnames: + # Check if the directory is named "data" + if dirname == "data": + parent_folder = os.path.basename(dirpath) + # Append the full path of the "data" directory to the list + bots_data_paths[parent_folder] = os.path.join(dirpath, dirname) + if "dashboard" in bots_data_paths: + # Create a new key "base folder" with the value of the old key + bots_data_paths["base folder"] = bots_data_paths["dashboard"] + # Delete the old key "dashboard" + del bots_data_paths["dashboard"] + return {key: value for key, value in bots_data_paths.items() if value is not None} + + def get_function_from_file(file_path: str, function_name: str): # Create a module specification from the file path and load it spec = importlib.util.spec_from_file_location("module.name", file_path) From 1a892358a859208dcecc1a85535dbba4e6506f1d Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 31 Oct 2023 17:30:15 -0300 Subject: [PATCH 2/9] (feat) get bots data paths from hummingbot_files/bots + general or uploaded data --- utils/database_manager.py | 2 +- utils/os_utils.py | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/utils/database_manager.py b/utils/database_manager.py index 80f8725e..8234fff1 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -12,7 +12,7 @@ class DatabaseManager: def __init__(self, db_name: str, executors_path: str = "data"): self.db_name = db_name # TODO: Create db path for all types of db - self.db_path = f'sqlite:///{os.path.join("data", db_name)}' + self.db_path = f'sqlite:///{os.path.join(db_name)}' self.executors_path = executors_path self.engine = create_engine(self.db_path, connect_args={'check_same_thread': False}) self.session_maker = sessionmaker(bind=self.engine) diff --git a/utils/os_utils.py b/utils/os_utils.py index e7b73ccb..c5cd6e2a 100644 --- a/utils/os_utils.py +++ b/utils/os_utils.py @@ -105,23 +105,20 @@ def load_controllers(path): def get_bots_data_paths(): - root_directory = os.getcwd() - bots_data_paths = {} - + root_directory = "hummingbot_files/bots" + bots_data_paths = {"General / Uploaded data": "data"} + reserved_word = "hummingbot-" # Walk through the directory tree for dirpath, dirnames, filenames in os.walk(root_directory): for dirname in dirnames: - # Check if the directory is named "data" if dirname == "data": parent_folder = os.path.basename(dirpath) - # Append the full path of the "data" directory to the list - bots_data_paths[parent_folder] = os.path.join(dirpath, dirname) + if parent_folder.startswith(reserved_word): + bots_data_paths[parent_folder] = os.path.join(dirpath, dirname) if "dashboard" in bots_data_paths: - # Create a new key "base folder" with the value of the old key - bots_data_paths["base folder"] = bots_data_paths["dashboard"] - # Delete the old key "dashboard" del bots_data_paths["dashboard"] - return {key: value for key, value in bots_data_paths.items() if value is not None} + data_sources = {key: value for key, value in bots_data_paths.items() if value is not None} + return data_sources def get_function_from_file(file_path: str, function_name: str): From 63d69f9cbcc0c1af70e890f4f69ce4e39e2c0f15 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 31 Oct 2023 18:13:25 -0300 Subject: [PATCH 3/9] (fix) correct accuracy --- utils/data_manipulation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index 80f62d18..aac458b0 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -246,8 +246,8 @@ def inventory_change_base_asset(self): @property def accuracy(self): - total_wins = len(self.trade_fill["net_realized_pnl"] >= 0) - total_losses = len(self.trade_fill["net_realized_pnl"] < 0) + total_wins = (self.trade_fill["net_realized_pnl"] >= 0).sum() + total_losses = (self.trade_fill["net_realized_pnl"] < 0).sum() return total_wins / (total_wins + total_losses) @property From b1de41675835a5df32891f1ff2641e12f68ee37a Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 31 Oct 2023 18:57:45 -0300 Subject: [PATCH 4/9] (fix) fix pnl --- pages/strategy_performance/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index b57b7219..cf5b6c0a 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -274,7 +274,7 @@ def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_ro cg = CandlesGraph(candles, show_volume=show_volume, extra_rows=extra_rows) cg.add_buy_trades(strat_data.buys) cg.add_sell_trades(strat_data.sells) - cg.add_pnl(strategy_data, row=2) + cg.add_pnl(strat_data, row=2) cg.add_quote_inventory_change(strat_data, row=3) return cg.figure() From 2e154f0fddbf302b38251b44b73cf07dae9ca60a Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 31 Oct 2023 19:01:19 -0300 Subject: [PATCH 5/9] (fix) remove margin volume --- utils/data_manipulation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index aac458b0..10a91ff0 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -24,7 +24,6 @@ def full_series(series): strategy_data = self.trade_fill.copy() strategy_data["volume"] = strategy_data["amount"] * strategy_data["price"] - strategy_data["margin_volume"] = strategy_data["amount"] * strategy_data["price"] / strategy_data["leverage"] strategy_summary = strategy_data.groupby(["strategy", "market", "symbol"]).agg({"order_id": "count", "volume": "sum", "margin_volume": "sum", From 062f9bd2adbc850b52001e57139b96d79dc84090 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 31 Oct 2023 19:02:02 -0300 Subject: [PATCH 6/9] (feat) replace duration hours by days --- pages/strategy_performance/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index cf5b6c0a..c54146d0 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -346,8 +346,8 @@ def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_ro st.metric(label="Profit Factor", value=round(strategy_data_filtered.profit_factor, 2)) with col5: - st.metric(label='Duration (Hours)', - value=round(strategy_data_filtered.duration_seconds / 3600, 2)) + st.metric(label='Duration (Days)', + value=round(strategy_data_filtered.duration_seconds / (60 * 60 * 24))) with col6: st.metric(label='Price change', value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") From 42b00296d67af5bc21714416fbad4ab1748864ff Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 31 Oct 2023 19:36:05 -0300 Subject: [PATCH 7/9] (fix) remove broken line --- pages/strategy_performance/app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index c54146d0..6b441c66 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -243,9 +243,6 @@ def hr_str(hr): return fig - return fig - - def returns_histogram(df: pd.DataFrame): fig = go.Figure() fig.add_trace(go.Histogram(name="Losses", From 305f3c2747e96e154ed32a104d1c32b9e524590e Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 31 Oct 2023 19:38:30 -0300 Subject: [PATCH 8/9] (fix) remove margin volume remaining lines --- utils/data_manipulation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index 10a91ff0..dcfc4048 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -26,7 +26,6 @@ def full_series(series): strategy_data["volume"] = strategy_data["amount"] * strategy_data["price"] strategy_summary = strategy_data.groupby(["strategy", "market", "symbol"]).agg({"order_id": "count", "volume": "sum", - "margin_volume": "sum", "net_realized_pnl": [full_series, "last"]}).reset_index() strategy_summary.columns = [f"{col[0]}_{col[1]}" if isinstance(col, tuple) and col[1] is not None else col for col in strategy_summary.columns] @@ -35,7 +34,6 @@ def full_series(series): "symbol_": "Trading Pair", "order_id_count": "# Trades", "volume_sum": "Volume", - "margin_volume_sum": "Margin volume", "net_realized_pnl_full_series": "PnL Over Time", "net_realized_pnl_last": "Realized PnL"}, inplace=True) strategy_summary.sort_values(["Realized PnL"], ascending=True, inplace=True) From ff3885983b7f54b9d4f494fbd65f200391ad3bef Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 31 Oct 2023 20:11:39 -0300 Subject: [PATCH 9/9] (fix) fix realized pnl calculation in trade fill --- utils/database_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/database_manager.py b/utils/database_manager.py index 8234fff1..ccd237c0 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -157,9 +157,9 @@ def get_trade_fills(self, config_file_path=None, start_date=None, end_date=None) trade_fills["inventory_cost"] = trade_fills["cum_net_amount"] * trade_fills["price"] trade_fills["realized_trade_pnl"] = trade_fills["unrealized_trade_pnl"] + trade_fills["inventory_cost"] trade_fills["net_realized_pnl"] = trade_fills["realized_trade_pnl"] - trade_fills["cum_fees_in_quote"] - trade_fills["realized_pnl"] = trade_fills["net_realized_pnl"].diff() - trade_fills["gross_pnl"] = trade_fills["realized_trade_pnl"].diff() - trade_fills["trade_fee"] = trade_fills["cum_fees_in_quote"].diff() + trade_fills["realized_pnl"] = trade_fills.groupby(groupers)["net_realized_pnl"].diff() + trade_fills["gross_pnl"] = trade_fills.groupby(groupers)["realized_trade_pnl"].diff() + trade_fills["trade_fee"] = trade_fills.groupby(groupers)["cum_fees_in_quote"].diff() trade_fills["timestamp"] = pd.to_datetime(trade_fills["timestamp"], unit="ms") trade_fills["market"] = trade_fills["market"].apply(lambda x: x.lower().replace("_papertrade", "")) trade_fills["quote_volume"] = trade_fills["price"] * trade_fills["amount"]