Merge pull request #94 from hummingbot/feat/strategy-performance-v-0.11

Feat/strategy performance v 0.11

pages/strategy_performance/app.py

@@ -4,6 +4,7 @@
 import plotly.graph_objects as go
 import math
 import plotly.express as px
+from utils.os_utils import get_bots_data_paths
 from utils.database_manager import DatabaseManager
 from utils.graphs import CandlesGraph
 from utils.st_utils import initialize_st_page
@@ -28,10 +29,16 @@
 
 
 def get_databases():
- sqlite_files = [db_name for db_name in os.listdir("data") if db_name.endswith(".sqlite")]
- databases_list = [DatabaseManager(db) for db in sqlite_files]
- if len(databases_list) > 0:
- return {database.db_name: database for database in databases_list}
+ databases = {}
+ bots_data_paths = get_bots_data_paths()
+ for source_name, source_path in bots_data_paths.items():
+ sqlite_files = {}
+ for db_name in os.listdir(source_path):
+ if db_name.endswith(".sqlite"):
+ sqlite_files[db_name] = os.path.join(source_path, db_name)
+ databases[source_name] = sqlite_files
+ if len(databases) > 0:
+ return {key: value for key, value in databases.items() if value}
  else:
  return None
 
@@ -233,7 +240,6 @@ def hr_str(hr):
  yanchor="bottom"
  ),
  )
-
  return fig
 
 
@@ -265,7 +271,7 @@ def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_ro
  cg = CandlesGraph(candles, show_volume=show_volume, extra_rows=extra_rows)
  cg.add_buy_trades(strat_data.buys)
  cg.add_sell_trades(strat_data.sells)
- cg.add_pnl(strategy_data, row=2)
+ cg.add_pnl(strat_data, row=2)
  cg.add_quote_inventory_change(strat_data, row=3)
  return cg.figure()
 
@@ -282,9 +288,12 @@ def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_ro
  selected_db = DatabaseManager(uploaded_db.name)
 dbs = get_databases()
 if dbs is not None:
- db_names = [x.db_name for x in dbs.values()]
+ bot_source = st.selectbox("Choose your database source:", dbs.keys())
+ db_names = [x for x in dbs[bot_source]]
  selected_db_name = st.selectbox("Select a database to start:", db_names)
- selected_db = dbs[selected_db_name]
+ executors_path = os.path.dirname(dbs[bot_source][selected_db_name])
+ selected_db = DatabaseManager(db_name=dbs[bot_source][selected_db_name],
+ executors_path=executors_path)
 else:
  st.warning("Ups! No databases were founded. Start uploading one")
  selected_db = None
@@ -334,8 +343,8 @@ def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_ro
  st.metric(label="Profit Factor",
  value=round(strategy_data_filtered.profit_factor, 2))
  with col5:
- st.metric(label='Duration (Hours)',
- value=round(strategy_data_filtered.duration_seconds / 3600, 2))
+ st.metric(label='Duration (Days)',
+ value=round(strategy_data_filtered.duration_seconds / (60 * 60 * 24)))
  with col6:
  st.metric(label='Price change',
  value=f"{round(strategy_data_filtered.price_change * 100, 2)} %")

utils/data_manipulation.py

@@ -24,10 +24,8 @@ def full_series(series):
 
  strategy_data = self.trade_fill.copy()
  strategy_data["volume"] = strategy_data["amount"] * strategy_data["price"]
- strategy_data["margin_volume"] = strategy_data["amount"] * strategy_data["price"] / strategy_data["leverage"]
  strategy_summary = strategy_data.groupby(["strategy", "market", "symbol"]).agg({"order_id": "count",
  "volume": "sum",
- "margin_volume": "sum",
  "net_realized_pnl": [full_series,
  "last"]}).reset_index()
  strategy_summary.columns = [f"{col[0]}_{col[1]}" if isinstance(col, tuple) and col[1] is not None else col for col in strategy_summary.columns]
@@ -36,7 +34,6 @@ def full_series(series):
  "symbol_": "Trading Pair",
  "order_id_count": "# Trades",
  "volume_sum": "Volume",
- "margin_volume_sum": "Margin volume",
  "net_realized_pnl_full_series": "PnL Over Time",
  "net_realized_pnl_last": "Realized PnL"}, inplace=True)
  strategy_summary.sort_values(["Realized PnL"], ascending=True, inplace=True)
@@ -246,8 +243,8 @@ def inventory_change_base_asset(self):
 
  @property
  def accuracy(self):
- total_wins = len(self.trade_fill["net_realized_pnl"] >= 0)
- total_losses = len(self.trade_fill["net_realized_pnl"] < 0)
+ total_wins = (self.trade_fill["net_realized_pnl"] >= 0).sum()
+ total_losses = (self.trade_fill["net_realized_pnl"] < 0).sum()
  return total_wins / (total_wins + total_losses)
 
  @property

utils/database_manager.py

@@ -12,7 +12,7 @@ class DatabaseManager:
  def __init__(self, db_name: str, executors_path: str = "data"):
  self.db_name = db_name
  # TODO: Create db path for all types of db
- self.db_path = f'sqlite:///{os.path.join("data", db_name)}'
+ self.db_path = f'sqlite:///{os.path.join(db_name)}'
  self.executors_path = executors_path
  self.engine = create_engine(self.db_path, connect_args={'check_same_thread': False})
  self.session_maker = sessionmaker(bind=self.engine)
@@ -157,9 +157,9 @@ def get_trade_fills(self, config_file_path=None, start_date=None, end_date=None)
  trade_fills["inventory_cost"] = trade_fills["cum_net_amount"] * trade_fills["price"]
  trade_fills["realized_trade_pnl"] = trade_fills["unrealized_trade_pnl"] + trade_fills["inventory_cost"]
  trade_fills["net_realized_pnl"] = trade_fills["realized_trade_pnl"] - trade_fills["cum_fees_in_quote"]
- trade_fills["realized_pnl"] = trade_fills["net_realized_pnl"].diff()
- trade_fills["gross_pnl"] = trade_fills["realized_trade_pnl"].diff()
- trade_fills["trade_fee"] = trade_fills["cum_fees_in_quote"].diff()
+ trade_fills["realized_pnl"] = trade_fills.groupby(groupers)["net_realized_pnl"].diff()
+ trade_fills["gross_pnl"] = trade_fills.groupby(groupers)["realized_trade_pnl"].diff()
+ trade_fills["trade_fee"] = trade_fills.groupby(groupers)["cum_fees_in_quote"].diff()
  trade_fills["timestamp"] = pd.to_datetime(trade_fills["timestamp"], unit="ms")
  trade_fills["market"] = trade_fills["market"].apply(lambda x: x.lower().replace("_papertrade", ""))
  trade_fills["quote_volume"] = trade_fills["price"] * trade_fills["amount"]

utils/os_utils.py

@@ -104,6 +104,23 @@ def load_controllers(path):
  return controllers
 
 
+def get_bots_data_paths():
+ root_directory = "hummingbot_files/bots"
+ bots_data_paths = {"General / Uploaded data": "data"}
+ reserved_word = "hummingbot-"
+ # Walk through the directory tree
+ for dirpath, dirnames, filenames in os.walk(root_directory):
+ for dirname in dirnames:
+ if dirname == "data":
+ parent_folder = os.path.basename(dirpath)
+ if parent_folder.startswith(reserved_word):
+ bots_data_paths[parent_folder] = os.path.join(dirpath, dirname)
+ if "dashboard" in bots_data_paths:
+ del bots_data_paths["dashboard"]
+ data_sources = {key: value for key, value in bots_data_paths.items() if value is not None}
+ return data_sources
+
+
 def get_function_from_file(file_path: str, function_name: str):
  # Create a module specification from the file path and load it
  spec = importlib.util.spec_from_file_location("module.name", file_path)