diff --git a/data_viz/__init__.py b/data_viz/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/data_viz/backtesting/__init__.py b/data_viz/backtesting/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/data_viz/backtesting/backtesting_candles.py b/data_viz/backtesting/backtesting_candles.py new file mode 100644 index 00000000..ea2def37 --- /dev/null +++ b/data_viz/backtesting/backtesting_candles.py @@ -0,0 +1,65 @@ +from data_viz.candles import CandlesBase +import pandas as pd +from typing import List +from data_viz.dtypes import IndicatorConfig +from quants_lab.strategy.strategy_analysis import StrategyAnalysis + + +class BacktestingCandles(CandlesBase): + def __init__(self, + strategy_analysis: StrategyAnalysis, + indicators_config: List[IndicatorConfig] = None, + line_mode: bool = False, + show_buys: bool = True, + show_sells: bool = True, + show_positions: bool = True, + show_indicators: bool = False): + self.candles_df = strategy_analysis.candles_df + super().__init__(candles_df=self.candles_df, + indicators_config=indicators_config, + line_mode=line_mode, + show_indicators=show_indicators) + self.positions = strategy_analysis.positions + if show_buys: + self.add_buy_trades(data=self.buys) + if show_sells: + self.add_sell_trades(data=self.sells) + if show_positions: + self.add_positions() + + def force_datetime_format(self): + datetime_columns = ["timestamp", "close_time", "tl", "stop_loss_time", "take_profit_time"] + for col in datetime_columns: + self.positions[col] = pd.to_datetime(self.positions[col], unit="ms") + + @property + def buys(self): + df = self.positions[["timestamp", "close", "close_price", "close_time", "side"]].copy() + df["price"] = df.apply(lambda row: row["close"] if row["side"] == "BUY" else row["close_price"], axis=1) + df["timestamp"] = df.apply(lambda row: row["timestamp"] if row["side"] == "BUY" else row["close_time"], axis=1) + df.set_index("timestamp", inplace=True) + return df["price"] + + @property + def sells(self): + df = self.positions[["timestamp", "close", "close_price", "close_time", "side"]].copy() + df["price"] = df.apply(lambda row: row["close"] if row["side"] == "SELL" else row["close_price"], axis=1) + df["timestamp"] = df.apply(lambda row: row["timestamp"] if row["side"] == "SELL" else row["close_time"], axis=1) + df.set_index("timestamp", inplace=True) + return df["price"] + + def add_positions(self): + i = 1 + for index, rown in self.positions.iterrows(): + i += 1 + self.base_figure.add_trace(self.tracer.get_positions_traces(position_number=i, open_time=rown["timestamp"], + close_time=rown["close_time"], + open_price=rown["close"], + close_price=rown["close_price"], + side=rown["side"], + close_type=rown["close_type"], + stop_loss=rown["sl"], take_profit=rown["tp"], + time_limit=rown["tl"], + net_pnl_quote=rown["net_pnl_quote"]), + row=1, col=1) + diff --git a/data_viz/backtesting/backtesting_charts.py b/data_viz/backtesting/backtesting_charts.py new file mode 100644 index 00000000..bfa15d79 --- /dev/null +++ b/data_viz/backtesting/backtesting_charts.py @@ -0,0 +1,32 @@ +from typing import Union +import pandas as pd +import plotly.graph_objects as go + +from data_viz.charts import ChartsBase +from data_viz.tracers import PerformancePlotlyTracer +from quants_lab.strategy.strategy_analysis import StrategyAnalysis + + +class BacktestingCharts(ChartsBase): + def __init__(self, + # TODO: Rename StrategyData as RealPerformanceData and StrategyAnalysis as BacktestingAnalysis + source: Union[StrategyAnalysis, None] = None): + super().__init__() + self.source = source + self.tracer = PerformancePlotlyTracer() + + @property + def realized_pnl_over_time_fig(self): + if self.source is not None: + data = self.source.positions.copy() + data.sort_values(by="timestamp", inplace=True) + return self.realized_pnl_over_time(data=data, + cum_realized_pnl_column="net_pnl_quote") + else: + return go.Figure() + + def pnl_vs_max_drawdown_fig(self, data: pd.DataFrame = None): + return self.pnl_vs_max_drawdown(data=data, + max_drawdown_pct_column="max_drawdown_pct", + net_pnl_pct_column="net_pnl_pct", + hovertext_column="hover_text") diff --git a/data_viz/candles.py b/data_viz/candles.py new file mode 100644 index 00000000..aee12d64 --- /dev/null +++ b/data_viz/candles.py @@ -0,0 +1,223 @@ +import pandas as pd +from plotly.subplots import make_subplots +import pandas_ta as ta # noqa: F401 +from typing import List +from data_viz.tracers import PandasTAPlotlyTracer +from data_viz.tracers import PerformancePlotlyTracer +from data_viz.dtypes import IndicatorConfig +import plotly.graph_objs as go + + +class CandlesBase: + def __init__(self, + candles_df: pd.DataFrame, + indicators_config: List[IndicatorConfig] = None, + show_annotations=True, + line_mode=False, + show_indicators=False, + main_height=0.7, + max_height=1000, + rows: int = None, + row_heights: list = None): + self.candles_df = candles_df + self.show_indicators = show_indicators + self.indicators_config = indicators_config + self.show_annotations = show_annotations + self.indicators_tracer = PandasTAPlotlyTracer(candles_df) + self.tracer = PerformancePlotlyTracer() + self.line_mode = line_mode + self.main_height = main_height + self.max_height = max_height + self.rows = rows + if rows is None: + rows, row_heights = self.get_n_rows_and_heights() + self.rows = rows + specs = [[{"secondary_y": True}]] * self.rows + self.base_figure = make_subplots(rows=self.rows, + cols=1, + shared_xaxes=True, + vertical_spacing=0.005, + row_heights=row_heights, + specs=specs) + if 'timestamp' in candles_df.columns: + candles_df.set_index("timestamp", inplace=True) + self.min_time = candles_df.index.min() + self.max_time = candles_df.index.max() + self.add_candles_graph() + if self.show_indicators and self.indicators_config is not None: + self.add_indicators() + self.update_layout() + + def get_n_rows_and_heights(self): + rows = 1 + if self.show_indicators and self.indicators_config is not None: + rows = max([config.row for config in self.indicators_config]) + complementary_height = 1 - self.main_height + row_heights = [self.main_height] + [complementary_height / (rows - 1)] * (rows - 1) if rows > 1 else [1] + return rows, row_heights + + def figure(self): + return self.base_figure + + def add_candles_graph(self): + if self.line_mode: + self.base_figure.add_trace( + go.Scatter(x=self.candles_df.index, + y=self.candles_df['close'], + name="Close", + mode='lines', + line=dict(color='blue')), + row=1, col=1, + ) + else: + hover_text = [] + if self.show_annotations: + for i in range(len(self.candles_df)): + hover_text.append( + f"Open: {self.candles_df['open'][i]}
" + f"High: {self.candles_df['high'][i]}
" + f"Low: {self.candles_df['low'][i]}
" + f"Close: {self.candles_df['close'][i]}
" + ) + self.base_figure.add_trace( + go.Candlestick( + x=self.candles_df.index, + open=self.candles_df['open'], + high=self.candles_df['high'], + low=self.candles_df['low'], + close=self.candles_df['close'], + name="OHLC", + hoverinfo="text", + hovertext=hover_text + ), + row=1, col=1, + ) + + def add_volume(self): + self.base_figure.add_trace( + go.Bar( + x=self.candles_df.index, + y=self.candles_df['volume'], + name="Volume", + opacity=0.5, + marker=dict(color='lightgreen'), + + ), + row=2, col=1, + ) + + def add_buy_trades(self, data: pd.Series): + buy_traces = self.tracer.get_buys_traces(data=data) if not data.empty else None + if bool(buy_traces): + self.base_figure.add_trace(buy_traces, + row=1, col=1) + + def add_sell_trades(self, data: pd.Series): + sell_traces = self.tracer.get_sells_traces(data=data) if not data.empty else None + print(f"Sell traces: {sell_traces}") + if bool(sell_traces): + self.base_figure.add_trace(sell_traces, + row=1, col=1) + + def add_quote_inventory_change(self, data: pd.DataFrame, quote_inventory_change_column: str, row_number: int = 3): + quote_inventory_change_trace = self.tracer.get_quote_inventory_change( + data=data, + quote_inventory_change_column=quote_inventory_change_column) + if quote_inventory_change_trace: + self.base_figure.add_trace(quote_inventory_change_trace, + row=row_number, col=1) + self.base_figure.update_yaxes(title_text='Quote Inventory Change', row=row_number, col=1) + + def add_pnl(self, data: pd.DataFrame, realized_pnl_column: str, fees_column: str, net_realized_pnl_column: str, + row_number: int = 2): + for trace in self.tracer.get_composed_pnl_traces(data=data, + realized_pnl_column=realized_pnl_column, + fees_column=fees_column, + net_realized_pnl_column=net_realized_pnl_column): + self.base_figure.add_trace(trace, row=row_number, col=1) + self.base_figure.update_yaxes(title_text='PNL', row=row_number, col=1) + + def add_positions(self): + """ + Depending on whether the data source is backtesting or performance, the name of the columns might change. + """ + pass + + def update_layout(self): + self.base_figure.update_layout( + legend=dict( + orientation="h", + x=0.5, + y=1.04, + xanchor="center", + yanchor="bottom" + ), + height=self.max_height, + xaxis=dict(rangeslider_visible=False, + range=[self.min_time, self.max_time]), + yaxis=dict(range=[self.candles_df.low.min(), self.candles_df.high.max()]), + hovermode='x unified' + ) + self.base_figure.update_yaxes(title_text="Price", row=1, col=1) + self.base_figure.update_xaxes(title_text="Time", row=self.rows, col=1) + + # ---------------------------- + # INDICATORS METHODS + # ---------------------------- + + def add_bollinger_bands(self, indicator_config: IndicatorConfig): + if indicator_config.visible: + bbu_trace, bbm_trace, bbl_trace = self.indicators_tracer.get_bollinger_bands_traces(indicator_config) + if all([bbu_trace, bbm_trace, bbl_trace]): + self.base_figure.add_trace(trace=bbu_trace, + row=indicator_config.row, + col=indicator_config.col) + self.base_figure.add_trace(trace=bbm_trace, + row=indicator_config.row, + col=indicator_config.col) + self.base_figure.add_trace(trace=bbl_trace, + row=indicator_config.row, + col=indicator_config.col) + + def add_ema(self, indicator_config: IndicatorConfig): + if indicator_config.visible: + ema_trace = self.indicators_tracer.get_ema_traces(indicator_config) + if ema_trace: + self.base_figure.add_trace(trace=ema_trace, + row=indicator_config.row, + col=indicator_config.col) + + def add_macd(self, indicator_config: IndicatorConfig): + if indicator_config.visible: + macd_trace, macd_signal_trace, macd_hist_trace = self.indicators_tracer.get_macd_traces(indicator_config) + if all([macd_trace, macd_signal_trace, macd_hist_trace]): + self.base_figure.add_trace(trace=macd_trace, + row=indicator_config.row, + col=indicator_config.col) + self.base_figure.add_trace(trace=macd_signal_trace, + row=indicator_config.row, + col=indicator_config.col) + self.base_figure.add_trace(trace=macd_hist_trace, + row=indicator_config.row, + col=indicator_config.col) + + def add_rsi(self, indicator_config: IndicatorConfig): + if indicator_config.visible: + rsi_trace = self.indicators_tracer.get_rsi_traces(indicator_config) + if rsi_trace: + self.base_figure.add_trace(trace=rsi_trace, + row=indicator_config.row, + col=indicator_config.col) + + def add_indicators(self): + for indicator in self.indicators_config: + if indicator.title == "bbands": + self.add_bollinger_bands(indicator) + elif indicator.title == "ema": + self.add_ema(indicator) + elif indicator.title == "macd": + self.add_macd(indicator) + elif indicator.title == "rsi": + self.add_rsi(indicator) + else: + raise ValueError(f"{indicator.title} is not a valid indicator. Choose from bbands, ema, macd, rsi") diff --git a/data_viz/charts.py b/data_viz/charts.py new file mode 100644 index 00000000..26bac8d8 --- /dev/null +++ b/data_viz/charts.py @@ -0,0 +1,143 @@ +import pandas as pd +import plotly.graph_objs as go +import plotly.express as px + +from data_viz.tracers import PerformancePlotlyTracer + + +class ChartsBase: + def __init__(self, + tracer: PerformancePlotlyTracer = PerformancePlotlyTracer()): + self.tracer = tracer + + def realized_pnl_over_trading_pair(self, data: pd.DataFrame(), trading_pair_column: str, realized_pnl_column: str, exchange: str): + """ + :param data: strategy dataframe with timestamp as index + :param trading_pair_column: column name of trading pair + :param realized_pnl_column: column name of realized pnl + :param exchange: column name of exchange + """ + fig = go.Figure() + for exchange in data[exchange].unique(): + fig.add_trace(self.tracer.get_realized_pnl_over_trading_pair_traces(data=data, + trading_pair=trading_pair_column, + realized_pnl=realized_pnl_column, + exchange=exchange)) + fig.update_traces(width=min(1.0, 0.1 * len(data))) + fig.update_layout(barmode='stack') + return fig + + def realized_pnl_over_time(self, data: pd.DataFrame, cum_realized_pnl_column: str): + fig = go.Figure() + fig.add_trace(self.tracer.get_realized_pnl_over_time_traces(data=data, + cum_realized_pnl_column=cum_realized_pnl_column)) + fig.update_layout(title=dict(text='Cummulative PnL', x=0.43, y=0.95), + plot_bgcolor='rgba(0,0,0,0)', + paper_bgcolor='rgba(0,0,0,0)') + return fig + + def pnl_vs_max_drawdown(self, data: pd.DataFrame(), max_drawdown_pct_column: str, net_pnl_pct_column: str, hovertext_column: str): + fig = go.Figure() + fig.add_trace(self.tracer.get_pnl_vs_max_drawdown_traces(data=data, + max_drawdown_pct_column=max_drawdown_pct_column, + net_pnl_pct_column=net_pnl_pct_column, + hovertext_column=hovertext_column)) + fig.update_layout(title="PnL vs Max Drawdown", + xaxis_title="Max Drawdown [%]", + yaxis_title="Net Profit [%]", + height=800) + return fig + + def intraday_performance(self, data: pd.DataFrame(), quote_volume_column: str, datetime_column: str, realized_pnl_column: str): + def hr2angle(hr): + return (hr * 15) % 360 + + def hr_str(hr): + # Normalize hr to be between 1 and 12 + hr_string = str(((hr - 1) % 12) + 1) + suffix = ' AM' if (hr % 24) < 12 else ' PM' + return hr_string + suffix + + data["hour"] = data[datetime_column].dt.hour + realized_pnl_per_hour = data.groupby("hour")[[realized_pnl_column, quote_volume_column]].sum().reset_index() + fig = go.Figure() + fig.add_trace(self.tracer.get_intraday_performance_traces(data=realized_pnl_per_hour, + quote_volume_column=quote_volume_column, + hour_column="hour", + realized_pnl_column=realized_pnl_column)) + fig.update_layout( + polar=dict( + radialaxis=dict( + visible=True, + showline=False, + ), + angularaxis=dict( + rotation=90, + direction="clockwise", + tickvals=[hr2angle(hr) for hr in range(24)], + ticktext=[hr_str(hr) for hr in range(24)], + ), + bgcolor='rgba(255, 255, 255, 0)', + + ), + legend=dict( + orientation="h", + x=0.5, + y=1.08, + xanchor="center", + yanchor="bottom" + ), + title=dict( + text='Intraday Performance', + x=0.5, + y=0.93, + xanchor="center", + yanchor="bottom" + ), + ) + return fig + + def returns_distribution(self, data: pd.DataFrame(), realized_pnl_column: str): + fig = go.Figure() + traces = self.tracer.get_returns_distribution_traces(data=data, + realized_pnl_column=realized_pnl_column) + for trace in traces: + fig.add_trace(trace) + # TODO: eventually, move this to the streamlit app + fig.update_layout( + title=dict( + text='Returns Distribution', + x=0.5, + xanchor="center", + ), + legend=dict( + orientation="h", + yanchor="bottom", + y=1.02, + xanchor="center", + x=.48 + )) + return fig + + @staticmethod + def positions_summary_sunburst(data: pd.DataFrame(), trading_pair_column: str, side_column: str, close_type_column: str, values_column: str): + fig = px.sunburst(data, + path=[trading_pair_column, side_column, close_type_column], + values=values_column, + color_continuous_scale='RdBu', + color_continuous_midpoint=0) + fig.update_layout( + title=dict( + text='Position Executor Summary', + x=0.5, + xanchor="center", + ), + legend=dict( + orientation="h", + yanchor="bottom", + y=1.02, + xanchor="center", + x=.48 + ) + ) + return fig diff --git a/data_viz/config/.gitignore b/data_viz/config/.gitignore new file mode 100644 index 00000000..f59ec20a --- /dev/null +++ b/data_viz/config/.gitignore @@ -0,0 +1 @@ +* \ No newline at end of file diff --git a/data_viz/dtypes.py b/data_viz/dtypes.py new file mode 100644 index 00000000..f4859e50 --- /dev/null +++ b/data_viz/dtypes.py @@ -0,0 +1,31 @@ +from pydantic import BaseModel + + +class IndicatorConfig(BaseModel): + visible: bool = True + title: str + row: int + col: int + color: str = 'black' + length: int = None + std: float = None + fast: int = None + slow: int = None + signal: int = None + + +class PositionsVisualConfig(BaseModel): + show: bool = True + color: str = 'black' + size: int = 20 + line_width: float = 0.7 + buy_symbol: str = 'triangle-up' + sell_symbol: str = 'triangle-down' + profitable_color: str = 'green' + non_profitable_color: str = 'red' + + +MACD_EXAMPLE_CONFIG = IndicatorConfig(visible=True, title="macd", row=1, col=1, color="red", fast=12, slow=26, signal=9) +RSI_EXAMPLE_CONFIG = IndicatorConfig(visible=True, title="rsi", row=2, col=3, color="green", length=14) +BBANDS_EXAMPLE_CONFIG = IndicatorConfig(visible=True, title="bbands", row=1, col=1, color="blue", length=20, std=2.0) +EMA_EXAMPLE_CONFIG = IndicatorConfig(visible=True, title="ema", row=1, col=1, color="yellow", length=20, culo="asd") diff --git a/data_viz/performance/__init__.py b/data_viz/performance/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/data_viz/performance/performance_candles.py b/data_viz/performance/performance_candles.py new file mode 100644 index 00000000..430e50d1 --- /dev/null +++ b/data_viz/performance/performance_candles.py @@ -0,0 +1,154 @@ +import pandas as pd +import json +from typing import Union, List + +from data_viz.candles import CandlesBase +from data_viz.dtypes import IndicatorConfig +from utils.data_manipulation import StrategyData, SingleMarketStrategyData + + +class PerformanceCandles(CandlesBase): + def __init__(self, + source: Union[StrategyData, SingleMarketStrategyData], + indicators_config: List[IndicatorConfig] = None, + candles_df: pd.DataFrame = None, + line_mode: bool = False, + show_buys: bool = False, + show_sells: bool = False, + show_positions: bool = False, + show_dca_prices: bool = False, + show_pnl: bool = True, + show_indicators: bool = False, + show_quote_inventory_change: bool = True, + show_annotations: bool = False, + executor_version: str = "v1", + main_height: float = 0.7): + self.candles_df = candles_df + + self.positions = source.executors if executor_version == "v2" else source.position_executor + self.executor_version = executor_version + self.show_buys = show_buys + self.show_sells = show_sells + self.show_positions = show_positions + self.show_pnl = show_pnl + self.show_quote_inventory_change = show_quote_inventory_change + self.show_indicators = show_indicators + self.indicators_config = indicators_config + self.main_height = main_height + + rows, row_heights = self.get_n_rows_and_heights() + super().__init__(candles_df=self.candles_df, + indicators_config=indicators_config, + line_mode=line_mode, + show_indicators=show_indicators, + rows=rows, + row_heights=row_heights, + main_height=main_height, + show_annotations=show_annotations) + if show_buys: + self.add_buy_trades(data=self.buys) + if show_sells: + self.add_sell_trades(data=self.sells) + if show_positions: + self.add_positions() + if show_pnl: + self.add_pnl(data=source.trade_fill, + realized_pnl_column="realized_trade_pnl", + fees_column="cum_fees_in_quote", + net_realized_pnl_column="net_realized_pnl", + row_number=rows - 1 if show_quote_inventory_change else rows) + if show_quote_inventory_change: + self.add_quote_inventory_change(data=source.trade_fill, + quote_inventory_change_column="inventory_cost", + row_number=rows) + if show_dca_prices: + self.add_dca_prices() + self.update_layout() + + @property + def buys(self): + df = self.positions[["datetime", "entry_price", "close_price", "close_datetime", "side"]].copy() + if len(df) > 0: + df["price"] = df.apply(lambda row: row["entry_price"] if row["side"] == 1 else row["close_price"], axis=1) + df["timestamp"] = df.apply(lambda row: row["datetime"] if row["side"] == 1 else row["close_datetime"], axis=1) + df.set_index("timestamp", inplace=True) + return df["price"] + + @property + def sells(self): + df = self.positions[["datetime", "entry_price", "close_price", "close_datetime", "side"]].copy() + if len(df) > 0: + df["price"] = df.apply(lambda row: row["entry_price"] if row["side"] == -1 else row["close_price"], axis=1) + df["timestamp"] = df.apply(lambda row: row["datetime"] if row["side"] == -1 else row["close_datetime"], axis=1) + df.set_index("timestamp", inplace=True) + return df["price"] + + def get_n_rows_and_heights(self): + rows = 1 + if self.show_indicators and self.indicators_config is not None: + rows = max([config.row for config in self.indicators_config]) + if self.show_pnl: + rows += 1 + if self.show_quote_inventory_change: + rows += 1 + complementary_height = 1 - self.main_height + row_heights = [self.main_height] + [round(complementary_height / (rows - 1), 3)] * (rows - 1) if rows > 1 else [1] + return rows, row_heights + + def add_positions(self): + if self.executor_version == "v1": + for index, rown in self.positions.iterrows(): + if abs(rown["net_pnl_quote"]) > 0: + self.base_figure.add_trace(self.tracer.get_positions_traces(position_number=rown["id"], + open_time=rown["datetime"], + close_time=rown["close_datetime"], + open_price=rown["entry_price"], + close_price=rown["close_price"], + side=rown["side"], + close_type=rown["close_type"], + stop_loss=rown["sl"], take_profit=rown["tp"], + time_limit=rown["tl"], + net_pnl_quote=rown["net_pnl_quote"]), + row=1, col=1) + elif self.executor_version == "v2": + for index, rown in self.positions.iterrows(): + if abs(rown["net_pnl_quote"]) > 0: + self.base_figure.add_trace(self.tracer.get_positions_traces(position_number=rown["id"], + open_time=rown["datetime"], + close_time=rown["close_datetime"], + open_price=rown["bep"], + close_price=rown["close_price"], + side=rown["side"], + close_type=rown["close_type"], + stop_loss=rown["sl"], take_profit=rown["tp"], + time_limit=rown["tl"], + net_pnl_quote=rown["net_pnl_quote"]), + row=1, col=1) + + def add_dca_prices(self): + if self.executor_version == "v2": + for index, rown in self.positions.iterrows(): + if abs(rown["net_pnl_quote"]) > 0: + data = list(json.loads(rown["config"])["prices"]) + data_series = pd.Series(data, index=[rown["datetime"]] * len(data)) + self.base_figure.add_trace(self.tracer.get_entry_traces(data=data_series)) + self.base_figure.add_vline(x=rown["datetime"], row=1, col=1, line_color="gray", line_dash="dash") + + + def update_layout(self): + self.base_figure.update_layout( + legend=dict( + orientation="h", + x=0.5, + y=1.04, + xanchor="center", + yanchor="bottom" + ), + height=1000, + xaxis=dict(rangeslider_visible=False, + range=[self.min_time, self.max_time]), + yaxis=dict(range=[self.candles_df.low.min(), self.candles_df.high.max()]), + hovermode='x unified' + ) + self.base_figure.update_yaxes(title_text="Price", row=1, col=1) + self.base_figure.update_xaxes(title_text="Time", row=self.rows, col=1) diff --git a/data_viz/performance/performance_charts.py b/data_viz/performance/performance_charts.py new file mode 100644 index 00000000..5fadba36 --- /dev/null +++ b/data_viz/performance/performance_charts.py @@ -0,0 +1,54 @@ +from typing import Union + +from data_viz.charts import ChartsBase +from data_viz.tracers import PerformancePlotlyTracer +from utils.data_manipulation import StrategyData, SingleMarketStrategyData + + +class PerformanceCharts(ChartsBase): + def __init__(self, + # TODO: Rename StrategyData as RealPerformanceData and StrategyAnalysis as BacktestingAnalysis + source: Union[StrategyData, SingleMarketStrategyData]): + super().__init__() + self.source = source + self.tracer = PerformancePlotlyTracer() + + @property + def realized_pnl_over_trading_pair_fig(self): + return self.realized_pnl_over_trading_pair(data=self.source.strategy_summary, + trading_pair_column="Trading Pair", + realized_pnl_column="Realized PnL", exchange="Exchange") + + @property + def realized_pnl_over_time_fig(self): + data = self.source.trade_fill.copy() + data.sort_values(by="timestamp", inplace=True) + return self.realized_pnl_over_time(data=data, + cum_realized_pnl_column="net_realized_pnl") + + @property + def intraday_performance_fig(self): + data = self.source.trade_fill.copy() + return self.intraday_performance(data=data, + quote_volume_column="quote_volume", + datetime_column="timestamp", + realized_pnl_column="realized_pnl") + + @property + def returns_distribution_fig(self): + data = self.source.trade_fill.copy() + return self.returns_distribution(data=data, + realized_pnl_column="realized_pnl") + + @property + def positions_summary_sunburst_fig(self): + if self.source.position_executor is not None: + df = self.source.position_executor.copy() + data = df.groupby(["trading_pair", "side", "close_type"]).size().reset_index(name="count") + return self.positions_summary_sunburst(data=data, + trading_pair_column="trading_pair", + side_column="side", + close_type_column="close_type", + values_column="count") + else: + return None diff --git a/data_viz/tracers.py b/data_viz/tracers.py new file mode 100644 index 00000000..d0a3ad84 --- /dev/null +++ b/data_viz/tracers.py @@ -0,0 +1,298 @@ +import plotly.graph_objs as go +import pandas as pd +from data_viz.dtypes import IndicatorConfig +from data_viz.dtypes import PositionsVisualConfig +from datetime import datetime + + +BULLISH_COLOR = "rgba(97, 199, 102, 0.9)" +BEARISH_COLOR = "rgba(255, 102, 90, 0.9)" +FEE_COLOR = "rgba(51, 0, 51, 0.9)" + + +class PandasTAPlotlyTracer: + def __init__(self, candles_df: pd.DataFrame): + """ + :param candles_df: candles dataframe with timestamp as index + """ + self.candles_df = candles_df + + @staticmethod + def raise_error_if_not_enough_data(indicator_title: str): + print(f"Not enough data to calculate {indicator_title}") + + def get_bollinger_bands_traces(self, indicator_config: IndicatorConfig): + self.candles_df.ta.bbands(length=indicator_config.length, std=indicator_config.std, append=True) + if len(self.candles_df) < indicator_config.length: + self.raise_error_if_not_enough_data(indicator_config.title) + return + else: + bbu_trace = go.Scatter(x=self.candles_df.index, + y=self.candles_df[f'BBU_{indicator_config.length}_{indicator_config.std}'], + name=f'BBU_{indicator_config.length}_{indicator_config.std}', + mode='lines', + line=dict(color=indicator_config.color, width=1)) + bbm_trace = go.Scatter(x=self.candles_df.index, + y=self.candles_df[f'BBM_{indicator_config.length}_{indicator_config.std}'], + name=f'BBM_{indicator_config.length}_{indicator_config.std}', + mode='lines', + line=dict(color=indicator_config.color, width=1)) + bbl_trace = go.Scatter(x=self.candles_df.index, + y=self.candles_df[f'BBL_{indicator_config.length}_{indicator_config.std}'], + name=f'BBL_{indicator_config.length}_{indicator_config.std}', + mode='lines', + line=dict(color=indicator_config.color, width=1)) + return bbu_trace, bbm_trace, bbl_trace + + def get_ema_traces(self, indicator_config: IndicatorConfig): + if len(self.candles_df) < indicator_config.length: + self.raise_error_if_not_enough_data(indicator_config.title) + else: + self.candles_df.ta.ema(length=indicator_config.length, append=True) + ema_trace = go.Scatter(x=self.candles_df.index, + y=self.candles_df[f'EMA_{indicator_config.length}'], + name=f'EMA_{indicator_config.length}', + mode='lines', + line=dict(color=indicator_config.color, width=1)) + return ema_trace + + def get_macd_traces(self, indicator_config): + fast = indicator_config.fast + slow = indicator_config.slow + signal = indicator_config.signal + if len(self.candles_df) < any([fast, slow, signal]): + self.raise_error_if_not_enough_data(indicator_config.title) + else: + self.candles_df.ta.macd(fast=fast, slow=slow, signal=signal, append=True) + macd_trace = go.Scatter(x=self.candles_df.index, + y=self.candles_df[f'MACD_{fast}_{slow}_{signal}'], + name=f'MACD_{fast}_{slow}_{signal}', + mode='lines', + line=dict(color=indicator_config.color, width=1)) + macd_signal_trace = go.Scatter(x=self.candles_df.index, + y=self.candles_df[f'MACDs_{fast}_{slow}_{signal}'], + name=f'MACDs_{fast}_{slow}_{signal}', + mode='lines', + line=dict(color=indicator_config.color, width=1)) + macd_hist_trace = go.Bar(x=self.candles_df.index, + y=self.candles_df[f'MACDh_{fast}_{slow}_{signal}'], + name=f'MACDh_{fast}_{slow}_{signal}', + marker=dict(color=indicator_config.color)) + return macd_trace, macd_signal_trace, macd_hist_trace + + def get_rsi_traces(self, indicator_config: IndicatorConfig): + length = indicator_config.length + if len(self.candles_df) < length: + self.raise_error_if_not_enough_data(indicator_config.title) + else: + self.candles_df.ta.rsi(length=length, append=True) + rsi_trace = go.Scatter(x=self.candles_df.index, + y=self.candles_df[f'RSI_{length}'], + name=f'RSI_{length}', + mode='lines', + line=dict(color=indicator_config.color, width=1)) + return rsi_trace + + +class PerformancePlotlyTracer: + def __init__(self, + positions_visual_config: PositionsVisualConfig = PositionsVisualConfig()): + self.positions_visual_config = positions_visual_config + + @staticmethod + def get_buys_traces(data: pd.Series): + buy_trades_trace = go.Scatter( + x=data.index, + y=data.values, + name="Buy Orders", + mode="markers", + marker=dict( + symbol="triangle-up", + color="green", + size=12, + line=dict(color="black", width=1), + opacity=0.7, + ), + hoverinfo="text", + hovertext=data.apply(lambda x: f"Buy Order: {x}
") + ) + return buy_trades_trace + + @staticmethod + def get_entry_traces(data: pd.Series): + entry_price_trace = go.Scatter( + x=data.index, + y=data.values, + name="Buy Orders", + mode="markers", + marker=dict( + symbol="triangle-right", + color="white", + size=12, + line=dict(color="black", width=1), + opacity=0.7, + ), + hoverinfo="text", + showlegend=False, + hovertext=data.apply(lambda x: f"Entry Price: {x}
") + ) + return entry_price_trace + + + @staticmethod + def get_sells_traces(data: pd.Series): + sell_trades_trace = go.Scatter( + x=data.index, + y=data.values, + name="Sell Orders", + mode="markers", + marker=dict( + symbol="triangle-down", + color="red", + size=12, + line=dict(color="black", width=1), + opacity=0.7, + ), + hoverinfo="text", + hovertext=data.apply(lambda x: f"Sell Order: {x}
") + ) + return sell_trades_trace + + @staticmethod + def get_positions_traces(position_number: int, + open_time: datetime, + close_time: datetime, + open_price: float, + close_price: float, + side: int, + close_type: str, + stop_loss: float, + take_profit: float, + time_limit: float, + net_pnl_quote: float): + """ + Draws a line between the open and close price of a position + """ + positions_trace = go.Scatter(name=f"Position {position_number}", + x=[open_time, close_time], + y=[open_price, close_price], + mode="lines", + line=dict(color="lightgreen" if net_pnl_quote > 0 else "red"), + hoverinfo="text", + hovertext=f"Position Nยฐ: {position_number}
" + f"Open datetime: {open_time}
" + f"Close datetime: {close_time}
" + f"Side: {side}
" + f"Entry price: {open_price}
" + f"Close price: {close_price}
" + f"Close type: {close_type}
" + f"Stop Loss: {100 * stop_loss:.2f}%
" + f"Take Profit: {100 * take_profit:.2f}%
" + f"Time Limit: {time_limit}
", + showlegend=False) + return positions_trace + + @staticmethod + def get_realized_pnl_over_trading_pair_traces(data: pd.DataFrame(), trading_pair: str, realized_pnl: str, exchange: str): + realized_pnl_over_trading_pair_traces = go.Bar(x=data[trading_pair], + y=data[realized_pnl], + name=exchange, + showlegend=True) + return realized_pnl_over_trading_pair_traces + + @staticmethod + def get_realized_pnl_over_time_traces(data: pd.DataFrame(), cum_realized_pnl_column: str): + realized_pnl_over_time_traces = go.Bar(name="Cum Realized PnL", + x=[x + 1 for x in range(len(data))], + y=data[cum_realized_pnl_column], + marker_color=data[cum_realized_pnl_column].apply(lambda x: BULLISH_COLOR if x > 0 else BEARISH_COLOR), + showlegend=False) + return realized_pnl_over_time_traces + + @staticmethod + def get_pnl_vs_max_drawdown_traces(data: pd.DataFrame(), max_drawdown_pct_column: str, net_pnl_pct_column: str, + hovertext_column: str): + pnl_vs_max_drawdown_traces = go.Scatter(name="Pnl vs Max Drawdown", + x=-100 * data[max_drawdown_pct_column], + y=100 * data[net_pnl_pct_column], + mode="markers", + text=None, + hovertext=data[hovertext_column]) + return pnl_vs_max_drawdown_traces + + @staticmethod + def get_composed_pnl_traces(data: pd.DataFrame(), realized_pnl_column: str, fees_column: str, + net_realized_pnl_column: str): + cum_profit_trace = go.Scatter( + x=data.timestamp, + y=[max(0, realized_pnl) for realized_pnl in data[realized_pnl_column].apply(lambda x: round(x, 4))], + name="Cum Profit", + mode='lines', + line=dict(shape="hv", color="rgba(1, 1, 1, 0.5)", dash="dash", width=0.1), + fill="tozeroy", + fillcolor="rgba(0, 255, 0, 0.5)" + ) + cum_loss_trace = go.Scatter( + x=data.timestamp, + y=[min(0, realized_pnl) for realized_pnl in data[realized_pnl_column].apply(lambda x: round(x, 4))], + name="Cum Loss", + mode='lines', + line=dict(shape="hv", color="rgba(1, 1, 1, 0.5)", dash="dash", width=0.3), + fill="tozeroy", + fillcolor="rgba(255, 0, 0, 0.5)", + ) + cum_fees_trace = go.Scatter( + x=data.timestamp, + y=data[fees_column].apply(lambda x: round(x, 4)), + name="Cum Fees", + mode='lines', + line=dict(shape="hv", color="rgba(1, 1, 1, 0.1)", dash="dash", width=0.1), + fill="tozeroy", + fillcolor="rgba(51, 0, 51, 0.5)" + ) + net_realized_profit_trace = go.Scatter(name="Net Realized Profit", + x=data.timestamp, + y=data[net_realized_pnl_column], + mode="lines", + line=dict(shape="hv")) + composed_pnl_traces = [cum_profit_trace, cum_loss_trace, cum_fees_trace, net_realized_profit_trace] + return composed_pnl_traces + + @staticmethod + def get_quote_inventory_change(data: pd.DataFrame, quote_inventory_change_column: str): + quote_inventory_change_trace = go.Scatter(name="Quote Inventory", + x=data.timestamp, + y=data[quote_inventory_change_column], + mode="lines", + line=dict(shape="hv")) + return quote_inventory_change_trace + + @staticmethod + def get_intraday_performance_traces(data: pd.DataFrame, quote_volume_column: str, hour_column: str, realized_pnl_column: str): + intraday_performance_traces = go.Barpolar( + name="Profits", + r=data[quote_volume_column], + theta=data[hour_column] * 15, + marker=dict( + color=data[realized_pnl_column], + colorscale="RdYlGn", + cmin=-(abs(data[realized_pnl_column]).max()), + cmid=0.0, + cmax=(abs(data[realized_pnl_column]).max()), + colorbar=dict( + title='Realized PnL', + x=0, + y=-0.5, + xanchor='left', + yanchor='bottom', + orientation='h' + ))) + return intraday_performance_traces + + @staticmethod + def get_returns_distribution_traces(data: pd.DataFrame(), realized_pnl_column: str): + losses = data.loc[data[realized_pnl_column] < 0, realized_pnl_column] + profits = data.loc[data[realized_pnl_column] > 0, realized_pnl_column] + returns_distribution_traces = [go.Histogram(name="Losses", x=losses, marker_color=BEARISH_COLOR), + go.Histogram(name="Profits", x=profits, marker_color=BULLISH_COLOR)] + return returns_distribution_traces diff --git a/data_viz/utils.py b/data_viz/utils.py new file mode 100644 index 00000000..d35c7cb4 --- /dev/null +++ b/data_viz/utils.py @@ -0,0 +1,37 @@ +from typing import List +import json +import os + +from data_viz.dtypes import IndicatorConfig + +example_case = [ + IndicatorConfig(visible=True, title="bbands", row=1, col=1, color="blue", length=20, std=2.0), + IndicatorConfig(visible=True, title="ema", row=1, col=1, color="yellow", length=20), + IndicatorConfig(visible=True, title="ema", row=1, col=1, color="yellow", length=40), + IndicatorConfig(visible=True, title="ema", row=1, col=1, color="yellow", length=60), + IndicatorConfig(visible=True, title="ema", row=1, col=1, color="yellow", length=80), + IndicatorConfig(visible=True, title="macd", row=2, col=1, color="red", fast=12, slow=26, signal=9), + IndicatorConfig(visible=True, title="rsi", row=3, col=1, color="green", length=14) +] +INDICATORS_CONFIG_PATH = "data_viz/config" + + +def dump_indicators_config(indicators_config: List[IndicatorConfig], name: str): + dump = [config.dict() for config in indicators_config] + path = os.path.join(INDICATORS_CONFIG_PATH, f"{name}.json") + with open(path, "w") as f: + json.dump(dump, f) + + +def load_indicators_config(path: str): + with open(path, "r") as f: + data = json.load(f) + return [IndicatorConfig.parse_obj(config) for config in data] + + +def get_indicators_config_paths(): + return [os.path.join(INDICATORS_CONFIG_PATH, file) for file in os.listdir(INDICATORS_CONFIG_PATH) if file != ".gitignore"] + + +if __name__ == "__main__": + dump_indicators_config(example_case, "config/example_case.json") diff --git a/main.py b/main.py index a627b3aa..3d25c4a9 100644 --- a/main.py +++ b/main.py @@ -22,6 +22,7 @@ def main_page(): Page("pages/launch_bot/app.py", "Deploy", "๐Ÿ™Œ"), Section("Community Pages", "๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ"), Page("pages/strategy_performance/app.py", "Strategy Performance", "๐Ÿš€"), + Page("pages/viz_lab/app.py", "Viz Lab", "๐Ÿฅผ"), Page("pages/db_inspector/app.py", "DB Inspector", "๐Ÿ”"), Page("pages/token_spreads/app.py", "Token Spreads", "๐Ÿง™"), Page("pages/tvl_vs_mcap/app.py", "TVL vs Market Cap", "๐Ÿฆ‰"), diff --git a/pages/backtest_analyze/analyze.py b/pages/backtest_analyze/analyze.py index 2a555735..0a70ad8f 100644 --- a/pages/backtest_analyze/analyze.py +++ b/pages/backtest_analyze/analyze.py @@ -11,7 +11,9 @@ from decimal import Decimal from quants_lab.strategy.strategy_analysis import StrategyAnalysis -from utils.graphs import BacktestingGraphs +from data_viz.backtesting.backtesting_charts import BacktestingCharts +from data_viz.backtesting.backtesting_candles import BacktestingCandles +import data_viz.utils as utils from utils.optuna_database_manager import OptunaDBManager from utils.os_utils import load_controllers from utils.st_utils import initialize_st_page @@ -60,7 +62,7 @@ def initialize_session_state_vars(): net_profit = st.slider("Net PNL (%)", min_value=merged_df["net_pnl_pct"].min(), max_value=merged_df["net_pnl_pct"].max(), value=[merged_df["net_pnl_pct"].min(), merged_df["net_pnl_pct"].max()], step=0.01) max_drawdown = st.slider("Max Drawdown (%)", min_value=merged_df["max_drawdown_pct"].min(), max_value=merged_df["max_drawdown_pct"].max(), - value=[merged_df["max_drawdown_pct"].min(), merged_df["max_drawdown_pct"].max()], step=0.01) + value=[merged_df["max_drawdown_pct"].min(), merged_df["max_drawdown_pct"].max()], step=0.01) total_positions = st.slider("Total Positions", min_value=merged_df["total_positions"].min(), max_value=merged_df["total_positions"].max(), value=[merged_df["total_positions"].min(), merged_df["total_positions"].max()], step=1) net_profit_filter = (merged_df["net_pnl_pct"] >= net_profit[0]) & (merged_df["net_pnl_pct"] <= net_profit[1]) @@ -68,9 +70,9 @@ def initialize_session_state_vars(): max_drawdown_filter = (merged_df["max_drawdown_pct"] >= max_drawdown[0]) & (merged_df["max_drawdown_pct"] <= max_drawdown[1]) total_positions_filter = (merged_df["total_positions"] >= total_positions[0]) & (merged_df["total_positions"] <= total_positions[1]) with scatter_column: - bt_graphs = BacktestingGraphs(merged_df[net_profit_filter & accuracy_filter & max_drawdown_filter & total_positions_filter]) - # Show and compare all of the study trials - st.plotly_chart(bt_graphs.pnl_vs_maxdrawdown(), use_container_width=True) + # Show and compare all the study trials + bt_main_charts = BacktestingCharts() + st.plotly_chart(bt_main_charts.pnl_vs_max_drawdown_fig(data=merged_df[net_profit_filter & accuracy_filter & max_drawdown_filter & total_positions_filter]), use_container_width=True) # Get study trials trials = studies[study_selected] # Choose trial @@ -188,19 +190,20 @@ def initialize_session_state_vars(): trade_cost = st.number_input("Trade cost", value=0.0006, min_value=0.0001, format="%.4f", ) - with col2: initial_portfolio_usd = st.number_input("Initial portfolio usd", value=10000.00, min_value=1.00, max_value=999999999.99) - with col3: + with col2: start = st.text_input("Start", value="2023-01-01") + indicators_config_path = st.selectbox("Indicators config path", utils.get_indicators_config_paths()) + with col3: end = st.text_input("End", value="2024-01-01") - c1, c2 = st.columns([1, 1]) with col4: - add_positions = st.checkbox("Add positions", value=True) - add_volume = st.checkbox("Add volume", value=True) - add_pnl = st.checkbox("Add PnL", value=True) + show_buys = st.checkbox("Buys", value=False) + show_sells = st.checkbox("Sells", value=False) + show_positions = st.checkbox("Positions", value=False) + show_indicators = st.checkbox("Indicators", value=False) save_config = st.button("๐Ÿ’พSave controller config!") config = controller["config"](**st.session_state["strategy_params"]) controller = controller["class"](config=config) @@ -220,10 +223,76 @@ def initialize_session_state_vars(): positions=backtesting_results["executors_df"], candles_df=backtesting_results["processed_data"], ) - metrics_container = BacktestingGraphs(backtesting_results["processed_data"]).get_trial_metrics( - strategy_analysis, - add_positions=add_positions, - add_volume=add_volume) + + backtesting_charts = BacktestingCharts(strategy_analysis) + backtesting_candles = BacktestingCandles(strategy_analysis, + indicators_config=utils.load_indicators_config(indicators_config_path), + line_mode=False, + show_buys=show_buys, + show_sells=show_sells, + show_indicators=show_indicators, + show_positions=show_positions) + col1, col2 = st.columns(2) + with col1: + st.subheader("๐Ÿฆ General") + with col2: + st.subheader("๐Ÿ“‹ General stats") + col1, col2, col3, col4 = st.columns(4) + with col1: + st.metric("Exchange", st.session_state["strategy_params"]["exchange"]) + with col2: + st.metric("Trading Pair", st.session_state["strategy_params"]["trading_pair"]) + with col3: + st.metric("Start date", strategy_analysis.start_date().strftime("%Y-%m-%d %H:%M")) + st.metric("End date", strategy_analysis.end_date().strftime("%Y-%m-%d %H:%M")) + with col4: + st.metric("Duration (hours)", f"{strategy_analysis.duration_in_minutes() / 60:.2f}") + st.metric("Price change", st.session_state["strategy_params"]["trading_pair"]) + st.subheader("๐Ÿ“ˆ Performance") + col1, col2, col3, col4, col5, col6, col7, col8 = st.columns(8) + with col1: + st.metric("Net PnL USD", + f"{strategy_analysis.net_profit_usd():.2f}", + delta=f"{100 * strategy_analysis.net_profit_pct():.2f}%", + help="The overall profit or loss achieved.") + with col2: + st.metric("Total positions", + f"{strategy_analysis.total_positions()}", + help="The total number of closed trades, winning and losing.") + with col3: + st.metric("Accuracy", + f"{100 * (len(strategy_analysis.win_signals()) / strategy_analysis.total_positions()):.2f} %", + help="The percentage of winning trades, the number of winning trades divided by the" + " total number of closed trades") + with col4: + st.metric("Profit factor", + f"{strategy_analysis.profit_factor():.2f}", + help="The amount of money the strategy made for every unit of money it lost, " + "gross profits divided by gross losses.") + with col5: + st.metric("Max Drawdown", + f"{strategy_analysis.max_drawdown_usd():.2f}", + delta=f"{100 * strategy_analysis.max_drawdown_pct():.2f}%", + help="The greatest loss drawdown, i.e., the greatest possible loss the strategy had compared " + "to its highest profits") + with col6: + st.metric("Avg Profit", + f"{strategy_analysis.avg_profit():.2f}", + help="The sum of money gained or lost by the average trade, Net Profit divided by " + "the overall number of closed trades.") + with col7: + st.metric("Avg Minutes", + f"{strategy_analysis.avg_trading_time_in_minutes():.2f}", + help="The average number of minutes that elapsed during trades for all closed trades.") + with col8: + st.metric("Sharpe Ratio", + f"{strategy_analysis.sharpe_ratio():.2f}", + help="The Sharpe ratio is a measure that quantifies the risk-adjusted return of an investment" + " or portfolio. It compares the excess return earned above a risk-free rate per unit of" + " risk taken.") + st.plotly_chart(backtesting_charts.realized_pnl_over_time_fig, use_container_width=True) + st.subheader("๐Ÿ’ฑ Market activity") + st.plotly_chart(backtesting_candles.figure(), use_container_width=True) except FileNotFoundError: st.warning(f"The requested candles could not be found.") diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index d7ea0c0e..c50c8995 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -5,6 +5,9 @@ from utils.os_utils import get_databases from utils.database_manager import DatabaseManager from utils.graphs import PerformanceGraphs +from data_viz.performance.performance_charts import PerformanceCharts +from data_viz.performance.performance_candles import PerformanceCandles +import data_viz.utils as utils from utils.st_utils import initialize_st_page, download_csv_button, style_metric_cards, db_error_message @@ -52,7 +55,9 @@ # Load strategy data strategy_data = selected_db.get_strategy_data() +executor_version = "v2" if strategy_data.executors is not None and not strategy_data.executors.empty else "v1" main_performance_charts = PerformanceGraphs(strategy_data) +performance_charts = PerformanceCharts(strategy_data) # Strategy summary section st.divider() @@ -64,7 +69,7 @@ else: main_tab, chart_tab = st.tabs(["Main", "Chart"]) with chart_tab: - st.plotly_chart(main_performance_charts.summary_chart(), use_container_width=True) + st.plotly_chart(performance_charts.realized_pnl_over_trading_pair_fig, use_container_width=True) with main_tab: selection = main_performance_charts.strategy_summary_table() if selection is None: @@ -95,6 +100,7 @@ single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) time_filtered_strategy_data = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) time_filtered_performance_charts = PerformanceGraphs(time_filtered_strategy_data) +time_performance_charts = PerformanceCharts(time_filtered_strategy_data) # Header metrics col1, col2, col3, col4, col5, col6, col7, col8 = st.columns(8) @@ -135,7 +141,7 @@ help="The total amount of quote asset sold.") # Cummulative pnl chart -st.plotly_chart(time_filtered_performance_charts.pnl_over_time(), use_container_width=True) +st.plotly_chart(time_performance_charts.realized_pnl_over_time_fig, use_container_width=True) # Market activity section st.subheader("๐Ÿ’ฑ Market activity") @@ -143,9 +149,37 @@ st.warning("Market data is not available so the candles graph is not going to be rendered." "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") else: + # Visibility options + with st.expander("Visual Options"): + col1, col2, col3, col4, col5, col6, col7 = st.columns(7) + with col1: + show_buys = st.checkbox("Buys", value=False) + if executor_version == "v2": + show_dca_prices = st.checkbox("DCA Prices", value=False) + else: + show_dca_prices = False + pass + with col2: + show_sells = st.checkbox("Sells", value=False) + show_annotations = st.checkbox("Annotations", value=False) + with col3: + show_positions = st.checkbox("Positions", value=False) + with col4: + show_pnl = st.checkbox("PNL", value=True) + with col5: + show_quote_inventory_change = st.checkbox("Quote Inventory Change", value=True) + with col6: + show_indicators = st.checkbox("Indicators", value=False) + with col7: + main_height = st.slider("Main Row Height", min_value=0.1, max_value=1.0, value=0.7, step=0.1) col1, col2 = st.columns([3, 1]) with col2: + st.markdown("### Candles config") # Set custom configs + if show_indicators: + indicators_config_path = st.selectbox("Indicators path", utils.get_indicators_config_paths()) + else: + indicators_config_path = None interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) rows_per_page = st.number_input("Candles per Page", value=1500, min_value=1, max_value=5000) @@ -165,14 +199,14 @@ # Get Page Filtered Strategy Data page_filtered_strategy_data = single_market_strategy_data.get_filtered_strategy_data(start_time_page, end_time_page) page_performance_charts = PerformanceGraphs(page_filtered_strategy_data) - candles_chart = page_performance_charts.candles_graph(candles_df, interval=interval) - + page_charts = PerformanceCharts(page_filtered_strategy_data) + # candles_chart = page_performance_charts.candles_graph(candles_df, interval=interval) # Show auxiliary charts intraday_tab, returns_tab, returns_data_tab, positions_tab, other_metrics_tab = st.tabs(["Intraday", "Returns", "Returns Data", "Positions", "Other Metrics"]) with intraday_tab: - st.plotly_chart(time_filtered_performance_charts.intraday_performance(), use_container_width=True) + st.plotly_chart(page_charts.intraday_performance_fig, use_container_width=True) with returns_tab: - st.plotly_chart(time_filtered_performance_charts.returns_histogram(), use_container_width=True) + st.plotly_chart(page_charts.returns_distribution_fig, use_container_width=True) with returns_data_tab: raw_returns_data = time_filtered_strategy_data.trade_fill[["timestamp", "gross_pnl", "trade_fee", "realized_pnl"]].dropna(subset="realized_pnl") st.dataframe(raw_returns_data, @@ -181,9 +215,8 @@ height=(min(len(time_filtered_strategy_data.trade_fill) * 39, 600))) download_csv_button(raw_returns_data, "raw_returns_data", "download-raw-returns") with positions_tab: - positions_sunburst = page_performance_charts.position_executor_summary_sunburst() - if positions_sunburst: - st.plotly_chart(page_performance_charts.position_executor_summary_sunburst(), use_container_width=True) + if page_charts.positions_summary_sunburst_fig is not None: + st.plotly_chart(page_charts.positions_summary_sunburst_fig, use_container_width=True) else: st.info("No position executor data found.") with other_metrics_tab: @@ -212,7 +245,20 @@ st.metric(label='Average Sell Price', value=round(time_filtered_strategy_data.average_sell_price, 4), help="The average price of the base asset sold.") with col1: - st.plotly_chart(candles_chart, use_container_width=True) + page_candles = PerformanceCandles(source=page_filtered_strategy_data, + indicators_config=utils.load_indicators_config(indicators_config_path) if show_indicators else None, + candles_df=candles_df, + show_dca_prices=show_dca_prices, + show_positions=show_positions, + show_buys=show_buys, + show_sells=show_sells, + show_pnl=show_pnl, + show_quote_inventory_change=show_quote_inventory_change, + show_indicators=show_indicators, + main_height=main_height, + executor_version=executor_version, + show_annotations=show_annotations) + st.plotly_chart(page_candles.figure(), use_container_width=True) # Tables section st.divider() @@ -234,3 +280,7 @@ with st.expander("๐Ÿค– Position executor"): st.write(strategy_data.position_executor) download_csv_button(strategy_data.position_executor, "position_executor", "download-position-executor") +if strategy_data.executors is not None and not strategy_data.executors.empty: + with st.expander("๐Ÿค– Executors"): + st.write(strategy_data.executors) + download_csv_button(strategy_data.executors, "executors", "download-executors") diff --git a/pages/viz_lab/app.py b/pages/viz_lab/app.py new file mode 100644 index 00000000..b2e29bbf --- /dev/null +++ b/pages/viz_lab/app.py @@ -0,0 +1,92 @@ +import os +import pandas as pd +import json +from typing import List +import streamlit as st +import plotly.graph_objs as go +from data_viz.dtypes import IndicatorConfig +import data_viz.utils as utils +import data_viz.dtypes as dtypes +from data_viz.candles import CandlesBase + + +def get_input_types(indicator_name: str): + if indicator_name == "bbands": + base_indicator_config = dtypes.BBANDS_EXAMPLE_CONFIG + elif indicator_name == "ema": + base_indicator_config = dtypes.EMA_EXAMPLE_CONFIG + elif indicator_name == "macd": + base_indicator_config = dtypes.MACD_EXAMPLE_CONFIG + elif indicator_name == "rsi": + base_indicator_config = dtypes.RSI_EXAMPLE_CONFIG + else: + raise ValueError(f"{indicator_name} is not a valid indicator. Choose from bbands, ema, macd, rsi") + + base_indicator_config_dict = vars(base_indicator_config) + indicator_config_dict = {} + st.markdown("### Customize your indicator") + for attr, value in base_indicator_config_dict.items(): + if attr in ["title", "col"] or value is None: + indicator_config_dict[attr] = value + elif attr == "color": + indicator_config_dict[attr] = st.color_picker(attr) + elif isinstance(value, bool): + indicator_config_dict[attr] = st.checkbox(attr, value=base_indicator_config_dict[attr]) + elif isinstance(value, int): + indicator_config_dict[attr] = st.number_input(attr, value=base_indicator_config_dict[attr]) + elif isinstance(value, str): + indicator_config_dict[attr] = st.text_input(attr, value=base_indicator_config_dict[attr]) + elif isinstance(value, float): + indicator_config_dict[attr] = st.number_input(attr, value=base_indicator_config_dict[attr]) + else: + raise ValueError(f"Type {value} not supported") + return indicator_config_dict + + +st.set_page_config(layout="wide") +if "indicator_config_list" not in st.session_state: + st.session_state.indicator_config_list = [] + +# Streamlit app +st.title("Viz Lab") + +tabs = st.tabs(["Technical Indicators"]) + +with tabs[0]: + st.subheader("Data Source") + if not bool(os.listdir("data/candles")): + st.info("No candles found in data/candles folder. Start generating a file from candles downloader page.") + else: + file = st.selectbox("Select candles to test:", + [file for file in os.listdir("data/candles") if file != ".gitignore" and file.endswith(".csv")]) + candles_df = pd.read_csv(f"data/candles/{file}") + col1, col2 = st.columns([1, 2]) + with col1: + st.subheader("Indicators") + name = st.selectbox("Select an indicator", ["bbands", "ema", "macd", "rsi"]) + indicator_config = get_input_types(name) + indicator = IndicatorConfig(**indicator_config) + print(st.session_state.indicator_config_list) + col1_a, col1_b = st.columns(2) + with col1_a: + if st.button("Add Indicator"): + st.session_state.indicator_config_list.append(indicator) + st.rerun() + with col1_b: + if st.button("Clear"): + st.session_state.indicator_config_list = [] + st.rerun() + with col2: + st.subheader("Preview") + if len(candles_df) > 0 and bool(st.session_state.indicator_config_list): + candles = CandlesBase(candles_df.tail(300), indicators_config=st.session_state.indicator_config_list, annotations=False, max_height=600, show_indicators=True) + st.plotly_chart(candles.figure(), use_container_width=True) + else: + st.info("Start adding one indicator!") + st.subheader("Save progress") + col1, col2, col3 = st.columns([1, 1, 2]) + with col1: + file_name = st.text_input("Save as:", value="example_case", label_visibility="collapsed") + with col2: + if st.button("Save"): + utils.dump_indicators_config(st.session_state.indicator_config_list, file_name) diff --git a/quants_lab/strategy/strategy_analysis.py b/quants_lab/strategy/strategy_analysis.py index bd4ca0b4..c4c0c460 100644 --- a/quants_lab/strategy/strategy_analysis.py +++ b/quants_lab/strategy/strategy_analysis.py @@ -16,133 +16,6 @@ def __init__(self, positions: pd.DataFrame, candles_df: Optional[pd.DataFrame] = self.positions["close_time"] = pd.to_datetime(self.positions["close_time"], unit="ms") self.base_figure = None - def create_base_figure(self, candlestick=True, volume=True, positions=False, trade_pnl=False, extra_rows=0): - rows, heights = self.get_n_rows_and_heights(extra_rows + trade_pnl, volume) - self.rows = rows - specs = [[{"secondary_y": True}]] * rows - self.base_figure = make_subplots(rows=rows, cols=1, shared_xaxes=True, vertical_spacing=0.01, - row_heights=heights, specs=specs) - if candlestick: - self.add_candles_graph(row=1, col=1) - if volume: - self.add_volume() - if positions: - self.add_positions() - if trade_pnl: - self.add_trade_pnl(row=rows) - self.update_layout(volume) - - def add_positions(self): - # Add long and short positions - active_signals = self.positions.copy() - active_signals.loc[active_signals["signal"] == -1, "symbol"] = "triangle-down" - active_signals.loc[active_signals["signal"] == 1, "symbol"] = "triangle-up" - active_signals.loc[active_signals["profitable"] == 1, "color"] = "lightgreen" - active_signals.loc[active_signals["profitable"] == -1, "color"] = "red" - self.base_figure.add_trace(go.Scatter(x=active_signals.loc[(active_signals["side"] != 0), "timestamp"], - y=active_signals.loc[active_signals["side"] != 0, "close"], - name="Entry Price: $", - mode="markers", - marker_color=active_signals.loc[(active_signals["side"] != 0), "color"], - marker_symbol=active_signals.loc[(active_signals["side"] != 0), "symbol"], - marker_size=20, - marker_line={"color": "black", "width": 0.7}), - row=1, col=1) - - for index, row in active_signals.iterrows(): - self.base_figure.add_shape(type="rect", - fillcolor="green", - opacity=0.5, - x0=row.timestamp, - y0=row.close, - x1=row.close_time, - y1=row.take_profit_price, - line=dict(color="green"), - row=1, col=1) - # Add SL - self.base_figure.add_shape(type="rect", - fillcolor="red", - opacity=0.5, - x0=row.timestamp, - y0=row.close, - x1=row.close_time, - y1=row.stop_loss_price, - line=dict(color="red"), - row=1, col=1) - - def get_n_rows_and_heights(self, extra_rows, volume=True): - rows = 1 + extra_rows + volume - row_heights = [0.5] * (extra_rows) - if volume: - row_heights.insert(0, 0.2) - row_heights.insert(0, 0.8) - return rows, row_heights - - def figure(self): - return self.base_figure - - def add_candles_graph(self, row, col, name_suffix='', timeframe_suffix=''): - self.base_figure.add_trace( - go.Candlestick( - x=self.candles_df[f"timestamp{timeframe_suffix}"], - open=self.candles_df[f"open{timeframe_suffix}"], - high=self.candles_df[f"high{timeframe_suffix}"], - low=self.candles_df[f"low{timeframe_suffix}"], - close=self.candles_df[f"close{timeframe_suffix}"], - name=f"OHLC_{name_suffix}" - ), - row=row, col=col, - ) - - def add_volume(self): - self.base_figure.add_trace( - go.Bar( - x=self.candles_df["timestamp"], - y=self.candles_df["volume"], - name="Volume", - opacity=0.5, - marker=dict(color="lightgreen") - ), - row=2, col=1, - ) - - def add_trade_pnl(self, row=2): - self.base_figure.add_trace( - go.Scatter( - x=self.positions["timestamp"], - y=self.positions["net_pnl_quote"].cumsum(), - name="Cumulative Trade PnL", - mode="lines", - line=dict(color="chocolate", width=2)), - row=row, col=1 - ) - self.base_figure.update_yaxes(title_text="Cum Trade PnL", row=row, col=1) - - def update_layout(self, volume=True): - self.base_figure.update_layout( - title={ - "text": "Backtesting Analysis", - "y": 0.95, - "x": 0.5, - "xanchor": "center", - "yanchor": "top" - }, - legend=dict( - orientation="h", - yanchor="bottom", - y=-0.2, - xanchor="right", - x=1 - ), - height=1500, - xaxis_rangeslider_visible=False, - hovermode="x unified" - ) - self.base_figure.update_yaxes(title_text="Price", row=1, col=1) - if volume: - self.base_figure.update_yaxes(title_text="Volume", row=2, col=1) - self.base_figure.update_xaxes(title_text="Time", row=self.rows, col=1) - def initial_portfolio(self): return self.positions["inventory"].dropna().values[0] @@ -150,7 +23,8 @@ def final_portfolio(self): return self.positions["inventory"].dropna().values[-1] def net_profit_usd(self): - return self.final_portfolio() - self.initial_portfolio() + # TODO: Fix inventory calculation, gives different to this + return self.positions["net_pnl_quote"].sum() def net_profit_pct(self): return self.net_profit_usd() / self.initial_portfolio() @@ -159,6 +33,7 @@ def returns(self): return self.positions["net_pnl_quote"] / self.initial_portfolio() def total_positions(self): + # TODO: Determine if it has to be shape[0] - 1 or just shape[0] return self.positions.shape[0] - 1 def win_signals(self): @@ -197,10 +72,10 @@ def avg_trading_time_in_minutes(self): return time_diff_minutes.mean() def start_date(self): - return pd.to_datetime(self.candles_df.timestamp.min(), unit="ms") + return pd.to_datetime(self.candles_df.index.min(), unit="ms") def end_date(self): - return pd.to_datetime(self.candles_df.timestamp.max(), unit="ms") + return pd.to_datetime(self.candles_df.index.max(), unit="ms") def avg_profit(self): return self.positions.net_pnl_quote.mean() @@ -222,8 +97,10 @@ def text_report(self): def pnl_over_time(self): fig = go.Figure() + positions = self.positions.copy() + positions.reset_index(inplace=True).sort_values("close_time", inplace=True) fig.add_trace(go.Scatter(name="PnL Over Time", - x=self.positions.index, + x=self.positions.close_time, y=self.positions.net_pnl_quote.cumsum())) # Update layout with the required attributes fig.update_layout( diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index 40605560..bad66b04 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -11,6 +11,7 @@ class StrategyData: trade_fill: pd.DataFrame market_data: pd.DataFrame = None position_executor: pd.DataFrame = None + executors: pd.DataFrame = None @property def strategy_summary(self): @@ -67,6 +68,30 @@ def full_series(series): strategy_summary["TIME_LIMIT"] = np.nan strategy_summary["total_positions"] = np.nan + # TODO: Improve executors parsing logic + if self.executors is not None: + executors_data = self.executors.copy() + grouped_executors = executors_data.groupby( + ["exchange", "trading_pair", "close_type"]).agg( + metric_count=("close_type", "count")).reset_index() + index_cols = ["exchange", "trading_pair"] + pivot_executors = pd.pivot_table(grouped_executors, values="metric_count", index=index_cols, + columns="close_type").reset_index() + result_cols = ["EARLY STOP", "STOP_LOSS", "TRAILING_STOP"] + pivot_executors = pivot_executors.reindex(columns=index_cols + result_cols, fill_value=0) + pivot_executors["total_positions"] = pivot_executors[result_cols].sum(axis=1) + strategy_summary = grouped_trade_fill.merge(pivot_executors, left_on=["market", "symbol"], + right_on=["exchange", "trading_pair"], + how="left") + strategy_summary.drop(columns=["exchange", "trading_pair"], inplace=True) + else: + strategy_summary = grouped_trade_fill.copy() + strategy_summary["TAKE_PROFIT"] = np.nan + strategy_summary["STOP_LOSS"] = np.nan + strategy_summary["TRAILING_STOP"] = np.nan + strategy_summary["TIME_LIMIT"] = np.nan + strategy_summary["total_positions"] = np.nan + strategy_summary.rename(columns=columns_dict, inplace=True) strategy_summary.sort_values(["Realized PnL"], ascending=True, inplace=True) strategy_summary["Explore"] = False @@ -89,6 +114,11 @@ def get_single_market_strategy_data(self, exchange: str, trading_pair: str): (self.position_executor["trading_pair"] == trading_pair)].copy() else: position_executor = None + if self.executors is not None: + executors = self.executors[(self.executors["exchange"] == exchange) & + (self.executors["trading_pair"] == trading_pair)].copy() + else: + executors = None return SingleMarketStrategyData( exchange=exchange, trading_pair=trading_pair, @@ -96,7 +126,8 @@ def get_single_market_strategy_data(self, exchange: str, trading_pair: str): order_status=order_status, trade_fill=trade_fill, market_data=market_data, - position_executor=position_executor + position_executor=position_executor, + executors=executors ) @property @@ -149,6 +180,7 @@ class SingleMarketStrategyData: trade_fill: pd.DataFrame market_data: pd.DataFrame = None position_executor: pd.DataFrame = None + executors: pd.DataFrame = None def get_filtered_strategy_data(self, start_date: datetime.datetime, end_date: datetime.datetime): orders = self.orders[ @@ -165,6 +197,11 @@ def get_filtered_strategy_data(self, start_date: datetime.datetime, end_date: da (self.position_executor.datetime <= end_date)].copy() else: position_executor = None + if self.executors is not None: + executors = self.executors[(self.executors.datetime >= start_date) & + (self.executors.datetime <= end_date)].copy() + else: + executors = None return SingleMarketStrategyData( exchange=self.exchange, trading_pair=self.trading_pair, @@ -172,7 +209,8 @@ def get_filtered_strategy_data(self, start_date: datetime.datetime, end_date: da order_status=order_status, trade_fill=trade_fill, market_data=market_data, - position_executor=position_executor + position_executor=position_executor, + executors=executors ) def get_market_data_resampled(self, interval): diff --git a/utils/database_manager.py b/utils/database_manager.py index 43dc57c9..e2bc06a9 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -1,7 +1,8 @@ import os import streamlit as st - +import json import pandas as pd +from hummingbot.smart_components.executors.position_executor.data_types import CloseType, TradeType from sqlalchemy import create_engine, text from sqlalchemy.orm import sessionmaker @@ -29,8 +30,9 @@ def load_data(table_loader): order_status = load_data(self.get_order_status) market_data = load_data(self.get_market_data) position_executor = load_data(self.get_position_executor_data) + executors = load_data(self.get_executors_data) - strategy_data = StrategyData(orders, order_status, trade_fills, market_data, position_executor) + strategy_data = StrategyData(orders, order_status, trade_fills, market_data, position_executor, executors) return strategy_data @staticmethod @@ -49,6 +51,7 @@ def status(self): "order_status": self._get_table_status(self.get_order_status), "market_data": self._get_table_status(self.get_market_data), "position_executor": self._get_table_status(self.get_position_executor_data), + "executors": self._get_table_status(self.get_executors_data) } return status @@ -142,6 +145,18 @@ def _get_position_executor_query(start_date=None, end_date=None): query += f" WHERE {' AND '.join(conditions)}" return query + @staticmethod + def _get_executors_query(start_date=None, end_date=None): + query = "SELECT * FROM Executors" + conditions = [] + if start_date: + conditions.append(f"timestamp >= '{start_date}'") + if end_date: + conditions.append(f"timestamp <= '{end_date}'") + if conditions: + query += f" WHERE {' AND '.join(conditions)}" + return query + def get_orders(self, config_file_path=None, start_date=None, end_date=None): with self.session_maker() as session: query = self._get_orders_query(config_file_path, start_date, end_date) @@ -199,5 +214,24 @@ def get_position_executor_data(self, start_date=None, end_date=None) -> pd.DataF position_executor = pd.read_sql_query(text(query), session.connection()) position_executor.set_index("timestamp", inplace=True) position_executor["datetime"] = pd.to_datetime(position_executor.index, unit="s") + position_executor["close_datetime"] = pd.to_datetime(position_executor["close_timestamp"], unit="s") position_executor["level"] = position_executor["order_level"].apply(lambda x: x.split("_")[1]) return position_executor + + def get_executors_data(self, start_date=None, end_date=None) -> pd.DataFrame: + with self.session_maker() as session: + query = self._get_executors_query(start_date, end_date) + executors = pd.read_sql_query(text(query), session.connection()) + executors.set_index("timestamp", inplace=True) + executors["datetime"] = pd.to_datetime(executors.index, unit="s") + executors["close_datetime"] = pd.to_datetime(executors["close_timestamp"], unit="s") + executors["trading_pair"] = executors["config"].apply(lambda x: json.loads(x)["trading_pair"]) + executors["exchange"] = executors["config"].apply(lambda x: json.loads(x)["connector_name"]) + executors["close_type"] = executors["close_type"].apply(lambda x: CloseType(x).name) + executors["side"] = executors["config"].apply(lambda x: TradeType(json.loads(x)["side"]).name) + executors["bep"] = executors["custom_info"].apply(lambda x: json.loads(x)["current_position_average_price"]) + executors["close_price"] = executors["custom_info"].apply(lambda x: json.loads(x)["close_price"]) + executors["sl"] = executors["config"].apply(lambda x: json.loads(x)["stop_loss"]).fillna(0) + executors["tp"] = executors["config"].apply(lambda x: json.loads(x)["take_profit"]).fillna(0) + executors["tl"] = executors["config"].apply(lambda x: json.loads(x)["time_limit"]).fillna(0) + return executors diff --git a/utils/graphs.py b/utils/graphs.py index 608e1e03..9ec9f9d5 100644 --- a/utils/graphs.py +++ b/utils/graphs.py @@ -6,7 +6,6 @@ from typing import Union from utils.data_manipulation import StrategyData, SingleMarketStrategyData -from quants_lab.strategy.strategy_analysis import StrategyAnalysis import plotly.graph_objs as go BULLISH_COLOR = "rgba(97, 199, 102, 0.9)" @@ -15,375 +14,6 @@ MIN_INTERVAL_RESOLUTION = "1m" -class CandlesGraph: - def __init__(self, candles_df: pd.DataFrame, line_mode=False, show_volume=True, extra_rows=1): - self.candles_df = candles_df - self.show_volume = show_volume - self.line_mode = line_mode - rows, heights = self.get_n_rows_and_heights(extra_rows) - self.rows = rows - specs = [[{"secondary_y": True}]] * rows - self.base_figure = make_subplots(rows=rows, cols=1, shared_xaxes=True, vertical_spacing=0.005, - row_heights=heights, specs=specs) - self.min_time = candles_df.reset_index().timestamp.min() - self.max_time = candles_df.reset_index().timestamp.max() - self.add_candles_graph() - if self.show_volume: - self.add_volume() - self.update_layout() - - def get_n_rows_and_heights(self, extra_rows): - rows = 1 + extra_rows + self.show_volume - row_heights = [0.4] * (extra_rows) - if self.show_volume: - row_heights.insert(0, 0.05) - row_heights.insert(0, 0.8) - return rows, row_heights - - def figure(self): - return self.base_figure - - def add_candles_graph(self): - if self.line_mode: - self.base_figure.add_trace( - go.Scatter(x=self.candles_df.index, - y=self.candles_df['close'], - name="Close", - mode='lines', - line=dict(color='blue')), - row=1, col=1, - ) - else: - hover_text = [] - for i in range(len(self.candles_df)): - hover_text.append( - f"Open: {self.candles_df['open'][i]}
" - f"High: {self.candles_df['high'][i]}
" - f"Low: {self.candles_df['low'][i]}
" - f"Close: {self.candles_df['close'][i]}
" - ) - self.base_figure.add_trace( - go.Candlestick( - x=self.candles_df.index, - open=self.candles_df['open'], - high=self.candles_df['high'], - low=self.candles_df['low'], - close=self.candles_df['close'], - name="OHLC", - hoverinfo="text", - hovertext=hover_text - ), - row=1, col=1, - ) - - def add_buy_trades(self, orders_data: pd.DataFrame): - self.base_figure.add_trace( - go.Scatter( - x=orders_data['timestamp'], - y=orders_data['price'], - name='Buy Orders', - mode='markers', - marker=dict( - symbol='triangle-up', - color='green', - size=12, - line=dict(color='black', width=1), - opacity=0.7, - ), - hoverinfo="text", - hovertext=orders_data["price"].apply(lambda x: f"Buy Order: {x}
")), - row=1, col=1, - ) - - def add_sell_trades(self, orders_data: pd.DataFrame): - self.base_figure.add_trace( - go.Scatter( - x=orders_data['timestamp'], - y=orders_data['price'], - name='Sell Orders', - mode='markers', - marker=dict(symbol='triangle-down', - color='red', - size=12, - line=dict(color='black', width=1), - opacity=0.7,), - hoverinfo="text", - hovertext=orders_data["price"].apply(lambda x: f"Sell Order: {x}
")), - row=1, col=1, - ) - - def add_bollinger_bands(self, length=20, std=2.0, row=1): - df = self.candles_df.copy() - if len(df) < length: - st.warning("Not enough data to calculate Bollinger Bands") - return - df.ta.bbands(length=length, std=std, append=True) - self.base_figure.add_trace( - go.Scatter( - x=df.index, - y=df[f'BBU_{length}_{std}'], - name='Bollinger Bands', - mode='lines', - line=dict(color='blue', width=1)), - row=row, col=1, - ) - self.base_figure.add_trace( - go.Scatter( - x=df.index, - y=df[f'BBM_{length}_{std}'], - name='Bollinger Bands', - mode='lines', - line=dict(color='blue', width=1)), - row=1, col=1, - ) - self.base_figure.add_trace( - go.Scatter( - x=df.index, - y=df[f'BBL_{length}_{std}'], - name='Bollinger Bands', - mode='lines', - line=dict(color='blue', width=1)), - row=1, col=1, - ) - - def add_volume(self): - self.base_figure.add_trace( - go.Bar( - x=self.candles_df.index, - y=self.candles_df['volume'], - name="Volume", - opacity=0.5, - marker=dict(color='lightgreen'), - - ), - row=2, col=1, - ) - - def add_ema(self, length=20, row=1): - df = self.candles_df.copy() - if len(df) < length: - st.warning("Not enough data to calculate EMA") - return - df.ta.ema(length=length, append=True) - self.base_figure.add_trace( - go.Scatter( - x=df.index, - y=df[f'EMA_{length}'], - name='EMA', - mode='lines', - line=dict(color='yellow', width=1)), - row=row, col=1, - ) - - def add_quote_inventory_change(self, strategy_data: StrategyData, row=3): - self.base_figure.add_trace( - go.Scatter( - x=strategy_data.trade_fill.timestamp, - y=strategy_data.trade_fill.inventory_cost, - name="Quote Inventory", - mode="lines", - line=dict(shape="hv"), - ), - row=row, col=1 - ) - self.base_figure.update_yaxes(title_text='Quote Inventory Change', row=row, col=1) - - def add_pnl(self, strategy_data: SingleMarketStrategyData, row=4): - self.base_figure.add_trace( - go.Scatter( - x=strategy_data.trade_fill.timestamp, - y=[max(0, realized_pnl) for realized_pnl in strategy_data.trade_fill["realized_trade_pnl"].apply(lambda x: round(x, 4))], - name="Cum Profit", - mode='lines', - line=dict(shape="hv", color="rgba(1, 1, 1, 0.5)", dash="dash", width=0.1), - fill="tozeroy", # Fill to the line below (trade pnl) - fillcolor="rgba(0, 255, 0, 0.5)" - ), - row=row, col=1 - ) - self.base_figure.add_trace( - go.Scatter( - x=strategy_data.trade_fill.timestamp, - y=[min(0, realized_pnl) for realized_pnl in strategy_data.trade_fill["realized_trade_pnl"].apply(lambda x: round(x, 4))], - name="Cum Loss", - mode='lines', - line=dict(shape="hv", color="rgba(1, 1, 1, 0.5)", dash="dash", width=0.3), - # marker=dict(symbol="arrow"), - fill="tozeroy", # Fill to the line below (trade pnl) - fillcolor="rgba(255, 0, 0, 0.5)", - ), - row=row, col=1 - ) - self.base_figure.add_trace( - go.Scatter( - x=strategy_data.trade_fill.timestamp, - y=strategy_data.trade_fill["cum_fees_in_quote"].apply(lambda x: round(x, 4)), - name="Cum Fees", - mode='lines', - line=dict(shape="hv", color="rgba(1, 1, 1, 0.1)", dash="dash", width=0.1), - fill="tozeroy", # Fill to the line below (trade pnl) - fillcolor="rgba(51, 0, 51, 0.5)" - ), - row=row, col=1 - ) - self.base_figure.add_trace(go.Scatter(name="Net Realized Profit", - x=strategy_data.trade_fill.timestamp, - y=strategy_data.trade_fill["net_realized_pnl"], - mode="lines", - line=dict(shape="hv")), - row=row, col=1 - ) - self.base_figure.update_yaxes(title_text='PNL', row=row, col=1) - - def add_positions(self, position_executor_data: pd.DataFrame, row=1): - position_executor_data["close_datetime"] = pd.to_datetime(position_executor_data["close_timestamp"], unit="s") - i = 1 - for index, rown in position_executor_data.iterrows(): - i += 1 - self.base_figure.add_trace(go.Scatter(name=f"Position {index}", - x=[rown.datetime, rown.close_datetime], - y=[rown.entry_price, rown.close_price], - mode="lines", - line=dict(color="lightgreen" if rown.net_pnl_quote > 0 else "red"), - hoverinfo="text", - hovertext=f"Position Nยฐ: {i}
" - f"Datetime: {rown.datetime}
" - f"Close datetime: {rown.close_datetime}
" - f"Side: {rown.side}
" - f"Entry price: {rown.entry_price}
" - f"Close price: {rown.close_price}
" - f"Close type: {rown.close_type}
" - f"Stop Loss: {100 * rown.sl:.2f}%
" - f"Take Profit: {100 * rown.tp:.2f}%
" - f"Time Limit: {100 * rown.tl:.2f}
" - f"Open Order Type: {rown.open_order_type}
" - f"Leverage: {rown.leverage}
" - f"Controller name: {rown.controller_name}
", - showlegend=False), - row=row, col=1) - - def update_layout(self): - self.base_figure.update_layout( - title={ - 'text': "Market activity", - 'y': 0.99, - 'x': 0.5, - 'xanchor': 'center', - 'yanchor': 'top' - }, - legend=dict( - orientation="h", - x=0.5, - y=1.04, - xanchor="center", - yanchor="bottom" - ), - height=1000, - xaxis=dict(rangeslider_visible=False, - range=[self.min_time, self.max_time]), - yaxis=dict(range=[self.candles_df.low.min(), self.candles_df.high.max()]), - hovermode='x unified' - ) - self.base_figure.update_yaxes(title_text="Price", row=1, col=1) - if self.show_volume: - self.base_figure.update_yaxes(title_text="Volume", row=2, col=1) - self.base_figure.update_xaxes(title_text="Time", row=self.rows, col=1) - - -class BacktestingGraphs: - def __init__(self, study_df: pd.DataFrame): - self.study_df = study_df - - def pnl_vs_maxdrawdown(self): - fig = go.Figure() - fig.add_trace(go.Scatter(name="Pnl vs Max Drawdown", - x=-100 * self.study_df["max_drawdown_pct"], - y=100 * self.study_df["net_pnl_pct"], - mode="markers", - text=None, - hovertext=self.study_df["hover_text"])) - fig.update_layout( - title="PnL vs Max Drawdown", - xaxis_title="Max Drawdown [%]", - yaxis_title="Net Profit [%]", - height=800 - ) - fig.data[0].text = [] - return fig - - @staticmethod - def get_trial_metrics(strategy_analysis: StrategyAnalysis, - add_volume: bool = True, - add_positions: bool = True, - add_pnl: bool = True): - """Isolated method because it needs to be called from analyze and simulate pages""" - metrics_container = st.container() - with metrics_container: - col1, col2 = st.columns(2) - with col1: - st.subheader("๐Ÿฆ Market") - with col2: - st.subheader("๐Ÿ“‹ General stats") - col1, col2, col3, col4 = st.columns(4) - with col1: - st.metric("Exchange", st.session_state["strategy_params"]["exchange"]) - with col2: - st.metric("Trading Pair", st.session_state["strategy_params"]["trading_pair"]) - with col3: - st.metric("Start date", strategy_analysis.start_date().strftime("%Y-%m-%d %H:%M")) - st.metric("End date", strategy_analysis.end_date().strftime("%Y-%m-%d %H:%M")) - with col4: - st.metric("Duration (hours)", f"{strategy_analysis.duration_in_minutes() / 60:.2f}") - st.metric("Price change", st.session_state["strategy_params"]["trading_pair"]) - st.subheader("๐Ÿ“ˆ Performance") - col1, col2, col3, col4, col5, col6, col7, col8 = st.columns(8) - with col1: - st.metric("Net PnL USD", - f"{strategy_analysis.net_profit_usd():.2f}", - delta=f"{100 * strategy_analysis.net_profit_pct():.2f}%", - help="The overall profit or loss achieved.") - with col2: - st.metric("Total positions", - f"{strategy_analysis.total_positions()}", - help="The total number of closed trades, winning and losing.") - with col3: - st.metric("Accuracy", - f"{100 * (len(strategy_analysis.win_signals()) / strategy_analysis.total_positions()):.2f} %", - help="The percentage of winning trades, the number of winning trades divided by the" - " total number of closed trades") - with col4: - st.metric("Profit factor", - f"{strategy_analysis.profit_factor():.2f}", - help="The amount of money the strategy made for every unit of money it lost, " - "gross profits divided by gross losses.") - with col5: - st.metric("Max Drawdown", - f"{strategy_analysis.max_drawdown_usd():.2f}", - delta=f"{100 * strategy_analysis.max_drawdown_pct():.2f}%", - help="The greatest loss drawdown, i.e., the greatest possible loss the strategy had compared " - "to its highest profits") - with col6: - st.metric("Avg Profit", - f"{strategy_analysis.avg_profit():.2f}", - help="The sum of money gained or lost by the average trade, Net Profit divided by " - "the overall number of closed trades.") - with col7: - st.metric("Avg Minutes", - f"{strategy_analysis.avg_trading_time_in_minutes():.2f}", - help="The average number of minutes that elapsed during trades for all closed trades.") - with col8: - st.metric("Sharpe Ratio", - f"{strategy_analysis.sharpe_ratio():.2f}", - help="The Sharpe ratio is a measure that quantifies the risk-adjusted return of an investment" - " or portfolio. It compares the excess return earned above a risk-free rate per unit of" - " risk taken.") - - st.plotly_chart(strategy_analysis.pnl_over_time(), use_container_width=True) - strategy_analysis.create_base_figure(volume=add_volume, positions=add_positions, trade_pnl=add_pnl) - st.plotly_chart(strategy_analysis.figure(), use_container_width=True) - return metrics_container - - class PerformanceGraphs: BULLISH_COLOR = "rgba(97, 199, 102, 0.9)" BEARISH_COLOR = "rgba(255, 102, 90, 0.9)" @@ -422,157 +52,6 @@ def strategy_summary_table(self): else: return None - def summary_chart(self): - fig = px.bar(self.strategy_data.strategy_summary, x="Trading Pair", y="Realized PnL", color="Exchange") - fig.update_traces(width=min(1.0, 0.1 * len(self.strategy_data.strategy_summary))) - return fig - - def pnl_over_time(self): - df = self.strategy_data.trade_fill.copy() - df.reset_index(drop=True, inplace=True) - df_above = df[df['net_realized_pnl'] >= 0] - df_below = df[df['net_realized_pnl'] < 0] - - fig = go.Figure() - fig.add_trace(go.Bar(name="Cum Realized PnL", - x=df_above.index, - y=df_above["net_realized_pnl"], - marker_color=BULLISH_COLOR, - # hoverdq - showlegend=False)) - fig.add_trace(go.Bar(name="Cum Realized PnL", - x=df_below.index, - y=df_below["net_realized_pnl"], - marker_color=BEARISH_COLOR, - showlegend=False)) - fig.update_layout(title=dict( - text='Cummulative PnL', # Your title text - x=0.43, - y=0.95, - ), - plot_bgcolor='rgba(0,0,0,0)', - paper_bgcolor='rgba(0,0,0,0)') - return fig - - def intraday_performance(self): - df = self.strategy_data.trade_fill.copy() - - def hr2angle(hr): - return (hr * 15) % 360 - - def hr_str(hr): - # Normalize hr to be between 1 and 12 - hr_string = str(((hr - 1) % 12) + 1) - suffix = ' AM' if (hr % 24) < 12 else ' PM' - return hr_string + suffix - - df["hour"] = df["timestamp"].dt.hour - realized_pnl_per_hour = df.groupby("hour")[["realized_pnl", "quote_volume"]].sum().reset_index() - fig = go.Figure() - fig.add_trace(go.Barpolar( - name="Profits", - r=realized_pnl_per_hour["quote_volume"], - theta=realized_pnl_per_hour["hour"] * 15, - marker=dict( - color=realized_pnl_per_hour["realized_pnl"], - colorscale="RdYlGn", - cmin=-(abs(realized_pnl_per_hour["realized_pnl"]).max()), - cmid=0.0, - cmax=(abs(realized_pnl_per_hour["realized_pnl"]).max()), - colorbar=dict( - title='Realized PnL', - x=0, - y=-0.5, - xanchor='left', - yanchor='bottom', - orientation='h' - ) - ))) - fig.update_layout( - polar=dict( - radialaxis=dict( - visible=True, - showline=False, - ), - angularaxis=dict( - rotation=90, - direction="clockwise", - tickvals=[hr2angle(hr) for hr in range(24)], - ticktext=[hr_str(hr) for hr in range(24)], - ), - bgcolor='rgba(255, 255, 255, 0)', - - ), - legend=dict( - orientation="h", - x=0.5, - y=1.08, - xanchor="center", - yanchor="bottom" - ), - title=dict( - text='Intraday Performance', - x=0.5, - y=0.93, - xanchor="center", - yanchor="bottom" - ), - ) - return fig - - def returns_histogram(self): - df = self.strategy_data.trade_fill.copy() - fig = go.Figure() - fig.add_trace(go.Histogram(name="Losses", - x=df.loc[df["realized_pnl"] < 0, "realized_pnl"], - marker_color=BEARISH_COLOR)) - fig.add_trace(go.Histogram(name="Profits", - x=df.loc[df["realized_pnl"] > 0, "realized_pnl"], - marker_color=BULLISH_COLOR)) - fig.update_layout( - title=dict( - text='Returns Distribution', - x=0.5, - xanchor="center", - ), - legend=dict( - orientation="h", - yanchor="bottom", - y=1.02, - xanchor="center", - x=.48 - )) - return fig - - def position_executor_summary_sunburst(self): - if self.strategy_data.position_executor is not None: - df = self.strategy_data.position_executor.copy() - grouped_df = df.groupby(["trading_pair", "side", "close_type"]).size().reset_index(name="count") - - fig = px.sunburst(grouped_df, - path=['trading_pair', 'side', 'close_type'], - values="count", - color_continuous_scale='RdBu', - color_continuous_midpoint=0) - - fig.update_layout( - title=dict( - text='Position Executor Summary', - x=0.5, - xanchor="center", - ), - legend=dict( - orientation="h", - yanchor="bottom", - y=1.02, - xanchor="center", - x=.48 - ) - ) - return fig - else: - return None - def candles_graph(self, candles: pd.DataFrame, interval="5m", show_volume=False, extra_rows=2): line_mode = interval == MIN_INTERVAL_RESOLUTION cg = CandlesGraph(candles, show_volume=show_volume, line_mode=line_mode, extra_rows=extra_rows) diff --git a/utils/os_utils.py b/utils/os_utils.py index e0ff6c39..f3f2dceb 100644 --- a/utils/os_utils.py +++ b/utils/os_utils.py @@ -109,14 +109,12 @@ def load_controllers(path): def get_bots_data_paths(): root_directory = "hummingbot_files/bots" bots_data_paths = {"General / Uploaded data": "data"} - reserved_word = "hummingbot-" - # Walk through the directory tree + # Walk through the directorxy tree for dirpath, dirnames, filenames in os.walk(root_directory): for dirname in dirnames: if dirname == "data": parent_folder = os.path.basename(dirpath) - if parent_folder.startswith(reserved_word): - bots_data_paths[parent_folder] = os.path.join(dirpath, dirname) + bots_data_paths[parent_folder] = os.path.join(dirpath, dirname) if "dashboard" in bots_data_paths: del bots_data_paths["dashboard"] data_sources = {key: value for key, value in bots_data_paths.items() if value is not None}