Improve perf of trades by changing data structures (#74)

* Change Trades type

* Change Trade type

* Refine make trades

* Refine make trades

* get_trade returns pd.Series

* Delete symbol from Trade

* Make Trade as Series

* Delete symbol from make_trade

* Add append_trade

* Add append_trade

* Delete add_transaction

* Delete make_trades

* Shorten flatten

* Delete flatten

* Delete from_series

* Rename from_tuple to make_trades

* Delete append_trade

* Delete Trade

* Add unittest

* Add filter_trade

* Add any and all

* Update version

* Update trade init

* Fix mypy

* Improve performance of trades

* Improve performance of exit

* Fix mypy

* Improve performance of trades

* Fix type

* Update docstring

src/backlight/__init__.py

@@ -1,4 +1,4 @@
 __author__ = "AlpacaJapan Co., Ltd."
-__version__ = "0.1.5"
-__release__ = "0.1.5"
+__version__ = "0.2.0"
+__release__ = "0.2.0"
 __license__ = "MIT"

src/backlight/metrics/position_metrics.py

@@ -6,7 +6,6 @@
 from backlight.datasource.marketdata import MarketData
 from backlight.positions import calc_positions
 from backlight.positions.positions import Positions
-from backlight.trades.trades import Trade, Trades
 
 
 def _sum(a: pd.Series) -> float:

src/backlight/metrics/trade_metrics.py

@@ -5,7 +5,7 @@
 
 import backlight.positions
 from backlight.datasource.marketdata import MarketData
-from backlight.trades.trades import Trade, Trades
+from backlight.trades.trades import Trades, make_trades
 from backlight.metrics.position_metrics import calc_pl, calc_position_performance
 
 
@@ -17,9 +17,10 @@ def _sum(a: pd.Series) -> float:
     return a.sum() if len(a) != 0 else 0.0
 
 
-def _calc_pl(trade: Trade, mkt: MarketData) -> float:
+def _calc_pl(trade: pd.Series, mkt: MarketData) -> float:
     mkt = mkt.loc[trade.index, :]
-    positions = backlight.positions.calc_positions((trade,), mkt)
+    trades = make_trades(mkt.symbol, [trade])
+    positions = backlight.positions.calc_positions(trades, mkt)
     pl = calc_pl(positions)
     return _sum(pl)
 
@@ -36,8 +37,12 @@ def count_trades(trades: Trades, mkt: MarketData) -> Tuple[int, int, int]:
     Returns:
         total count, win count, lose count
     """
-    pls = [_calc_pl(t, mkt) for t in trades if len(t.index) > 1]
-    total = len(trades)
+    pls = [
+        _calc_pl(trades.get_trade(i), mkt)
+        for i in trades.ids
+        if len(trades.get_trade(i).index) > 1
+    ]
+    total = len(trades.ids)
     win = sum([pl > 0.0 for pl in pls])
     lose = sum([pl < 0.0 for pl in pls])
     return total, win, lose

src/backlight/positions/positions.py

@@ -3,8 +3,7 @@
 from typing import Type, Callable
 
 from backlight.datasource.marketdata import MarketData, MidMarketData, AskBidMarketData
-from backlight.trades import flatten
-from backlight.trades.trades import Trade, Trades
+from backlight.trades.trades import Trades
 
 
 def _freq(idx: pd.Index) -> pd.Timedelta:
@@ -44,14 +43,15 @@ def _constructor(self) -> Type["Positions"]:
         return Positions
 
 
-def _pricer(trade: Trade, mkt: MarketData, principal: float) -> Positions:
+def _pricer(trades: Trades, mkt: MarketData, principal: float) -> pd.DataFrame:
+    trade = trades.amount
 
     # historical data
     idx = mkt.index[trade.index[0] <= mkt.index]  # only after first trades
     positions = pd.DataFrame(index=idx)
-    positions.loc[:, "amount"] = trade.amount.cumsum()
+    positions.loc[:, "amount"] = trade.cumsum()
     positions.loc[:, "price"] = mkt.mid.loc[idx]
-    fee = mkt.fee(trade.amount)
+    fee = mkt.fee(trade)
     positions.loc[:, "principal"] = -fee.cumsum() + principal
     positions = positions.ffill()
 
@@ -61,10 +61,7 @@ def _pricer(trade: Trade, mkt: MarketData, principal: float) -> Positions:
     positions.loc[initial_idx, "price"] = 0.0
     positions.loc[initial_idx, "principal"] = principal
 
-    pos = Positions(positions.sort_index())
-    pos.reset_cols()
-    pos.symbol = trade.symbol
-    return pos
+    return positions.sort_index()
 
 
 def calc_positions(
@@ -78,11 +75,10 @@ def calc_positions(
         mkt: Market data.
         principal: The initial principal value.
     """
-    trade = flatten(trades)
-
-    assert trade.symbol == mkt.symbol
-    assert (trade.index.isin(mkt.index)).all()
+    assert trades.symbol == mkt.symbol
+    assert trades.index.isin(mkt.index).all()
 
-    positions = _pricer(trade, mkt, principal)
-    positions.symbol = trade.symbol
-    return positions
+    pos = Positions(_pricer(trades, mkt, principal))
+    pos.reset_cols()
+    pos.symbol = trades.symbol
+    return pos

src/backlight/strategies/amount_based.py

@@ -6,7 +6,7 @@
 from backlight.datasource.marketdata import MarketData
 from backlight.signal.signal import Signal
 from backlight.trades import make_trade
-from backlight.trades.trades import Trade, Trades, Transaction, from_series
+from backlight.trades.trades import Trades, make_trades
 from backlight.labelizer.common import TernaryDirection
 from backlight.strategies.common import Action
 from backlight.strategies.entry import direction_based_entry
@@ -35,8 +35,8 @@ def direction_based_trades(
     amount = pd.Series(index=df.index, name="amount").astype(np.float64)
     for direction, action in direction_action_dict.items():
         amount.loc[df["pred"] == direction.value] = action.act_on_amount()
-    trade = from_series(amount, df.symbol)
-    return (trade,)
+    trade = amount
+    return make_trades(df.symbol, [trade])
 
 
 def only_take_long(mkt: MarketData, sig: Signal) -> Trades:
@@ -79,8 +79,8 @@ def _entry_and_exit_at_max_holding_time(
         sig: Signal data
         direction_action_dict: Dictionary from signals to actions
         max_holding_time: maximum holding time
-        exit_condition: The entry is closed most closest time which 
-                        condition is `True`.
+        exit_condition: The entry is closed most closest time which condition is `True`.
+
     Result:
         Trades
     """

src/backlight/strategies/entry.py

@@ -1,18 +1,14 @@
 import pandas as pd
 
+from typing import List
+
 from backlight.datasource.marketdata import MarketData
 from backlight.signal.signal import Signal
 from backlight.trades import make_trade
-from backlight.trades.trades import Transaction, Trade, Trades
+from backlight.trades.trades import Trades, from_dataframe
 from backlight.strategies.common import Action
 
 
-def _entry(amount: float, idx: pd.Timestamp, symbol: str) -> Trade:
-    trade = make_trade(symbol)
-    trade.add(Transaction(timestamp=idx, amount=amount))
-    return trade
-
-
 def direction_based_entry(
     mkt: MarketData, sig: Signal, direction_action_dict: dict
 ) -> Trades:
@@ -28,12 +24,23 @@ def direction_based_entry(
     assert all([idx in mkt.index for idx in sig.index])
     df = sig
 
-    trades = ()  # type: Trades
+    trades = []  # type: List[pd.Dataframe]
     for direction, action in direction_action_dict.items():
+
         amount = action.act_on_amount()
         if amount == 0.0:
             continue
-        target_index = df[df["pred"] == direction.value].index
-        trades += tuple(_entry(amount, idx, df.symbol) for idx in target_index)
 
-    return trades
+        trades.append(
+            pd.DataFrame(
+                index=df[df["pred"] == direction.value].index,
+                data=direction.value,
+                columns=["amount"],
+            )
+        )
+
+    df_trades = pd.concat(trades, axis=0).sort_index()
+    df_trades.loc[:, "_id"] = range(len(df_trades.index))
+
+    t = from_dataframe(df_trades, df.symbol)
+    return t

src/backlight/strategies/exit.py

@@ -1,13 +1,13 @@
 import numpy as np
 import pandas as pd
 
-from typing import Callable, List, Optional
+from typing import Callable, List, Optional, Tuple
 
 from backlight.datasource.marketdata import MarketData
 from backlight.labelizer.common import TernaryDirection
 from backlight.signal.signal import Signal
 from backlight.trades import make_trade
-from backlight.trades.trades import Transaction, Trade, Trades
+from backlight.trades.trades import Transaction, Trades, concat, from_dataframe
 from backlight.strategies.common import Action
 
 
@@ -25,28 +25,28 @@ def _concat(mkt: MarketData, sig: Optional[Signal]) -> pd.DataFrame:
 
 def _exit_transaction(
     df: pd.DataFrame,
-    trade: Trade,
-    exit_condition: Callable[[pd.DataFrame, Trade], pd.Series],
+    trade: pd.Series,
+    exit_condition: Callable[[pd.DataFrame, pd.Series], pd.Series],
 ) -> Transaction:
     exit_indices = df[exit_condition(df, trade)].index
     if exit_indices.empty:
         exit_index = df.index[-1]
     else:
         exit_index = exit_indices[0]
-    return Transaction(timestamp=exit_index, amount=-trade.amount.sum())
+    return Transaction(timestamp=exit_index, amount=-trade.sum())
 
 
-def _no_exit_condition(df: pd.DataFrame, trade: Trade) -> pd.Series:
+def _no_exit_condition(df: pd.DataFrame, trade: pd.Series) -> pd.Series:
     return pd.Series(index=df.index, data=False)
 
 
 def exit(
     mkt: MarketData,
     sig: Optional[Signal],
     entries: Trades,
-    exit_condition: Callable[[pd.DataFrame, Trade], pd.Series],
+    exit_condition: Callable[[pd.DataFrame, pd.Series], pd.Series],
 ) -> Trades:
-    """Exit trade at max holding time or satisfying condition.
+    """Exit trade when satisfying condition.
 
     Args:
         mkt: Market data
@@ -61,29 +61,40 @@ def exit(
     df = _concat(mkt, sig)
 
     def _exit(
-        trade: Trade,
+        trades: Trades,
         df: pd.DataFrame,
-        exit_condition: Callable[[pd.DataFrame, Trade], pd.Series],
-    ) -> Trade:
-        if trade.amount.sum() == 0:
-            return trade
+        exit_condition: Callable[[pd.DataFrame, pd.Series], pd.Series],
+    ) -> pd.Series:
+
+        indices = []  # type: List[pd.Timestamp]
+        exits = []  # type: List[Tuple[float, int]]
+        for i in trades.ids:
+            trade = trades.get_trade(i)
+            if trade.sum() == 0:
+                continue
+
+            idx = trade.index[0]
+            df_exit = df[idx <= df.index]
+            transaction = _exit_transaction(df_exit, trade, exit_condition)
+
+            indices.append(transaction.timestamp)
+            exits.append((transaction.amount, i))
+
+        df = pd.DataFrame(index=indices, data=exits, columns=["amount", "_id"])
 
-        idx = trade.index[0]
-        df_exit = df[idx <= df.index]
-        transaction = _exit_transaction(df_exit, trade, exit_condition)
-        trade.add(transaction)
-        return trade
+        return from_dataframe(df, symbol)
 
-    trades = tuple(_exit(trade, df, exit_condition) for trade in entries)
-    return trades
+    symbol = entries.symbol
+    exits = _exit(entries, df, exit_condition)
+    return concat([entries, exits])
 
 
 def exit_by_max_holding_time(
     mkt: MarketData,
     sig: Optional[Signal],
     entries: Trades,
     max_holding_time: pd.Timedelta,
-    exit_condition: Callable[[pd.DataFrame, Trade], pd.Series],
+    exit_condition: Callable[[pd.DataFrame, pd.Series], pd.Series],
 ) -> Trades:
     """Exit trade at max holding time or satisfying condition.
 
@@ -100,22 +111,32 @@ def exit_by_max_holding_time(
     df = _concat(mkt, sig)
 
     def _exit_by_max_holding_time(
-        trade: Trade,
+        trades: Trades,
         df: pd.DataFrame,
         max_holding_time: pd.Timedelta,
-        exit_condition: Callable[[pd.DataFrame, Trade], pd.Series],
-    ) -> Trade:
-        idx = trade.index[0]
-        df_exit = df[(idx <= df.index) & (df.index <= idx + max_holding_time)]
-        transaction = _exit_transaction(df_exit, trade, exit_condition)
-        trade.add(transaction)
-        return trade
-
-    trades = tuple(
-        _exit_by_max_holding_time(trade, df, max_holding_time, exit_condition)
-        for trade in entries
-    )
-    return trades
+        exit_condition: Callable[[pd.DataFrame, pd.Series], pd.Series],
+    ) -> Trades:
+
+        indices = []  # type: List[pd.Timestamp]
+        exits = []  # type: List[Tuple[float, int]]
+        for i in trades.ids:
+            trade = trades.get_trade(i)
+            if trade.sum() == 0:
+                continue
+
+            idx = trade.index[0]
+            df_exit = df[(idx <= df.index) & (df.index <= idx + max_holding_time)]
+            transaction = _exit_transaction(df_exit, trade, exit_condition)
+
+            indices.append(transaction.timestamp)
+            exits.append((transaction.amount, i))
+
+        df = pd.DataFrame(index=indices, data=exits, columns=["amount", "_id"])
+        return from_dataframe(df, symbol)
+
+    symbol = entries.symbol
+    exits = _exit_by_max_holding_time(entries, df, max_holding_time, exit_condition)
+    return concat([entries, exits])
 
 
 def exit_at_max_holding_time(
@@ -162,7 +183,7 @@ def _exit_at_opposite_signals_condition(
         opposite_signals = opposite_signals_dict[current_signal]
         return df["pred"].isin(opposite_signals)
 
-    def _exit_condition(df: pd.DataFrame, trade: Trade) -> pd.Series:
+    def _exit_condition(df: pd.DataFrame, trade: pd.Series) -> pd.Series:
         return _exit_at_opposite_signals_condition(df, opposite_signals_dict)
 
     return exit_by_max_holding_time(
@@ -184,7 +205,7 @@ def exit_by_expectation(
         Trades
     """
 
-    def _exit_by_expectation_condition(df: pd.DataFrame, trade: Trade) -> pd.Series:
+    def _exit_by_expectation_condition(df: pd.DataFrame, trade: pd.Series) -> pd.Series:
         current_signal = TernaryDirection(df["pred"][0])
         v = np.array([1.0, 0.0, -1.0])
         expectation = np.dot(df[["up", "neutral", "down"]].values, v)
@@ -219,10 +240,10 @@ def exit_by_trailing_stop(
     assert initial_stop >= 0.0
     assert trailing_stop >= 0.0
 
-    def _exit_by_trailing_stop(df: pd.DataFrame, trade: Trade) -> pd.Series:
+    def _exit_by_trailing_stop(df: pd.DataFrame, trade: pd.Series) -> pd.Series:
         prices = df.mid
 
-        amount = trade.amount.sum()
+        amount = trade.sum()
         entry_price = prices.iloc[0]
         pl_per_amount = np.sign(amount) * (prices - entry_price)
         is_initial_stop = pl_per_amount <= -initial_stop

src/backlight/trades/__init__.py

@@ -1 +1 @@
-from backlight.trades.trades import flatten, make_trade  # noqa
+from backlight.trades.trades import make_trade, make_trades  # noqa