diff --git a/.coverargerc b/.coverargerc index 488b75db..e5372b9c 100644 --- a/.coverargerc +++ b/.coverargerc @@ -18,6 +18,7 @@ exclude_lines = # Don't complain about missing debug-only code: def __repr__ def __str__ + pass # Don't complain if abstractmethods are skipped @abstractmethod diff --git a/demo_scripts/backtester/run_alpha_model_backtest_demo.py b/demo_scripts/backtester/run_alpha_model_backtest_demo.py index ed773caf..dadc5760 100644 --- a/demo_scripts/backtester/run_alpha_model_backtest_demo.py +++ b/demo_scripts/backtester/run_alpha_model_backtest_demo.py @@ -56,7 +56,7 @@ def main(): # ----- build models ----- # model = MovingAverageAlphaModel(fast_time_period=5, slow_time_period=20, risk_estimation_factor=1.25, - data_provider=ts.data_handler) + data_provider=ts.data_provider) model_tickers = [DummyTicker('AAA'), DummyTicker('BBB')] model_tickers_dict = {model: model_tickers} diff --git a/demo_scripts/strategies/alpha_model_strategy_with_data_checksum.py b/demo_scripts/strategies/alpha_model_strategy_with_data_checksum.py index 39cf4c0a..22ed2d9b 100644 --- a/demo_scripts/strategies/alpha_model_strategy_with_data_checksum.py +++ b/demo_scripts/strategies/alpha_model_strategy_with_data_checksum.py @@ -52,7 +52,7 @@ def main(): # ----- build models ----- # model = MovingAverageAlphaModel(fast_time_period=5, slow_time_period=20, risk_estimation_factor=1.25, - data_provider=ts.data_handler) + data_provider=ts.data_provider) model_tickers = [DummyTicker('AAA'), DummyTicker('BBB'), DummyTicker('CCC'), DummyTicker('DDD'), DummyTicker('EEE'), DummyTicker('FFF')] model_tickers_dict = {model: model_tickers} diff --git a/demo_scripts/strategies/intraday_strategy.py b/demo_scripts/strategies/intraday_strategy.py index d1ac683f..0114a87c 100644 --- a/demo_scripts/strategies/intraday_strategy.py +++ b/demo_scripts/strategies/intraday_strategy.py @@ -45,13 +45,14 @@ class IntradayMAStrategy(AbstractStrategy): 10:00 and 13:00, and creates a buy order in case if the short moving average is greater or equal to the long moving average. """ + def __init__(self, ts: BacktestTradingSession, ticker: Ticker): super().__init__(ts) self.broker = ts.broker self.order_factory = ts.order_factory - self.data_handler = ts.data_handler + self.data_provider = ts.data_provider self.position_sizer = ts.position_sizer - self.timer = ts.timer + self.timer = ts.data_provider.timer self.ticker = ticker self.logger = qf_logger.getChild(self.__class__.__name__) @@ -64,8 +65,8 @@ def calculate_and_place_orders(self): short_ma_len = 5 # Use data handler to download last 20 daily close prices and use them to compute the moving averages - long_ma_series = self.data_handler.historical_price(self.ticker, PriceField.Close, long_ma_len, - frequency=Frequency.MIN_1) + long_ma_series = self.data_provider.historical_price(self.ticker, PriceField.Close, long_ma_len, + frequency=Frequency.MIN_1) long_ma_price = long_ma_series.mean() short_ma_series = long_ma_series.tail(short_ma_len) diff --git a/demo_scripts/strategies/simple_ma_strategy.py b/demo_scripts/strategies/simple_ma_strategy.py index b1b7ff98..cfe24841 100644 --- a/demo_scripts/strategies/simple_ma_strategy.py +++ b/demo_scripts/strategies/simple_ma_strategy.py @@ -41,11 +41,12 @@ class SimpleMAStrategy(AbstractStrategy): short - 5 days) and creates a buy order in case if the short moving average is greater or equal to the long moving average. """ + def __init__(self, ts: BacktestTradingSession, ticker: Ticker): super().__init__(ts) self.broker = ts.broker self.order_factory = ts.order_factory - self.data_handler = ts.data_handler + self.data_provider = ts.data_provider self.ticker = ticker def calculate_and_place_orders(self): @@ -54,7 +55,7 @@ def calculate_and_place_orders(self): short_ma_len = 5 # Use data handler to download last 20 daily close prices and use them to compute the moving averages - long_ma_series = self.data_handler.historical_price(self.ticker, PriceField.Close, long_ma_len) + long_ma_series = self.data_provider.historical_price(self.ticker, PriceField.Close, long_ma_len) long_ma_price = long_ma_series.mean() short_ma_series = long_ma_series.tail(short_ma_len) diff --git a/docs/source/backtesting.rst b/docs/source/backtesting.rst index d21fa78b..177c4f85 100644 --- a/docs/source/backtesting.rst +++ b/docs/source/backtesting.rst @@ -27,20 +27,6 @@ contract contract_to_ticker_conversion.simulated_contract_ticker_mapper.SimulatedContractTickerMapper contract_to_ticker_conversion.ib_contract_ticker_mapper.IBContractTickerMapper - -data_handler -================== -.. currentmodule:: qf_lib.backtesting.data_handler - -.. autosummary:: - :nosignatures: - :toctree: _autosummary - :template: short_class.rst - - data_handler.DataHandler - daily_data_handler.DailyDataHandler - intraday_data_handler.IntradayDataHandler - events ======== .. currentmodule:: qf_lib.backtesting.events.time_event diff --git a/docs/source/data_providers.rst b/docs/source/data_providers.rst index 68cd4383..6d4ef0be 100644 --- a/docs/source/data_providers.rst +++ b/docs/source/data_providers.rst @@ -11,6 +11,7 @@ data_providers data_provider.DataProvider abstract_price_data_provider.AbstractPriceDataProvider + futures_data_provider.FuturesDataProvider bloomberg.bloomberg_data_provider.BloombergDataProvider bloomberg_beap_hapi.bloomberg_beap_hapi_data_provider.BloombergBeapHapiDataProvider preset_data_provider.PresetDataProvider diff --git a/qf_lib/analysis/signals_analysis/signals_plotter.py b/qf_lib/analysis/signals_analysis/signals_plotter.py index 917bd56d..5af242ef 100644 --- a/qf_lib/analysis/signals_analysis/signals_plotter.py +++ b/qf_lib/analysis/signals_analysis/signals_plotter.py @@ -21,7 +21,6 @@ from qf_lib.analysis.common.abstract_document import AbstractDocument from qf_lib.backtesting.alpha_model.alpha_model import AlphaModel from qf_lib.backtesting.alpha_model.exposure_enum import Exposure -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent from qf_lib.common.enums.frequency import Frequency @@ -35,6 +34,7 @@ from qf_lib.containers.futures.futures_adjustment_method import FuturesAdjustmentMethod from qf_lib.containers.futures.futures_chain import FuturesChain from qf_lib.containers.series.qf_series import QFSeries +from qf_lib.data_providers.data_provider import DataProvider from qf_lib.documents_utils.document_exporting.element.chart import ChartElement from qf_lib.documents_utils.document_exporting.element.custom import CustomElement from qf_lib.documents_utils.document_exporting.element.heading import HeadingElement @@ -69,8 +69,8 @@ class SignalsPlotter(AbstractDocument): starting date of the prices data frame end_date: datetime last date of the prices data frame - data_handler: DataHandler - data handler used to download prices + data_provider: DataProvider + data provider used to download prices alpha_models: AlphaModel, Sequence[AlphaModel] instances of alpha models which signals will be evaluated. Each plot in the document is described using the alpha_models __str__ function. @@ -87,7 +87,7 @@ class SignalsPlotter(AbstractDocument): """ def __init__(self, tickers: Union[Ticker, Sequence[Ticker]], start_date: datetime, end_date: datetime, - data_handler: DataHandler, alpha_models: Union[AlphaModel, Sequence[AlphaModel]], settings: Settings, + data_provider: DataProvider, alpha_models: Union[AlphaModel, Sequence[AlphaModel]], settings: Settings, pdf_exporter: PDFExporter, title: str = "Signals Plotter", signal_frequency: Frequency = Frequency.DAILY, data_frequency: Frequency = Frequency.DAILY): @@ -99,10 +99,8 @@ def __init__(self, tickers: Union[Ticker, Sequence[Ticker]], start_date: datetim self.start_date = start_date self.end_date = end_date - self.data_handler = data_handler - - assert isinstance(self.data_handler.timer, SettableTimer) - self.timer: SettableTimer = self.data_handler.timer + self.data_provider = data_provider + self.data_provider.set_timer(SettableTimer(end_date)) self.signal_frequency = signal_frequency self.data_frequency = data_frequency @@ -110,7 +108,7 @@ def __init__(self, tickers: Union[Ticker, Sequence[Ticker]], start_date: datetim for ticker in self.tickers: if isinstance(ticker, FutureTicker): # use a new timer that allows to look until the end date - ticker.initialize_data_provider(SettableTimer(end_date), self.data_handler.data_provider) + ticker.initialize_data_provider(self.data_provider) def build_document(self): self._add_header() @@ -139,8 +137,8 @@ def create_tickers_analysis(self, ticker: Ticker): prev_exposure = Exposure.OUT for date in self._get_signals_dates(): try: - self.timer.set_current_time(date) - new_exposure = alpha_model.get_signal(ticker, prev_exposure, date, self.data_frequency)\ + self.data_provider.set_current_time(date) + new_exposure = alpha_model.get_signal(ticker, prev_exposure, date, self.data_frequency) \ .suggested_exposure exposures.append(new_exposure.value) dates.append(date) @@ -174,17 +172,17 @@ def get_prices(self, ticker: Ticker): Here we can use data provider as we do not worry about look-ahead """ if isinstance(ticker, FutureTicker): - futures_chain = FuturesChain(ticker, self.data_handler.data_provider, FuturesAdjustmentMethod.NTH_NEAREST) + futures_chain = FuturesChain(ticker, self.data_provider, FuturesAdjustmentMethod.NTH_NEAREST) prices_df = futures_chain.get_price(PriceField.ohlc(), start_date=self.start_date, end_date=self.end_date, frequency=self.data_frequency) else: - prices_df = self.data_handler.data_provider.get_price(ticker, - PriceField.ohlc(), - start_date=self.start_date, - end_date=self.end_date, - frequency=self.data_frequency) + prices_df = self.data_provider.get_price(ticker, + PriceField.ohlc(), + start_date=self.start_date, + end_date=self.end_date, + frequency=self.data_frequency) return prices_df def add_models_implementation(self): diff --git a/qf_lib/analysis/strategy_monitoring/assets_monitoring_sheet.py b/qf_lib/analysis/strategy_monitoring/assets_monitoring_sheet.py index bd41b3fd..3024a26f 100644 --- a/qf_lib/analysis/strategy_monitoring/assets_monitoring_sheet.py +++ b/qf_lib/analysis/strategy_monitoring/assets_monitoring_sheet.py @@ -41,7 +41,7 @@ from qf_lib.containers.series.prices_series import PricesSeries from qf_lib.containers.series.qf_series import QFSeries from qf_lib.containers.series.simple_returns_series import SimpleReturnsSeries -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider from qf_lib.documents_utils.document_exporting.element.df_table import DFTable from qf_lib.documents_utils.document_exporting.element.heading import HeadingElement from qf_lib.documents_utils.document_exporting.element.new_page import NewPageElement @@ -70,7 +70,7 @@ class AssetPerfAndDrawdownSheet(AbstractDocument): start_date: datetime end_date: datetime Dates to used as start and end date for the statistics - data_provider: DataProvider + data_provider: AbstractPriceDataProvider Data provider used to download the prices and future contracts information, necessary to compute Buy and Hold benchmark performance settings: Settings @@ -87,7 +87,7 @@ class AssetPerfAndDrawdownSheet(AbstractDocument): """ def __init__(self, category_to_model_tickers: Dict[str, List[Ticker]], transactions: Union[List[Transaction], str], - start_date: datetime, end_date: datetime, data_provider: DataProvider, settings: Settings, + start_date: datetime, end_date: datetime, data_provider: AbstractPriceDataProvider, settings: Settings, pdf_exporter: PDFExporter, title: str = "Assets Monitoring Sheet", initial_cash: int = 10000000, frequency: Frequency = Frequency.YEARLY): @@ -104,6 +104,7 @@ def __init__(self, category_to_model_tickers: Dict[str, List[Ticker]], transacti self._start_date = start_date self._end_date = end_date self._data_provider = data_provider + self._data_provider.set_timer(SettableTimer(self._end_date)) self._initial_cash = initial_cash if frequency not in (Frequency.MONTHLY, Frequency.YEARLY): @@ -190,7 +191,7 @@ def _generate_buy_and_hold_returns(self, ticker: Ticker) -> SimpleReturnsSeries: """ Computes series of simple returns, which would be returned by the Buy and Hold strategy. """ if isinstance(ticker, FutureTicker): try: - ticker.initialize_data_provider(SettableTimer(self._end_date), self._data_provider) + ticker.initialize_data_provider(self._data_provider) futures_chain = FuturesChain(ticker, self._data_provider, FuturesAdjustmentMethod.BACK_ADJUSTED) prices_series = futures_chain.get_price(PriceField.Close, self._start_date, self._end_date) except NoValidTickerException: diff --git a/qf_lib/analysis/strategy_monitoring/pnl_calculator.py b/qf_lib/analysis/strategy_monitoring/pnl_calculator.py index 07c81870..782c5a13 100644 --- a/qf_lib/analysis/strategy_monitoring/pnl_calculator.py +++ b/qf_lib/analysis/strategy_monitoring/pnl_calculator.py @@ -30,11 +30,11 @@ from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker from qf_lib.containers.series.prices_series import PricesSeries from qf_lib.containers.series.qf_series import QFSeries -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class PnLCalculator: - def __init__(self, data_provider: DataProvider): + def __init__(self, data_provider: AbstractPriceDataProvider): """ The purpose of this class is the computation of Profit and Loss for a given ticker, based on a list of Transaction objects. The PnL is computed every day, at the AfterMarketCloseEvent time. The calculation requires @@ -45,10 +45,11 @@ def __init__(self, data_provider: DataProvider): Parameters ----------- - data_provider: DataProvider + data_provider: AbstractPriceDataProvider data provider used to download prices data """ self._data_provider = data_provider + self._data_provider.set_timer(SettableTimer()) def compute_pnl(self, ticker: Ticker, transactions: List[Transaction], start_date: datetime, end_date: datetime) \ -> PricesSeries: @@ -83,7 +84,9 @@ def compute_pnl(self, ticker: Ticker, transactions: List[Transaction], start_dat def _get_prices_df(self, ticker: Ticker, start_date: datetime, end_date: datetime) -> PricesDataFrame: """ Returns non-adjusted open and close prices, indexed with the Market Open and Market Close time.""" if isinstance(ticker, FutureTicker): - ticker.initialize_data_provider(SettableTimer(end_date), self._data_provider) + assert isinstance(self._data_provider.timer, SettableTimer) + self._data_provider.timer.set_current_time(end_date) + ticker.initialize_data_provider(self._data_provider) tickers_chain = ticker.get_expiration_dates() if start_date >= tickers_chain.index[-1] or end_date <= tickers_chain.index[0]: diff --git a/qf_lib/analysis/tearsheets/portfolio_analysis_sheet.py b/qf_lib/analysis/tearsheets/portfolio_analysis_sheet.py index b4e84a0c..9063f213 100644 --- a/qf_lib/analysis/tearsheets/portfolio_analysis_sheet.py +++ b/qf_lib/analysis/tearsheets/portfolio_analysis_sheet.py @@ -238,7 +238,7 @@ def _add_avg_time_in_the_market_per_ticker(self): self.document.add_element(HeadingElement(level=2, text="Average time in the market per asset")) start_time = self.backtest_result.start_date - end_time = self.backtest_result.portfolio.timer.now() + end_time = self.backtest_result.portfolio.data_provider.timer.now() backtest_duration = pd.Timedelta(end_time - start_time) / pd.Timedelta(minutes=1) # backtest duration in min positions_list = self.backtest_result.portfolio.closed_positions() + \ list(self.backtest_result.portfolio.open_positions_dict.values()) diff --git a/qf_lib/backtesting/alpha_model/alpha_model.py b/qf_lib/backtesting/alpha_model/alpha_model.py index 4eccea94..adc22378 100644 --- a/qf_lib/backtesting/alpha_model/alpha_model.py +++ b/qf_lib/backtesting/alpha_model/alpha_model.py @@ -24,7 +24,7 @@ from qf_lib.common.tickers.tickers import Ticker from qf_lib.common.utils.logging.qf_parent_logger import qf_logger from qf_lib.common.utils.miscellaneous.average_true_range import average_true_range -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class AlphaModel(metaclass=ABCMeta): @@ -36,12 +36,11 @@ class AlphaModel(metaclass=ABCMeta): risk_estimation_factor float value which estimates the risk level of the specific AlphaModel. Corresponds to the level at which the stop-loss should be placed. - data_provider: DataProvider - DataProvider which provides data for the ticker. For the backtesting purposes, in order to avoid looking into - the future, use DataHandler wrapper. + data_provider: AbstractPriceDataProvider + DataProvider which provides data for the ticker. """ - def __init__(self, risk_estimation_factor: float, data_provider: DataProvider): + def __init__(self, risk_estimation_factor: float, data_provider: AbstractPriceDataProvider): self.risk_estimation_factor = risk_estimation_factor self.data_provider = data_provider self.logger = qf_logger.getChild(self.__class__.__name__) @@ -83,7 +82,7 @@ def calculate_exposure(self, ticker: Ticker, current_exposure: Exposure, current """ Returns the expected Exposure, which is the key part of a generated Signal. Exposure suggests the trend direction for managing the trading position. - Uses DataHandler passed when the AlphaModel (child) is initialized - all required data is provided in the child + Uses DataProvider passed when the AlphaModel (child) is initialized - all required data is provided in the child class. Parameters diff --git a/qf_lib/backtesting/alpha_model/futures_model.py b/qf_lib/backtesting/alpha_model/futures_model.py index 8eb0a13a..9a2a7824 100644 --- a/qf_lib/backtesting/alpha_model/futures_model.py +++ b/qf_lib/backtesting/alpha_model/futures_model.py @@ -35,7 +35,7 @@ from qf_lib.containers.futures.futures_adjustment_method import FuturesAdjustmentMethod from qf_lib.containers.futures.futures_chain import FuturesChain from qf_lib.containers.series.qf_series import QFSeries -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class FuturesModel(AlphaModel, metaclass=abc.ABCMeta): @@ -51,14 +51,14 @@ class FuturesModel(AlphaModel, metaclass=abc.ABCMeta): risk_estimation_factor: float float value which estimates the risk level of the specific AlphaModel. Corresponds to the level at which the stop-loss should be placed. - data_provider: DataProvider - DataProvider which provides data for the ticker. For backtesting purposes the Data Handler should be used. + data_provider: AbstractPriceDataProvider + DataProvider which provides data for the ticker cache_path: Optional[str] path to a directory, which could be used by the model for caching purposes. If provided, the model will cache the outputs of get_data function. """ - def __init__(self, num_of_bars_needed: int, risk_estimation_factor: float, data_provider: DataProvider, + def __init__(self, num_of_bars_needed: int, risk_estimation_factor: float, data_provider: AbstractPriceDataProvider, cache_path: Optional[str] = None): super().__init__(risk_estimation_factor, data_provider) diff --git a/qf_lib/backtesting/alpha_model/random_trades_alpha_model.py b/qf_lib/backtesting/alpha_model/random_trades_alpha_model.py index 60d09d52..d569ecc0 100644 --- a/qf_lib/backtesting/alpha_model/random_trades_alpha_model.py +++ b/qf_lib/backtesting/alpha_model/random_trades_alpha_model.py @@ -17,12 +17,12 @@ from qf_lib.backtesting.alpha_model.alpha_model import AlphaModel from qf_lib.backtesting.alpha_model.exposure_enum import Exposure from qf_lib.backtesting.alpha_model.futures_model import FuturesModel -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.fast_alpha_model_tester.scenarios_generator import ScenariosGenerator from qf_lib.common.enums.frequency import Frequency from qf_lib.common.tickers.tickers import Ticker from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame +from qf_lib.data_providers.data_provider import DataProvider class RandomTradesAlphaModel(AlphaModel): @@ -34,8 +34,8 @@ class RandomTradesAlphaModel(AlphaModel): risk_estimation_factor float value which estimates the risk level of the specific AlphaModel. Corresponds to the level at which the stop-loss should be placed. - data_provider: DataHandler - DataHandler which provides data for the ticker. + data_provider: DataProvider + DataProvider which provides data for the ticker. start_date: datetime first date considered in the returned series end_date: datetime @@ -51,12 +51,11 @@ class RandomTradesAlphaModel(AlphaModel): seed: Optional[int] seed used to make the scenarios deterministic """ - def __init__(self, risk_estimation_factor: float, data_provider: DataHandler, start_date: datetime, + def __init__(self, risk_estimation_factor: float, data_provider: DataProvider, start_date: datetime, end_date: datetime, tickers: Sequence[Ticker], number_of_trades: int, time_in_the_market: float, exposure: Exposure = Exposure.LONG, frequency: Frequency = Frequency.DAILY, seed: Optional[int] = None): super().__init__(risk_estimation_factor, data_provider) - self.timer = data_provider.timer self.start_date = start_date self.end_date = end_date self.frequency = frequency @@ -88,8 +87,8 @@ class RandomTradesFuturesAlphaModel(FuturesModel): risk_estimation_factor float value which estimates the risk level of the specific AlphaModel. Corresponds to the level at which the stop-loss should be placed. - data_provider: DataHandler - DataHandler which provides data for the ticker. + data_provider: DataProvider + DataProvider which provides data for the ticker. start_date: datetime first date considered in the returned series end_date: datetime @@ -105,13 +104,12 @@ class RandomTradesFuturesAlphaModel(FuturesModel): seed: Optional[int] seed used to make the scenarios deterministic """ - def __init__(self, risk_estimation_factor: float, data_provider: DataHandler, start_date: datetime, + def __init__(self, risk_estimation_factor: float, data_provider: DataProvider, start_date: datetime, end_date: datetime, tickers: Sequence[Ticker], number_of_trades: int, time_in_the_market: float, num_of_bars_needed: int = 180, exposure: Exposure = Exposure.LONG, frequency: Frequency = Frequency.DAILY, seed: Optional[int] = None): super().__init__(num_of_bars_needed, risk_estimation_factor, data_provider) - self.timer = data_provider.timer self.start_date = start_date self.end_date = end_date self.frequency = frequency diff --git a/qf_lib/backtesting/data_handler/__init__.py b/qf_lib/backtesting/data_handler/__init__.py deleted file mode 100644 index f31f5e3d..00000000 --- a/qf_lib/backtesting/data_handler/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2016-present CERN – European Organization for Nuclear Research -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/qf_lib/backtesting/data_handler/daily_data_handler.py b/qf_lib/backtesting/data_handler/daily_data_handler.py deleted file mode 100644 index b5a703cb..00000000 --- a/qf_lib/backtesting/data_handler/daily_data_handler.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2016-present CERN – European Organization for Nuclear Research -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from datetime import datetime -from typing import Optional, Union, Sequence, Type - -from numpy import nan -from pandas import concat - -from qf_lib.backtesting.data_handler.data_handler import DataHandler -from qf_lib.backtesting.events.time_event.regular_time_event.regular_market_event import RegularMarketEvent -from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent -from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent -from qf_lib.common.enums.frequency import Frequency -from qf_lib.common.enums.price_field import PriceField -from qf_lib.common.tickers.tickers import Ticker -from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta -from qf_lib.common.utils.dateutils.timer import Timer -from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list -from qf_lib.containers.series.prices_series import PricesSeries -from qf_lib.containers.series.qf_series import QFSeries -from qf_lib.data_providers.data_provider import DataProvider - - -class DailyDataHandler(DataHandler): - def __init__(self, data_provider: DataProvider, timer: Timer): - super().__init__(data_provider, timer) - self.default_frequency = data_provider.frequency if data_provider.frequency is not None else Frequency.DAILY - - def _check_frequency(self, frequency): - if frequency and frequency > Frequency.DAILY: - raise ValueError("Frequency higher than daily is not supported by DailyDataHandler.") - - def _get_end_date_without_look_ahead(self, end_date: Optional[datetime], frequency: Frequency): - """ Points always to the latest market close for which the data could be retrieved. """ - return self._get_last_available_market_event(end_date, MarketCloseEvent) - - def _get_last_available_market_event(self, end_date: Optional[datetime], event: Type[RegularMarketEvent]): - current_datetime = self.timer.now() + RelativeDelta(second=0, microsecond=0) - end_date = end_date or current_datetime - end_date += RelativeDelta(days=1, hour=0, minute=0, second=0, microsecond=0, microseconds=-1) - - today_market_event = current_datetime + event.trigger_time() - yesterday_market_event = today_market_event - RelativeDelta(days=1) - latest_available_market_event = yesterday_market_event if current_datetime < today_market_event \ - else today_market_event - - latest_market_event = min(latest_available_market_event, end_date) - return datetime(latest_market_event.year, latest_market_event.month, latest_market_event.day) - - def get_last_available_price(self, tickers: Union[Ticker, Sequence[Ticker]], frequency: Frequency = None, - end_time: Optional[datetime] = None) -> Union[float, QFSeries]: - tickers, got_single_ticker = convert_to_list(tickers, Ticker) - if not tickers: - return nan if got_single_ticker else PricesSeries() - - frequency = frequency or self.default_frequency - end_time = end_time or self.timer.now() - end_date_without_look_ahead = self._get_end_date_without_look_ahead(end_time, frequency) - - last_prices = self.data_provider.get_last_available_price(tickers, frequency, end_date_without_look_ahead) - - latest_market_open = self._get_last_available_market_event(end_time, MarketOpenEvent) - if end_date_without_look_ahead < latest_market_open: - - current_open_prices = self.data_provider.get_price(tickers, PriceField.Open, latest_market_open, - latest_market_open, frequency) - last_prices = concat([last_prices, current_open_prices], axis=1).ffill(axis=1) - last_prices = last_prices.iloc[:, -1] - - return last_prices.iloc[0] if got_single_ticker else last_prices diff --git a/qf_lib/backtesting/data_handler/data_handler.py b/qf_lib/backtesting/data_handler/data_handler.py deleted file mode 100644 index d2a114c8..00000000 --- a/qf_lib/backtesting/data_handler/data_handler.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright 2016-present CERN – European Organization for Nuclear Research -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from abc import abstractmethod -from datetime import datetime -from typing import Union, Sequence, Optional, Dict - -from pandas import date_range - -from qf_lib.common.enums.expiration_date_field import ExpirationDateField -from qf_lib.common.enums.frequency import Frequency -from qf_lib.common.enums.price_field import PriceField -from qf_lib.common.tickers.tickers import Ticker -from qf_lib.common.utils.dateutils.timer import Timer -from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list -from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame -from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame -from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker -from qf_lib.containers.qf_data_array import QFDataArray -from qf_lib.containers.series.prices_series import PricesSeries -from qf_lib.containers.series.qf_series import QFSeries -from qf_lib.data_providers.data_provider import DataProvider -from qf_lib.data_providers.helpers import normalize_data_array -from qf_lib.data_providers.prefetching_data_provider import PrefetchingDataProvider - - -class DataHandler(DataProvider): - """ - DataHandler is a wrapper which can be used with any AbstractPriceDataProvider in both live and backtest - environment. It makes sure that data "from the future" is not passed into components in the backtest environment. - - DataHandler should be used by all the Backtester's components (even in the live trading setup). - - The goal of a DataHandler is to provide backtester's components with financial data. It makes sure that - no data from the future (relative to a "current" time of a backtester) is being accessed, that is: that there - is no look-ahead bias. - - Parameters - ----------- - data_provider: DataProvider - the underlying data provider - timer: Timer - timer used to keep track of the data "from the future" - """ - - def __init__(self, data_provider: DataProvider, timer: Timer): - super().__init__() - self.data_provider = data_provider - - self._check_frequency(data_provider.frequency) - self.default_frequency = data_provider.frequency # type: Frequency - - self.timer = timer - self.is_optimised = False - - def use_data_bundle(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[PriceField, Sequence[PriceField]], - start_date: datetime, end_date: datetime, frequency: Frequency = Frequency.DAILY): - """ - Optimises running of the backtest. All the data will be downloaded before the backtest. - Note that requesting during the backtest any other ticker or price field than the ones in the params - of this function will result in an Exception. - - Parameters - ---------- - tickers: Ticker, Sequence[Ticker] - ticker or sequence of tickers of the securities - fields: PriceField, Sequence[PriceField] - PriceField or sequence of PriceFields of the securities - start_date: datetime - initial date that should be downloaded - end_date: datetime - last date that should be downloaded - frequency - frequency of the data - """ - assert not self.is_optimised, "Multiple calls on use_data_bundle() are forbidden" - - tickers, _ = convert_to_list(tickers, Ticker) - fields, _ = convert_to_list(fields, PriceField) - - self._check_frequency(frequency) - self.default_frequency = frequency - - self.data_provider = PrefetchingDataProvider(self.data_provider, tickers, fields, start_date, end_date, - frequency) - self.is_optimised = True - - def historical_price(self, tickers: Union[Ticker, Sequence[Ticker]], - fields: Union[PriceField, Sequence[PriceField]], - nr_of_bars: int, end_date: Optional[datetime] = None, frequency: Frequency = None) -> \ - Union[PricesSeries, PricesDataFrame, QFDataArray]: - - frequency = frequency or self.default_frequency - end_date = self._get_end_date_without_look_ahead(end_date, frequency) - return self.data_provider.historical_price(tickers, fields, nr_of_bars, end_date, frequency) - - def get_price(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[PriceField, Sequence[PriceField]], - start_date: datetime, end_date: datetime = None, frequency: Frequency = None, **kwargs) -> \ - Union[PricesSeries, PricesDataFrame, QFDataArray]: - """ - Runs DataProvider.get_price(...) but before makes sure that the query doesn't concern data from - the future. It always returns the fully available bars (e.g. it will return a full bar for a day only after - the market close). - - Parameters - ---------- - tickers: Ticker, Sequence[Ticker] - tickers for securities which should be retrieved - fields: PriceField, Sequence[PriceField] - fields of securities which should be retrieved - start_date: datetime - date representing the beginning of historical period from which data should be retrieved - end_date: datetime - date representing the end of historical period from which data should be retrieved; - if no end_date was provided, by default the current date will be used - frequency: Frequency - frequency of the data - - Returns - ------- - None, PricesSeries, PricesDataFrame, QFDataArray - :param **kwargs: - """ - frequency = frequency or self.default_frequency - assert frequency is not None, "Frequency cannot be equal to None" - - start_date = self._adjust_start_date(start_date, frequency) - end_date = end_date or self.timer.now() - got_single_date = self._got_single_date(start_date, end_date, frequency) - - end_date_without_look_ahead = self._get_end_date_without_look_ahead(end_date, frequency) - got_single_date_without_look_ahead = self._got_single_date(start_date, end_date_without_look_ahead, frequency) - - if start_date > end_date_without_look_ahead: - prices_data = self._empty_container(tickers, fields, start_date, end_date, frequency) - elif got_single_date != got_single_date_without_look_ahead: - prices_data = self.data_provider.get_price(tickers, fields, start_date, - end_date_without_look_ahead + frequency.time_delta(), frequency) - prices_data = prices_data.loc[start_date:end_date_without_look_ahead] - else: - prices_data = self.data_provider.get_price(tickers, fields, start_date, end_date_without_look_ahead, - frequency) - - return prices_data - - def get_history(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[str, Sequence[str]], - start_date: datetime, end_date: datetime = None, frequency: Frequency = None, **kwargs) -> \ - Union[QFSeries, QFDataFrame, QFDataArray]: - """ - Runs DataProvider.get_history(...) but before makes sure that the query doesn't concern data from the future. - - It accesses the latest fully available bar as of "today", that is: if a bar wasn't closed for today yet, - then all the PriceFields (e.g. OPEN) will concern data from yesterday. - - See Also - -------- - DataProvider.get_history - """ - frequency = frequency or self.default_frequency - assert frequency is not None, "Frequency cannot be equal to None" - - start_date = self._adjust_start_date(start_date, frequency) - single_date = self._got_single_date(start_date, end_date, frequency) - end_date_without_look_ahead = self._get_end_date_without_look_ahead(end_date, frequency) - - if start_date > end_date_without_look_ahead: - data = self._empty_container(tickers, fields, start_date, end_date, frequency) - elif start_date.date() == end_date_without_look_ahead.date() and not single_date: - data = self.data_provider.get_history(tickers, fields, start_date, - end_date_without_look_ahead + frequency.time_delta(), frequency) - data = data.loc[start_date:end_date_without_look_ahead] - else: - data = self.data_provider.get_history(tickers, fields, start_date, end_date_without_look_ahead, - frequency) - - return data - - def get_futures_chain_tickers(self, tickers: Union[FutureTicker, Sequence[FutureTicker]], - expiration_date_fields: Union[ExpirationDateField, Sequence[ExpirationDateField]]) \ - -> Dict[FutureTicker, Union[QFSeries, QFDataFrame]]: - return self.data_provider.get_futures_chain_tickers(tickers, expiration_date_fields) - - def supported_ticker_types(self): - return self.data_provider.supported_ticker_types() - - @abstractmethod - def get_last_available_price(self, tickers: Union[Ticker, Sequence[Ticker]], frequency: Frequency = None, - end_time: Optional[datetime] = None) -> Union[float, QFSeries]: - pass - - @abstractmethod - def _get_end_date_without_look_ahead(self, end_date: datetime, frequency: Frequency): - pass - - @abstractmethod - def _check_frequency(self, frequency): - """ Verify if the provided frequency is compliant with the type of Data Handler used. """ - pass - - def _empty_container(self, tickers, fields, start_date, end_date, frequency): - tickers, got_single_ticker = convert_to_list(tickers, Ticker) - fields, got_single_field = convert_to_list(fields, (str, PriceField)) - got_single_date = self._got_single_date(start_date, end_date, frequency) - - dates = date_range(start_date, end_date, freq=frequency.to_pandas_freq()) - data_array = QFDataArray.create(dates, tickers, fields, data=None) - return normalize_data_array(data_array, tickers, fields, got_single_date, got_single_ticker, - got_single_field, True) diff --git a/qf_lib/backtesting/data_handler/intraday_data_handler.py b/qf_lib/backtesting/data_handler/intraday_data_handler.py deleted file mode 100644 index 5e6a793c..00000000 --- a/qf_lib/backtesting/data_handler/intraday_data_handler.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2016-present CERN – European Organization for Nuclear Research -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import math -from datetime import datetime -from typing import Union, Sequence, Optional - -from numpy import nan -from pandas import concat -from pandas._libs.tslibs.offsets import to_offset -from pandas._libs.tslibs.timestamps import Timestamp - -from qf_lib.backtesting.data_handler.data_handler import DataHandler -from qf_lib.common.enums.frequency import Frequency -from qf_lib.common.enums.price_field import PriceField -from qf_lib.common.tickers.tickers import Ticker -from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta -from qf_lib.common.utils.dateutils.timer import Timer -from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list -from qf_lib.containers.series.prices_series import PricesSeries -from qf_lib.containers.series.qf_series import QFSeries -from qf_lib.data_providers.data_provider import DataProvider - - -class IntradayDataHandler(DataHandler): - def __init__(self, data_provider: DataProvider, timer: Timer): - super().__init__(data_provider, timer) - self.default_frequency = data_provider.frequency if data_provider.frequency is not None else Frequency.MIN_1 - - def _check_frequency(self, frequency): - if frequency and frequency <= Frequency.DAILY: - raise ValueError("Only frequency higher than daily is supported by IntradayDataHandler.") - - def _get_end_date_without_look_ahead(self, end_date: Optional[datetime], frequency: Frequency): - """ If end_date is None, current time is taken as end_date. The function returns the end of latest full bar - (get_price, get_history etc. functions always include the end_date e.g. in case of 1 minute frequency: - current_time = 16:20 and end_date = 16:06 the latest returned bar is the [16:06, 16:07)). - - Examples: - - current_time = 20:00, end_time = 17:01, frequency = 1h, - => end_date_without_look_ahead = 17:00 - - - current_time = 20:00, end_time = 19:58, frequency = 1h, - => end_date_without_look_ahead = 19:00 - - - current_time = 20:00, end_time = 20:01, frequency = 1h , - => end_date_without_look_ahead = 19:00 - - - current_time = 20:00, end_time = 20:00, frequency = 1h, - => end_date_without_look_ahead = 19 - - - current_time = 20:10, end_time = 22:10, frequency = 1h, - => end_date_without_look_ahead = 19 - - - current_time = 19:58, end_time = 19:56 , frequency = 1m, - => end_date_without_look_ahead = 19:56 - - - current_time = 19:56, end_time = 19:58 , frequency = 1m, - => end_date_without_look_ahead = 19:55 - """ - - current_time = self.timer.now() + RelativeDelta(second=0, microsecond=0) - end_date = end_date or current_time - end_date += RelativeDelta(second=0, microsecond=0) - - frequency_delta = to_offset(frequency.to_pandas_freq()).delta.value - if current_time <= end_date: - end_date_without_lookahead = Timestamp(math.floor(Timestamp(current_time).value / frequency_delta) * - frequency_delta).to_pydatetime() - frequency.time_delta() - else: - end_date_without_lookahead = Timestamp(math.floor(Timestamp(end_date).value / frequency_delta) * - frequency_delta).to_pydatetime() - return end_date_without_lookahead - - def get_last_available_price(self, tickers: Union[Ticker, Sequence[Ticker]], frequency: Frequency = None, - end_time: Optional[datetime] = None) -> Union[float, QFSeries]: - - tickers, got_single_ticker = convert_to_list(tickers, Ticker) - if not tickers: - return nan if got_single_ticker else PricesSeries() - frequency = frequency or self.default_frequency - current_time = self.timer.now() + RelativeDelta(second=0, microsecond=0) - end_time = end_time or current_time - end_date_without_look_ahead = self._get_end_date_without_look_ahead(end_time, frequency) - - if current_time <= end_time: - last_prices = self.data_provider.get_last_available_price(tickers, frequency, end_date_without_look_ahead) - current_open_prices = self.data_provider.get_price(tickers, PriceField.Open, - start_date=end_date_without_look_ahead + frequency.time_delta(), - end_date=end_date_without_look_ahead + frequency.time_delta(), - frequency=frequency) - else: - last_prices = self.data_provider.get_last_available_price(tickers, frequency, end_date_without_look_ahead - frequency.time_delta()) - current_open_prices = self.data_provider.get_price(tickers, PriceField.Open, - start_date=end_date_without_look_ahead, - end_date=end_date_without_look_ahead, - frequency=frequency) - - last_prices = concat([last_prices, current_open_prices], axis=1).ffill(axis=1) - last_prices = last_prices.iloc[:, -1] - - return last_prices.iloc[0] if got_single_ticker else last_prices diff --git a/qf_lib/backtesting/execution_handler/market_orders_executor.py b/qf_lib/backtesting/execution_handler/market_orders_executor.py index 20c53157..2a4d4a02 100644 --- a/qf_lib/backtesting/execution_handler/market_orders_executor.py +++ b/qf_lib/backtesting/execution_handler/market_orders_executor.py @@ -71,8 +71,8 @@ def _get_orders_with_fill_prices_without_slippage(self, market_orders_list, tick def _get_current_prices(self, tickers: Sequence[Ticker]): """ Function used to obtain the current prices for the tickers in order to further calculate fill prices for orders. - The function uses data provider and not data handler, as it is necessary to get the current bar at each point - in time to compute the fill prices. + The function uses data provider with look ahead bias = True, as it is necessary to get the current bar at each + point in time to compute the fill prices. """ if not tickers: return QFSeries() @@ -81,7 +81,7 @@ def _get_current_prices(self, tickers: Sequence[Ticker]): " executor" # Compute the time ranges, used further by the get_price function - current_datetime = self._timer.now() + current_datetime = self._data_provider.timer.now() market_close_time = current_datetime + MarketCloseEvent.trigger_time() == current_datetime market_open_time = current_datetime + MarketOpenEvent.trigger_time() == current_datetime @@ -104,7 +104,8 @@ def _get_current_prices(self, tickers: Sequence[Ticker]): start_time_range = current_datetime price_field = PriceField.Close if market_close_time else PriceField.Open - prices = self._data_provider.get_price(tickers, price_field, start_time_range, start_time_range, self._frequency) + prices = self._data_provider.get_price(tickers, price_field, start_time_range, start_time_range, + self._frequency, look_ahead_bias=True) return prices def _check_order_validity(self, order): diff --git a/qf_lib/backtesting/execution_handler/simulated_execution_handler.py b/qf_lib/backtesting/execution_handler/simulated_execution_handler.py index 3fa41e89..7c47d11b 100644 --- a/qf_lib/backtesting/execution_handler/simulated_execution_handler.py +++ b/qf_lib/backtesting/execution_handler/simulated_execution_handler.py @@ -16,7 +16,6 @@ from itertools import count, groupby from typing import List, Sequence, Dict -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.events.time_event.periodic_event.intraday_bar_event import IntradayBarEvent from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent @@ -41,8 +40,8 @@ from qf_lib.common.enums.price_field import PriceField from qf_lib.common.exceptions.broker_exceptions import OrderCancellingException from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta -from qf_lib.common.utils.dateutils.timer import Timer from qf_lib.common.utils.logging.qf_parent_logger import qf_logger +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class SimulatedExecutionHandler(ExecutionHandler): @@ -52,7 +51,7 @@ class SimulatedExecutionHandler(ExecutionHandler): StopOrders are executed at the MarketClose (if applicable) with the Low price. """ - def __init__(self, data_handler: DataHandler, timer: Timer, scheduler: Scheduler, monitor: AbstractMonitor, + def __init__(self, data_provider: AbstractPriceDataProvider, scheduler: Scheduler, monitor: AbstractMonitor, commission_model: CommissionModel, portfolio: Portfolio, slippage_model: Slippage, scheduling_time_delay: RelativeDelta = RelativeDelta(minutes=1), frequency: Frequency = Frequency.DAILY) -> None: @@ -70,26 +69,25 @@ def __init__(self, data_handler: DataHandler, timer: Timer, scheduler: Scheduler if self.intraday_trading: scheduler.subscribe(IntradayBarEvent, self) - self.data_handler = data_handler + self.data_provider = data_provider self.commission_model = commission_model self.portfolio = portfolio self.monitor = monitor - self.timer = timer self.scheduling_time_delay = scheduling_time_delay order_id_generator = count(start=1) - self._market_orders_executor = MarketOrdersExecutor(data_handler, monitor, portfolio, timer, order_id_generator, + self._market_orders_executor = MarketOrdersExecutor(data_provider, monitor, portfolio, order_id_generator, commission_model, slippage_model, frequency) - self._stop_orders_executor = StopOrdersExecutor(data_handler, monitor, portfolio, timer, order_id_generator, + self._stop_orders_executor = StopOrdersExecutor(data_provider, monitor, portfolio, order_id_generator, commission_model, slippage_model, frequency) - self._market_on_close_orders_executor = MarketOnCloseOrdersExecutor(data_handler, monitor, portfolio, timer, + self._market_on_close_orders_executor = MarketOnCloseOrdersExecutor(data_provider, monitor, portfolio, order_id_generator, commission_model, slippage_model, frequency) - self._market_on_open_orders_executor = MarketOnOpenOrdersExecutor(data_handler, monitor, portfolio, timer, + self._market_on_open_orders_executor = MarketOnOpenOrdersExecutor(data_provider, monitor, portfolio, order_id_generator, commission_model, slippage_model, frequency) @@ -102,7 +100,7 @@ def on_market_close(self, _: MarketCloseEvent): # this was in the past done after market close self._remove_acquired_or_not_active_positions() self.portfolio.update(record=True) - self.monitor.end_of_day_update(self.timer.now()) + self.monitor.end_of_day_update(self.data_provider.timer.now()) def on_market_open(self, _: MarketOpenEvent): self._market_orders_executor.execute_orders(market_open=True) @@ -113,7 +111,7 @@ def on_new_bar(self, _: IntradayBarEvent): self._stop_orders_executor.execute_orders() def on_orders_accept(self, event: ScheduleOrderExecutionEvent): - executors_to_orders_dict = event.get_executors_to_orders_dict(self.timer.now()) # type: Dict[SimulatedExecutor, List[Order]] + executors_to_orders_dict = event.get_executors_to_orders_dict(self.data_provider.timer.now()) # type: Dict[SimulatedExecutor, List[Order]] for executor in executors_to_orders_dict.keys(): executor.accept_orders(executors_to_orders_dict[executor]) @@ -151,7 +149,7 @@ def assign_order_ids(self, orders: Sequence[Order]) -> List[int]: order_id_list += partial_order_id_list # Schedule the orders execution - scheduled_time = self.timer.now() + self.scheduling_time_delay + scheduled_time = self.data_provider.timer.now() + self.scheduling_time_delay ScheduleOrderExecutionEvent.schedule_new_event(scheduled_time, scheduled_event_data) return order_id_list @@ -196,16 +194,16 @@ def _remove_acquired_or_not_active_positions(self): last price of the asset recorded by the portfolio and the commission is set to 0, as no real order is created. """ all_tickers_in_portfolio = list(self.portfolio.open_positions_dict.keys()) - start_date = self.timer.now() - RelativeDelta(days=7) + start_date = self.data_provider.timer.now() - RelativeDelta(days=7) for ticker in all_tickers_in_portfolio: # Double check if there was no price for the past 7 days for the given ticker - prices = self.data_handler.get_price(ticker, PriceField.ohlc(), start_date).dropna(how="all", axis=1) + prices = self.data_provider.get_price(ticker, PriceField.ohlc(), start_date).dropna(how="all", axis=1) if prices.empty: position = self.portfolio.open_positions_dict[ticker] - closing_transaction = Transaction(self.timer.now(), ticker, -position.quantity(), + closing_transaction = Transaction(self.data_provider.timer.now(), ticker, -position.quantity(), position.current_price, 0) self.monitor.record_transaction(closing_transaction) self.portfolio.transact_transaction(closing_transaction) - self.logger.warning(f"{self.timer.now()}: position assigned to Ticker {position.ticker()} " + self.logger.warning(f"{self.data_provider.timer.now()}: position assigned to Ticker {position.ticker()} " f"removed due to incomplete price data.") diff --git a/qf_lib/backtesting/execution_handler/simulated_executor.py b/qf_lib/backtesting/execution_handler/simulated_executor.py index 7287985e..7c1e2907 100644 --- a/qf_lib/backtesting/execution_handler/simulated_executor.py +++ b/qf_lib/backtesting/execution_handler/simulated_executor.py @@ -16,7 +16,6 @@ from itertools import count from typing import List, Sequence, Optional, Dict, Tuple -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.execution_handler.commission_models.commission_model import CommissionModel from qf_lib.backtesting.execution_handler.slippage.base import Slippage from qf_lib.backtesting.monitoring.abstract_monitor import AbstractMonitor @@ -25,22 +24,20 @@ from qf_lib.backtesting.portfolio.transaction import Transaction from qf_lib.common.enums.frequency import Frequency from qf_lib.common.tickers.tickers import Ticker -from qf_lib.common.utils.dateutils.timer import Timer from qf_lib.common.utils.numberutils.is_finite_number import is_finite_number +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class SimulatedExecutor(metaclass=abc.ABCMeta): - def __init__(self, data_handler: DataHandler, monitor: AbstractMonitor, portfolio: Portfolio, timer: Timer, + def __init__(self, data_provider: AbstractPriceDataProvider, monitor: AbstractMonitor, portfolio: Portfolio, order_id_generator: count, commission_model: CommissionModel, slippage_model: Slippage, frequency: Frequency): - self._data_handler = data_handler - self._data_provider = data_handler.data_provider + self._data_provider = data_provider self._frequency = frequency self._monitor = monitor self._portfolio = portfolio - self._timer = timer self._order_id_generator = order_id_generator self._commission_model = commission_model self._slippage_model = slippage_model @@ -92,7 +89,7 @@ def execute_orders(self, market_open=False, market_close=False): self._get_orders_with_fill_prices_without_slippage(open_orders_list, tickers, market_open, market_close) if len(to_be_executed_orders) > 0: - current_time = self._timer.now() + current_time = self._data_provider.timer.now() fill_prices, fill_volumes = self._slippage_model.process_orders(current_time, to_be_executed_orders, no_slippage_fill_prices_list) @@ -113,7 +110,7 @@ def _execute_order(self, order: Order, fill_price: float, fill_volume: int): """ Simulates execution of a single Order by converting the Order into Transaction. """ - timestamp = self._timer.now() + timestamp = self._data_provider.timer.now() commission = self._commission_model.calculate_commission(fill_volume, fill_price) transaction = Transaction(timestamp, order.ticker, fill_volume, fill_price, commission) diff --git a/qf_lib/backtesting/execution_handler/slippage/base.py b/qf_lib/backtesting/execution_handler/slippage/base.py index 06794d4a..73d61c23 100644 --- a/qf_lib/backtesting/execution_handler/slippage/base.py +++ b/qf_lib/backtesting/execution_handler/slippage/base.py @@ -27,7 +27,7 @@ from qf_lib.common.tickers.tickers import Ticker from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta from qf_lib.common.utils.logging.qf_parent_logger import qf_logger -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class Slippage(metaclass=ABCMeta): @@ -38,7 +38,7 @@ class Slippage(metaclass=ABCMeta): Parameters ---------- - data_provider: DataProvider + data_provider: AbstractPriceDataProvider DataProvider component max_volume_share_limit: float, None number from range [0,1] which denotes how big (volume-wise) the Order can be i.e. if it's 0.5 and a daily @@ -46,7 +46,7 @@ class Slippage(metaclass=ABCMeta): volume checks are performed. """ - def __init__(self, data_provider: DataProvider, max_volume_share_limit: Optional[float] = None): + def __init__(self, data_provider: AbstractPriceDataProvider, max_volume_share_limit: Optional[float] = None): self.max_volume_share_limit = max_volume_share_limit self._data_provider = data_provider @@ -98,7 +98,8 @@ def _volumes_traded_today(self, date: datetime, tickers: Sequence[Ticker]) -> Se end_date = start_date + RelativeDelta(days=1) # Look into the future in order to see the total volume traded today - volume_df = self._data_provider.get_price(tickers, PriceField.Volume, start_date, end_date, Frequency.DAILY) + volume_df = self._data_provider.get_price(tickers, PriceField.Volume, start_date, end_date, Frequency.DAILY, + look_ahead_bias=True) volume_df = volume_df.fillna(0.0) try: volumes = volume_df.loc[start_date, tickers].values diff --git a/qf_lib/backtesting/execution_handler/stop_orders_executor.py b/qf_lib/backtesting/execution_handler/stop_orders_executor.py index dd5b6bd9..e743941d 100644 --- a/qf_lib/backtesting/execution_handler/stop_orders_executor.py +++ b/qf_lib/backtesting/execution_handler/stop_orders_executor.py @@ -31,7 +31,7 @@ def assign_order_ids(self, orders: Sequence[Order]) -> List[int]: tickers = [order.ticker for order in orders] unique_tickers_list = list(set(tickers)) - prices_at_acceptance_time = self._data_handler.get_last_available_price(unique_tickers_list) + prices_at_acceptance_time = self._data_provider.get_last_available_price(unique_tickers_list) order_id_list = [] for order, ticker in zip(orders, tickers): @@ -114,7 +114,7 @@ def _get_latest_available_bars(self, tickers: Sequence[Ticker]) -> QFDataFrame: assert self._frequency >= Frequency.DAILY, "Lower than daily frequency is not supported by the simulated " \ "executor" - current_datetime = self._timer.now() + current_datetime = self._data_provider.timer.now() market_close_time = current_datetime + MarketCloseEvent.trigger_time() == current_datetime if self._frequency == Frequency.DAILY: @@ -129,7 +129,7 @@ def _get_latest_available_bars(self, tickers: Sequence[Ticker]) -> QFDataFrame: # In case of intraday trading the current full bar is always indexed by the left side of the time range start_date = current_datetime - self._frequency.time_delta() - return self._data_handler.get_price(tickers, PriceField.ohlcv(), start_date, start_date, self._frequency) + return self._data_provider.get_price(tickers, PriceField.ohlcv(), start_date, start_date, self._frequency) def _calculate_no_slippage_fill_price(self, current_bar, order): """ diff --git a/qf_lib/backtesting/fast_alpha_model_tester/fast_alpha_models_tester.py b/qf_lib/backtesting/fast_alpha_model_tester/fast_alpha_models_tester.py index f11abbc4..873eeec5 100644 --- a/qf_lib/backtesting/fast_alpha_model_tester/fast_alpha_models_tester.py +++ b/qf_lib/backtesting/fast_alpha_model_tester/fast_alpha_models_tester.py @@ -133,7 +133,7 @@ def __init__(self, alpha_model_configs: Sequence[FastAlphaModelTesterConfig], self._start_date = start_date self._end_date = end_date self._data_provider = data_provider - self._timer = SettableTimer(start_date) if timer is None else timer + self._data_provider.set_timer(SettableTimer(start_date)) self._n_jobs = n_jobs self._frequency = frequency self._start_time = start_time @@ -176,7 +176,7 @@ def _get_valid_tickers(self, original_ticker: Sequence[Ticker]) -> List[Ticker]: for ticker in original_ticker: try: if isinstance(ticker, FutureTicker): - ticker.initialize_data_provider(self._timer, self._data_provider) + ticker.initialize_data_provider(self._data_provider) ticker = ticker.get_current_specific_ticker() tickers.append(ticker) except NoValidTickerException: @@ -189,7 +189,7 @@ def _get_data_for_backtest(self) -> QFDataArray: Creates a QFDataArray containing OHLCV values for all tickers passes to Fast Alpha Models Tester. """ self.logger.info("\nLoading all price values of tickers:") - self._timer.set_current_time(self._end_date) + self._data_provider.timer.set_current_time(self._end_date) tickers_dict = {} for ticker in self._tickers: @@ -428,14 +428,14 @@ def _generate_exposure_values(self, config: FastAlphaModelTesterConfig, data_pro if isinstance(ticker, FutureTicker): # Even if the tickers were already initialized, during pickling process, the data provider and timer # information is lost - ticker.initialize_data_provider(self._timer, data_provider) + ticker.initialize_data_provider(data_provider) for i, curr_datetime in enumerate(backtest_dates.index): if i % 1000 == 0: self.logger.info('{} / {} of Exposure dates processed'.format(i, len(backtest_dates))) new_exposures = QFSeries(index=tickers) - self._timer.set_current_time(curr_datetime) + self._data_provider.timer.set_current_time(curr_datetime) for j, ticker, curr_exp_value in zip(count(), tickers, current_exposures_values): curr_exp = Exposure(curr_exp_value) if is_finite_number(curr_exp_value) else Exposure.OUT diff --git a/qf_lib/backtesting/order/order_factory.py b/qf_lib/backtesting/order/order_factory.py index 3ff18737..6b441b47 100644 --- a/qf_lib/backtesting/order/order_factory.py +++ b/qf_lib/backtesting/order/order_factory.py @@ -27,7 +27,7 @@ from qf_lib.common.utils.miscellaneous.constants import ISCLOSE_REL_TOL, ISCLOSE_ABS_TOL from qf_lib.common.utils.miscellaneous.function_name import get_function_name from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class OrderFactory: @@ -37,11 +37,11 @@ class OrderFactory: ---------- broker: Broker broker used to access the portfolio - data_provider: DataProvider - data provider used to download prices. In case of backtesting, the DataHandler wrapper should be used. + data_provider: AbstractPriceDataProvider + data provider used to download prices. """ - def __init__(self, broker: Broker, data_provider: DataProvider): + def __init__(self, broker: Broker, data_provider: AbstractPriceDataProvider): self.broker = broker self.data_provider = data_provider self.logger = qf_logger.getChild(self.__class__.__name__) @@ -309,7 +309,7 @@ def _calculate_target_shares_and_tolerances( """ tickers = list(ticker_to_amount_of_money.keys()) # In case of live trading the get_last_available_price will use datetime.now() as the current time to obtain - # last price and in case of a backtest - it will use the data handlers timer to compute the date + # last price and in case of a backtest - it will use the data providers timer to compute the date current_prices = self.data_provider.get_last_available_price(tickers, frequency) # Ticker -> target number of shares diff --git a/qf_lib/backtesting/orders_filter/orders_filter.py b/qf_lib/backtesting/orders_filter/orders_filter.py index b1e63de1..ad558fe6 100644 --- a/qf_lib/backtesting/orders_filter/orders_filter.py +++ b/qf_lib/backtesting/orders_filter/orders_filter.py @@ -16,12 +16,12 @@ from qf_lib.backtesting.order.order import Order from qf_lib.common.utils.logging.qf_parent_logger import qf_logger -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class OrdersFilter(metaclass=abc.ABCMeta): """Adjusts final orders list to meet various requirements e.g. volume limitations.""" - def __init__(self, data_provider: DataProvider): + def __init__(self, data_provider: AbstractPriceDataProvider): self._data_provider = data_provider self.logger = qf_logger.getChild(self.__class__.__name__) diff --git a/qf_lib/backtesting/orders_filter/volume_orders_filter.py b/qf_lib/backtesting/orders_filter/volume_orders_filter.py index 161ab037..29d2f0d4 100644 --- a/qf_lib/backtesting/orders_filter/volume_orders_filter.py +++ b/qf_lib/backtesting/orders_filter/volume_orders_filter.py @@ -24,7 +24,7 @@ from qf_lib.common.tickers.tickers import Ticker from qf_lib.common.utils.numberutils.is_finite_number import is_finite_number from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class VolumeOrdersFilter(OrdersFilter): @@ -38,7 +38,7 @@ class VolumeOrdersFilter(OrdersFilter): defines the maximum percentage of the volume value, that the orders size should not exceed """ - def __init__(self, data_provider: DataProvider, volume_percentage_limit: float): + def __init__(self, data_provider: AbstractPriceDataProvider, volume_percentage_limit: float): super().__init__(data_provider) self._volume_percentage_limit = volume_percentage_limit diff --git a/qf_lib/backtesting/portfolio/portfolio.py b/qf_lib/backtesting/portfolio/portfolio.py index 40660172..4e4a3454 100644 --- a/qf_lib/backtesting/portfolio/portfolio.py +++ b/qf_lib/backtesting/portfolio/portfolio.py @@ -14,28 +14,26 @@ from datetime import datetime from typing import List, Dict -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.portfolio.backtest_position import BacktestPosition, BacktestPositionSummary from qf_lib.backtesting.portfolio.position_factory import BacktestPositionFactory from qf_lib.backtesting.portfolio.transaction import Transaction from qf_lib.backtesting.portfolio.utils import split_transaction_if_needed from qf_lib.common.tickers.tickers import Ticker -from qf_lib.common.utils.dateutils.timer import Timer from qf_lib.common.utils.logging.qf_parent_logger import qf_logger from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame from qf_lib.containers.series.prices_series import PricesSeries from qf_lib.containers.series.qf_series import QFSeries +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class Portfolio: - def __init__(self, data_handler: DataHandler, initial_cash: float, timer: Timer): + def __init__(self, data_provider: AbstractPriceDataProvider, initial_cash: float): """ On creation, the Portfolio object contains no positions and all values are "reset" to the initial cash, with no PnL. """ self.initial_cash = initial_cash - self.data_handler = data_handler - self.timer = timer + self.data_provider = data_provider self.net_liquidation = initial_cash """ Cash value includes futures P&L + stock value + securities options value + bond value + fund value. """ @@ -103,7 +101,7 @@ def update(self, record=False): self.gross_exposure_of_positions = 0 tickers = list(self.open_positions_dict.keys()) - current_prices_series = self.data_handler.get_last_available_price(tickers=tickers) + current_prices_series = self.data_provider.get_last_available_price(tickers=tickers) current_positions = {} for ticker, position in self.open_positions_dict.items(): @@ -117,7 +115,7 @@ def update(self, record=False): current_positions[ticker] = BacktestPositionSummary(position) if record: - self._dates.append(self.timer.now()) + self._dates.append(self.data_provider.timer.now()) self._portfolio_values.append(self.net_liquidation) self._leverage_list.append(self.gross_exposure_of_positions / self.net_liquidation) self._positions_history.append(current_positions) diff --git a/qf_lib/backtesting/strategies/abstract_strategy.py b/qf_lib/backtesting/strategies/abstract_strategy.py index 272884b3..619f3798 100644 --- a/qf_lib/backtesting/strategies/abstract_strategy.py +++ b/qf_lib/backtesting/strategies/abstract_strategy.py @@ -25,7 +25,7 @@ class AbstractStrategy(metaclass=abc.ABCMeta): """ Basic interface used to create a generic strategy. """ def __init__(self, ts: TradingSession): - self.timer = ts.timer + self.timer = ts.data_provider.timer self.notifiers = ts.notifiers @abc.abstractmethod diff --git a/qf_lib/backtesting/strategies/alpha_model_strategy.py b/qf_lib/backtesting/strategies/alpha_model_strategy.py index 46187b6a..63a70718 100644 --- a/qf_lib/backtesting/strategies/alpha_model_strategy.py +++ b/qf_lib/backtesting/strategies/alpha_model_strategy.py @@ -28,7 +28,6 @@ from qf_lib.backtesting.trading_session.trading_session import TradingSession from qf_lib.common.exceptions.future_contracts_exceptions import NoValidTickerException from qf_lib.common.tickers.tickers import Ticker -from qf_lib.common.utils.dateutils.timer import Timer from qf_lib.common.utils.logging.qf_parent_logger import qf_logger from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker from qf_lib.containers.futures.futures_rolling_orders_generator import FuturesRollingOrdersGenerator @@ -65,13 +64,14 @@ def __init__(self, ts: TradingSession, model_tickers_dict: Dict[AlphaModel, Sequ for ticker in tickers_for_model if isinstance(ticker, FutureTicker)] self._futures_rolling_orders_generator = self._get_futures_rolling_orders_generator(all_future_tickers, - ts.timer, ts.data_provider, + ts.data_provider, ts.broker, ts.order_factory) self._broker = ts.broker self._order_factory = ts.order_factory self._position_sizer = ts.position_sizer self._orders_filters = ts.orders_filters self._frequency = ts.frequency + self.timer = ts.data_provider.timer assert ts.frequency is not None, "Trading Session does not have the frequency parameter set. You need to set " \ "it before using the Alpha Model Strategy." @@ -84,13 +84,13 @@ def __init__(self, ts: TradingSession, model_tickers_dict: Dict[AlphaModel, Sequ self.logger = qf_logger.getChild(self.__class__.__name__) self._log_configuration() - def _get_futures_rolling_orders_generator(self, future_tickers: Sequence[FutureTicker], timer: Timer, + def _get_futures_rolling_orders_generator(self, future_tickers: Sequence[FutureTicker], data_provider: DataProvider, broker: Broker, order_factory: OrderFactory): # Initialize timer and data provider in case of FutureTickers for future_ticker in future_tickers: - future_ticker.initialize_data_provider(timer, data_provider) + future_ticker.initialize_data_provider(data_provider) - return FuturesRollingOrdersGenerator(future_tickers, timer, broker, order_factory) + return FuturesRollingOrdersGenerator(future_tickers, data_provider.timer, broker, order_factory) def calculate_and_place_orders(self): date = self.timer.now().date() diff --git a/qf_lib/backtesting/trading_session/backtest_trading_session.py b/qf_lib/backtesting/trading_session/backtest_trading_session.py index 55735263..7529294b 100644 --- a/qf_lib/backtesting/trading_session/backtest_trading_session.py +++ b/qf_lib/backtesting/trading_session/backtest_trading_session.py @@ -15,7 +15,6 @@ from qf_lib.backtesting.broker.backtest_broker import BacktestBroker from qf_lib.backtesting.contract.contract_to_ticker_conversion.base import ContractTickerMapper -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.events.event_manager import EventManager from qf_lib.backtesting.events.notifiers import Notifiers from qf_lib.backtesting.monitoring.backtest_monitor import BacktestMonitor @@ -29,10 +28,11 @@ from qf_lib.common.enums.price_field import PriceField from qf_lib.common.tickers.tickers import Ticker from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta -from qf_lib.common.utils.dateutils.timer import SettableTimer from qf_lib.common.utils.logging.qf_parent_logger import qf_logger from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list from qf_lib.containers.helpers import compute_container_hash +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider +from qf_lib.data_providers.prefetching_data_provider import PrefetchingDataProvider class BacktestTradingSession(TradingSession): @@ -41,13 +41,12 @@ class BacktestTradingSession(TradingSession): """ def __init__(self, contract_ticker_mapper: ContractTickerMapper, start_date, end_date, - position_sizer: PositionSizer, orders_filters: Sequence[OrdersFilter], data_handler: DataHandler, - timer: SettableTimer, notifiers: Notifiers, portfolio: Portfolio, events_manager: EventManager, - monitor: BacktestMonitor, broker: BacktestBroker, order_factory: OrderFactory, frequency: Frequency, - backtest_result: BacktestResult): + position_sizer: PositionSizer, orders_filters: Sequence[OrdersFilter], + data_provider: AbstractPriceDataProvider, notifiers: Notifiers, + portfolio: Portfolio, events_manager: EventManager, monitor: BacktestMonitor, broker: BacktestBroker, + order_factory: OrderFactory, frequency: Frequency, backtest_result: BacktestResult): """ Set up the backtest variables according to what has been passed in. - The data_provider parameter of the BacktestTradingSession points to a Data Handler object. """ super().__init__() self.logger = qf_logger.getChild(self.__class__.__name__) @@ -59,14 +58,12 @@ def __init__(self, contract_ticker_mapper: ContractTickerMapper, start_date, end self.notifiers = notifiers self.event_manager = events_manager - self.data_handler = data_handler - self.data_provider = data_handler # type: DataHandler + self.data_provider = data_provider self.portfolio = portfolio self.position_sizer = position_sizer self.orders_filters = orders_filters self.monitor = monitor - self.timer = timer self.order_factory = order_factory self.broker = broker self.frequency = frequency @@ -82,9 +79,12 @@ def use_data_preloading(self, tickers: Union[Ticker, Sequence[Ticker]], time_del # The tickers and price fields are sorted in order to always return the same hash of the data bundle for # the same set of tickers and fields tickers, _ = convert_to_list(tickers, Ticker) - self.data_handler.use_data_bundle(sorted(tickers), sorted(PriceField.ohlcv()), data_start, self.end_date, - self.frequency) - self._hash_of_data_bundle = compute_container_hash(self.data_handler.data_provider.data_bundle) + + self.data_provider = PrefetchingDataProvider(self.data_provider, sorted(tickers), sorted(PriceField.ohlcv()), + data_start, self.end_date, self.frequency, + timer=self.data_provider.timer) + + self._hash_of_data_bundle = compute_container_hash(self.data_provider.data_bundle) self.logger.info("Preloaded data hash value {}".format(self._hash_of_data_bundle)) def get_preloaded_data_checksum(self) -> str: diff --git a/qf_lib/backtesting/trading_session/backtest_trading_session_builder.py b/qf_lib/backtesting/trading_session/backtest_trading_session_builder.py index 25f7d504..cc0650d0 100644 --- a/qf_lib/backtesting/trading_session/backtest_trading_session_builder.py +++ b/qf_lib/backtesting/trading_session/backtest_trading_session_builder.py @@ -18,8 +18,6 @@ from qf_lib.backtesting.broker.backtest_broker import BacktestBroker from qf_lib.backtesting.contract.contract_to_ticker_conversion.simulated_contract_ticker_mapper import \ SimulatedContractTickerMapper -from qf_lib.backtesting.data_handler.daily_data_handler import DailyDataHandler -from qf_lib.backtesting.data_handler.intraday_data_handler import IntradayDataHandler from qf_lib.backtesting.events.event_manager import EventManager from qf_lib.backtesting.events.notifiers import Notifiers from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent @@ -307,7 +305,7 @@ def set_slippage_model(self, slippage_model_type: Type[Slippage], **kwargs): @ConfigExporter.update_config def set_position_sizer(self, position_sizer_type: Type[PositionSizer], **kwargs): """Sets the position sizer. The parameters to initialize the PositionSizer should be passed as keyword - arguments. Parameters corresponding to the broker, data handler, contract ticker mapper or signals register + arguments. Parameters corresponding to the broker, data provider, contract ticker mapper or signals register should not be provided, as all these parameters are setup by the backtest trading session builder. For example to set position sizer with initial risk = 0.3 and tolerance percentage = 0.1 the following command should be called on the session builder: @@ -336,7 +334,7 @@ def set_position_sizer(self, position_sizer_type: Type[PositionSizer], **kwargs) @ConfigExporter.append_config def add_orders_filter(self, orders_filter_type: Type[OrdersFilter], **kwargs): """Adds orders filter to the pipeline. Ths parameters to initialize the OrdersFilter should be passed as keyword - arguments. Parameters corresponding to data handler and contract ticker mapper should not be provided, as + arguments. Parameters corresponding to data provider and contract ticker mapper should not be provided, as they are setup by the backtest trading session builder. For example to set orders filter with volume_percentage_limit = 0.3 the following command should be called on the session builder: @@ -371,21 +369,6 @@ def _create_event_manager(timer, notifiers: Notifiers): ]) return event_manager - def _create_data_handler(self, data_provider, timer): - assert data_provider is not None, "Data provider is None. Set data_provider using set_data_provider() " \ - "method before building BacktestTradingSession" - if self._frequency == Frequency.MIN_1: - data_handler = IntradayDataHandler(data_provider, timer) - elif self._frequency == Frequency.DAILY: - data_handler = DailyDataHandler(data_provider, timer) - else: - raise ValueError("Invalid frequency parameter. The only frequencies supported by the DataHandler are " - "Frequency.DAILY and Frequency.MIN_1. " - "\nMake sure you set the frequency in the session builder for example: " - "\n\t-> 'session_builder.set_frequency(Frequency.DAILY)'") - - return data_handler - def build(self, start_date: datetime, end_date: datetime) -> BacktestTradingSession: """Builds a backtest trading session. @@ -402,13 +385,14 @@ def build(self, start_date: datetime, end_date: datetime) -> BacktestTradingSess trading session containing all the necessary parameters """ self._timer = SettableTimer(start_date) + self._data_provider.set_timer(self._timer) + self._data_provider.frequency = self._frequency self._notifiers = Notifiers(self._timer) self._events_manager = self._create_event_manager(self._timer, self._notifiers) - self._data_handler = self._create_data_handler(self._data_provider, self._timer) signals_register = self._signals_register if self._signals_register else BacktestSignalsRegister() - self._portfolio = Portfolio(self._data_handler, self._initial_cash, self._timer) + self._portfolio = Portfolio(self._data_provider, self._initial_cash) self._backtest_result = BacktestResult(self._portfolio, signals_register, self._backtest_name, start_date, end_date, self._initial_risk) @@ -417,7 +401,7 @@ def build(self, start_date: datetime, end_date: datetime) -> BacktestTradingSess self._slippage_model = self._slippage_model_setup() self._commission_model = self._commission_model_setup() self._execution_handler = SimulatedExecutionHandler( - self._data_handler, self._timer, self._notifiers.scheduler, self._monitor, self._commission_model, + self._data_provider, self._notifiers.scheduler, self._monitor, self._commission_model, self._portfolio, self._slippage_model, scheduling_time_delay=self._scheduling_time_delay, frequency=self._frequency) @@ -426,7 +410,7 @@ def build(self, start_date: datetime, end_date: datetime) -> BacktestTradingSess self._notifiers.empty_queue_event_notifier, end_date) self._broker = BacktestBroker(self._contract_ticker_mapper, self._portfolio, self._execution_handler) - self._order_factory = OrderFactory(self._broker, self._data_handler) + self._order_factory = OrderFactory(self._broker, self._data_provider) self._position_sizer = self._position_sizer_setup(signals_register) self._orders_filters = self._orders_filter_setup() @@ -446,8 +430,7 @@ def build(self, start_date: datetime, end_date: datetime) -> BacktestTradingSess "\n".join([ "Configuration of components:", "\tPosition sizer: {:s}".format(self._position_sizer.__class__.__name__), - "\tTimer: {:s}".format(self._timer.__class__.__name__), - "\tData Handler: {:s}".format(self._data_handler.__class__.__name__), + "\tData Provider: {:s}".format(self._data_provider.__class__.__name__), "\tBacktest Result: {:s}".format(self._backtest_result.__class__.__name__), "\tMonitor: {:s}".format(self._monitor.__class__.__name__), "\tExecution Handler: {:s}".format(self._execution_handler.__class__.__name__), @@ -472,8 +455,7 @@ def build(self, start_date: datetime, end_date: datetime) -> BacktestTradingSess end_date=end_date, position_sizer=self._position_sizer, orders_filters=self._orders_filters, - data_handler=self._data_handler, - timer=self._timer, + data_provider=self._data_provider, notifiers=self._notifiers, portfolio=self._portfolio, events_manager=self._events_manager, @@ -493,12 +475,12 @@ def _monitor_setup(self) -> BacktestMonitor: def _position_sizer_setup(self, signals_register: SignalsRegister): return self._position_sizer_type( - self._broker, self._data_handler, self._order_factory, signals_register, **self._position_sizer_kwargs) + self._broker, self._data_provider, self._order_factory, signals_register, **self._position_sizer_kwargs) def _orders_filter_setup(self): orders_filters = [] for orders_filter_type, kwargs in self._orders_filter_types_params: - orders_filter = orders_filter_type(self._data_handler, **kwargs) + orders_filter = orders_filter_type(self._data_provider, **kwargs) orders_filters.append(orders_filter) return orders_filters diff --git a/qf_lib/backtesting/trading_session/trading_session.py b/qf_lib/backtesting/trading_session/trading_session.py index 85098cad..cb7b9c4a 100644 --- a/qf_lib/backtesting/trading_session/trading_session.py +++ b/qf_lib/backtesting/trading_session/trading_session.py @@ -40,7 +40,6 @@ def __init__(self): self.settings = None # type: Settings self.data_provider = None # type: DataProvider - self.timer = None # type: Timer self.monitor = None # type: AbstractMonitor self.broker = None # type: Broker diff --git a/qf_lib/common/tickers/tickers.py b/qf_lib/common/tickers/tickers.py index b2fe86d6..e503ba36 100644 --- a/qf_lib/common/tickers/tickers.py +++ b/qf_lib/common/tickers/tickers.py @@ -278,6 +278,10 @@ def as_string(self) -> str: else: raise TypeError("Incorrect database type: {}".format(self.database_type)) + @property + def name(self) -> str: + return self.as_string() + def field_to_column_name(self, field: str): return self.as_string() + ' - ' + field diff --git a/qf_lib/containers/futures/future_tickers/future_ticker.py b/qf_lib/containers/futures/future_tickers/future_ticker.py index a69283aa..04fba6fc 100644 --- a/qf_lib/containers/futures/future_tickers/future_ticker.py +++ b/qf_lib/containers/futures/future_tickers/future_ticker.py @@ -21,7 +21,6 @@ from qf_lib.common.exceptions.future_contracts_exceptions import NoValidTickerException from qf_lib.common.tickers.tickers import Ticker from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta -from qf_lib.common.utils.dateutils.timer import Timer from qf_lib.containers.series.qf_series import QFSeries @@ -30,7 +29,7 @@ class FutureTicker(Ticker, metaclass=abc.ABCMeta): The FutureTicker class extends the standard Ticker class. It allows the user to use only a Ticker abstraction, which provides all of the standard Ticker functionalities (e.g. just as standard tickers, it can be used along with - DataHandler functions get_price, get_last_available_price, get_current_price etc.), without the need to manually + DataProvider functions get_price, get_last_available_price, get_current_price etc.), without the need to manually manage the rolling of the contracts or to select a certain specific Ticker. Notes @@ -66,6 +65,7 @@ class FutureTicker(Ticker, metaclass=abc.ABCMeta): March, June, September and December, even if contracts for any other months exist and are returned by the DataProvider get_futures_chain_tickers function. """ + def __init__(self, name: str, family_id: str, N: int, days_before_exp_date: int, point_value: int = 1, designated_contracts: Optional[str] = None, security_type: SecurityType = SecurityType.FUTURE): super().__init__(family_id, security_type, point_value) @@ -78,7 +78,6 @@ def __init__(self, name: str, family_id: str, N: int, days_before_exp_date: int, self._days_before_exp_date = days_before_exp_date self._exp_dates = None # type: QFSeries - self._timer = None # type: Timer self._data_provider = None # type: "DataProvider" self._ticker_initialized = False # type: bool @@ -87,14 +86,14 @@ def __init__(self, name: str, family_id: str, N: int, days_before_exp_date: int, self._last_cached_datetime = None self._expiration_hour = RelativeDelta(hour=0, minute=0, second=0, microsecond=0) - def initialize_data_provider(self, timer: Timer, data_provider: "DataProvider"): + def initialize_data_provider(self, data_provider: "FuturesDataProvider"): """ Initialize the future ticker with data provider and ticker. Parameters ---------- timer: Timer Timer which is used further when computing the current ticker. - data_provider: DataProvider + data_provider: FuturesDataProvider Data provider which is used to download symbols of tickers, belonging to the given future ticker family """ if self._ticker_initialized: @@ -102,7 +101,6 @@ def initialize_data_provider(self, timer: Timer, data_provider: "DataProvider"): f"Provider. The previous Timer and Data Provider references will be overwritten") self._data_provider = data_provider - self._timer = timer # Download and validate expiration dates for the future ticker exp_dates = self._get_futures_chain_tickers() @@ -164,7 +162,7 @@ def _get_current_specific_ticker() -> Ticker: _exp_dates = _exp_dates.sort_index() date_index = _exp_dates.index - pd.Timedelta(days=self._days_before_exp_date - 1) date_index = pd.DatetimeIndex([dt + self._expiration_hour for dt in date_index]) - date_index_loc = date_index.get_indexer([self._timer.now()], method="pad")[0] + date_index_loc = date_index.get_indexer([self._data_provider.timer.now()], method="pad")[0] return _exp_dates.iloc[date_index_loc:].iloc[self.N] def cached_ticker_still_valid(caching_time: datetime, current_time: datetime, @@ -186,9 +184,10 @@ def cached_ticker_still_valid(caching_time: datetime, current_time: datetime, return cached_time_start <= current_time < cached_time_end - if not cached_ticker_still_valid(self._last_cached_datetime, self._timer.now(), self._expiration_hour): + if not cached_ticker_still_valid(self._last_cached_datetime, self._data_provider.timer.now(), + self._expiration_hour): self._ticker = _get_current_specific_ticker() - self._last_cached_datetime = self._timer.now() + self._last_cached_datetime = self._data_provider.timer.now() return self._ticker except (LookupError, ValueError): @@ -198,12 +197,12 @@ def cached_ticker_still_valid(caching_time: datetime, current_time: datetime, # Therefore, in case if the data with expiration dates is not available for the current date, the # _get_current_specific_ticker function will raise a LookupError. raise NoValidTickerException(f"No valid ticker for the FutureTicker {self._name} found on " - f"{self._timer.now()}") from None + f"{self._data_provider.timer.now()}") from None def get_expiration_dates(self) -> QFSeries: """ Returns QFSeries containing the list of specific future contracts Tickers, indexed by their expiration - dates. The index contains original expiration dates, as returned by the data handler, without shifting it by the + dates. The index contains original expiration dates, as returned by the data provider, without shifting it by the days_before_exp_date days (it is important to store the original values, instead of shifted ones, as this function is public and used by multiple other components). @@ -316,7 +315,7 @@ def __lt__(self, other): other_class_name = other.__class__.__name__ return (class_name, self._name, self.family_id, self.N, self._days_before_exp_date) < \ - (other_class_name, other.name, other.family_id, other.get_N(), other.get_days_before_exp_date()) + (other_class_name, other.name, other.family_id, other.get_N(), other.get_days_before_exp_date()) def __getstate__(self): """ @@ -325,7 +324,7 @@ def __getstate__(self): """ self.logger = None self._data_provider = None - self._timer = None + self._data_provider.timer = None self._ticker_initialized = False return self.__dict__ diff --git a/qf_lib/containers/futures/futures_chain.py b/qf_lib/containers/futures/futures_chain.py index 10f81e79..383c7609 100644 --- a/qf_lib/containers/futures/futures_chain.py +++ b/qf_lib/containers/futures/futures_chain.py @@ -41,7 +41,6 @@ class FuturesChain(pd.Series): result of get_price function. data_provider: DataProvider Reference to the data provider, necessary to download latest prices, returned by the get_price function. - In case of backtests, the DataHandler wrapper should be used to avoid looking into the future. method: FuturesAdjustmentMethod FuturesAdjustmentMethod corresponding to one of two available methods of chaining the futures contracts. """ diff --git a/qf_lib/data_providers/__init__.py b/qf_lib/data_providers/__init__.py index f31f5e3d..9bcfdb09 100644 --- a/qf_lib/data_providers/__init__.py +++ b/qf_lib/data_providers/__init__.py @@ -11,3 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from qf_lib.data_providers.binance_dp.binance_data_provider import BinanceDataProvider +from qf_lib.data_providers.bloomberg import BloombergDataProvider +from qf_lib.data_providers.bloomberg_beap_hapi.bloomberg_beap_hapi_data_provider import BloombergBeapHapiDataProvider +from qf_lib.data_providers.csv.csv_data_provider import CSVDataProvider +from qf_lib.data_providers.haver import HaverDataProvider +from qf_lib.data_providers.quandl.quandl_data_provider import QuandlDataProvider diff --git a/qf_lib/data_providers/abstract_price_data_provider.py b/qf_lib/data_providers/abstract_price_data_provider.py index 84777503..791e3fcb 100644 --- a/qf_lib/data_providers/abstract_price_data_provider.py +++ b/qf_lib/data_providers/abstract_price_data_provider.py @@ -11,22 +11,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import math from abc import abstractmethod, ABCMeta from datetime import datetime -from typing import Union, Sequence, Dict +from typing import Union, Sequence, Dict, Optional + +from numpy import nan +from pandas import concat +from pandas._libs.tslibs.offsets import to_offset +from pandas._libs.tslibs.timestamps import Timestamp -from qf_lib.common.enums.expiration_date_field import ExpirationDateField +from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent from qf_lib.common.enums.frequency import Frequency from qf_lib.common.enums.price_field import PriceField from qf_lib.common.tickers.tickers import Ticker from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta +from qf_lib.common.utils.dateutils.timer import RealTimer, SettableTimer from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame -from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker from qf_lib.containers.qf_data_array import QFDataArray from qf_lib.containers.series.prices_series import PricesSeries +from qf_lib.containers.series.qf_series import QFSeries from qf_lib.data_providers.data_provider import DataProvider from qf_lib.data_providers.helpers import normalize_data_array @@ -49,11 +55,39 @@ class AbstractPriceDataProvider(DataProvider, metaclass=ABCMeta): """ def get_price(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[PriceField, Sequence[PriceField]], - start_date: datetime, end_date: datetime = None, frequency: Frequency = Frequency.DAILY, **kwargs) -> \ + start_date: datetime, end_date: datetime = None, frequency: Frequency = Frequency.DAILY, + look_ahead_bias: Optional[bool] = False, **kwargs) -> \ Union[None, PricesSeries, PricesDataFrame, QFDataArray]: + """ + Gets adjusted historical Prices (Open, High, Low, Close) and Volume + + Parameters + ---------- + tickers: Ticker, Sequence[Ticker] + tickers for securities which should be retrieved + fields: PriceField, Sequence[PriceField] + fields of securities which should be retrieved + start_date: datetime + date representing the beginning of historical period from which data should be retrieved + end_date: datetime + date representing the end of historical period from which data should be retrieved; + if no end_date was provided, by default the current date will be used + frequency: Frequency + frequency of the data + look_ahead_bias: False + if set to False, no future data will be ever returned - end_date = end_date or datetime.now() - end_date = end_date + RelativeDelta(second=0, microsecond=0) + Returns + ------- + None, PricesSeries, PricesDataFrame, QFDataArray + If possible the result will be squeezed so that instead of returning QFDataArray (3-D structure), + data of lower dimensionality will be returned. The results will be either an QFDataArray (with 3 dimensions: + dates, tickers, fields), PricesDataFrame (with 2 dimensions: dates, tickers or fields. + It is also possible to get 2 dimensions ticker and field if single date was provided), or PricesSeries + with 1 dimension: dates. All the containers will be indexed with PriceField whenever possible + (for example: instead of 'Close' column in the PricesDataFrame there will be PriceField.Close) + """ + end_date = (end_date or self.timer.now()) + RelativeDelta(second=0, microsecond=0) start_date = self._adjust_start_date(start_date, frequency) got_single_date = self._got_single_date(start_date, end_date, frequency) @@ -61,13 +95,14 @@ def get_price(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[Pric fields, got_single_field = convert_to_list(fields, PriceField) fields_str = self._map_field_to_str(fields) - container = self.get_history(tickers, fields_str, start_date, end_date, frequency, **kwargs) - + container = self.get_history(tickers, fields_str, start_date, end_date, frequency, + look_ahead_bias=look_ahead_bias, **kwargs) str_to_field_dict = self.str_to_price_field_map() # Map the specific fields onto the fields given by the str_to_field_dict if isinstance(container, QFDataArray): - container = container.assign_coords(fields=[str_to_field_dict[field_str] for field_str in container.fields.values]) + container = container.assign_coords( + fields=[str_to_field_dict[field_str] for field_str in container.fields.values]) normalized_result = normalize_data_array( container, tickers, fields, got_single_date, got_single_ticker, got_single_field, use_prices_types=True ) @@ -81,7 +116,7 @@ def get_price(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[Pric return normalized_result @abstractmethod - def price_field_to_str_map(self) -> Dict[PriceField, str]: + def price_field_to_str_map(self, *args) -> Dict[PriceField, str]: """ Method has to be implemented in each data provider in order to be able to use get_price. Returns dictionary containing mapping between PriceField and corresponding string that has to be used by @@ -94,72 +129,108 @@ def price_field_to_str_map(self) -> Dict[PriceField, str]: """ raise NotImplementedError() - @abstractmethod - def expiration_date_field_str_map(self, ticker: Ticker = None) -> Dict[ExpirationDateField, str]: + def historical_price(self, tickers: Union[Ticker, Sequence[Ticker]], + fields: Union[PriceField, Sequence[PriceField]], + nr_of_bars: int, end_date: Optional[datetime] = None, + frequency: Frequency = None, **kwargs) -> Union[PricesSeries, PricesDataFrame, QFDataArray]: """ - Method has to be implemented in each data provider in order to be able to use get_futures_chain_tickers. - Returns dictionary containing mapping between ExpirationDateField and corresponding string that has to be used - by get_futures_chain_tickers method. + Returns the latest available data samples, which simply correspond to the last available number + of bars. + + In case of intraday data and N minutes frequency, the most recent data may not represent exactly N minutes + (if the whole bar was not available at this time). The time ranges are always aligned to the market open time. + Non-zero seconds and microseconds are in the above case omitted (the output at 11:05:10 will be exactly + the same as at 11:05). Parameters - ----------- - ticker: None, Ticker - ticker is optional and might be uses by particular data providers to create appropriate dictionary + ---------- + tickers: Ticker, Sequence[Ticker] + ticker or sequence of tickers of the securities + fields: PriceField, Sequence[PriceField] + PriceField or sequence of PriceFields of the securities + nr_of_bars: int + number of data samples (bars) to be returned. + Note: while requesting more than one ticker, some tickers may have fewer than n_of_bars data points + end_date: Optional[datetime] + last date which should be considered in the query, the nr_of_bars that should be returned will always point + to the time before end_date. The parameter is optional and if not provided, the end_date will point to the + current user time. + frequency + frequency of the data Returns - ------- - Dict[ExpirationDateField, str] - mapping between ExpirationDateField and corresponding strings + -------- + PricesSeries, PricesDataFrame, QFDataArray """ - pass - - def get_futures_chain_tickers(self, tickers: Union[FutureTicker, Sequence[FutureTicker]], - expiration_date_fields: Union[ExpirationDateField, Sequence[ExpirationDateField]]) \ - -> Dict[FutureTicker, QFDataFrame]: - - expiration_date_fields, got_single_expiration_date_field = convert_to_list(expiration_date_fields, - ExpirationDateField) - mapping_dict = self.expiration_date_field_str_map() - expiration_date_fields_str = [mapping_dict[field] for field in expiration_date_fields] - exp_dates_dict = self._get_futures_chain_dict(tickers, expiration_date_fields_str) - - for future_ticker, exp_dates in exp_dates_dict.items(): - exp_dates = exp_dates.rename(columns=self.str_to_expiration_date_field_map()) - for ticker in exp_dates.index: - ticker.security_type = future_ticker.security_type - ticker.point_value = future_ticker.point_value - ticker.set_name(future_ticker.name) - if got_single_expiration_date_field: - exp_dates = exp_dates.squeeze() - exp_dates_dict[future_ticker] = exp_dates - - return exp_dates_dict + frequency = frequency or self.frequency or Frequency.DAILY + assert frequency >= Frequency.DAILY, "Frequency lower than daily is not supported by the Data Provider" + assert nr_of_bars > 0, "Numbers of data samples should be a positive integer" + end_date = self.get_end_date_without_look_ahead(end_date, frequency) - @abstractmethod - def _get_futures_chain_dict(self, tickers: Union[FutureTicker, Sequence[FutureTicker]], - expiration_date_fields: Union[str, Sequence[str]]) -> Dict[FutureTicker, QFDataFrame]: + # Add additional days to ensure that any data absence will not impact the number of bars which will be returned + start_date = self._compute_start_date(nr_of_bars, end_date, frequency) + start_date = self._adjust_start_date(start_date, frequency) + + container = self.get_price(tickers, fields, start_date, end_date, frequency) + missing_bars = nr_of_bars - container.shape[0] + + # In case if a bigger margin is necessary to get the historical price, shift the start date and download prices + # once again + if missing_bars > 0: + start_date = self._compute_start_date(missing_bars, start_date, frequency) + container = self.get_price(tickers, fields, start_date, end_date, frequency) + + num_of_dates_available = container.shape[0] + if num_of_dates_available < nr_of_bars: + if isinstance(tickers, Ticker): + tickers_as_strings = tickers.as_string() + else: + tickers_as_strings = ", ".join(ticker.as_string() for ticker in tickers) + raise ValueError(f"Not enough data points for \ntickers: {tickers_as_strings} \ndate: {end_date}." + f"\n{nr_of_bars} Data points requested, \n{num_of_dates_available} Data points available.") + + if isinstance(container, QFDataArray): + return container.isel(dates=slice(-nr_of_bars, None)) + else: + return container.tail(nr_of_bars) + + def get_last_available_price(self, tickers: Union[Ticker, Sequence[Ticker]], frequency: Optional[Frequency] = None, + end_time: Optional[datetime] = None) -> Union[float, QFSeries]: """ - Returns a dictionary, which maps Tickers to QFSeries, consisting of the expiration dates of Future - Contracts: Dict[FutureTicker, Union[QFSeries, QFDataFrame]]]. + Gets the latest available price for given assets as of end_time. Parameters - ---------- + ----------- tickers: Ticker, Sequence[Ticker] - tickers for securities which should be retrieved - expiration_date_fields: str, Sequence[str] - expiration date fields of securities which should be retrieved. Specific for each data provider, - the mapping between strings and corresponding ExpirationDateField enum values should be implemented as - str_to_expiration_date_field_map function. - """ - pass + tickers of the securities which prices should be downloaded + frequency: Frequency + frequency of the data + end_time: datetime + date which should be used as a base to compute the last available price. The parameter is optional and if + not provided, the end_date will point to the current user time. - def str_to_expiration_date_field_map(self, ticker: Ticker = None) -> Dict[str, ExpirationDateField]: - """ - Inverse of str_to_expiration_date_field_map. + Returns + ------- + float, pandas.Series + last_prices series where: + - last_prices.name contains a date of current prices, + - last_prices.index contains tickers + - last_prices.data contains latest available prices for given tickers """ - field_str_dict = self.expiration_date_field_str_map(ticker) - inv_dict = {v: k for k, v in field_str_dict.items()} - return inv_dict + frequency = frequency or self.frequency + if isinstance(self.timer, RealTimer): + return self._last_available_price(tickers, frequency, end_time) + + if frequency is None: + raise AttributeError(f"Data provider {self.__class__.__name__} has no set frequency. Please set it before " + f"running a simulation with SettableTimer.") + + if isinstance(self.timer, SettableTimer) and frequency == Frequency.DAILY: + return self._last_available_price_settable_timer_daily(tickers, frequency, end_time) + if isinstance(self.timer, SettableTimer) and frequency > Frequency.DAILY: + return self._last_available_price_settable_timer_intraday(tickers, frequency, end_time) + else: + raise NotImplementedError("TODO") def str_to_price_field_map(self) -> Dict[str, PriceField]: """ @@ -169,6 +240,60 @@ def str_to_price_field_map(self) -> Dict[str, PriceField]: inv_dict = {v: k for k, v in field_str_dict.items()} return inv_dict + @staticmethod + def _get_valid_latest_available_prices(start_date: datetime, tickers: Sequence[Ticker], open_prices: QFDataFrame, + close_prices: QFDataFrame) -> QFSeries: + latest_available_prices = [] + for ticker in tickers: + last_valid_open_price_date = open_prices.loc[:, ticker].last_valid_index() or start_date + last_valid_close_price_date = close_prices.loc[:, ticker].last_valid_index() or start_date + + try: + if last_valid_open_price_date > last_valid_close_price_date: + price = open_prices.loc[last_valid_open_price_date, ticker] + else: + price = close_prices.loc[last_valid_close_price_date, ticker] + except KeyError: + price = nan + + latest_available_prices.append(price) + + latest_available_prices_series = PricesSeries(data=latest_available_prices, index=tickers) + return latest_available_prices_series + + @staticmethod + def _compute_start_date(nr_of_bars_needed: int, end_date: datetime, frequency: Frequency): + margin = 10 if frequency <= Frequency.DAILY else 1 + nr_of_days_to_go_back = math.ceil(nr_of_bars_needed * 365 / frequency.value) + margin + + # In case if the end_date points to saturday, sunday or monday shift it to saturday midnight + if end_date.weekday() in (0, 5, 6): + end_date = end_date - RelativeDelta(weeks=1, weekday=5, hour=0, minute=0, microsecond=0) + + # Remove the time part and leave only days in order to align the start date to match the market open time + # in case of intraday data, e.g. if the market opens at 13:30, the bars will also start at 13:30 + start_date = end_date - RelativeDelta(days=nr_of_days_to_go_back, hour=0, minute=0, second=0, microsecond=0) + return start_date + + def _adjust_start_date(self, start_date: datetime, frequency: Frequency): + if frequency > Frequency.DAILY: + frequency_delta = to_offset(frequency.to_pandas_freq()).delta.value + new_start_date = Timestamp(math.ceil(Timestamp(start_date).value / frequency_delta) * frequency_delta) \ + .to_pydatetime() + if new_start_date != start_date: + self.logger.info(f"Adjusting the starting date to {new_start_date} from {start_date}.") + else: + new_start_date = start_date + RelativeDelta(hour=0, minute=0, second=0, microsecond=0) + if new_start_date.date() != start_date.date(): + self.logger.info(f"Adjusting the starting date to {new_start_date} from {start_date}.") + + return new_start_date + + @staticmethod + def _got_single_date(start_date: datetime, end_date: datetime, frequency: Frequency): + return start_date.date() == end_date.date() if frequency <= Frequency.DAILY else \ + (start_date + frequency.time_delta() > end_date) + def _map_field_to_str( self, fields: Union[None, PriceField, Sequence[PriceField]]) \ -> Union[None, str, Sequence[str]]: @@ -214,16 +339,77 @@ def _map_field_to_str( def _is_single_price_field(fields: Union[None, PriceField, Sequence[PriceField]]): return fields is not None and isinstance(fields, PriceField) - @staticmethod - def _is_single_ticker(value): - if isinstance(value, Ticker): - return True + def _last_available_price(self, tickers: Union[Ticker, Sequence[Ticker]], frequency: Frequency, + end_time: Optional[datetime] = None): - return False + end_time = self.get_end_date_without_look_ahead(end_time, frequency) + tickers, got_single_ticker = convert_to_list(tickers, Ticker) + if not tickers: + return nan if got_single_ticker else PricesSeries() + + frequency = frequency or Frequency.DAILY + assert frequency >= Frequency.DAILY, "Frequency lower then daily is not supported by the " \ + "get_last_available_price function" + + start_date = self._compute_start_date(5, end_time, frequency) + start_date = self._adjust_start_date(start_date, frequency) + + open_prices = self.get_price(tickers, PriceField.Open, start_date, end_time, frequency) + close_prices = self.get_price(tickers, PriceField.Close, start_date, end_time, frequency) + + latest_available_prices_series = self._get_valid_latest_available_prices(start_date, tickers, open_prices, + close_prices) + return latest_available_prices_series.iloc[0] if got_single_ticker else latest_available_prices_series + + def _last_available_price_settable_timer_daily(self, tickers: Union[Ticker, Sequence[Ticker]], + frequency: Frequency = None, + end_time: Optional[datetime] = None) -> Union[float, QFSeries]: + tickers, got_single_ticker = convert_to_list(tickers, Ticker) + if not tickers: + return nan if got_single_ticker else PricesSeries() - def _get_first_ticker(self, tickers): - if self._is_single_ticker(tickers): - ticker = tickers + frequency = frequency or Frequency.DAILY + end_time = end_time or self.timer.now() + end_date_without_look_ahead = self.get_end_date_without_look_ahead(end_time, frequency) + + last_prices = self._last_available_price(tickers, frequency, end_date_without_look_ahead) + + latest_market_open = self._get_last_available_market_event(end_time, MarketOpenEvent) + if end_date_without_look_ahead < latest_market_open: + current_open_prices = self.get_price(tickers, PriceField.Open, latest_market_open, + latest_market_open, frequency, look_ahead_bias=True) + last_prices = concat([last_prices, current_open_prices], axis=1).ffill(axis=1) + last_prices = last_prices.iloc[:, -1] + + return last_prices.iloc[0] if got_single_ticker else last_prices + + def _last_available_price_settable_timer_intraday(self, tickers: Union[Ticker, Sequence[Ticker]], + frequency: Frequency = None, + end_time: Optional[datetime] = None) -> Union[float, QFSeries]: + + tickers, got_single_ticker = convert_to_list(tickers, Ticker) + if not tickers: + return nan if got_single_ticker else PricesSeries() + frequency = frequency or Frequency.MIN_1 + current_time = self.timer.now() + RelativeDelta(second=0, microsecond=0) + end_time = end_time or current_time + end_date_without_look_ahead = self.get_end_date_without_look_ahead(end_time, frequency) + + if current_time <= end_time: + last_prices = self._last_available_price(tickers, frequency, end_date_without_look_ahead) + current_open_prices = self.get_price(tickers, PriceField.Open, + start_date=end_date_without_look_ahead + frequency.time_delta(), + end_date=end_date_without_look_ahead + frequency.time_delta(), + frequency=frequency, look_ahead_bias=True) else: - ticker = tickers[0] - return ticker + last_prices = self._last_available_price(tickers, frequency, + end_date_without_look_ahead - frequency.time_delta()) + current_open_prices = self.get_price(tickers, PriceField.Open, + start_date=end_date_without_look_ahead, + end_date=end_date_without_look_ahead, + frequency=frequency, look_ahead_bias=True) + + last_prices = concat([last_prices, current_open_prices], axis=1).ffill(axis=1) + last_prices = last_prices.iloc[:, -1] + + return last_prices.iloc[0] if got_single_ticker else last_prices diff --git a/qf_lib/data_providers/binance_dp/binance_data_provider.py b/qf_lib/data_providers/binance_dp/binance_data_provider.py index d4eaf3d2..c0b53822 100644 --- a/qf_lib/data_providers/binance_dp/binance_data_provider.py +++ b/qf_lib/data_providers/binance_dp/binance_data_provider.py @@ -11,13 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import warnings from datetime import datetime from typing import Union, Sequence +import os + import pytz import pandas as pd -from binance import Client from numpy import float64 from qf_lib.brokers.binance_broker.binance_contract_ticker_mapper import BinanceContractTickerMapper @@ -28,12 +29,20 @@ from qf_lib.common.tickers.tickers import Ticker from qf_lib.common.utils.dateutils.date_format import DateFormat from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta -import os from qf_lib.common.utils.logging.qf_parent_logger import qf_logger from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame from qf_lib.data_providers.csv.csv_data_provider import CSVDataProvider +try: + from binance import Client + + is_binance_installed = True +except ImportError: + is_binance_installed = False + warnings.warn( + "No binance installed. If you would like to use BinanceDataProvider first install the binance library.") + class BinanceDataProvider(CSVDataProvider): """ @@ -60,14 +69,16 @@ class BinanceDataProvider(CSVDataProvider): frequency of the data """ - def __init__(self, path: str, filename: str, tickers: Union[Ticker, Sequence[Ticker]], start_date: datetime, end_date: datetime, + def __init__(self, path: str, filename: str, tickers: Union[Ticker, Sequence[Ticker]], start_date: datetime, + end_date: datetime, contract_ticker_mapper: BinanceContractTickerMapper, frequency: Frequency = Frequency.MIN_1): + self.logger = qf_logger.getChild(self.__class__.__name__) + if frequency not in [Frequency.DAILY, Frequency.MIN_1]: raise NotImplementedError("Only 1m and DAILY freq is supported now") self.contract_ticker_mapper = contract_ticker_mapper - tickers, _ = convert_to_list(tickers, Ticker) self.frequency_mapping = { @@ -81,20 +92,21 @@ def __init__(self, path: str, filename: str, tickers: Union[Ticker, Sequence[Tic fields = ['Open', 'High', 'Low', 'Close', 'Volume'] ticker_col = 'Ticker' - self.logger = qf_logger.getChild(self.__class__.__name__) - - self.logger.info("creating BinanceDataProvider") - self.client = Client() - filepath = os.path.join(path, filename) - self._load_data(filepath, tickers, fields, start_date, end_date, frequency, index_col, ticker_col) + if is_binance_installed: + self.client = Client() - super().__init__(filepath, tickers, index_col, field_to_price_field_dict, fields, start_date, end_date, frequency, ticker_col=ticker_col) + self._load_data(filepath, tickers, fields, start_date, end_date, frequency, index_col, ticker_col) + super().__init__(filepath, tickers, index_col, field_to_price_field_dict, fields, start_date, end_date, + frequency, ticker_col=ticker_col) + else: + self.logger.warning("Couldn't import the Binance API. Check if the necessary dependencies are installed.") def _load_data(self, filepath, tickers, fields, start_date, end_date, frequency, index_col, ticker_col): if not os.path.isfile(filepath): - list_of_dfs = [self._download_binance_data_df(ticker, start_date, end_date, frequency, ticker_col) for ticker in tickers] + list_of_dfs = [self._download_binance_data_df(ticker, start_date, end_date, frequency, ticker_col) for + ticker in tickers] else: list_of_dfs = [] @@ -103,7 +115,8 @@ def _load_data(self, filepath, tickers, fields, start_date, end_date, frequency, infer_freq = Frequency.infer_freq(df.index) if infer_freq != frequency: - raise ValueError(f'Requested frequency: {frequency} is different from the one in the file: {infer_freq}') + raise ValueError( + f'Requested frequency: {frequency} is different from the one in the file: {infer_freq}') for ticker in tickers: @@ -120,14 +133,16 @@ def _load_data(self, filepath, tickers, fields, start_date, end_date, frequency, df_to_append = self._download_binance_data_df(ticker, current_end_date, end_date, frequency, ticker_col) combined_df = pd.concat([current_df, df_to_append]) - combined_df = combined_df[~combined_df.index.duplicated(keep='last')] # to have the most recent bar data updated + combined_df = combined_df[ + ~combined_df.index.duplicated(keep='last')] # to have the most recent bar data updated list_of_dfs.append(combined_df) df = pd.concat(list_of_dfs) df[fields] = df[fields].astype(float64) df.to_csv(filepath) - def _download_binance_data_df(self, ticker, start_time: datetime, end_time: datetime, frequency, ticker_col) -> QFDataFrame: + def _download_binance_data_df(self, ticker, start_time: datetime, end_time: datetime, frequency, + ticker_col) -> QFDataFrame: start_time = start_time + RelativeDelta(second=0, microsecond=0) end_time = end_time + RelativeDelta(second=0, microsecond=0) @@ -156,7 +171,8 @@ def _download_binance_data_df(self, ticker, start_time: datetime, end_time: date df = QFDataFrame(res_dict).set_index('Dates') df.index = pd.to_datetime(df.index, format=str(DateFormat.FULL_ISO)) - missing_dates = pd.date_range(start=start_time, end=end_time, freq=frequency.to_pandas_freq()).difference(df.index) + missing_dates = pd.date_range(start=start_time, end=end_time, freq=frequency.to_pandas_freq()).difference( + df.index) if not missing_dates.empty: self.logger.info(f'Missing dates: {missing_dates} for ticker: {ticker}') diff --git a/qf_lib/data_providers/bloomberg/bloomberg_data_provider.py b/qf_lib/data_providers/bloomberg/bloomberg_data_provider.py index 4ddcdcf3..c2ae40e8 100644 --- a/qf_lib/data_providers/bloomberg/bloomberg_data_provider.py +++ b/qf_lib/data_providers/bloomberg/bloomberg_data_provider.py @@ -23,6 +23,7 @@ from qf_lib.common.enums.security_type import SecurityType from qf_lib.common.tickers.tickers import BloombergTicker, Ticker from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta +from qf_lib.common.utils.dateutils.timer import Timer from qf_lib.common.utils.logging.qf_parent_logger import qf_logger from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame @@ -30,6 +31,7 @@ from qf_lib.containers.qf_data_array import QFDataArray from qf_lib.containers.series.qf_series import QFSeries from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider +from qf_lib.data_providers.futures_data_provider import FuturesDataProvider from qf_lib.data_providers.helpers import normalize_data_array, cast_dataframe_to_proper_type from qf_lib.data_providers.tickers_universe_provider import TickersUniverseProvider from qf_lib.settings import Settings @@ -37,7 +39,7 @@ try: import blpapi - from qf_lib.data_providers.bloomberg.futures_data_provider import FuturesDataProvider + from qf_lib.data_providers.bloomberg.futures_data_provider import BloombergFuturesDataProvider from qf_lib.data_providers.bloomberg.historical_data_provider import HistoricalDataProvider from qf_lib.data_providers.bloomberg.reference_data_provider import ReferenceDataProvider from qf_lib.data_providers.bloomberg.tabular_data_provider import TabularDataProvider @@ -52,13 +54,13 @@ " library") -class BloombergDataProvider(AbstractPriceDataProvider, TickersUniverseProvider): +class BloombergDataProvider(AbstractPriceDataProvider, TickersUniverseProvider, FuturesDataProvider): """ Data Provider which provides financial data from Bloomberg. """ - def __init__(self, settings: Settings): - super().__init__() + def __init__(self, settings: Settings, timer: Optional[Timer] = None): + super().__init__(timer) self.settings = settings self.host = settings.bloomberg.host @@ -75,7 +77,7 @@ def __init__(self, settings: Settings): self._historical_data_provider = HistoricalDataProvider(self.session) self._reference_data_provider = ReferenceDataProvider(self.session) self._tabular_data_provider = TabularDataProvider(self.session) - self._futures_data_provider = FuturesDataProvider(self.session) + self._futures_data_provider = BloombergFuturesDataProvider(self.session) else: self.session = None self._historical_data_provider = None @@ -210,8 +212,9 @@ def get_current_values(self, tickers: Union[BloombergTicker, Sequence[BloombergT return casted_result def get_history(self, tickers: Union[BloombergTicker, Sequence[BloombergTicker]], fields: Union[str, Sequence[str]], - start_date: datetime, end_date: datetime = None, frequency: Frequency = Frequency.DAILY, - currency: str = None, override_name: str = None, override_value: str = None) \ + start_date: datetime, end_date: datetime = None, frequency: Frequency = None, + currency: str = None, override_name: str = None, override_value: str = None, + look_ahead_bias: bool = False, **kwargs) \ -> Union[QFSeries, QFDataFrame, QFDataArray]: """ Gets historical data from Bloomberg from the (start_date - end_date) time range. In case of frequency, which is @@ -231,11 +234,11 @@ def get_history(self, tickers: Union[BloombergTicker, Sequence[BloombergTicker]] date representing the end of historical period from which data should be retrieved; if no end_date was provided, by default the current date will be used frequency: Frequency - frequency of the data + frequency of the data. It defaults to DAILY. currency: str override_name: str override_value: str - + look_ahead_bias: bool Returns ------- QFSeries, QFDataFrame, QFDataArray @@ -252,11 +255,13 @@ def get_history(self, tickers: Union[BloombergTicker, Sequence[BloombergTicker]] self._connect_if_needed() self._assert_is_connected() - end_date = end_date or datetime.now() - end_date = end_date + RelativeDelta(second=0, microsecond=0) + frequency = frequency or self.frequency or Frequency.DAILY + original_end_date = (end_date or self.timer.now()) + RelativeDelta(second=0, microsecond=0) + end_date = original_end_date if look_ahead_bias else self.get_end_date_without_look_ahead(original_end_date, frequency) start_date = self._adjust_start_date(start_date, frequency) - got_single_date = self._got_single_date(start_date, end_date, frequency) + got_single_date = self._got_single_date(start_date, original_end_date, frequency) + tickers, got_single_ticker = convert_to_list(tickers, BloombergTicker) fields, got_single_field = convert_to_list(fields, (PriceField, str)) diff --git a/qf_lib/data_providers/bloomberg/futures_data_provider.py b/qf_lib/data_providers/bloomberg/futures_data_provider.py index 9d8af7e0..86d37342 100644 --- a/qf_lib/data_providers/bloomberg/futures_data_provider.py +++ b/qf_lib/data_providers/bloomberg/futures_data_provider.py @@ -25,7 +25,7 @@ FIELD_EXCEPTIONS, SECURITY_ERROR -class FuturesDataProvider: +class BloombergFuturesDataProvider: def __init__(self, session): self._session = session diff --git a/qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_data_provider.py b/qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_data_provider.py index b762b57a..e58c1ab3 100644 --- a/qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_data_provider.py +++ b/qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_data_provider.py @@ -26,6 +26,7 @@ from qf_lib.common.enums.security_type import SecurityType from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker +from qf_lib.data_providers.futures_data_provider import FuturesDataProvider from qf_lib.data_providers.tickers_universe_provider import TickersUniverseProvider try: @@ -61,7 +62,7 @@ from qf_lib.starting_dir import get_starting_dir_abs_path -class BloombergBeapHapiDataProvider(AbstractPriceDataProvider, TickersUniverseProvider): +class BloombergBeapHapiDataProvider(AbstractPriceDataProvider, TickersUniverseProvider, FuturesDataProvider): """ Data Provider which provides financial data from Bloomberg BEAP HAPI. diff --git a/qf_lib/data_providers/csv/csv_data_provider.py b/qf_lib/data_providers/csv/csv_data_provider.py index 684063d5..e7f48585 100644 --- a/qf_lib/data_providers/csv/csv_data_provider.py +++ b/qf_lib/data_providers/csv/csv_data_provider.py @@ -99,6 +99,7 @@ class CSVDataProvider(PresetDataProvider): data_provider = CSVDataProvider(path, tickers, index_column, field_to_price_field_dict, start_date, end_date, Frequency.MIN_1) """ + def __init__(self, path: str, tickers: Union[Ticker, Sequence[Ticker]], index_col: str, field_to_price_field_dict: Optional[Dict[str, PriceField]] = None, fields: Optional[Union[str, List[str]]] = None, start_date: Optional[datetime] = None, diff --git a/qf_lib/data_providers/data_provider.py b/qf_lib/data_providers/data_provider.py index bd70cc16..98e984b2 100644 --- a/qf_lib/data_providers/data_provider.py +++ b/qf_lib/data_providers/data_provider.py @@ -14,73 +14,43 @@ import math from abc import ABCMeta, abstractmethod from datetime import datetime -from typing import Union, Sequence, Type, Set, Dict, Optional +from typing import Union, Sequence, Type, Set, Optional -from numpy import nan -from pandas._libs.tslibs.offsets import to_offset -from pandas._libs.tslibs.timestamps import Timestamp +from pandas import Timestamp +from pandas._libs.tslibs import to_offset -from qf_lib.common.enums.expiration_date_field import ExpirationDateField +from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent +from qf_lib.backtesting.events.time_event.regular_time_event.regular_market_event import RegularMarketEvent from qf_lib.common.enums.frequency import Frequency -from qf_lib.common.enums.price_field import PriceField from qf_lib.common.tickers.tickers import Ticker from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta +from qf_lib.common.utils.dateutils.timer import Timer, RealTimer from qf_lib.common.utils.logging.qf_parent_logger import qf_logger -from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list -from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame -from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker from qf_lib.containers.qf_data_array import QFDataArray -from qf_lib.containers.series.prices_series import PricesSeries from qf_lib.containers.series.qf_series import QFSeries -class DataProvider(object, metaclass=ABCMeta): +class DataProvider(metaclass=ABCMeta): """ An interface for data providers (for example AbstractPriceDataProvider or GeneralPriceProvider). """ frequency = None - def __init__(self): + def __init__(self, timer: Optional[Timer] = None): self.logger = qf_logger.getChild(self.__class__.__name__) + self.timer = timer or RealTimer() - @abstractmethod - def get_price(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[PriceField, Sequence[PriceField]], - start_date: datetime, end_date: datetime = None, frequency: Frequency = None, **kwargs) -> Union[ - None, PricesSeries, PricesDataFrame, QFDataArray]: - """ - Gets adjusted historical Prices (Open, High, Low, Close) and Volume - - Parameters - ---------- - tickers: Ticker, Sequence[Ticker] - tickers for securities which should be retrieved - fields: PriceField, Sequence[PriceField] - fields of securities which should be retrieved - start_date: datetime - date representing the beginning of historical period from which data should be retrieved - end_date: datetime - date representing the end of historical period from which data should be retrieved; - if no end_date was provided, by default the current date will be used - frequency: Frequency - frequency of the data + def set_timer(self, timer: Timer): + self.timer = timer - Returns - ------- - None, PricesSeries, PricesDataFrame, QFDataArray - If possible the result will be squeezed so that instead of returning QFDataArray (3-D structure), - data of lower dimensionality will be returned. The results will be either an QFDataArray (with 3 dimensions: - dates, tickers, fields), PricesDataFrame (with 2 dimensions: dates, tickers or fields. - It is also possible to get 2 dimensions ticker and field if single date was provided), or PricesSeries - with 1 dimension: dates. All the containers will be indexed with PriceField whenever possible - (for example: instead of 'Close' column in the PricesDataFrame there will be PriceField.Close) - """ - pass + def set_frequency(self, frequency): + self.frequency = frequency @abstractmethod def get_history( self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[None, str, Sequence[str]], - start_date: datetime, end_date: datetime = None, frequency: Frequency = None, **kwargs) -> Union[ - QFSeries, QFDataFrame, QFDataArray]: + start_date: datetime, end_date: datetime = None, frequency: Frequency = None, look_ahead_bias: bool = False, + **kwargs) -> Union[QFSeries, QFDataFrame, QFDataArray]: """ Gets historical attributes (fields) of different securities (tickers). @@ -106,6 +76,8 @@ def get_history( if no end_date was provided, by default the current date will be used frequency: Frequency frequency of the data + look_ahead_bias: bool + if set to False, the look-ahead bias will be taken care of to make sure no future data is returned kwargs kwargs should not be used on the level of AbstractDataProvider. They are here to provide a common interface for all data providers since some of the specific data providers accept additional arguments @@ -118,7 +90,7 @@ def get_history( field), a QFDataFrame (with 2 dimensions: date, ticker or field; it is also possible to get 2 dimensions ticker and field if single date was provided) or QFSeries (with 1 dimensions: date). If no data is available in the database or an non existing ticker was provided an empty structure - (QFSeries, QFDataFrame or QFDataArray) will be returned returned. + (QFSeries, QFDataFrame or QFDataArray) will be returned. """ pass @@ -129,189 +101,72 @@ def supported_ticker_types(self) -> Set[Type[Ticker]]: """ pass - @abstractmethod - def get_futures_chain_tickers(self, tickers: Union[FutureTicker, Sequence[FutureTicker]], - expiration_date_fields: Union[ExpirationDateField, Sequence[ExpirationDateField]]) \ - -> Dict[FutureTicker, Union[QFSeries, QFDataFrame]]: - """ - Returns tickers of futures contracts, which belong to the same futures contract chain as the provided ticker - (tickers), along with their expiration dates in form of a QFSeries or QFDataFrame. + def get_end_date_without_look_ahead(self, end_date: Optional[datetime], frequency: Frequency): + end_date = end_date or self.timer.now() + end_date = end_date + RelativeDelta(second=0, microsecond=0) + if isinstance(self.timer, RealTimer): + return end_date - Parameters - ---------- - tickers: FutureTicker, Sequence[FutureTicker] - tickers for which should the future chain tickers be retrieved - expiration_date_fields: ExpirationDateField, Sequence[ExpirationDateField] - field that should be downloaded as the expiration date field, by default last tradeable date + frequency = frequency or self.frequency + if frequency == Frequency.DAILY: + return self._get_last_available_market_event(end_date, MarketCloseEvent) + else: + return self._get_end_date_without_look_ahead_intraday(end_date, frequency) - Returns - ------- - Dict[FutureTicker, Union[QFSeries, QFDataFrame]] - Returns a dictionary, which maps Tickers to QFSeries, consisting of the expiration dates of Future - Contracts: Dict[FutureTicker, Union[QFSeries, QFDataFrame]]]. The QFSeries' / QFDataFrames contain the - specific Tickers, which belong to the corresponding futures family, same as the FutureTicker, and are - indexed by the expiration dates of the specific future contracts. - """ - pass + def _get_last_available_market_event(self, end_date: Optional[datetime], event: Type[RegularMarketEvent]): + current_datetime = self.timer.now() + RelativeDelta(second=0, microsecond=0) + end_date = end_date or current_datetime + end_date += RelativeDelta(days=1, hour=0, minute=0, second=0, microsecond=0, microseconds=-1) - def historical_price(self, tickers: Union[Ticker, Sequence[Ticker]], - fields: Union[PriceField, Sequence[PriceField]], - nr_of_bars: int, end_date: Optional[datetime] = None, - frequency: Frequency = None) -> Union[PricesSeries, PricesDataFrame, QFDataArray]: - """ - Returns the latest available data samples, which simply correspond to the last available number - of bars. + today_market_event = current_datetime + event.trigger_time() + yesterday_market_event = today_market_event - RelativeDelta(days=1) + latest_available_market_event = yesterday_market_event if current_datetime < today_market_event \ + else today_market_event - In case of intraday data and N minutes frequency, the most recent data may not represent exactly N minutes - (if the whole bar was not available at this time). The time ranges are always aligned to the market open time. - Non-zero seconds and microseconds are in the above case omitted (the output at 11:05:10 will be exactly - the same as at 11:05). + latest_market_event = min(latest_available_market_event, end_date) + return datetime(latest_market_event.year, latest_market_event.month, latest_market_event.day) - Parameters - ---------- - tickers: Ticker, Sequence[Ticker] - ticker or sequence of tickers of the securities - fields: PriceField, Sequence[PriceField] - PriceField or sequence of PriceFields of the securities - nr_of_bars: int - number of data samples (bars) to be returned. - Note: while requesting more than one ticker, some tickers may have fewer than n_of_bars data points - end_date: Optional[datetime] - last date which should be considered in the query, the nr_of_bars that should be returned will always point - to the time before end_date. The parameter is optional and if not provided, the end_date will point to the - current user time. - frequency - frequency of the data - - Returns - -------- - PricesSeries, PricesDataFrame, QFDataArray + def _get_end_date_without_look_ahead_intraday(self, end_date: Optional[datetime], frequency: Frequency): """ - # frequency = frequency or self.default_frequency - assert frequency >= Frequency.DAILY, "Frequency lower than daily is not supported by the Data Provider" - assert nr_of_bars > 0, "Numbers of data samples should be a positive integer" - end_date = datetime.now() if end_date is None else end_date - - # Add additional days to ensure that any data absence will not impact the number of bars which will be returned - start_date = self._compute_start_date(nr_of_bars, end_date, frequency) - start_date = self._adjust_start_date(start_date, frequency) + If end_date is None, current time is taken as end_date. The function returns the end of latest full bar + (get_price, get_history etc. functions always include the end_date e.g. in case of 1 minute frequency: + current_time = 16:20 and end_date = 16:06 the latest returned bar is the [16:06, 16:07)). - container = self.get_price(tickers, fields, start_date, end_date, frequency) - missing_bars = nr_of_bars - container.shape[0] + Examples: + - current_time = 20:00, end_time = 17:01, frequency = 1h, + => end_date_without_look_ahead = 17:00 - # In case if a bigger margin is necessary to get the historical price, shift the start date and download prices - # once again - if missing_bars > 0: - start_date = self._compute_start_date(missing_bars, start_date, frequency) - container = self.get_price(tickers, fields, start_date, end_date, frequency) + - current_time = 20:00, end_time = 19:58, frequency = 1h, + => end_date_without_look_ahead = 19:00 - num_of_dates_available = container.shape[0] - if num_of_dates_available < nr_of_bars: - if isinstance(tickers, Ticker): - tickers_as_strings = tickers.as_string() - else: - tickers_as_strings = ", ".join(ticker.as_string() for ticker in tickers) - raise ValueError(f"Not enough data points for \ntickers: {tickers_as_strings} \ndate: {end_date}." - f"\n{nr_of_bars} Data points requested, \n{num_of_dates_available} Data points available.") + - current_time = 20:00, end_time = 20:01, frequency = 1h , + => end_date_without_look_ahead = 19:00 - if isinstance(container, QFDataArray): - return container.isel(dates=slice(-nr_of_bars, None)) - else: - return container.tail(nr_of_bars) + - current_time = 20:00, end_time = 20:00, frequency = 1h, + => end_date_without_look_ahead = 19 - def get_last_available_price(self, tickers: Union[Ticker, Sequence[Ticker]], frequency: Frequency, - end_time: Optional[datetime] = None) -> Union[float, QFSeries]: - """ - Gets the latest available price for given assets as of end_time. + - current_time = 20:10, end_time = 22:10, frequency = 1h, + => end_date_without_look_ahead = 19 - Parameters - ----------- - tickers: Ticker, Sequence[Ticker] - tickers of the securities which prices should be downloaded - frequency: Frequency - frequency of the data - end_time: datetime - date which should be used as a base to compute the last available price. The parameter is optional and if - not provided, the end_date will point to the current user time. + - current_time = 19:58, end_time = 19:56 , frequency = 1m, + => end_date_without_look_ahead = 19:56 - Returns - ------- - float, pandas.Series - last_prices series where: - - last_prices.name contains a date of current prices, - - last_prices.index contains tickers - - last_prices.data contains latest available prices for given tickers + - current_time = 19:56, end_time = 19:58 , frequency = 1m, + => end_date_without_look_ahead = 19:55 """ - end_time = datetime.now() if end_time is None else end_time - tickers, got_single_ticker = convert_to_list(tickers, Ticker) - if not tickers: - return nan if got_single_ticker else PricesSeries() - - assert frequency >= Frequency.DAILY, "Frequency lower then daily is not supported by the " \ - "get_last_available_price function" - - start_date = self._compute_start_date(5, end_time, frequency) - start_date = self._adjust_start_date(start_date, frequency) - - open_prices = self.get_price(tickers, PriceField.Open, start_date, end_time, frequency) - close_prices = self.get_price(tickers, PriceField.Close, start_date, end_time, frequency) - - latest_available_prices_series = self._get_valid_latest_available_prices(start_date, tickers, open_prices, close_prices) - return latest_available_prices_series.iloc[0] if got_single_ticker else latest_available_prices_series - - @staticmethod - def _get_valid_latest_available_prices(start_date: datetime, tickers: Sequence[Ticker], open_prices: QFDataFrame, - close_prices: QFDataFrame) -> QFSeries: - latest_available_prices = [] - for ticker in tickers: - last_valid_open_price_date = open_prices.loc[:, ticker].last_valid_index() or start_date - last_valid_close_price_date = close_prices.loc[:, ticker].last_valid_index() or start_date - - try: - if last_valid_open_price_date > last_valid_close_price_date: - price = open_prices.loc[last_valid_open_price_date, ticker] - else: - price = close_prices.loc[last_valid_close_price_date, ticker] - except KeyError: - price = nan - latest_available_prices.append(price) + current_time = self.timer.now() + RelativeDelta(second=0, microsecond=0) + end_date = end_date or current_time + end_date += RelativeDelta(second=0, microsecond=0) - latest_available_prices_series = PricesSeries(data=latest_available_prices, index=tickers) - return latest_available_prices_series - - @staticmethod - def _compute_start_date(nr_of_bars_needed: int, end_date: datetime, frequency: Frequency): - margin = 10 if frequency <= Frequency.DAILY else 1 - nr_of_days_to_go_back = math.ceil(nr_of_bars_needed * 365 / frequency.value) + margin - - # In case if the end_date points to saturday, sunday or monday shift it to saturday midnight - if end_date.weekday() in (0, 5, 6): - end_date = end_date - RelativeDelta(weeks=1, weekday=5, hour=0, minute=0, microsecond=0) - - # Remove the time part and leave only days in order to align the start date to match the market open time - # in case of intraday data, e.g. if the market opens at 13:30, the bars will also start at 13:30 - start_date = end_date - RelativeDelta(days=nr_of_days_to_go_back, hour=0, minute=0, second=0, microsecond=0) - return start_date - - def _adjust_start_date(self, start_date: datetime, frequency: Frequency): - if frequency > Frequency.DAILY: - frequency_delta = to_offset(frequency.to_pandas_freq()).delta.value - new_start_date = Timestamp(math.ceil(Timestamp(start_date).value / frequency_delta) * frequency_delta) \ - .to_pydatetime() - if new_start_date != start_date: - self.logger.info(f"Adjusting the starting date to {new_start_date} from {start_date}.") + frequency_delta = to_offset(self.frequency.to_pandas_freq()).delta.value + if current_time <= end_date: + end_date_without_lookahead = Timestamp(math.floor(Timestamp(current_time).value / frequency_delta) * + frequency_delta).to_pydatetime() - self.frequency.time_delta() else: - new_start_date = start_date + RelativeDelta(hour=0, minute=0, second=0, microsecond=0) - if new_start_date.date() != start_date.date(): - self.logger.info(f"Adjusting the starting date to {new_start_date} from {start_date}.") - - return new_start_date - - @staticmethod - def _got_single_date(start_date: datetime, end_date: datetime, frequency: Frequency): - return start_date.date() == end_date.date() if frequency <= Frequency.DAILY else \ - (start_date + frequency.time_delta() > end_date) + end_date_without_lookahead = Timestamp(math.floor(Timestamp(end_date).value / frequency_delta) * + frequency_delta).to_pydatetime() + return end_date_without_lookahead def __str__(self): return self.__class__.__name__ diff --git a/qf_lib/data_providers/futures_data_provider.py b/qf_lib/data_providers/futures_data_provider.py new file mode 100644 index 00000000..96f82e3e --- /dev/null +++ b/qf_lib/data_providers/futures_data_provider.py @@ -0,0 +1,111 @@ +# Copyright 2016-present CERN – European Organization for Nuclear Research +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABCMeta, abstractmethod +from typing import Union, Sequence, Dict + +from qf_lib.common.enums.expiration_date_field import ExpirationDateField +from qf_lib.common.tickers.tickers import Ticker +from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list +from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame +from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker + + +class FuturesDataProvider(metaclass=ABCMeta): + """ + An interface for providers of futures' data. + """ + + @abstractmethod + def expiration_date_field_str_map(self, ticker: Ticker = None) -> Dict[ExpirationDateField, str]: + """ + Method has to be implemented in each data provider in order to be able to use get_futures_chain_tickers. + Returns dictionary containing mapping between ExpirationDateField and corresponding string that has to be used + by get_futures_chain_tickers method. + + Parameters + ----------- + ticker: None, Ticker + ticker is optional and might be uses by particular data providers to create appropriate dictionary + + Returns + ------- + Dict[ExpirationDateField, str] + mapping between ExpirationDateField and corresponding strings + """ + pass + + def get_futures_chain_tickers(self, tickers: Union[FutureTicker, Sequence[FutureTicker]], + expiration_date_fields: Union[ExpirationDateField, Sequence[ExpirationDateField]]) \ + -> Dict[FutureTicker, QFDataFrame]: + """ + Returns tickers of futures contracts, which belong to the same futures contract chain as the provided ticker + (tickers), along with their expiration dates in form of a QFSeries or QFDataFrame. + + Parameters + ---------- + tickers: FutureTicker, Sequence[FutureTicker] + tickers for which should the future chain tickers be retrieved + expiration_date_fields: ExpirationDateField, Sequence[ExpirationDateField] + field that should be downloaded as the expiration date field, by default last tradeable date + + Returns + ------- + Dict[FutureTicker, QFDataFrame] + Returns a dictionary, which maps Tickers to QFDataFrame, consisting of the expiration dates of Future + Contracts: Dict[FutureTicker, QFDataFrame]]. The QFDataFrames contain the + specific Tickers, which belong to the corresponding futures family, same as the FutureTicker, and are + indexed by the expiration dates of the specific future contracts. + """ + expiration_date_fields, got_single_expiration_date_field = convert_to_list(expiration_date_fields, + ExpirationDateField) + mapping_dict = self.expiration_date_field_str_map() + expiration_date_fields_str = [mapping_dict[field] for field in expiration_date_fields] + exp_dates_dict = self._get_futures_chain_dict(tickers, expiration_date_fields_str) + + for future_ticker, exp_dates in exp_dates_dict.items(): + exp_dates = exp_dates.rename(columns=self.str_to_expiration_date_field_map()) + for ticker in exp_dates.index: + ticker.security_type = future_ticker.security_type + ticker.point_value = future_ticker.point_value + ticker.set_name(future_ticker.name) + exp_dates_dict[future_ticker] = exp_dates + + return exp_dates_dict + + def str_to_expiration_date_field_map(self, ticker: Ticker = None) -> Dict[str, ExpirationDateField]: + """ + Inverse of str_to_expiration_date_field_map. + """ + field_str_dict = self.expiration_date_field_str_map(ticker) + return {v: k for k, v in field_str_dict.items()} + + @abstractmethod + def _get_futures_chain_dict(self, tickers: Union[FutureTicker, Sequence[FutureTicker]], + expiration_date_fields: Union[str, Sequence[str]]) -> Dict[FutureTicker, QFDataFrame]: + """ + Returns a dictionary, which maps Tickers to QFDataFrame, consisting of the expiration date(s) of Future + Contracts: Dict[FutureTicker, QFDataFrame]]. The frame is indexed by the specific + tickers belonging to the futures family. + + Parameters + ---------- + tickers: Ticker, Sequence[Ticker] + tickers for securities which should be retrieved + expiration_date_fields: str, Sequence[str] + expiration date fields of securities which should be retrieved. Specific for each data provider, + the mapping between strings and corresponding ExpirationDateField enum values should be implemented as + str_to_expiration_date_field_map function. + """ + pass diff --git a/qf_lib/data_providers/general_price_provider.py b/qf_lib/data_providers/general_price_provider.py index 00e594ce..a3327895 100644 --- a/qf_lib/data_providers/general_price_provider.py +++ b/qf_lib/data_providers/general_price_provider.py @@ -15,6 +15,7 @@ from datetime import datetime from itertools import groupby from typing import Sequence, Union, Dict, Type +from warnings import warn import pandas as pd @@ -30,22 +31,27 @@ from qf_lib.containers.qf_data_array import QFDataArray from qf_lib.containers.series.prices_series import PricesSeries from qf_lib.containers.series.qf_series import QFSeries +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider +from qf_lib.data_providers.futures_data_provider import FuturesDataProvider from qf_lib.data_providers.haver import HaverDataProvider from qf_lib.data_providers.helpers import normalize_data_array -from qf_lib.data_providers.data_provider import DataProvider from qf_lib.data_providers.quandl.quandl_data_provider import QuandlDataProvider from qf_lib.data_providers.bloomberg.bloomberg_data_provider import BloombergDataProvider -class GeneralPriceProvider(DataProvider): +class GeneralPriceProvider(AbstractPriceDataProvider, FuturesDataProvider): """ The main class that should be used in order to access prices of financial instruments. + Deprecated. """ def __init__(self, bloomberg: BloombergDataProvider = None, quandl: QuandlDataProvider = None, haver: HaverDataProvider = None): super().__init__() - self._ticker_type_to_data_provider_dict = {} # type: Dict[Type[Ticker], DataProvider] + warn('GeneralPriceProvider is deprecated and will be removed in a future version of qf_lib', + DeprecationWarning, stacklevel=2) + + self._ticker_type_to_data_provider_dict = {} # type: Dict[Type[Ticker], AbstractPriceDataProvider] for provider in [bloomberg, quandl, haver]: if provider is not None: @@ -86,7 +92,7 @@ def get_history( self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[str, Sequence[str]], start_date: datetime, end_date: datetime = None, frequency: Frequency = Frequency.DAILY, **kwargs) -> Union[QFSeries, QFDataFrame, QFDataArray]: """ - Implements the functionality of DataProvider using duck-typing. + Implements the functionality of AbstractPriceDataProvider using duck-typing. Parameters ---------- @@ -122,7 +128,7 @@ def get_futures_chain_tickers(self, tickers: Union[FutureTicker, Sequence[Future expiration_date_fields: Union[ExpirationDateField, Sequence[ExpirationDateField]]) \ -> Dict[FutureTicker, Union[QFSeries, QFDataFrame]]: """ - Implements the functionality of DataProvider using duck-typing. + Implements the functionality of AbstractPriceDataProvider using duck-typing. Returns tickers of futures contracts, which belong to the same futures contract chain as the provided ticker (tickers), along with their expiration dates in form of a QFSeries. @@ -143,7 +149,7 @@ def get_futures_chain_tickers(self, tickers: Union[FutureTicker, Sequence[Future tickers, got_single_ticker = convert_to_list(tickers, Ticker) results = {} - def get_data_func(data_prov: DataProvider, tickers_for_single_data_provider) -> Dict[FutureTicker, QFSeries]: + def get_data_func(data_prov: AbstractPriceDataProvider, tickers_for_single_data_provider) -> Dict[FutureTicker, QFSeries]: return data_prov.get_futures_chain_tickers(tickers_for_single_data_provider, ExpirationDateField.all_dates()) for ticker_class, ticker_group in groupby(tickers, lambda t: type(t)): @@ -161,14 +167,14 @@ def _get_data_for_multiple_tickers(self, tickers, fields, start_date, end_date, if use_prices_types: type_of_field = PriceField - def get_data_func(data_prov: DataProvider, tickers_for_single_data_provider): + def get_data_func(data_prov: AbstractPriceDataProvider, tickers_for_single_data_provider): prices = data_prov.get_price(tickers_for_single_data_provider, fields, start_date, end_date, frequency) return prices else: type_of_field = str - def get_data_func(data_prov: DataProvider, tickers_for_single_data_provider): + def get_data_func(data_prov: AbstractPriceDataProvider, tickers_for_single_data_provider): prices = data_prov.get_history(tickers_for_single_data_provider, fields, start_date, end_date, frequency) return prices @@ -196,11 +202,11 @@ def get_data_func(data_prov: DataProvider, tickers_for_single_data_provider): return result - def _register_data_provider(self, price_provider: DataProvider): + def _register_data_provider(self, price_provider: AbstractPriceDataProvider): for ticker_class in price_provider.supported_ticker_types(): self._ticker_type_to_data_provider_dict[ticker_class] = price_provider - def _identify_data_provider(self, ticker_class: Type[Ticker]) -> DataProvider: + def _identify_data_provider(self, ticker_class: Type[Ticker]) -> AbstractPriceDataProvider: """ Defines the association between ticker type and data provider. """ @@ -210,3 +216,13 @@ def _identify_data_provider(self, ticker_class: Type[Ticker]) -> DataProvider: "Unknown ticker type: {}. No appropriate data provider found".format(str(ticker_class))) return data_provider + + def price_field_to_str_map(self) -> Dict[PriceField, str]: + raise NotImplementedError("price_field_to_str_map is not supported by the GeneralPriceProvider.") + + def expiration_date_field_str_map(self, ticker: Ticker = None) -> Dict[ExpirationDateField, str]: + raise NotImplementedError("expiration_date_field_str_map is not supported by the GeneralPriceProvider.") + + def _get_futures_chain_dict(self, tickers: Union[FutureTicker, Sequence[FutureTicker]], + expiration_date_fields: Union[str, Sequence[str]]) -> Dict[FutureTicker, QFDataFrame]: + pass diff --git a/qf_lib/data_providers/haver/haver_data_provider.py b/qf_lib/data_providers/haver/haver_data_provider.py index 4cafe4c8..0798ae2a 100644 --- a/qf_lib/data_providers/haver/haver_data_provider.py +++ b/qf_lib/data_providers/haver/haver_data_provider.py @@ -19,13 +19,10 @@ from typing import Union, Sequence, Dict, Optional from pandas import PeriodIndex, DataFrame -from qf_lib.common.enums.expiration_date_field import ExpirationDateField from qf_lib.common.enums.price_field import PriceField -from qf_lib.common.tickers.tickers import HaverTicker, Ticker -from qf_lib.common.utils.logging.qf_parent_logger import qf_logger +from qf_lib.common.tickers.tickers import HaverTicker from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame -from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker from qf_lib.containers.series.qf_series import QFSeries from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider from qf_lib.settings import Settings @@ -51,11 +48,10 @@ class HaverDataProvider(AbstractPriceDataProvider): get_lock = threading.Lock() def __init__(self, settings: Settings): + super().__init__() self.db_location = settings.haver_path self.connected = False - self.logger = qf_logger.getChild(self.__class__.__name__) - def get_history(self, tickers: Union[HaverTicker, Sequence[HaverTicker]], fields=None, start_date: datetime = None, end_date: datetime = None, **kwargs) -> Union[QFSeries, QFDataFrame]: """ Gets historical fields for Haver tickers. @@ -143,10 +139,3 @@ def _connect_if_needed(self): if not self.connected: raise ConnectionError("No Haver connection.") - - def _get_futures_chain_dict(self, tickers: Union[FutureTicker, Sequence[FutureTicker]], - expiration_date_fields: Union[str, Sequence[str]]) -> Dict[FutureTicker, QFDataFrame]: - raise NotImplementedError("Downloading Future Chain Tickers in HaverDataProvider is not supported yet") - - def expiration_date_field_str_map(self, ticker: Ticker = None) -> Dict[ExpirationDateField, str]: - pass diff --git a/qf_lib/data_providers/helpers.py b/qf_lib/data_providers/helpers.py index 5751b3f1..cfca3d59 100644 --- a/qf_lib/data_providers/helpers.py +++ b/qf_lib/data_providers/helpers.py @@ -81,7 +81,6 @@ def normalize_data_array( def squeeze_data_array_and_cast_to_proper_type(original_data_array: QFDataArray, got_single_date: bool, got_single_ticker: bool, got_single_field: bool, use_prices_types: bool): - if isinstance(original_data_array, DataArray) and not isinstance(original_data_array, QFDataArray): warnings.warn("data_array to be normalized should be a QFDataFrame instance. " "Transforming data_array to QFDataArray. Please check types in the future.") diff --git a/qf_lib/data_providers/interactive_brokers/ib_figi_contracts_mapper.py b/qf_lib/data_providers/interactive_brokers/ib_figi_contracts_mapper.py index 51703f73..8e515dfd 100644 --- a/qf_lib/data_providers/interactive_brokers/ib_figi_contracts_mapper.py +++ b/qf_lib/data_providers/interactive_brokers/ib_figi_contracts_mapper.py @@ -11,14 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import warnings from threading import Thread, Event, Lock from typing import Optional, Dict -from ibapi.client import EClient -from qf_lib.backtesting.contract.contract_to_ticker_conversion.ib_contract_ticker_mapper import IBContractTickerMapper -from qf_lib.brokers.ib_broker.ib_contract import IBContract from qf_lib.common.tickers.tickers import Ticker -from qf_lib.common.utils.logging.qf_parent_logger import ib_logger -from qf_lib.brokers.ib_broker.ib_wrapper import IBWrapper + +try: + from ibapi.client import EClient + from qf_lib.brokers.ib_broker.ib_contract import IBContract + from qf_lib.common.utils.logging.qf_parent_logger import ib_logger + from qf_lib.brokers.ib_broker.ib_wrapper import IBWrapper + from qf_lib.backtesting.contract.contract_to_ticker_conversion.ib_contract_ticker_mapper import \ + IBContractTickerMapper + + is_ibapi_installed = True +except ImportError: + is_ibapi_installed = False + warnings.warn( + "No ibapi installed. If you would like to use IBFIGItoIBContractMapper first install the ibapi library.") class IBFIGItoIBContractMapper: @@ -54,25 +64,31 @@ def __init__(self, clientId: int = 0, host: str = "127.0.0.1", port: int = 7497) self.waiting_time = 30 # expressed in seconds # Lock that informs us that wrapper received the response - self.action_event_lock = Event() - self.wrapper = IBWrapper(self.action_event_lock, IBContractTickerMapper({})) # not necessary to have configured IBContractTickerMapper for FIGI contracts mapping - self.client = EClient(wrapper=self.wrapper) - self.clientId = clientId - self.client.connect(host, port, self.clientId) - - # Run the client in the separate thread so that the execution of the program can go on - # now we will have 3 threads: - # - thread of the main program - # - thread of the client - # - thread of the wrapper - thread = Thread(target=self.client.run) - thread.start() - - # This will be released after the client initialises and wrapper receives the nextValidOrderId - if not self._wait_for_results(): - raise ConnectionError("IB IBFIGItoIBContractMapper was not initialized correctly") - - def get_ticker_to_contract_mapping_from_figi_contracts(self, ticker_to_contract: Dict[Ticker, IBContract]) -> Dict[Ticker, IBContract]: + + if is_ibapi_installed: + self.action_event_lock = Event() + self.wrapper = IBWrapper(self.action_event_lock, IBContractTickerMapper( + {})) # not necessary to have configured IBContractTickerMapper for FIGI contracts mapping + self.client = EClient(wrapper=self.wrapper) + self.clientId = clientId + self.client.connect(host, port, self.clientId) + + # Run the client in the separate thread so that the execution of the program can go on + # now we will have 3 threads: + # - thread of the main program + # - thread of the client + # - thread of the wrapper + thread = Thread(target=self.client.run) + thread.start() + + # This will be released after the client initialises and wrapper receives the nextValidOrderId + if not self._wait_for_results(): + raise ConnectionError("IB IBFIGItoIBContractMapper was not initialized correctly") + else: + self.logger.warning("Couldn't import the IB API. Check if the necessary dependencies are installed.") + + def get_ticker_to_contract_mapping_from_figi_contracts(self, ticker_to_contract: Dict[Ticker, IBContract]) -> ( + Dict)[Ticker, IBContract]: """" Function to map dictionary: ticker -> ib_figi_contract diff --git a/qf_lib/data_providers/portara/portara_data_provider.py b/qf_lib/data_providers/portara/portara_data_provider.py index d49024b9..a2da1b18 100644 --- a/qf_lib/data_providers/portara/portara_data_provider.py +++ b/qf_lib/data_providers/portara/portara_data_provider.py @@ -13,7 +13,7 @@ # limitations under the License. from datetime import datetime -from typing import Sequence, Union, List +from typing import Sequence, Union, List, Optional from pathlib import Path import pandas as pd @@ -22,6 +22,7 @@ from qf_lib.common.enums.frequency import Frequency from qf_lib.common.enums.price_field import PriceField from qf_lib.common.tickers.tickers import Ticker, PortaraTicker +from qf_lib.common.utils.dateutils.timer import Timer from qf_lib.common.utils.logging.qf_parent_logger import qf_logger from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame @@ -74,8 +75,7 @@ class PortaraDataProvider(PresetDataProvider): """ def __init__(self, path: str, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[PriceField, List[PriceField]], - start_date: datetime, end_date: datetime, frequency: Frequency): - + start_date: datetime, end_date: datetime, frequency: Frequency, timer: Optional[Timer] = None): self.logger = qf_logger.getChild(self.__class__.__name__) if frequency not in [Frequency.DAILY, Frequency.MIN_1]: @@ -110,7 +110,8 @@ def __init__(self, path: str, tickers: Union[Ticker, Sequence[Ticker]], fields: exp_dates=exp_dates, start_date=start_date, end_date=end_date, - frequency=frequency) + frequency=frequency, + timer=timer) def get_contracts_df(self) -> QFDataFrame: """ Returns contracts information. A non empty data frame is returned only if the pricing data files contain diff --git a/qf_lib/data_providers/prefetching_data_provider.py b/qf_lib/data_providers/prefetching_data_provider.py index 33abe3a5..9184e2c1 100644 --- a/qf_lib/data_providers/prefetching_data_provider.py +++ b/qf_lib/data_providers/prefetching_data_provider.py @@ -12,18 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. from datetime import datetime -from typing import Sequence, Union +from typing import Sequence, Union, Optional from qf_lib.common.enums.expiration_date_field import ExpirationDateField from qf_lib.common.enums.frequency import Frequency from qf_lib.common.enums.price_field import PriceField from qf_lib.common.tickers.tickers import Ticker +from qf_lib.common.utils.dateutils.timer import Timer +from qf_lib.common.utils.logging.qf_parent_logger import qf_logger from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider +from qf_lib.data_providers.futures_data_provider import FuturesDataProvider from qf_lib.data_providers.helpers import chain_tickers_within_range from qf_lib.data_providers.preset_data_provider import PresetDataProvider -from qf_lib.data_providers.data_provider import DataProvider class PrefetchingDataProvider(PresetDataProvider): @@ -34,7 +37,7 @@ class PrefetchingDataProvider(PresetDataProvider): Parameters ----------- - data_provider: DataProvider + data_provider: AbstractPriceDataProvider data provider used to download the data tickers: Ticker, Sequence[Ticker] one or a list of tickers, used further to download the futures contracts related data. @@ -51,11 +54,13 @@ class PrefetchingDataProvider(PresetDataProvider): frequency of the data """ - def __init__(self, data_provider: DataProvider, + def __init__(self, data_provider: AbstractPriceDataProvider, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[PriceField, Sequence[PriceField]], start_date: datetime, end_date: datetime, - frequency: Frequency): + frequency: Frequency, timer: Optional[Timer] = None): + self.logger = qf_logger.getChild(self.__class__.__name__) + # Convert fields into list in order to return a QFDataArray as the result of get_price function fields, _ = convert_to_list(fields, PriceField) @@ -70,16 +75,21 @@ def __init__(self, data_provider: DataProvider, all_tickers = non_future_tickers if future_tickers: - exp_dates = data_provider.get_futures_chain_tickers(future_tickers, ExpirationDateField.all_dates()) + if not isinstance(data_provider, FuturesDataProvider): + self.logger.error("The passed data provider does not support future tickers. All future tickers will " + "be ignored in the process.") + else: + exp_dates = data_provider.get_futures_chain_tickers(future_tickers, ExpirationDateField.all_dates()) - # Filter out all theses specific future contracts, which expired before start_date - for ft in future_tickers: - all_tickers.extend(chain_tickers_within_range(ft, exp_dates[ft], start_date, end_date)) + # Filter out all these specific future contracts, which expired before start_date + for ft in future_tickers: + all_tickers.extend(chain_tickers_within_range(ft, exp_dates[ft], start_date, end_date)) - data_array = data_provider.get_price(all_tickers, fields, start_date, end_date, frequency) + data_array = data_provider.get_price(all_tickers, fields, start_date, end_date, frequency, timer) super().__init__(data=data_array, exp_dates=exp_dates, start_date=start_date, end_date=end_date, - frequency=frequency) + frequency=frequency, + timer=timer) diff --git a/qf_lib/data_providers/preset_data_provider.py b/qf_lib/data_providers/preset_data_provider.py index cbaa5b0a..10515b5c 100644 --- a/qf_lib/data_providers/preset_data_provider.py +++ b/qf_lib/data_providers/preset_data_provider.py @@ -23,6 +23,7 @@ from qf_lib.common.enums.price_field import PriceField from qf_lib.common.tickers.tickers import Ticker from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta +from qf_lib.common.utils.dateutils.timer import Timer from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame @@ -31,11 +32,12 @@ from qf_lib.containers.qf_data_array import QFDataArray from qf_lib.containers.series.prices_series import PricesSeries from qf_lib.containers.series.qf_series import QFSeries -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider +from qf_lib.data_providers.futures_data_provider import FuturesDataProvider from qf_lib.data_providers.helpers import normalize_data_array -class PresetDataProvider(DataProvider): +class PresetDataProvider(AbstractPriceDataProvider, FuturesDataProvider): """ Wrapper on QFDataArray which makes it a DataProvider. @@ -55,10 +57,10 @@ class PresetDataProvider(DataProvider): """ def __init__(self, data: QFDataArray, start_date: datetime, end_date: datetime, frequency: Frequency, - exp_dates: Dict[FutureTicker, QFDataFrame] = None): - super().__init__() + exp_dates: Dict[FutureTicker, QFDataFrame] = None, timer: Optional[Timer] = None): + super().__init__(timer) self._data_bundle = data - self._frequency = frequency + self.frequency = frequency self._exp_dates = exp_dates self._tickers_cached_set = frozenset(data.tickers.values) @@ -73,10 +75,6 @@ def __init__(self, data: QFDataArray, start_date: datetime, end_date: datetime, def data_bundle(self) -> QFDataArray: return self._data_bundle - @property - def frequency(self) -> Frequency: - return self._frequency - @property def exp_dates(self) -> Dict[FutureTicker, QFDataFrame]: return self._exp_dates @@ -105,30 +103,33 @@ def supported_ticker_types(self) -> Set[Type[Ticker]]: return self._ticker_types def get_price(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[PriceField, Sequence[PriceField]], - start_date: datetime, end_date: datetime = None, frequency: Frequency = Frequency.DAILY, **kwargs) -> \ + start_date: datetime, end_date: datetime = None, frequency: Frequency = None, + look_ahead_bias: bool = False, **kwargs) -> \ Union[None, PricesSeries, PricesDataFrame, QFDataArray]: # The passed desired data frequency should be at most equal to the frequency of the initially loaded data # (in case of downsampling the data may be aggregated, but no data upsampling is supported). - assert frequency <= self._frequency, "The passed data frequency should be at most equal to the frequency of " \ - "the initially loaded data" + frequency = frequency or self.frequency or Frequency.DAILY + assert frequency <= self.frequency, "The passed data frequency should be at most equal to the frequency of " \ + "the initially loaded data" # The PresetDataProvider does not support data aggregation for frequency lower than daily frequency - if frequency < self._frequency and frequency <= Frequency.DAILY: + if frequency < self.frequency and frequency <= Frequency.DAILY: self.logger.warning("aggregating intraday data to frequency Daily or lower is based on the time of " "underlying intrady data and might not be identical to getting daily data form the " "data provider.") + original_end_date = (end_date or self.timer.now()) + RelativeDelta(second=0, microsecond=0) start_date = self._adjust_start_date(start_date, frequency) - end_date = self._adjust_end_date(end_date) + end_date = end_date if look_ahead_bias else self.get_end_date_without_look_ahead(end_date, frequency) + got_single_date = self._got_single_date(start_date, original_end_date, frequency) tickers, specific_tickers, tickers_mapping, got_single_ticker = self._tickers_mapping(tickers) fields, got_single_field = convert_to_list(fields, PriceField) - got_single_date = self._got_single_date(start_date, end_date, frequency) self._check_if_cached_data_available(specific_tickers, fields, start_date, end_date) data_array = self._data_bundle.loc[start_date:end_date, specific_tickers, fields] # Data aggregation - if frequency < self._frequency and data_array.shape[0] > 0: + if frequency < self.frequency and data_array.shape[0] > 0: data_array = self._aggregate_bars(data_array, fields, frequency) normalized_result = normalize_data_array( @@ -143,10 +144,10 @@ def get_price(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[Pric def historical_price(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[PriceField, Sequence[PriceField]], nr_of_bars: int, end_date: Optional[datetime] = None, - frequency: Frequency = None) -> Union[PricesSeries, PricesDataFrame, QFDataArray]: - + frequency: Frequency = None, **kwargs) -> Union[PricesSeries, PricesDataFrame, QFDataArray]: + frequency = frequency or self.frequency or Frequency.DAILY assert nr_of_bars > 0, "Numbers of data samples should be a positive integer" - end_date = datetime.now() if end_date is None else end_date + end_date = self.get_end_date_without_look_ahead(end_date, frequency) tickers, specific_tickers, tickers_mapping, got_single_ticker = self._tickers_mapping(tickers) fields, got_single_field = convert_to_list(fields, PriceField) @@ -155,7 +156,7 @@ def historical_price(self, tickers: Union[Ticker, Sequence[Ticker]], start_date = self._compute_start_date(nr_of_bars, end_date, frequency) data_bundle = self._data_bundle.loc[start_date:end_date, specific_tickers, fields].dropna(DATES, how='all') - if frequency < self._frequency and data_bundle.shape[0] > 0: # Aggregate bars to desired frequency + if frequency < self.frequency and data_bundle.shape[0] > 0: # Aggregate bars to desired frequency data_bundle = self._aggregate_bars(data_bundle, fields, frequency) self._check_data_availibility(data_bundle, end_date, nr_of_bars, tickers) @@ -169,11 +170,10 @@ def historical_price(self, tickers: Union[Ticker, Sequence[Ticker]], self._check_data_availibility(normalized_result, end_date, nr_of_bars, tickers) return normalized_result - def get_last_available_price(self, tickers: Union[Ticker, Sequence[Ticker]], frequency: Frequency, - end_time: Optional[datetime] = None) -> Union[float, PricesSeries]: - end_time = datetime.now() if end_time is None else end_time - end_time += RelativeDelta(second=0, microsecond=0) - + def _last_available_price(self, tickers: Union[Ticker, Sequence[Ticker]], frequency: Optional[Frequency] = None, + end_time: Optional[datetime] = None) -> Union[float, PricesSeries]: + frequency = frequency or self.frequency or Frequency.DAILY + end_time = self.get_end_date_without_look_ahead(end_time, frequency) assert frequency >= Frequency.DAILY, "Frequency lower then daily is not supported by the " \ "get_last_available_price function" @@ -206,7 +206,8 @@ def get_last_available_price(self, tickers: Union[Ticker, Sequence[Ticker]], fre latest_available_prices_series = self._get_valid_latest_available_prices(start_date, specific_tickers, open_prices, close_prices) - latest_available_prices_series = self._map_normalized_result(latest_available_prices_series, tickers_mapping, tickers) + latest_available_prices_series = self._map_normalized_result(latest_available_prices_series, tickers_mapping, + tickers) return latest_available_prices_series.iloc[0] if got_single_ticker else latest_available_prices_series def _tickers_mapping(self, tickers: Union[Ticker, Sequence[Ticker]]) -> \ @@ -232,13 +233,13 @@ def _check_if_cached_data_available(self, tickers, fields, start_date, end_date) def remove_time_part(date: datetime): return datetime(date.year, date.month, date.day) - start_date_not_included = start_date < self._start_date if self._frequency > Frequency.DAILY else \ + start_date_not_included = start_date < self._start_date if self.frequency > Frequency.DAILY else \ remove_time_part(start_date) < remove_time_part(self._start_date) if start_date_not_included: raise ValueError("Requested start date {} is before data bundle start date {}". format(start_date, self._start_date)) - end_date_not_included = end_date > self._end_date if self._frequency > Frequency.DAILY else \ + end_date_not_included = end_date > self._end_date if self.frequency > Frequency.DAILY else \ remove_time_part(end_date) > remove_time_part(self._end_date) if end_date_not_included: raise ValueError("Requested end date {} is after data bundle end date {}". @@ -246,14 +247,19 @@ def remove_time_part(date: datetime): def get_history(self, tickers: Union[Ticker, Sequence[Ticker]], fields: Union[Any, Sequence[Any]], - start_date: datetime, end_date: datetime = None, frequency: Frequency = Frequency.DAILY, **kwargs + start_date: datetime, end_date: datetime = None, frequency: Frequency = None, + look_ahead_bias: bool = False, **kwargs ) -> Union[QFSeries, QFDataFrame, QFDataArray]: + frequency = frequency or self.frequency or Frequency.DAILY # Verify whether the passed frequency parameter is correct and can be used with the preset data - assert frequency == self._frequency, "Currently, for the get history does not support data sampling" + assert frequency == self.frequency, "Currently, for the get history does not support data sampling" start_date = self._adjust_start_date(start_date, frequency) - end_date = self._adjust_end_date(end_date) + original_end_date = (end_date or self.timer.now()) + RelativeDelta(second=0, microsecond=0) + end_date = original_end_date if look_ahead_bias else self.get_end_date_without_look_ahead(original_end_date, + frequency) + got_single_date = self._got_single_date(start_date, original_end_date, frequency) # In order to be able to return data for FutureTickers create a mapping between tickers and corresponding # specific tickers (in case of non FutureTickers it will be an identity mapping) @@ -265,7 +271,6 @@ def get_history(self, tickers: Union[Ticker, Sequence[Ticker]], fields_type = {type(field) for field in fields} if isinstance(fields, Sequence) else {type(fields)} fields, got_single_field = convert_to_list(fields, tuple(fields_type)) - got_single_date = self._got_single_date(start_date, end_date, frequency) self._check_if_cached_data_available(specific_tickers, fields, start_date, end_date) data_array = self._data_bundle.loc[start_date:end_date, specific_tickers, fields] @@ -357,3 +362,13 @@ def _aggregate_bars(self, data_array, fields, frequency: Frequency): def _adjust_end_date(end_date: Optional[datetime]) -> datetime: end_date = end_date or datetime.now() return end_date + RelativeDelta(second=0, microsecond=0) + + def price_field_to_str_map(self) -> Dict[PriceField, str]: + pass + + def expiration_date_field_str_map(self, ticker: Ticker = None) -> Dict[ExpirationDateField, str]: + pass + + def _get_futures_chain_dict(self, tickers: Union[FutureTicker, Sequence[FutureTicker]], + expiration_date_fields: Union[str, Sequence[str]]) -> Dict[FutureTicker, QFDataFrame]: + pass diff --git a/qf_lib/data_providers/quandl/quandl_data_provider.py b/qf_lib/data_providers/quandl/quandl_data_provider.py index 66f70eab..c4037c80 100644 --- a/qf_lib/data_providers/quandl/quandl_data_provider.py +++ b/qf_lib/data_providers/quandl/quandl_data_provider.py @@ -17,22 +17,20 @@ from typing import Union, Sequence, Dict import pandas as pd -from qf_lib.common.enums.expiration_date_field import ExpirationDateField from qf_lib.common.enums.frequency import Frequency from qf_lib.common.enums.price_field import PriceField from qf_lib.common.enums.quandl_db_type import QuandlDBType -from qf_lib.common.tickers.tickers import QuandlTicker, Ticker +from qf_lib.common.tickers.tickers import QuandlTicker from qf_lib.common.utils.dateutils.date_to_string import date_to_str from qf_lib.common.utils.logging.qf_parent_logger import qf_logger from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame -from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker from qf_lib.containers.qf_data_array import QFDataArray from qf_lib.containers.series.qf_series import QFSeries +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider from qf_lib.data_providers.helpers import tickers_dict_to_data_array, \ normalize_data_array, get_fields_from_tickers_data_dict -from qf_lib.data_providers.data_provider import DataProvider from qf_lib.settings import Settings try: @@ -43,7 +41,7 @@ warnings.warn("No quandl installed. If you would like to use QuandlDataProvider first install the quandl library.") -class QuandlDataProvider(DataProvider): +class QuandlDataProvider(AbstractPriceDataProvider): """ Class providing the Quandl data. The table database: WIKI/PRICES offers stock prices, dividends and splits for 3000 US publicly-traded companies. @@ -151,17 +149,17 @@ def supported_ticker_types(self): return {QuandlTicker} def _map_fields_to_str(self, fields: Sequence[PriceField], database_name: str, database_type: QuandlDBType): - field_to_str = self._price_field_to_str_map(database_name, database_type) + field_to_str = self.price_field_to_str_map(database_name, database_type) fields_as_strings = [field_to_str[field] for field in fields] return fields_as_strings def _str_to_price_field_map(self, database_name: str, database_type: QuandlDBType): - field_to_str = self._price_field_to_str_map(database_name, database_type) + field_to_str = self.price_field_to_str_map(database_name, database_type) str_to_field = {field_str: field for field, field_str in field_to_str.items()} return str_to_field - def _price_field_to_str_map(self, database_name: str, database_type: QuandlDBType) -> Dict[PriceField, str]: + def price_field_to_str_map(self, database_name: str, database_type: QuandlDBType) -> Dict[PriceField, str]: if database_type == QuandlDBType.Table and database_name == 'WIKI/PRICES': price_field_dict = { PriceField.Open: 'adj_open', @@ -299,11 +297,3 @@ def _format_single_ticker_table(table: pd.DataFrame, start_date: datetime, end_d table = table.loc[start_date:end_date] return table - - def get_futures_chain_tickers(self, tickers: Union[FutureTicker, Sequence[FutureTicker]], - expiration_date_fields: Union[ExpirationDateField, Sequence[ExpirationDateField]]) \ - -> Dict[FutureTicker, Union[QFSeries, QFDataFrame]]: - raise NotImplementedError("Downloading Future Chain Tickers in QuandlDataProvider is not supported yet") - - def expiration_date_field_str_map(self, ticker: Ticker = None) -> Dict[ExpirationDateField, str]: - pass diff --git a/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_alpha_model_for_limiting_open_positions.py b/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_alpha_model_for_limiting_open_positions.py index 799fe2ec..35eb2627 100644 --- a/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_alpha_model_for_limiting_open_positions.py +++ b/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_alpha_model_for_limiting_open_positions.py @@ -19,7 +19,6 @@ from qf_lib.backtesting.alpha_model.alpha_model import AlphaModel from qf_lib.backtesting.alpha_model.exposure_enum import Exposure -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.events.time_event.regular_time_event.calculate_and_place_orders_event import \ CalculateAndPlaceOrdersRegularEvent from qf_lib.backtesting.portfolio.portfolio import Portfolio @@ -33,6 +32,7 @@ from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame from qf_lib.containers.futures.future_tickers.bloomberg_future_ticker import BloombergFutureTicker from qf_lib.containers.qf_data_array import QFDataArray +from qf_lib.data_providers.data_provider import DataProvider from qf_lib.data_providers.preset_data_provider import PresetDataProvider from qf_lib.tests.integration_tests.backtesting.trading_session_for_tests import TradingSessionForTests @@ -66,7 +66,7 @@ def setUp(self): ) # --- Build the model --- # - model = DummyAlphaModel(risk_estimation_factor=0.05, data_provider=self.ts.data_handler) + model = DummyAlphaModel(risk_estimation_factor=0.05, data_provider=self.ts.data_provider) self.model_tickers_dict = {model: self.tickers} def _mock_data_provider(self): @@ -178,7 +178,7 @@ def test_limiting_open_positions_2_position(self): class DummyAlphaModel(AlphaModel): - def __init__(self, risk_estimation_factor: float, data_provider: DataHandler): + def __init__(self, risk_estimation_factor: float, data_provider: DataProvider): super().__init__(0.0, data_provider) def calculate_exposure(self, ticker: Ticker, current_exposure: Exposure, current_time: datetime, diff --git a/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_alpha_model_strategy_for_stop_losses.py b/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_alpha_model_strategy_for_stop_losses.py index f3227ae3..855af248 100644 --- a/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_alpha_model_strategy_for_stop_losses.py +++ b/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_alpha_model_strategy_for_stop_losses.py @@ -64,8 +64,8 @@ def setUp(self): risk_estimation_factor = 0.05 - ts = self._test_trading_session_init() - alpha_model = self.DummyAlphaModel(risk_estimation_factor, ts.data_handler) + ts = self._test_trading_session_init(self._price_provider_mock) + alpha_model = self.DummyAlphaModel(risk_estimation_factor, ts.data_provider) # Mock the backtest result in order to be able to compare transactions self.transactions = [] @@ -84,7 +84,7 @@ def test_stop_losses(self): expected_transactions_quantities = \ [8130, -127, 1, -8004, 7454, -58, -7396, 6900, -6900, 6390, -44, -6346, 5718, -36] result_transactions_quantities = [t.quantity for t in self.transactions] - assert_equal(expected_transactions_quantities, result_transactions_quantities) + assert_equal(result_transactions_quantities, expected_transactions_quantities) expected_transactions_prices = [125, 130, 135, 235.6, 255, 260, 259.35, 280, 264.1, 285, 290, 282, 315, 320] result_transactions_prices = [t.price for t in self.transactions] @@ -127,9 +127,9 @@ def _add_test_cases(self, mocked_result, tickers): mocked_result.loc[str_to_date('2015-02-23'), tickers[0], PriceField.Open] = \ mocked_result.loc[str_to_date('2015-02-23'), tickers[0], PriceField.Low] - def _test_trading_session_init(self): + def _test_trading_session_init(self, data_provider): ts = TradingSessionForTests( - data_provider=self._price_provider_mock, + data_provider=data_provider, start_date=self.test_start_date, end_date=self.test_end_date, initial_cash=1000000, diff --git a/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_fast_alpha_models_tester.py b/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_fast_alpha_models_tester.py index 02221b97..6c89a91a 100644 --- a/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_fast_alpha_models_tester.py +++ b/qf_lib/tests/integration_tests/backtesting/alpha_model_strategy_testers/test_fast_alpha_models_tester.py @@ -23,8 +23,6 @@ from qf_lib.backtesting.alpha_model.alpha_model import AlphaModel from qf_lib.backtesting.alpha_model.exposure_enum import Exposure -from qf_lib.backtesting.data_handler.daily_data_handler import DailyDataHandler -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent from qf_lib.backtesting.fast_alpha_model_tester.fast_alpha_models_tester import FastAlphaModelTester, \ @@ -37,6 +35,7 @@ from qf_lib.containers.qf_data_array import QFDataArray from qf_lib.containers.series.qf_series import QFSeries from qf_lib.containers.series.simple_returns_series import SimpleReturnsSeries +from qf_lib.data_providers.data_provider import DataProvider from qf_lib.data_providers.preset_data_provider import PresetDataProvider from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_series_equal @@ -61,10 +60,9 @@ def setUp(self): all_fields = PriceField.ohlcv() self._mocked_prices_arr = self._make_mock_data_array(self.tickers, all_fields) - price_provider_mock = PresetDataProvider(self._mocked_prices_arr, self.data_start_date, - self.data_end_date, self.frequency) self.timer = SettableTimer() - self.data_handler = DailyDataHandler(price_provider_mock, self.timer) + self.data_provider = PresetDataProvider(self._mocked_prices_arr, self.data_start_date, + self.data_end_date, self.frequency, timer=self.timer) self.alpha_model_type = DummyAlphaModel @classmethod @@ -88,7 +86,7 @@ def _make_mock_data_array(cls, tickers, fields): def test_alpha_models_tester(self): first_param_set = (10, Exposure.LONG) second_param_set = (5, Exposure.SHORT) - data_handler = self.data_handler + data_provider = self.data_provider params = [FastAlphaModelTesterConfig(self.alpha_model_type, {"period_length": 10, "first_suggested_exposure": Exposure.LONG, @@ -99,7 +97,7 @@ def test_alpha_models_tester(self): "risk_estimation_factor": None}, ("period_length", "first_suggested_exposure"))] - tester = FastAlphaModelTester(params, self.tickers, self.test_start_date, self.test_end_date, data_handler, + tester = FastAlphaModelTester(params, self.tickers, self.test_start_date, self.test_end_date, data_provider, self.timer) backtest_summary = tester.test_alpha_models() @@ -175,7 +173,7 @@ def test_alpha_models_tester(self): class DummyAlphaModel(AlphaModel): def __init__(self, period_length: int, first_suggested_exposure: Exposure, - risk_estimation_factor: float, data_provider: DataHandler = None): + risk_estimation_factor: float, data_provider: DataProvider = None): super().__init__(risk_estimation_factor, data_provider) assert first_suggested_exposure != Exposure.OUT diff --git a/qf_lib/tests/integration_tests/backtesting/data_handler/__init__.py b/qf_lib/tests/integration_tests/backtesting/data_handler/__init__.py deleted file mode 100644 index f31f5e3d..00000000 --- a/qf_lib/tests/integration_tests/backtesting/data_handler/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2016-present CERN – European Organization for Nuclear Research -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/qf_lib/tests/integration_tests/backtesting/data_handler/test_data_handler.py b/qf_lib/tests/integration_tests/backtesting/data_handler/test_data_handler.py deleted file mode 100644 index ccef1819..00000000 --- a/qf_lib/tests/integration_tests/backtesting/data_handler/test_data_handler.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2016-present CERN – European Organization for Nuclear Research -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from unittest import TestCase - -import pandas as pd - -from qf_lib.backtesting.data_handler.daily_data_handler import DailyDataHandler -from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent -from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent -from qf_lib.common.enums.price_field import PriceField -from qf_lib.common.tickers.tickers import BloombergTicker -from qf_lib.common.utils.dateutils.date_format import DateFormat -from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta -from qf_lib.common.utils.dateutils.string_to_date import str_to_date -from qf_lib.common.utils.dateutils.timer import SettableTimer -from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame -from qf_lib.containers.dimension_names import DATES, TICKERS -from qf_lib.containers.qf_data_array import QFDataArray -from qf_lib.containers.series.prices_series import PricesSeries -from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_series_equal, assert_same_index -from qf_lib.tests.integration_tests.connect_to_data_provider import get_data_provider - - -class TestDataHandler(TestCase): - @classmethod - def setUpClass(cls): - cls.spx_index_ticker = BloombergTicker("SPX Index") - cls.google_ticker = BloombergTicker("GOOGL US Equity") - cls.microsoft_ticker = BloombergTicker("MSFT US Equity") - - cls.start_date = str_to_date("2018-01-02") - cls.end_date = str_to_date("2018-01-31") - cls.end_date_trimmed = str_to_date("2018-01-30") - cls.get_history_field = "PX_TO_BOOK_RATIO" - - def setUp(self): - try: - self.price_data_provider = get_data_provider() - except Exception as e: - raise self.skipTest(e) - - self.timer = SettableTimer() - self.data_handler = DailyDataHandler(self.price_data_provider, self.timer) - - MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 30, "second": 0, "microsecond": 0}) - MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0}) - - def test_get_price_when_end_date_is_in_the_past(self): - self.timer.set_current_time(str_to_date("2018-02-12 00:00:00.000000", DateFormat.FULL_ISO)) - prices_tms = self.data_handler.get_price(self.spx_index_ticker, PriceField.Close, self.start_date, - self.end_date) - - self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime()) - self.assertEqual(self.end_date, prices_tms.index[-1].to_pydatetime()) - - def test_get_price_when_end_date_is_today_after_market_close(self): - self.timer.set_current_time( - str_to_date("2018-01-31") + MarketCloseEvent.trigger_time() + RelativeDelta(hours=1)) - prices_tms = self.data_handler.get_price(self.spx_index_ticker, PriceField.Close, self.start_date, - self.end_date) - - self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime()) - self.assertEqual(self.end_date, prices_tms.index[-1].to_pydatetime()) - - def test_get_price_when_end_date_is_today_before_market_close(self): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) - close_prices_tms = self.data_handler.get_price(self.spx_index_ticker, PriceField.Close, self.start_date, - self.end_date) - - self.assertEqual(self.start_date, close_prices_tms.index[0].to_pydatetime()) - self.assertEqual(self.end_date_trimmed, close_prices_tms.index[-1].to_pydatetime()) - - def test_get_open_price_when_end_date_is_today_before_market_close__single_ticker(self): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) - open_prices_tms = self.data_handler.get_price(self.spx_index_ticker, PriceField.Open, self.start_date) - - self.assertEqual(self.start_date, open_prices_tms.index[0].to_pydatetime()) - self.assertEqual(str_to_date("2018-01-30"), open_prices_tms.index[-1].to_pydatetime()) - - def test_get_open_price_when_end_date_is_today_before_market_close__multiple_tickers(self): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) - tickers = [self.spx_index_ticker, self.microsoft_ticker] - open_prices_tms = self.data_handler.get_price(tickers, PriceField.Open, self.start_date, self.timer.now()) - - self.assertEqual(self.start_date, open_prices_tms.index[0].to_pydatetime()) - self.assertEqual(str_to_date("2018-01-30"), open_prices_tms.index[-1].to_pydatetime()) - - def test_get_price_when_end_date_is_tomorrow(self): - self.timer.set_current_time( - str_to_date("2018-01-30") + MarketCloseEvent.trigger_time() + RelativeDelta(hours=1)) - prices_tms = self.data_handler.get_price(self.spx_index_ticker, PriceField.Close, self.start_date, - self.end_date_trimmed) - - self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime()) - self.assertEqual(self.end_date_trimmed, prices_tms.index[-1].to_pydatetime()) - - def test_get_last_price_single_ticker(self): - with self.subTest("Test if getting single ticker value works, when a single ticker is passed"): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() + - RelativeDelta(hours=1)) - single_price = self.data_handler.get_last_available_price(self.spx_index_ticker) - self.assertTrue(isinstance(single_price, float)) - - with self.subTest("Test at market open"): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time()) - at_market_open = self.data_handler.get_last_available_price([self.spx_index_ticker]) - - self.assertEqual(self.spx_index_ticker, at_market_open.index[0]) - self.assertEqual(single_price, at_market_open[0]) - - with self.subTest("Test during the trading session"): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) - during_the_day_last_prices = self.data_handler.get_last_available_price([self.spx_index_ticker]) - - self.assertEqual(self.spx_index_ticker, during_the_day_last_prices.index[0]) - self.assertEqual(single_price, during_the_day_last_prices[0]) - - with self.subTest("Test after the trading session"): - self.timer.set_current_time( - str_to_date("2018-01-31") + MarketCloseEvent.trigger_time() + RelativeDelta(hours=1)) - after_close_last_prices = self.data_handler.get_last_available_price([self.spx_index_ticker]) - - self.assertEqual(self.spx_index_ticker, after_close_last_prices.index[0]) - self.assertNotEqual(during_the_day_last_prices[0], after_close_last_prices[0]) - - with self.subTest("Test before the trading session"): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() - RelativeDelta(hours=1)) - before_trading_session_prices = self.data_handler.get_last_available_price([self.spx_index_ticker]) - - self.assertEqual(self.spx_index_ticker, before_trading_session_prices.index[0]) - self.assertNotEqual(during_the_day_last_prices[0], before_trading_session_prices[0]) - self.assertNotEqual(after_close_last_prices[0], before_trading_session_prices[0]) - - def test_get_last_price_with_multiple_tickers_when_current_data_is_unavailable(self): - self.timer.set_current_time(str_to_date("2018-01-01") + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) - last_prices = self.data_handler.get_last_available_price([self.spx_index_ticker, self.google_ticker]) - - self.assertEqual(self.spx_index_ticker, last_prices.index[0]) - self.assertEqual(self.google_ticker, last_prices.index[1]) - - def test_get_last_price_with_empty_tickers_list(self): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) - last_prices = self.data_handler.get_last_available_price([]) - assert_series_equal(PricesSeries(), last_prices) - - def test_get_history_when_end_date_is_in_the_past(self): - self.timer.set_current_time(str_to_date("2018-02-12 00:00:00.000000", DateFormat.FULL_ISO)) - prices_tms = self.data_handler.get_history(self.spx_index_ticker, self.get_history_field, - self.start_date, self.end_date) - - self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime()) - self.assertEqual(self.end_date, prices_tms.index[-1].to_pydatetime()) - - def test_get_history_when_end_date_is_today_after_market_close(self): - self.timer.set_current_time( - str_to_date("2018-01-31") + MarketCloseEvent.trigger_time() + RelativeDelta(hours=1)) - prices_tms = self.data_handler.get_history(self.spx_index_ticker, self.get_history_field, - self.start_date, self.end_date) - - self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime()) - self.assertEqual(self.end_date, prices_tms.index[-1].to_pydatetime()) - - def test_get_history_when_end_date_is_today_before_market_close(self): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) - prices_tms = self.data_handler.get_history(self.spx_index_ticker, self.get_history_field, - self.start_date, self.end_date) - - self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime()) - self.assertEqual(self.end_date_trimmed, prices_tms.index[-1].to_pydatetime()) - - def test_get_history_when_end_date_is_tomorrow(self): - self.timer.set_current_time( - str_to_date("2018-01-30") + MarketCloseEvent.trigger_time() + RelativeDelta(hours=1)) - prices_tms = self.data_handler.get_history(self.spx_index_ticker, self.get_history_field, - self.start_date, self.end_date_trimmed) - - self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime()) - self.assertEqual(self.end_date_trimmed, prices_tms.index[-1].to_pydatetime()) - - def test_get_history_with_multiple_tickers(self): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) - resilt_df = self.data_handler.get_history([self.microsoft_ticker, self.google_ticker], self.get_history_field, - self.start_date, self.end_date_trimmed) - - self.assertEqual(self.microsoft_ticker, resilt_df.columns[0]) - self.assertEqual(self.google_ticker, resilt_df.columns[1]) - self.assertEqual(self.start_date, resilt_df.index[0].to_pydatetime()) - self.assertEqual(self.end_date_trimmed, resilt_df.index[-1].to_pydatetime()) - self.assertEqual(resilt_df.shape, (20, 2)) - - def test_historical_price_many_tickers_many_fields(self): - self.timer.set_current_time(str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) - result_array = self.data_handler.historical_price([self.microsoft_ticker], [PriceField.Open, PriceField.Close], - nr_of_bars=5) - - self.assertEqual(QFDataArray, type(result_array)) - self.assertEqual((5, 1, 2), result_array.shape) - - expected_dates_str = ["2018-01-24", "2018-01-25", "2018-01-26", "2018-01-29", "2018-01-30"] - expected_dates = [str_to_date(date_str) for date_str in expected_dates_str] - assert_same_index(pd.DatetimeIndex(expected_dates, name=DATES), result_array.dates.to_index(), - check_index_type=True, check_names=True) - - def test_historical_price_many_tickers_one_field(self): - self.timer.set_current_time(str_to_date("2018-01-04") + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) - result_df = self.data_handler.historical_price([self.microsoft_ticker], PriceField.Open, nr_of_bars=5) - - self.assertEqual(PricesDataFrame, type(result_df)) - - expected_dates_idx = pd.DatetimeIndex( - ['2017-12-27', '2017-12-28', '2017-12-29', '2018-01-02', '2018-01-03'], name=DATES - ) - assert_same_index(expected_dates_idx, result_df.index, check_index_type=True, check_names=True) - - expected_tickers_idx = pd.Index([self.microsoft_ticker], name=TICKERS) - assert_same_index(expected_tickers_idx, result_df.columns, check_index_type=True, check_names=True) diff --git a/qf_lib/tests/integration_tests/backtesting/test_backtester.py b/qf_lib/tests/integration_tests/backtesting/test_backtester.py index 2774c850..290ca42c 100644 --- a/qf_lib/tests/integration_tests/backtesting/test_backtester.py +++ b/qf_lib/tests/integration_tests/backtesting/test_backtester.py @@ -27,7 +27,6 @@ from qf_lib.common.enums.price_field import PriceField from qf_lib.common.tickers.tickers import BloombergTicker from qf_lib.common.utils.dateutils.string_to_date import str_to_date -from qf_lib.data_providers.general_price_provider import GeneralPriceProvider from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_series_equal from qf_lib.tests.integration_tests.backtesting.trading_session_for_tests import TradingSessionForTests from qf_lib.tests.integration_tests.connect_to_data_provider import get_data_provider @@ -63,22 +62,22 @@ class TestBacktester(TestCase): def setUp(self): try: self.data_provider = get_data_provider() + self.data_provider.set_frequency(Frequency.DAILY) except Exception as e: raise self.skipTest(e) def test_backtester_with_buy_and_hold_strategy(self): start_date = str_to_date("2010-01-01") end_date = str_to_date("2010-02-01") - data_provider = GeneralPriceProvider(self.data_provider, None, None) - msft_prices = data_provider.get_price( + msft_prices = self.data_provider.get_price( BuyAndHoldStrategy.MICROSOFT_TICKER, fields=[PriceField.Open, PriceField.Close], start_date=str_to_date("2009-12-28"), end_date=str_to_date("2010-02-01") ) first_trade_date = str_to_date("2010-01-04") initial_cash = msft_prices.loc[first_trade_date, PriceField.Open] - ts = TradingSessionForTests(data_provider, start_date, end_date, initial_cash, frequency=Frequency.DAILY) + ts = TradingSessionForTests(self.data_provider, start_date, end_date, initial_cash, frequency=Frequency.DAILY) strategy = BuyAndHoldStrategy(ts) diff --git a/qf_lib/tests/integration_tests/backtesting/trading_session_for_tests.py b/qf_lib/tests/integration_tests/backtesting/trading_session_for_tests.py index 3fa03bb5..edc0fa74 100644 --- a/qf_lib/tests/integration_tests/backtesting/trading_session_for_tests.py +++ b/qf_lib/tests/integration_tests/backtesting/trading_session_for_tests.py @@ -16,8 +16,6 @@ from qf_lib.backtesting.broker.backtest_broker import BacktestBroker from qf_lib.backtesting.contract.contract_to_ticker_conversion.simulated_contract_ticker_mapper import \ SimulatedContractTickerMapper -from qf_lib.backtesting.data_handler.daily_data_handler import DailyDataHandler -from qf_lib.backtesting.data_handler.intraday_data_handler import IntradayDataHandler from qf_lib.backtesting.events.notifiers import Notifiers from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent @@ -36,7 +34,7 @@ from qf_lib.common.utils.dateutils.date_to_string import date_to_str from qf_lib.common.utils.dateutils.timer import SettableTimer from qf_lib.common.utils.logging.qf_parent_logger import qf_logger -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class TradingSessionForTests(TradingSession): @@ -48,7 +46,7 @@ class TradingSessionForTests(TradingSession): MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 30, "second": 0, "microsecond": 0}) MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0}) - def __init__(self, data_provider: DataProvider, start_date, end_date, initial_cash, + def __init__(self, data_provider: AbstractPriceDataProvider, start_date, end_date, initial_cash, frequency: Frequency = Frequency.MIN_1): """ Set up the backtest variables according to what has been passed in. @@ -67,13 +65,10 @@ def __init__(self, data_provider: DataProvider, start_date, end_date, initial_ca ) timer = SettableTimer(start_date) - notifiers = Notifiers(timer) - if frequency <= Frequency.DAILY: - data_handler = DailyDataHandler(data_provider, timer) - else: - data_handler = IntradayDataHandler(data_provider, timer) + data_provider.set_timer(timer) - portfolio = Portfolio(data_handler, initial_cash, timer) + notifiers = Notifiers(timer) + portfolio = Portfolio(data_provider, initial_cash) signals_register = BacktestSignalsRegister() backtest_result = BacktestResult(portfolio=portfolio, backtest_name="Testing the Backtester", start_date=start_date, end_date=end_date, signals_register=signals_register) @@ -83,18 +78,18 @@ def __init__(self, data_provider: DataProvider, start_date, end_date, initial_ca slippage_model = PriceBasedSlippage(0.0, data_provider) execution_handler = SimulatedExecutionHandler( - data_handler, timer, notifiers.scheduler, monitor, commission_model, + data_provider, notifiers.scheduler, monitor, commission_model, portfolio, slippage_model, frequency=frequency) contract_ticker_mapper = SimulatedContractTickerMapper() broker = BacktestBroker(contract_ticker_mapper, portfolio, execution_handler) - order_factory = OrderFactory(broker, data_handler) + order_factory = OrderFactory(broker, data_provider) event_manager = self._create_event_manager(timer, notifiers) time_flow_controller = BacktestTimeFlowController( notifiers.scheduler, event_manager, timer, notifiers.empty_queue_event_notifier, end_date ) - position_sizer = SimplePositionSizer(broker, data_handler, order_factory, signals_register) + position_sizer = SimplePositionSizer(broker, data_provider, order_factory, signals_register) self.logger.info( "\n".join([ @@ -117,14 +112,12 @@ def __init__(self, data_provider: DataProvider, start_date, end_date, initial_ca self.end_date = end_date self.event_manager = event_manager self.contract_ticker_mapper = contract_ticker_mapper - self.data_handler = data_handler - self.data_provider = data_handler + self.data_provider = data_provider self.portfolio = portfolio self.execution_handler = execution_handler self.position_sizer = position_sizer self.orders_filters = [] self.monitor = monitor - self.timer = timer self.order_factory = order_factory self.time_flow_controller = time_flow_controller self.frequency = frequency diff --git a/qf_lib/tests/integration_tests/connect_to_data_provider.py b/qf_lib/tests/integration_tests/connect_to_data_provider.py index 0cce29d4..89f5e01b 100644 --- a/qf_lib/tests/integration_tests/connect_to_data_provider.py +++ b/qf_lib/tests/integration_tests/connect_to_data_provider.py @@ -11,16 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import Optional + +from qf_lib.common.utils.dateutils.timer import Timer from qf_lib.data_providers.bloomberg import BloombergDataProvider from qf_lib.tests.unit_tests.config.test_settings import get_test_settings -def get_data_provider(): +def get_data_provider(timer: Optional[Timer] = None): """ Connects to Bloomberg data provider using the test settings. """ settings = get_test_settings() - bbg_provider = BloombergDataProvider(settings) + bbg_provider = BloombergDataProvider(settings, timer) bbg_provider.connect() if not bbg_provider.connected: raise ConnectionError("No Bloomberg connection") diff --git a/qf_lib/tests/integration_tests/data_providers/bloomberg/test_bbg_data_handler.py b/qf_lib/tests/integration_tests/data_providers/bloomberg/test_bbg_data_handler.py deleted file mode 100644 index 8f4090e0..00000000 --- a/qf_lib/tests/integration_tests/data_providers/bloomberg/test_bbg_data_handler.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2016-present CERN – European Organization for Nuclear Research -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from datetime import datetime -from unittest import TestCase - -from qf_lib.backtesting.data_handler.daily_data_handler import DailyDataHandler -from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent -from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent -from qf_lib.common.enums.price_field import PriceField -from qf_lib.common.tickers.tickers import BloombergTicker -from qf_lib.common.utils.dateutils.timer import SettableTimer -from qf_lib.common.utils.numberutils.is_finite_number import is_finite_number -from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame -from qf_lib.tests.integration_tests.connect_to_data_provider import get_data_provider - - -class TestBloombergDataHandler(TestCase): - """Class which tests Data Handler encapsulation with Bloomberg Data Provider.""" - - def setUp(self) -> None: - try: - bbg_provider = get_data_provider() - self.timer = SettableTimer() - self.daily_data_handler = DailyDataHandler(bbg_provider, self.timer) - - MarketOpenEvent.set_trigger_time({"hour": 8, "minute": 30, "second": 0, "microsecond": 0}) - MarketCloseEvent.set_trigger_time({"hour": 16, "minute": 0, "second": 0, "microsecond": 0}) - - except Exception as e: - raise self.skipTest(e) - - def test_market_open_price_before_market_open(self): - self.timer.set_current_time(datetime(2022, 4, 26, 1)) - prices = self.daily_data_handler.get_price(BloombergTicker("SPX Index"), PriceField.ohlcv(), - datetime(2022, 4, 24), datetime(2022, 4, 26)) - - self.assertTrue(is_finite_number(prices.loc[datetime(2022, 4, 25), PriceField.Open])) - self.assertTrue(is_finite_number(prices.loc[datetime(2022, 4, 25), PriceField.Close])) - - self.assertEqual(prices.index[-1], datetime(2022, 4, 25)) - - def test_market_open_price_before_market_open__single_date(self): - self.timer.set_current_time(datetime(2022, 4, 26, 1)) - prices = self.daily_data_handler.get_price(BloombergTicker("SPX Index"), PriceField.Open, datetime(2022, 4, 26), - datetime(2022, 4, 26)) - self.assertFalse(is_finite_number(prices)) - - def test_market_open_price_before_market_close__single_date(self): - self.timer.set_current_time(datetime(2022, 4, 26, 15)) - prices = self.daily_data_handler.get_price(BloombergTicker("SPX Index"), [PriceField.Open, PriceField.Close], - datetime(2022, 4, 26), datetime(2022, 4, 26)) - self.assertTrue(all(prices.isna())) - - self.timer.set_current_time(datetime(2022, 4, 29, 15)) - prices = self.daily_data_handler.get_price(BloombergTicker("SPX Index"), [PriceField.Open, PriceField.Close], - datetime(2022, 4, 29), datetime(2022, 4, 30)) - self.assertTrue(prices.empty) - - def test_market_open_price_after_market_close__single_date(self): - self.timer.set_current_time(datetime(2022, 4, 26, 23)) - prices = self.daily_data_handler.get_price(BloombergTicker("SPX Index"), [PriceField.Open, PriceField.Close], - datetime(2022, 4, 26), datetime(2022, 4, 26)) - self.assertTrue(is_finite_number(prices.loc[PriceField.Open])) - self.assertTrue(is_finite_number(prices.loc[PriceField.Close])) - - def test_market_open_price_before_market_close(self): - """Test if the Open price is returned in case if current time is after market open - and before market close on the end_date day. The low, high, close prices and volume - should be set to Nan. """ - - self.timer.set_current_time(datetime(2022, 4, 26, 15)) - prices = self.daily_data_handler.get_price(BloombergTicker("SPX Index"), PriceField.ohlcv(), - datetime(2022, 4, 25), datetime(2022, 4, 26)) - - self.assertEqual(prices.index[-1], datetime(2022, 4, 25)) - self.assertEqual(type(prices), PricesDataFrame) - - def test_market_open_price_after_market_close(self): - self.timer.set_current_time(datetime(2022, 4, 26, 23)) - prices = self.daily_data_handler.get_price(BloombergTicker("SPX Index"), PriceField.ohlcv(), - datetime(2022, 4, 25), datetime(2022, 4, 26)) - - self.assertTrue(is_finite_number(prices.loc[datetime(2022, 4, 26), PriceField.Open])) - self.assertTrue(is_finite_number(prices.loc[datetime(2022, 4, 26), PriceField.Close])) - - self.assertEqual(prices.index[-1], datetime(2022, 4, 26)) - - def test_market_open_price_after_market_close_before_end_date(self): - """Test if the Open price is returned in case if the current time is after market open - but before end_date. """ - - self.timer.set_current_time(datetime(2022, 4, 26, 15)) - prices = self.daily_data_handler.get_price(BloombergTicker("SPX Index"), PriceField.ohlcv(), - datetime(2022, 4, 25), datetime(2022, 4, 30)) - self.assertEqual(prices.index[-1], datetime(2022, 4, 25)) - self.assertEqual(type(prices), PricesDataFrame) - - def test_day_without_data__single_date(self): - self.timer.set_current_time(datetime(2022, 4, 30, 15)) - prices = self.daily_data_handler.get_price(BloombergTicker("SPX Index"), [PriceField.Open, PriceField.Close], - datetime(2022, 4, 30), datetime(2022, 4, 30)) - self.assertFalse(is_finite_number(prices.loc[PriceField.Open])) - self.assertFalse(is_finite_number(prices.loc[PriceField.Close])) diff --git a/qf_lib/tests/integration_tests/data_providers/bloomberg/test_bbg_look_ahead_bias.py b/qf_lib/tests/integration_tests/data_providers/bloomberg/test_bbg_look_ahead_bias.py new file mode 100644 index 00000000..64c01975 --- /dev/null +++ b/qf_lib/tests/integration_tests/data_providers/bloomberg/test_bbg_look_ahead_bias.py @@ -0,0 +1,300 @@ +# Copyright 2016-present CERN – European Organization for Nuclear Research +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from datetime import datetime +from unittest import TestCase + +import pandas as pd + +from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent +from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent +from qf_lib.common.enums.frequency import Frequency +from qf_lib.common.enums.price_field import PriceField +from qf_lib.common.tickers.tickers import BloombergTicker +from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta +from qf_lib.common.utils.dateutils.string_to_date import str_to_date +from qf_lib.common.utils.dateutils.timer import SettableTimer +from qf_lib.common.utils.numberutils.is_finite_number import is_finite_number +from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame +from qf_lib.containers.dimension_names import DATES, TICKERS +from qf_lib.containers.qf_data_array import QFDataArray +from qf_lib.containers.series.prices_series import PricesSeries +from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_same_index, assert_series_equal +from qf_lib.tests.integration_tests.connect_to_data_provider import get_data_provider + + +class TestBloombergDataProvider(TestCase): + """Class which tests look_ahead_bias flag with Bloomberg Data Provider.""" + + def setUp(self) -> None: + try: + self.timer = SettableTimer() + self.bbg_provider = get_data_provider(self.timer) + self.bbg_provider.frequency = Frequency.DAILY + MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 30, "second": 0, "microsecond": 0}) + MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0}) + + except Exception as e: + raise self.skipTest(e) + + def test_market_open_price_before_market_open(self): + self.timer.set_current_time(datetime(2022, 4, 26, 1)) + prices = self.bbg_provider.get_price(BloombergTicker("SPX Index"), PriceField.ohlcv(), + datetime(2022, 4, 24), datetime(2022, 4, 26)) + + self.assertTrue(is_finite_number(prices.loc[datetime(2022, 4, 25), PriceField.Open])) + self.assertTrue(is_finite_number(prices.loc[datetime(2022, 4, 25), PriceField.Close])) + + self.assertEqual(prices.index[-1], datetime(2022, 4, 25)) + + def test_market_open_price_before_market_open__single_date(self): + self.timer.set_current_time(datetime(2022, 4, 26, 1)) + prices = self.bbg_provider.get_price(BloombergTicker("SPX Index"), PriceField.Open, + datetime(2022, 4, 26), + datetime(2022, 4, 26)) + self.assertFalse(is_finite_number(prices)) + + def test_market_open_price_before_market_close__single_date(self): + self.timer.set_current_time(datetime(2022, 4, 26, 15)) + prices = self.bbg_provider.get_price(BloombergTicker("SPX Index"), [PriceField.Open, PriceField.Close], + datetime(2022, 4, 26), datetime(2022, 4, 26)) + self.assertTrue(all(prices.isna())) + + self.timer.set_current_time(datetime(2022, 4, 29, 15)) + prices = self.bbg_provider.get_price(BloombergTicker("SPX Index"), [PriceField.Open, PriceField.Close], + datetime(2022, 4, 29), datetime(2022, 4, 30)) + self.assertTrue(prices.empty) + + def test_market_open_price_after_market_close__single_date(self): + self.timer.set_current_time(datetime(2022, 4, 26, 23)) + prices = self.bbg_provider.get_price(BloombergTicker("SPX Index"), [PriceField.Open, PriceField.Close], + datetime(2022, 4, 26), datetime(2022, 4, 26)) + self.assertTrue(is_finite_number(prices.loc[PriceField.Open])) + self.assertTrue(is_finite_number(prices.loc[PriceField.Close])) + + def test_market_open_price_before_market_close(self): + """Test if the Open price is returned in case if current time is after market open + and before market close on the end_date day. The low, high, close prices and volume + should be set to Nan. """ + + self.timer.set_current_time(datetime(2022, 4, 26, 15)) + prices = self.bbg_provider.get_price(BloombergTicker("SPX Index"), PriceField.ohlcv(), + datetime(2022, 4, 25), datetime(2022, 4, 26)) + + self.assertEqual(prices.index[-1], datetime(2022, 4, 25)) + self.assertEqual(type(prices), PricesDataFrame) + + def test_market_open_price_after_market_close(self): + self.timer.set_current_time(datetime(2022, 4, 26, 23)) + prices = self.bbg_provider.get_price(BloombergTicker("SPX Index"), PriceField.ohlcv(), + datetime(2022, 4, 25), datetime(2022, 4, 26)) + + self.assertTrue(is_finite_number(prices.loc[datetime(2022, 4, 26), PriceField.Open])) + self.assertTrue(is_finite_number(prices.loc[datetime(2022, 4, 26), PriceField.Close])) + + self.assertEqual(prices.index[-1], datetime(2022, 4, 26)) + + def test_market_open_price_after_market_close_before_end_date(self): + """Test if the Open price is returned in case if the current time is after market open + but before end_date. """ + + self.timer.set_current_time(datetime(2022, 4, 26, 15)) + prices = self.bbg_provider.get_price(BloombergTicker("SPX Index"), PriceField.ohlcv(), + datetime(2022, 4, 25), datetime(2022, 4, 30)) + self.assertEqual(prices.index[-1], datetime(2022, 4, 25)) + self.assertEqual(type(prices), PricesDataFrame) + + def test_day_without_data__single_date(self): + self.timer.set_current_time(datetime(2022, 4, 30, 15)) + prices = self.bbg_provider.get_price(BloombergTicker("SPX Index"), [PriceField.Open, PriceField.Close], + datetime(2022, 4, 30), datetime(2022, 4, 30)) + self.assertFalse(is_finite_number(prices.loc[PriceField.Open])) + self.assertFalse(is_finite_number(prices.loc[PriceField.Close])) + + def test_get_price_when_end_date_is_in_the_past(self): + self.timer.set_current_time(datetime(2018, 2, 12)) + prices_tms = self.bbg_provider.get_price(BloombergTicker("SPX Index"), PriceField.Close, datetime(2018, 1, 2), + datetime(2018, 1, 31)) + + self.assertEqual(datetime(2018, 1, 2), prices_tms.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 31), prices_tms.index[-1].to_pydatetime()) + + def test_get_price_when_end_date_is_today_after_market_close(self): + self.timer.set_current_time( + datetime(2018, 1, 31) + MarketCloseEvent.trigger_time() + RelativeDelta(hours=1)) + prices_tms = self.bbg_provider.get_price(BloombergTicker("SPX Index"), PriceField.Close, datetime(2018, 1, 2), + datetime(2018, 1, 31)) + + self.assertEqual(datetime(2018, 1, 2), prices_tms.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 31), prices_tms.index[-1].to_pydatetime()) + + def test_get_price_when_end_date_is_today_before_market_close(self): + self.timer.set_current_time(datetime(2018, 1, 31) + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) + close_prices_tms = self.bbg_provider.get_price(BloombergTicker("SPX Index"), PriceField.Close, + datetime(2018, 1, 2), + datetime(2018, 1, 31)) + + self.assertEqual(datetime(2018, 1, 2), close_prices_tms.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 30), close_prices_tms.index[-1].to_pydatetime()) + + def test_get_open_price_when_end_date_is_today_before_market_close__single_ticker(self): + self.timer.set_current_time(datetime(2018, 1, 31) + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) + open_prices_tms = self.bbg_provider.get_price(BloombergTicker("SPX Index"), PriceField.Open, + datetime(2018, 1, 2)) + + self.assertEqual(datetime(2018, 1, 2), open_prices_tms.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 30), open_prices_tms.index[-1].to_pydatetime()) + + def test_get_open_price_when_end_date_is_today_before_market_close__multiple_tickers(self): + self.timer.set_current_time(datetime(2018, 1, 31) + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) + tickers = [BloombergTicker("SPX Index"), BloombergTicker("MSFT US Equity")] + open_prices_tms = self.bbg_provider.get_price(tickers, PriceField.Open, datetime(2018, 1, 2), self.timer.now()) + + self.assertEqual(datetime(2018, 1, 2), open_prices_tms.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 30), open_prices_tms.index[-1].to_pydatetime()) + + def test_get_price_when_end_date_is_tomorrow(self): + self.timer.set_current_time( + datetime(2018, 1, 30) + MarketCloseEvent.trigger_time() + RelativeDelta(hours=1)) + prices_tms = self.bbg_provider.get_price(BloombergTicker("SPX Index"), PriceField.Close, datetime(2018, 1, 2), + datetime(2018, 1, 30)) + + self.assertEqual(datetime(2018, 1, 2), prices_tms.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 30), prices_tms.index[-1].to_pydatetime()) + + def test_get_last_price_single_ticker(self): + with self.subTest("Test if getting single ticker value works, when a single ticker is passed"): + self.timer.set_current_time(datetime(2018, 1, 31) + MarketOpenEvent.trigger_time() + + RelativeDelta(hours=1)) + single_price = self.bbg_provider.get_last_available_price(BloombergTicker("SPX Index")) + self.assertTrue(isinstance(single_price, float)) + + with self.subTest("Test at market open"): + self.timer.set_current_time(datetime(2018, 1, 31) + MarketOpenEvent.trigger_time()) + at_market_open = self.bbg_provider.get_last_available_price([BloombergTicker("SPX Index")]) + + self.assertEqual(BloombergTicker("SPX Index"), at_market_open.index[0]) + self.assertEqual(single_price, at_market_open[0]) + + with self.subTest("Test during the trading session"): + self.timer.set_current_time( + datetime(2018, 1, 31) + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) + during_the_day_last_prices = self.bbg_provider.get_last_available_price([BloombergTicker("SPX Index")]) + + self.assertEqual(BloombergTicker("SPX Index"), during_the_day_last_prices.index[0]) + self.assertEqual(single_price, during_the_day_last_prices[0]) + + with self.subTest("Test after the trading session"): + self.timer.set_current_time( + datetime(2018, 1, 31) + MarketCloseEvent.trigger_time() + RelativeDelta(hours=1)) + after_close_last_prices = self.bbg_provider.get_last_available_price([BloombergTicker("SPX Index")]) + + self.assertEqual(BloombergTicker("SPX Index"), after_close_last_prices.index[0]) + self.assertNotEqual(during_the_day_last_prices[0], after_close_last_prices[0]) + + with self.subTest("Test before the trading session"): + self.timer.set_current_time( + datetime(2018, 1, 31) + MarketOpenEvent.trigger_time() - RelativeDelta(hours=1)) + before_trading_session_prices = self.bbg_provider.get_last_available_price([BloombergTicker("SPX Index")]) + + self.assertEqual(BloombergTicker("SPX Index"), before_trading_session_prices.index[0]) + self.assertNotEqual(during_the_day_last_prices[0], before_trading_session_prices[0]) + self.assertNotEqual(after_close_last_prices[0], before_trading_session_prices[0]) + + def test_get_last_price_with_multiple_tickers_when_current_data_is_unavailable(self): + self.timer.set_current_time(datetime(2018, 1, 1) + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) + last_prices = self.bbg_provider.get_last_available_price( + [BloombergTicker("SPX Index"), BloombergTicker("GOOGL US Equity")]) + + self.assertEqual(BloombergTicker("SPX Index"), last_prices.index[0]) + self.assertEqual(BloombergTicker("GOOGL US Equity"), last_prices.index[1]) + + def test_get_last_price_with_empty_tickers_list(self): + self.timer.set_current_time(datetime(2018, 1, 31) + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) + last_prices = self.bbg_provider.get_last_available_price([]) + assert_series_equal(PricesSeries(), last_prices) + + def test_get_history_when_end_date_is_in_the_past(self): + self.timer.set_current_time(datetime(2018, 2, 12)) + prices_tms = self.bbg_provider.get_history(BloombergTicker("SPX Index"), "PX_TO_BOOK_RATIO", + datetime(2018, 1, 2), datetime(2018, 1, 31)) + + self.assertEqual(datetime(2018, 1, 2), prices_tms.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 31), prices_tms.index[-1].to_pydatetime()) + + def test_get_history_when_end_date_is_today_after_market_close(self): + self.timer.set_current_time( + datetime(2018, 1, 31) + MarketCloseEvent.trigger_time() + RelativeDelta(hours=1)) + prices_tms = self.bbg_provider.get_history(BloombergTicker("SPX Index"), "PX_TO_BOOK_RATIO", + datetime(2018, 1, 2), datetime(2018, 1, 31)) + + self.assertEqual(datetime(2018, 1, 2), prices_tms.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 31), prices_tms.index[-1].to_pydatetime()) + + def test_get_history_when_end_date_is_today_before_market_close(self): + self.timer.set_current_time(datetime(2018, 1, 31) + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) + prices_tms = self.bbg_provider.get_history(BloombergTicker("SPX Index"), "PX_TO_BOOK_RATIO", + datetime(2018, 1, 2), datetime(2018, 1, 31)) + + self.assertEqual(datetime(2018, 1, 2), prices_tms.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 30), prices_tms.index[-1].to_pydatetime()) + + def test_get_history_when_end_date_is_tomorrow(self): + self.timer.set_current_time( + datetime(2018, 1, 30) + MarketCloseEvent.trigger_time() + RelativeDelta(hours=1)) + prices_tms = self.bbg_provider.get_history(BloombergTicker("SPX Index"), "PX_TO_BOOK_RATIO", + datetime(2018, 1, 2), datetime(2018, 1, 30)) + + self.assertEqual(datetime(2018, 1, 2), prices_tms.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 30), prices_tms.index[-1].to_pydatetime()) + + def test_get_history_with_multiple_tickers(self): + self.timer.set_current_time(datetime(2018, 1, 31) + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) + resilt_df = self.bbg_provider.get_history( + [BloombergTicker("MSFT US Equity"), BloombergTicker("GOOGL US Equity")], "PX_TO_BOOK_RATIO", + datetime(2018, 1, 2), datetime(2018, 1, 30)) + + self.assertEqual(BloombergTicker("MSFT US Equity"), resilt_df.columns[0]) + self.assertEqual(BloombergTicker("GOOGL US Equity"), resilt_df.columns[1]) + self.assertEqual(datetime(2018, 1, 2), resilt_df.index[0].to_pydatetime()) + self.assertEqual(datetime(2018, 1, 30), resilt_df.index[-1].to_pydatetime()) + self.assertEqual(resilt_df.shape, (20, 2)) + + def test_historical_price_many_tickers_many_fields(self): + self.timer.set_current_time(datetime(2018, 1, 31) + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) + result_array = self.bbg_provider.historical_price([BloombergTicker("MSFT US Equity")], + [PriceField.Open, PriceField.Close], + nr_of_bars=5) + + self.assertEqual(QFDataArray, type(result_array)) + self.assertEqual((5, 1, 2), result_array.shape) + + expected_dates_str = ["2018-01-24", "2018-01-25", "2018-01-26", "2018-01-29", "2018-01-30"] + expected_dates = [str_to_date(date_str) for date_str in expected_dates_str] + assert_same_index(pd.DatetimeIndex(expected_dates, name=DATES), result_array.dates.to_index(), + check_index_type=True, check_names=True) + + def test_historical_price_many_tickers_one_field(self): + self.timer.set_current_time(datetime(2018, 1, 4) + MarketOpenEvent.trigger_time() + RelativeDelta(hours=1)) + result_df = self.bbg_provider.historical_price([BloombergTicker("MSFT US Equity")], PriceField.Open, + nr_of_bars=5) + + self.assertEqual(PricesDataFrame, type(result_df)) + + expected_dates_idx = pd.DatetimeIndex( + ['2017-12-27', '2017-12-28', '2017-12-29', '2018-01-02', '2018-01-03'], name=DATES + ) + assert_same_index(expected_dates_idx, result_df.index, check_index_type=True, check_names=True) + + expected_tickers_idx = pd.Index([BloombergTicker("MSFT US Equity")], name=TICKERS) + assert_same_index(expected_tickers_idx, result_df.columns, check_index_type=True, check_names=True) diff --git a/qf_lib/tests/integration_tests/data_providers/bloomberg/test_bloomberg.py b/qf_lib/tests/integration_tests/data_providers/bloomberg/test_bloomberg.py index 10f894c7..e9e4cf73 100644 --- a/qf_lib/tests/integration_tests/data_providers/bloomberg/test_bloomberg.py +++ b/qf_lib/tests/integration_tests/data_providers/bloomberg/test_bloomberg.py @@ -48,6 +48,7 @@ class TestBloomberg(unittest.TestCase): def setUp(self): try: self.bbg_provider = get_data_provider() + self.bbg_provider.frequency = Frequency.DAILY except Exception as e: raise self.skipTest(e) diff --git a/qf_lib/tests/integration_tests/data_providers/futures/test_bloomberg_futures.py b/qf_lib/tests/integration_tests/data_providers/futures/test_bloomberg_futures.py index 9f0eacb7..21fedadf 100644 --- a/qf_lib/tests/integration_tests/data_providers/futures/test_bloomberg_futures.py +++ b/qf_lib/tests/integration_tests/data_providers/futures/test_bloomberg_futures.py @@ -33,9 +33,6 @@ def setUpClass(cls): cls.start_date = str_to_date('2008-10-08') cls.end_date = str_to_date('2018-12-20') - cls.timer = SettableTimer() - cls.timer.set_current_time(cls.end_date) - cls.frequency = Frequency.DAILY cls.ticker_1 = BloombergFutureTicker("Euroswiss", "ES{} Index", 1, 3, 100, "HMUZ") cls.ticker_2 = BloombergFutureTicker("Corn", "C {} Comdty", 1, 3, 100, "HKNUZ") @@ -45,13 +42,15 @@ def setUpClass(cls): def setUp(self): try: self.data_provider = get_data_provider() + self.data_provider.set_timer(SettableTimer()) + except Exception as e: raise self.skipTest(e) - self.ticker_1.initialize_data_provider(self.timer, self.data_provider) - self.ticker_2.initialize_data_provider(self.timer, self.data_provider) + self.ticker_1.initialize_data_provider(self.data_provider) + self.ticker_2.initialize_data_provider(self.data_provider) - self.timer.set_current_time(str_to_date("2017-12-20 00:00:00.000000", DateFormat.FULL_ISO)) + self.data_provider.timer.set_current_time(str_to_date("2017-12-20 00:00:00.000000", DateFormat.FULL_ISO)) # =========================== Test get_futures and get_ticker with multiple PriceFields ============================ @@ -62,23 +61,23 @@ def test_get_ticker_1st_contract_1_day_before_exp_date(self): } future_ticker = BloombergFutureTicker("Euroswiss", "ES{} Index", 1, 1, 100, "HMUZ") - future_ticker.initialize_data_provider(self.timer, self.data_provider) + future_ticker.initialize_data_provider(self.data_provider) # Check dates before 2016-12-16 - self.timer.set_current_time(str_to_date('2016-11-11')) + self.data_provider.timer.set_current_time(str_to_date('2016-11-11')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-12-16")]) - self.timer.set_current_time(str_to_date('2016-12-15')) + self.data_provider.timer.set_current_time(str_to_date('2016-12-15')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-12-16")]) - self.timer.set_current_time(str_to_date('2016-12-15 23:55:00.0', DateFormat.FULL_ISO)) + self.data_provider.timer.set_current_time(str_to_date('2016-12-15 23:55:00.0', DateFormat.FULL_ISO)) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-12-16")]) # On the expiry day, the next contract should be returned - self.timer.set_current_time(str_to_date('2016-12-16')) + self.data_provider.timer.set_current_time(str_to_date('2016-12-16')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2017-03-17")]) @@ -89,22 +88,22 @@ def test_get_ticker_1st_contract_6_days_before_exp_date(self): } future_ticker = BloombergFutureTicker("Euroswiss", "ES{} Index", 1, 6, 100, "HMUZ") - future_ticker.initialize_data_provider(self.timer, self.data_provider) + future_ticker.initialize_data_provider(self.data_provider) # Check dates before 2016-12-16 - self.timer.set_current_time(str_to_date('2016-11-11')) + self.data_provider.timer.set_current_time(str_to_date('2016-11-11')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-12-16")]) - self.timer.set_current_time(str_to_date('2016-12-10')) + self.data_provider.timer.set_current_time(str_to_date('2016-12-10')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-12-16")]) - self.timer.set_current_time(str_to_date('2016-12-10 23:55:00.0', DateFormat.FULL_ISO)) + self.data_provider.timer.set_current_time(str_to_date('2016-12-10 23:55:00.0', DateFormat.FULL_ISO)) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-12-16")]) - self.timer.set_current_time(str_to_date('2016-12-16')) + self.data_provider.timer.set_current_time(str_to_date('2016-12-16')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2017-03-17")]) @@ -116,21 +115,21 @@ def test_get_ticker_2nd_contract_1_day_before_exp_date(self): } future_ticker = BloombergFutureTicker("Corn", "C {} Comdty", 2, 1, 100, "HKNUZ") - future_ticker.initialize_data_provider(self.timer, self.data_provider) + future_ticker.initialize_data_provider(self.data_provider) - self.timer.set_current_time(str_to_date('2016-06-03')) + self.data_provider.timer.set_current_time(str_to_date('2016-06-03')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-08-31")]) - self.timer.set_current_time(str_to_date('2016-06-29')) + self.data_provider.timer.set_current_time(str_to_date('2016-06-29')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-08-31")]) - self.timer.set_current_time(str_to_date('2016-06-29 23:59:59.0', DateFormat.FULL_ISO)) + self.data_provider.timer.set_current_time(str_to_date('2016-06-29 23:59:59.0', DateFormat.FULL_ISO)) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-08-31")]) - self.timer.set_current_time(str_to_date('2016-06-30')) + self.data_provider.timer.set_current_time(str_to_date('2016-06-30')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-11-30")]) @@ -142,17 +141,17 @@ def test_get_ticker_2nd_contract_6_days_before_exp_date(self): } future_ticker = BloombergFutureTicker("Corn", "C {} Comdty", 2, 6, 100, "HKNUZ") - future_ticker.initialize_data_provider(self.timer, self.data_provider) + future_ticker.initialize_data_provider(self.data_provider) - self.timer.set_current_time(str_to_date('2016-06-03')) + self.data_provider.timer.set_current_time(str_to_date('2016-06-03')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-08-31")]) - self.timer.set_current_time(str_to_date('2016-06-24 23:59:59.0', DateFormat.FULL_ISO)) + self.data_provider.timer.set_current_time(str_to_date('2016-06-24 23:59:59.0', DateFormat.FULL_ISO)) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-08-31")]) - self.timer.set_current_time(str_to_date('2016-07-09')) + self.data_provider.timer.set_current_time(str_to_date('2016-07-09')) self.assertEqual(future_ticker.get_current_specific_ticker(), exp_dates_to_ticker_str[str_to_date("2016-11-30")]) diff --git a/qf_lib/tests/integration_tests/data_providers/futures/test_general_price_provider.py b/qf_lib/tests/integration_tests/data_providers/futures/test_general_price_provider.py index 850d5170..8807dbdd 100644 --- a/qf_lib/tests/integration_tests/data_providers/futures/test_general_price_provider.py +++ b/qf_lib/tests/integration_tests/data_providers/futures/test_general_price_provider.py @@ -40,10 +40,11 @@ def setUpClass(cls): BloombergFutureTicker("Cotton", "CT{} Comdty", 1, 3), BloombergFutureTicker("Corn", 'C {} Comdty', 1, 5, 50, "HMUZ")] + data_provider.set_timer(timer) timer.set_current_time(str_to_date('2017-12-20')) for ticker in cls.tickers: - ticker.initialize_data_provider(timer, bbg_provider) + ticker.initialize_data_provider(bbg_provider) cls.start_date = str_to_date('2015-10-08') cls.end_date = str_to_date('2017-12-20') diff --git a/qf_lib/tests/integration_tests/data_providers/futures/test_preset_data_provider_futures.py b/qf_lib/tests/integration_tests/data_providers/futures/test_preset_data_provider_futures.py index 01748bc1..3e65e9aa 100644 --- a/qf_lib/tests/integration_tests/data_providers/futures/test_preset_data_provider_futures.py +++ b/qf_lib/tests/integration_tests/data_providers/futures/test_preset_data_provider_futures.py @@ -13,7 +13,8 @@ # limitations under the License. import unittest -from qf_lib.backtesting.data_handler.daily_data_handler import DailyDataHandler +from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent +from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent from qf_lib.common.enums.expiration_date_field import ExpirationDateField from qf_lib.common.enums.frequency import Frequency from qf_lib.common.enums.price_field import PriceField @@ -33,8 +34,6 @@ def setUpClass(cls) -> None: cls.end_date = str_to_date('2015-10-08') cls.start_date = cls.end_date - RelativeDelta(years=2) - cls.timer = SettableTimer(cls.end_date) - cls.frequency = Frequency.DAILY cls.TICKER_1 = BloombergFutureTicker("Cotton", "CT{} Comdty", 1, 3) cls.TICKER_2 = BloombergFutureTicker("Corn", 'C {} Comdty', 1, 5) @@ -42,29 +41,33 @@ def setUpClass(cls) -> None: def setUp(self): try: self.data_provider = get_data_provider() + self.data_provider.set_timer(SettableTimer(self.end_date)) except Exception as e: raise self.skipTest(e) - self.TICKER_1.initialize_data_provider(self.timer, self.data_provider) - self.TICKER_2.initialize_data_provider(self.timer, self.data_provider) - data_provider = PrefetchingDataProvider(self.data_provider, - self.TICKER_2, - PriceField.ohlcv(), - self.start_date, - self.end_date, - self.frequency) + self.TICKER_1.initialize_data_provider(self.data_provider) + self.TICKER_2.initialize_data_provider(self.data_provider) + + MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 30, "second": 0, "microsecond": 0}) + MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0}) + + self.data_provider = PrefetchingDataProvider(self.data_provider, + self.TICKER_2, + PriceField.ohlcv(), + self.start_date, + self.end_date, + self.frequency, timer=SettableTimer()) - self.timer.set_current_time(self.end_date) - self.data_handler = DailyDataHandler(data_provider, self.timer) + self.data_provider.timer.set_current_time(self.end_date) def test_data_provider_init(self): - self.assertCountEqual(self.data_handler.data_provider.supported_ticker_types(), + self.assertCountEqual(self.data_provider.supported_ticker_types(), {BloombergTicker}) def test_get_futures_chain_1_ticker(self): bbg_fut_chain_tickers = self.data_provider.get_futures_chain_tickers( self.TICKER_2, ExpirationDateField.all_dates()) - preset_fut_chain_tickers = self.data_handler.data_provider.get_futures_chain_tickers( + preset_fut_chain_tickers = self.data_provider.get_futures_chain_tickers( self.TICKER_2, ExpirationDateField.all_dates()) self.assertCountEqual(bbg_fut_chain_tickers[self.TICKER_2], preset_fut_chain_tickers[self.TICKER_2]) @@ -72,7 +75,7 @@ def test_get_futures_chain_1_ticker(self): def test_get_futures_chain_multiple_tickers(self): tickers = [self.TICKER_2] bbg_fut_chain_tickers = self.data_provider.get_futures_chain_tickers(tickers, ExpirationDateField.all_dates()) - preset_fut_chain_tickers = self.data_handler.data_provider.get_futures_chain_tickers( + preset_fut_chain_tickers = self.data_provider.get_futures_chain_tickers( tickers, ExpirationDateField.all_dates()) for ticker in tickers: diff --git a/qf_lib/tests/manual_tests/futures_strategy.py b/qf_lib/tests/manual_tests/futures_strategy.py index 8ed524ca..83cae4f5 100644 --- a/qf_lib/tests/manual_tests/futures_strategy.py +++ b/qf_lib/tests/manual_tests/futures_strategy.py @@ -17,7 +17,6 @@ from qf_lib.backtesting.alpha_model.alpha_model import AlphaModel from qf_lib.backtesting.alpha_model.exposure_enum import Exposure -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.events.time_event.regular_time_event.calculate_and_place_orders_event import \ CalculateAndPlaceOrdersRegularEvent from qf_lib.backtesting.monitoring.backtest_monitor import BacktestMonitorSettings @@ -40,14 +39,12 @@ class SimpleFuturesModel(AlphaModel): def __init__(self, fast_time_period: int, slow_time_period: int, - risk_estimation_factor: float, data_provider: DataHandler): + risk_estimation_factor: float, data_provider: DataProvider): super().__init__(risk_estimation_factor, data_provider) self.fast_time_period = fast_time_period self.slow_time_period = slow_time_period - self.timer = data_provider.timer - self.futures_chain = None # type: FuturesChain self.time_of_opening_position = None # type: datetime self.average_true_range = None # type: float @@ -156,7 +153,7 @@ def run_strategy(data_provider: DataProvider) -> Tuple[float, str]: # ----- build models ----- # model = SimpleFuturesModel(fast_time_period=50, slow_time_period=100, risk_estimation_factor=3, - data_provider=ts.data_handler) + data_provider=ts.data_provider) model_tickers_dict = {model: model_tickers} # ----- start trading ----- # diff --git a/qf_lib/tests/manual_tests/simple_ma_strategy.py b/qf_lib/tests/manual_tests/simple_ma_strategy.py index fb18bc5b..428b3475 100644 --- a/qf_lib/tests/manual_tests/simple_ma_strategy.py +++ b/qf_lib/tests/manual_tests/simple_ma_strategy.py @@ -48,9 +48,8 @@ def __init__(self, ts: BacktestTradingSession): super().__init__(ts) self.broker = ts.broker self.order_factory = ts.order_factory - self.data_handler = ts.data_handler + self.data_provider = ts.data_provider self.position_sizer = ts.position_sizer - self.timer = ts.timer def calculate_and_place_orders(self): self.calculate_signals() @@ -59,7 +58,7 @@ def calculate_signals(self): long_ma_len = 20 short_ma_len = 5 - long_ma_series = self.data_handler.historical_price(self.ticker, PriceField.Close, long_ma_len) + long_ma_series = self.data_provider.historical_price(self.ticker, PriceField.Close, long_ma_len) long_ma_price = long_ma_series.mean() short_ma_series = long_ma_series.tail(short_ma_len) diff --git a/qf_lib/tests/manual_tests/spx_with_stop_loss.py b/qf_lib/tests/manual_tests/spx_with_stop_loss.py index 453ee278..27b5cbe7 100644 --- a/qf_lib/tests/manual_tests/spx_with_stop_loss.py +++ b/qf_lib/tests/manual_tests/spx_with_stop_loss.py @@ -44,15 +44,14 @@ def __init__(self, ts: BacktestTradingSession): super().__init__(ts) self.broker = ts.broker self.order_factory = ts.order_factory - self.data_handler = ts.data_handler + self.data_provider = ts.data_provider self.position_sizer = ts.position_sizer - self.timer = ts.timer def calculate_and_place_orders(self): self.calculate_signals() def calculate_signals(self): - last_price = self.data_handler.get_last_available_price(self.ticker, Frequency.DAILY) + last_price = self.data_provider.get_last_available_price(self.ticker, Frequency.DAILY) orders = self.order_factory.target_percent_orders({self.ticker: 1.0}, MarketOrder(), time_in_force=TimeInForce.OPG, tolerance_percentage=0.02) diff --git a/qf_lib/tests/unit_tests/backtesting/alpha_model/test_alpha_model.py b/qf_lib/tests/unit_tests/backtesting/alpha_model/test_alpha_model.py index 86554a57..f3fa5247 100644 --- a/qf_lib/tests/unit_tests/backtesting/alpha_model/test_alpha_model.py +++ b/qf_lib/tests/unit_tests/backtesting/alpha_model/test_alpha_model.py @@ -37,15 +37,15 @@ def setUp(self) -> None: self.frequency = Frequency.DAILY def test_alpha_model__calculate_fraction_at_risk(self): - data_handler = MagicMock() + data_provider = MagicMock() prices_df = PricesDataFrame.from_records(data=[(6.0, 4.0, 5.0) for _ in range(10)], index=date_range(self.start_time, self.end_time), columns=[PriceField.High, PriceField.Low, PriceField.Close]) - data_handler.historical_price.return_value = prices_df + data_provider.historical_price.return_value = prices_df atr = average_true_range(prices_df, normalized=True) risk_estimation_factor = 3 - alpha_model = AlphaModel(risk_estimation_factor=risk_estimation_factor, data_provider=data_handler) + alpha_model = AlphaModel(risk_estimation_factor=risk_estimation_factor, data_provider=data_provider) fraction_at_risk = alpha_model.calculate_fraction_at_risk(self.ticker, self.end_time, self.frequency) self.assertEqual(risk_estimation_factor * atr, fraction_at_risk) diff --git a/qf_lib/tests/unit_tests/backtesting/alpha_model/test_alpha_model_strategy.py b/qf_lib/tests/unit_tests/backtesting/alpha_model/test_alpha_model_strategy.py index 2907156a..64177d29 100644 --- a/qf_lib/tests/unit_tests/backtesting/alpha_model/test_alpha_model_strategy.py +++ b/qf_lib/tests/unit_tests/backtesting/alpha_model/test_alpha_model_strategy.py @@ -43,7 +43,9 @@ def setUp(self) -> None: # Mock trading session self.ts = MagicMock() - self.ts.timer = SettableTimer(str_to_date("2000-01-04 08:00:00.0", DateFormat.FULL_ISO)) + data_provider = MagicMock() + data_provider.timer = SettableTimer(str_to_date("2000-01-04 08:00:00.0", DateFormat.FULL_ISO)) + self.ts.data_provider = data_provider self.ts.frequency = Frequency.DAILY self.positions_in_portfolio = [] # type: List[BacktestPosition] @@ -62,7 +64,7 @@ def test__get_current_exposure(self): use_stop_losses=False) # In case of empty portfolio get_signal function should have current exposure set to OUT alpha_model_strategy.calculate_and_place_orders() - self.alpha_model.get_signal.assert_called_with(self.ticker, Exposure.OUT, self.ts.timer.now(), Frequency.DAILY) + self.alpha_model.get_signal.assert_called_with(self.ticker, Exposure.OUT, self.ts.data_provider.timer.now(), Frequency.DAILY) # Open long position in the portfolio self.positions_in_portfolio = [Mock(spec=BacktestPosition, **{ @@ -71,7 +73,7 @@ def test__get_current_exposure(self): 'start_time': str_to_date("2000-01-01") })] alpha_model_strategy.calculate_and_place_orders() - self.alpha_model.get_signal.assert_called_with(self.ticker, Exposure.LONG, self.ts.timer.now(), Frequency.DAILY) + self.alpha_model.get_signal.assert_called_with(self.ticker, Exposure.LONG, self.ts.data_provider.timer.now(), Frequency.DAILY) # Open short position in the portfolio self.positions_in_portfolio = [Mock(spec=BacktestPosition, **{ @@ -80,7 +82,7 @@ def test__get_current_exposure(self): 'start_time': str_to_date("2000-01-01") })] alpha_model_strategy.calculate_and_place_orders() - self.alpha_model.get_signal.assert_called_with(self.ticker, Exposure.SHORT, self.ts.timer.now(), Frequency.DAILY) + self.alpha_model.get_signal.assert_called_with(self.ticker, Exposure.SHORT, self.ts.data_provider.timer.now(), Frequency.DAILY) # Verify if in case of two positions for the same ticker an exception will be raised by the strategy self.positions_in_portfolio = [BacktestPositionFactory.create_position(c) for c in ( @@ -102,7 +104,7 @@ def test__get_current_exposure__future_ticker(self): # In case of empty portfolio get_signal function should have current exposure set to OUT futures_alpha_model_strategy.calculate_and_place_orders() expected_current_exposure_values.append(Exposure.OUT) - self.alpha_model.get_signal.assert_called_with(self.future_ticker, Exposure.OUT, self.ts.timer.now(), + self.alpha_model.get_signal.assert_called_with(self.future_ticker, Exposure.OUT, self.ts.data_provider.timer.now(), Frequency.DAILY) self.positions_in_portfolio = [Mock(spec=BacktestPosition, **{ @@ -111,7 +113,7 @@ def test__get_current_exposure__future_ticker(self): 'start_time': str_to_date("2000-01-01") })] futures_alpha_model_strategy.calculate_and_place_orders() - self.alpha_model.get_signal.assert_called_with(self.future_ticker, Exposure.LONG, self.ts.timer.now(), + self.alpha_model.get_signal.assert_called_with(self.future_ticker, Exposure.LONG, self.ts.data_provider.timer.now(), Frequency.DAILY) self.positions_in_portfolio = [BacktestPositionFactory.create_position(c) for c in (ticker, ticker)] @@ -136,7 +138,7 @@ def test__get_current_exposure__future_ticker_rolling(self, generate_close_order })] futures_alpha_model_strategy.calculate_and_place_orders() - self.alpha_model.get_signal.assert_called_once_with(self.future_ticker, Exposure.LONG, self.ts.timer.now(), + self.alpha_model.get_signal.assert_called_once_with(self.future_ticker, Exposure.LONG, self.ts.data_provider.timer.now(), Frequency.DAILY) self.positions_in_portfolio = [Mock(spec=BacktestPosition, **{ @@ -173,7 +175,7 @@ def test__get_current_exposure__future_ticker_rolling_2(self, generate_close_ord 'start_time': str_to_date("2000-01-02") })] futures_alpha_model_strategy.calculate_and_place_orders() - self.alpha_model.get_signal.assert_called_once_with(self.future_ticker, Exposure.LONG, self.ts.timer.now(), + self.alpha_model.get_signal.assert_called_once_with(self.future_ticker, Exposure.LONG, self.ts.data_provider.timer.now(), Frequency.DAILY) @patch.object(FuturesRollingOrdersGenerator, 'generate_close_orders') diff --git a/qf_lib/tests/unit_tests/backtesting/data_handler/test_daily_data_handler.py b/qf_lib/tests/unit_tests/backtesting/data_handler/test_daily_data_handler.py index e76a1411..6cf89cda 100644 --- a/qf_lib/tests/unit_tests/backtesting/data_handler/test_daily_data_handler.py +++ b/qf_lib/tests/unit_tests/backtesting/data_handler/test_daily_data_handler.py @@ -14,40 +14,35 @@ from datetime import datetime from unittest import TestCase -from unittest.mock import Mock from numpy import nan -from pandas import date_range, isnull, Index, DatetimeIndex, concat +from pandas import date_range, isnull, Index, DatetimeIndex -from qf_lib.backtesting.data_handler.daily_data_handler import DailyDataHandler from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent from qf_lib.common.enums.frequency import Frequency from qf_lib.common.enums.price_field import PriceField -from qf_lib.common.tickers.tickers import QuandlTicker, Ticker +from qf_lib.common.tickers.tickers import QuandlTicker from qf_lib.common.utils.dateutils.date_format import DateFormat -from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta from qf_lib.common.utils.dateutils.string_to_date import str_to_date from qf_lib.common.utils.dateutils.timer import SettableTimer -from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame from qf_lib.containers.dimension_names import FIELDS, TICKERS, DATES from qf_lib.containers.qf_data_array import QFDataArray from qf_lib.containers.series.prices_series import PricesSeries from qf_lib.containers.series.qf_series import QFSeries -from qf_lib.data_providers.data_provider import DataProvider from qf_lib.data_providers.helpers import normalize_data_array +from qf_lib.data_providers.preset_data_provider import PresetDataProvider from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_series_equal, assert_dataframes_equal, \ assert_dataarrays_equal -class TestDailyDataHandler(TestCase): +class TestDataProviderDailyFrequencyForLookaheadBias(TestCase): def setUp(self): self.timer = SettableTimer() self.tickers = [QuandlTicker("MSFT", "WIKI"), QuandlTicker("AAPL", "WIKI")] - price_data_provider_mock = self._create_price_provider_mock(self.tickers) - self.data_handler = DailyDataHandler(price_data_provider_mock, self.timer) + self.data_provider = self._create_price_provider_mock(self.tickers, self.timer) MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 30, "second": 0, "microsecond": 0}) MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0}) @@ -583,14 +578,14 @@ def _assert_last_prices_are_correct(self, curr_time_str, expected_values): current_time = str_to_date(curr_time_str, DateFormat.FULL_ISO) self.timer.set_current_time(current_time) expected_series = PricesSeries(data=expected_values, index=self.tickers) - actual_series = self.data_handler.get_last_available_price(self.tickers) + actual_series = self.data_provider.get_last_available_price(self.tickers) assert_series_equal(expected_series, actual_series, check_names=False) def _assert_get_prices_are_correct(self, curr_time_str, start_date, end_date, tickers, fields, expected_result): current_time = str_to_date(curr_time_str, DateFormat.FULL_ISO) self.timer.set_current_time(current_time) - actual_prices = self.data_handler.get_price(tickers, fields, start_date, end_date) + actual_prices = self.data_provider.get_price(tickers, fields, start_date, end_date) self.assertEqual(type(expected_result), type(actual_prices)) if isinstance(expected_result, QFSeries): @@ -604,7 +599,7 @@ def _assert_get_prices_are_correct(self, curr_time_str, start_date, end_date, ti else: self.assertEqual(expected_result, actual_prices) - def _create_price_provider_mock(self, tickers): + def _create_price_provider_mock(self, tickers, timer): mock_data_array = QFDataArray.create( dates=date_range(start='2009-12-28', end='2010-01-02', freq='D'), tickers=tickers, @@ -649,27 +644,4 @@ def _create_price_provider_mock(self, tickers): ] ) - def get_price(t, fields, start_time, end_time, _): - got_single_date = start_time.date() == end_time.date() - fields, got_single_field = convert_to_list(fields, PriceField) - t, got_single_ticker = convert_to_list(t, Ticker) - - return normalize_data_array(mock_data_array.loc[start_time:end_time, t, fields], t, fields, - got_single_date, got_single_ticker, got_single_field, use_prices_types=True) - - def get_last_available_price(t, _, end_time: datetime = None): - open_prices = mock_data_array.loc[:, t, PriceField.Open].to_pandas() - open_prices.index = [ind + MarketOpenEvent.trigger_time() for ind in open_prices.index] - close_prices = mock_data_array.loc[:, t, PriceField.Close].to_pandas() - close_prices.index = [ind + MarketCloseEvent.trigger_time() for ind in close_prices.index] - prices = PricesDataFrame(concat([open_prices, close_prices])).sort_index().ffill() - - end_date = end_time + RelativeDelta(days=1, hour=0, minute=0, second=0, microsecond=0, microseconds=-1) - prices = prices.loc[:end_date] - return prices.iloc[-1] if not prices.empty else PricesSeries(index=t, data=nan) - - price_data_provider_mock = Mock(spec=DataProvider, frequency=Frequency.DAILY) - price_data_provider_mock.get_price.side_effect = get_price - price_data_provider_mock.get_last_available_price.side_effect = get_last_available_price - - return price_data_provider_mock + return PresetDataProvider(mock_data_array, datetime(2009, 12, 1), datetime(2010, 1, 2), Frequency.DAILY, timer=timer) diff --git a/qf_lib/tests/unit_tests/backtesting/data_handler/test_intraday_data_handler.py b/qf_lib/tests/unit_tests/backtesting/data_handler/test_intraday_data_handler.py index d6bac7ad..46d0a293 100644 --- a/qf_lib/tests/unit_tests/backtesting/data_handler/test_intraday_data_handler.py +++ b/qf_lib/tests/unit_tests/backtesting/data_handler/test_intraday_data_handler.py @@ -11,52 +11,43 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import unittest from datetime import datetime from unittest import TestCase -from unittest.mock import Mock from numpy import nan, concatenate -from pandas import date_range, isnull, concat, Index, DatetimeIndex +from pandas import date_range, isnull, Index, DatetimeIndex -from qf_lib.backtesting.data_handler.intraday_data_handler import IntradayDataHandler from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent from qf_lib.common.enums.frequency import Frequency from qf_lib.common.enums.price_field import PriceField -from qf_lib.common.tickers.tickers import QuandlTicker, Ticker +from qf_lib.common.tickers.tickers import QuandlTicker from qf_lib.common.utils.dateutils.date_format import DateFormat -from qf_lib.common.utils.dateutils.relative_delta import RelativeDelta from qf_lib.common.utils.dateutils.string_to_date import str_to_date from qf_lib.common.utils.dateutils.timer import SettableTimer -from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame from qf_lib.containers.dimension_names import TICKERS, DATES, FIELDS from qf_lib.containers.qf_data_array import QFDataArray from qf_lib.containers.series.prices_series import PricesSeries from qf_lib.containers.series.qf_series import QFSeries -from qf_lib.data_providers.data_provider import DataProvider from qf_lib.data_providers.helpers import normalize_data_array +from qf_lib.data_providers.preset_data_provider import PresetDataProvider from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_series_equal, assert_dataframes_equal, \ assert_dataarrays_equal -class TestIntradayDataHandler(TestCase): +class TestDataProviderIntradayForLookaheadBias(TestCase): def setUp(self): self.timer = SettableTimer() self.tickers = [QuandlTicker("MSFT", "WIKI"), QuandlTicker("AAPL", "WIKI")] - price_data_provider_mock_MIN_1 = self._create_price_provider_mock(self.tickers, frequency=Frequency.MIN_1) - price_data_provider_mock_MIN_5 = self._create_price_provider_mock(self.tickers, frequency=Frequency.MIN_5) - price_data_provider_mock_MIN_60 = self._create_price_provider_mock(self.tickers, frequency=Frequency.MIN_60) + self.data_provider_MIN_1 = self._create_price_provider_mock(self.tickers, Frequency.MIN_1, self.timer) + self.data_provider_MIN_5 = self._create_price_provider_mock(self.tickers, Frequency.MIN_5, self.timer) + self.data_provider_MIN_60 = self._create_price_provider_mock(self.tickers, Frequency.MIN_60, self.timer) - self.data_handler_MIN_1 = IntradayDataHandler(price_data_provider_mock_MIN_1, self.timer) - self.data_handler_MIN_5 = IntradayDataHandler(price_data_provider_mock_MIN_5, self.timer) - self.data_handler_MIN_60 = IntradayDataHandler(price_data_provider_mock_MIN_60, self.timer) - - self.data_handlers = {Frequency.MIN_1: self.data_handler_MIN_1, - Frequency.MIN_5: self.data_handler_MIN_5, - Frequency.MIN_60: self.data_handler_MIN_60} + self.data_providers = {Frequency.MIN_1: self.data_provider_MIN_1, + Frequency.MIN_5: self.data_provider_MIN_5, + Frequency.MIN_60: self.data_provider_MIN_60} MarketOpenEvent.set_trigger_time({"hour": 8, "minute": 0, "second": 0, "microsecond": 0}) MarketCloseEvent.set_trigger_time({"hour": 16, "minute": 0, "second": 0, "microsecond": 0}) @@ -250,7 +241,8 @@ def test_get_price_after_market_close_single_date_single_field_multiple_tickers( self._assert_get_prices_are_correct("2009-12-30 16:03:00.000000", start_date, end_date, self.tickers, PriceField.Open, expected_result, frequency=Frequency.MIN_1) - expected_result = PricesSeries(index=Index(self.tickers, name=TICKERS), name=PriceField.Close, data=[45.65, nan]) + expected_result = PricesSeries(index=Index(self.tickers, name=TICKERS), name=PriceField.Close, + data=[45.65, nan]) self._assert_get_prices_are_correct("2009-12-30 16:03:00.000000", start_date, end_date, self.tickers, PriceField.Close, expected_result, frequency=Frequency.MIN_1) @@ -270,7 +262,8 @@ def test_get_price_before_market_close_during_bar_single_date_single_field_multi self._assert_get_prices_are_correct("2009-12-30 15:56:00.100000", start_date, end_date, self.tickers, PriceField.Open, expected_result, frequency=Frequency.MIN_1) - def test_get_price_before_market_close_during_bar_single_date_single_field_multiple_tickers_empty_container_freq5(self): + def test_get_price_before_market_close_during_bar_single_date_single_field_multiple_tickers_empty_container_freq5( + self): start_date = datetime(2009, 12, 30, 15, 56) end_date = datetime(2009, 12, 30, 15, 56) @@ -278,7 +271,8 @@ def test_get_price_before_market_close_during_bar_single_date_single_field_multi self._assert_get_prices_are_correct("2009-12-30 15:57:00.100000", start_date, end_date, self.tickers, PriceField.Open, expected_result, frequency=Frequency.MIN_5) - def test_get_price_before_market_close_during_bar_single_date_single_field_multiple_tickers_empty_container_freq60(self): + def test_get_price_before_market_close_during_bar_single_date_single_field_multiple_tickers_empty_container_freq60( + self): start_date = datetime(2009, 12, 30, 15, 56) end_date = datetime(2009, 12, 30, 15, 56) @@ -303,7 +297,8 @@ def test_get_price_before_market_close_single_date_multiple_fields_single_ticker expected_result = PricesSeries(index=Index([PriceField.Open, PriceField.Close], name=FIELDS), name=self.tickers[0].as_string(), data=[45.1, 45.3]) self._assert_get_prices_are_correct("2009-12-30 15:58:00.000000", start_date, end_date, self.tickers[0], - [PriceField.Open, PriceField.Close], expected_result, frequency=Frequency.MIN_1) + [PriceField.Open, PriceField.Close], expected_result, + frequency=Frequency.MIN_1) def test_get_price_after_market_close_single_date_multiple_fields_single_ticker(self): start_date = datetime(2009, 12, 30, 16, 2) @@ -312,7 +307,8 @@ def test_get_price_after_market_close_single_date_multiple_fields_single_ticker( expected_result = PricesSeries(index=Index([PriceField.Open, PriceField.Close], name=FIELDS), name=self.tickers[0].as_string(), data=[45.7, 45.65]) self._assert_get_prices_are_correct("2009-12-30 16:03:00.000000", start_date, end_date, self.tickers[0], - [PriceField.Open, PriceField.Close], expected_result, frequency=Frequency.MIN_1) + [PriceField.Open, PriceField.Close], expected_result, + frequency=Frequency.MIN_1) def test_get_price_after_market_close_single_date_multiple_fields_single_ticker_empty_container(self): start_date = datetime(2009, 12, 30, 16, 4) @@ -321,7 +317,8 @@ def test_get_price_after_market_close_single_date_multiple_fields_single_ticker_ expected_result = PricesSeries(index=Index([PriceField.Open, PriceField.Close], name=FIELDS), name=self.tickers[0].as_string()) self._assert_get_prices_are_correct("2009-12-30 16:04:00.000000", start_date, end_date, self.tickers[0], - [PriceField.Open, PriceField.Close], expected_result, frequency=Frequency.MIN_1) + [PriceField.Open, PriceField.Close], expected_result, + frequency=Frequency.MIN_1) """ Test get_price function - single date, multiple fields, multiple tickers """ @@ -333,7 +330,8 @@ def test_get_price_before_market_close_single_date_multiple_fields_multiple_tick index=Index(self.tickers, name=TICKERS), data=[[45.1, 45.3], [nan, nan]]) self._assert_get_prices_are_correct("2009-12-30 15:58:00.000000", start_date, end_date, self.tickers, - [PriceField.Open, PriceField.Close], expected_result, frequency=Frequency.MIN_1) + [PriceField.Open, PriceField.Close], expected_result, + frequency=Frequency.MIN_1) def test_get_price_before_market_close_single_date_multiple_fields_multiple_tickers_empty_container(self): start_date = datetime(2009, 12, 30, 16, 4) @@ -343,7 +341,8 @@ def test_get_price_before_market_close_single_date_multiple_fields_multiple_tick index=Index(self.tickers, name=TICKERS), data=None) self._assert_get_prices_are_correct("2009-12-30 16:04:00.000000", start_date, end_date, self.tickers, - [PriceField.Open, PriceField.Close], expected_result, frequency=Frequency.MIN_1) + [PriceField.Open, PriceField.Close], expected_result, + frequency=Frequency.MIN_1) def test_get_price_after_market_close_single_date_multiple_fields_multiple_tickers(self): start_date = datetime(2009, 12, 30, 16, 2) @@ -353,7 +352,8 @@ def test_get_price_after_market_close_single_date_multiple_fields_multiple_ticke index=Index(self.tickers, name=TICKERS), data=[[45.7, 45.65], [nan, nan]]) self._assert_get_prices_are_correct("2009-12-30 16:03:00.000000", start_date, end_date, self.tickers, - [PriceField.Open, PriceField.Close], expected_result, frequency=Frequency.MIN_1) + [PriceField.Open, PriceField.Close], expected_result, + frequency=Frequency.MIN_1) def test_get_price_after_market_close_during_bar_single_date_multiple_fields_multiple_tickers(self): start_date = datetime(2009, 12, 30, 16, 2) @@ -363,7 +363,8 @@ def test_get_price_after_market_close_during_bar_single_date_multiple_fields_mul index=Index(self.tickers, name=TICKERS), data=[[45.7, 45.65], [nan, nan]]) self._assert_get_prices_are_correct("2009-12-30 16:03:00.100000", start_date, end_date, self.tickers, - [PriceField.Open, PriceField.Close], expected_result, frequency=Frequency.MIN_1) + [PriceField.Open, PriceField.Close], expected_result, + frequency=Frequency.MIN_1) def test_get_price_after_market_close_during_bar_single_date_multiple_fields_multiple_tickers_freq5(self): start_date = datetime(2009, 12, 30, 16, 10) @@ -373,7 +374,8 @@ def test_get_price_after_market_close_during_bar_single_date_multiple_fields_mul index=Index(self.tickers, name=TICKERS), data=[[45.7, 45.65], [nan, nan]]) self._assert_get_prices_are_correct("2009-12-30 16:15:00.100000", start_date, end_date, self.tickers, - [PriceField.Open, PriceField.Close], expected_result, frequency=Frequency.MIN_5) + [PriceField.Open, PriceField.Close], expected_result, + frequency=Frequency.MIN_5) def test_get_price_after_market_close_during_bar_single_date_multiple_fields_multiple_tickers_freq60(self): start_date = datetime(2009, 12, 30, 17) @@ -383,7 +385,8 @@ def test_get_price_after_market_close_during_bar_single_date_multiple_fields_mul index=Index(self.tickers, name=TICKERS), data=[[45.6, 46.2], [30.1, 30.3]]) self._assert_get_prices_are_correct("2009-12-30 18:00:00.100000", start_date, end_date, self.tickers, - [PriceField.Open, PriceField.Close], expected_result, frequency=Frequency.MIN_60) + [PriceField.Open, PriceField.Close], expected_result, + frequency=Frequency.MIN_60) def test_get_price_after_market_close_single_date_multiple_fields_multiple_tickers_empty_container(self): start_date = datetime(2009, 12, 30, 16, 4) @@ -393,7 +396,8 @@ def test_get_price_after_market_close_single_date_multiple_fields_multiple_ticke index=Index(self.tickers, name=TICKERS), data=None) self._assert_get_prices_are_correct("2009-12-30 16:04:00.000000", start_date, end_date, self.tickers, - [PriceField.Open, PriceField.Close], expected_result, frequency=Frequency.MIN_1) + [PriceField.Open, PriceField.Close], expected_result, + frequency=Frequency.MIN_1) """ Test get_price function - multiple dates, single field, single ticker """ @@ -502,7 +506,8 @@ def test_get_price_after_market_close_multiple_dates_single_field_multiple_ticke end_date = datetime(2009, 12, 30, 16, 10) fields = PriceField.Open - expected_result = PricesDataFrame(columns=Index(self.tickers, name=TICKERS), index=DatetimeIndex([], name=DATES)) + expected_result = PricesDataFrame(columns=Index(self.tickers, name=TICKERS), + index=DatetimeIndex([], name=DATES)) expected_result.name = fields self._assert_get_prices_are_correct("2009-12-30 16:15:00.000000", start_date, end_date, self.tickers, fields, expected_result, frequency=Frequency.MIN_1) @@ -527,7 +532,8 @@ def test_get_price_before_data_during_bar_multiple_dates_multiple_fields_single_ end_date = datetime(2009, 12, 30, 16, 0) expected_result = PricesDataFrame(columns=Index([PriceField.Open], name=FIELDS), - index=DatetimeIndex([datetime(2009, 12, 30, 15, 57)], name=DATES), data=[45.1]) + index=DatetimeIndex([datetime(2009, 12, 30, 15, 57)], name=DATES), + data=[45.1]) expected_result.name = self.tickers[0].as_string() self._assert_get_prices_are_correct("2009-12-30 15:58:59.000000", start_date, end_date, self.tickers[0], [PriceField.Open], expected_result, frequency=Frequency.MIN_1) @@ -537,7 +543,8 @@ def test_get_price_before_data_during_bar_multiple_dates_multiple_fields_single_ end_date = datetime(2009, 12, 30, 16, 0) expected_result = PricesDataFrame(columns=Index([PriceField.Open], name=FIELDS), - index=DatetimeIndex([datetime(2009, 12, 30, 15, 45)], name=DATES), data=[45.1]) + index=DatetimeIndex([datetime(2009, 12, 30, 15, 45)], name=DATES), + data=[45.1]) expected_result.name = self.tickers[0].as_string() self._assert_get_prices_are_correct("2009-12-30 15:51:59.000000", start_date, end_date, self.tickers[0], [PriceField.Open], expected_result, frequency=Frequency.MIN_5) @@ -654,7 +661,7 @@ def _assert_last_prices_are_correct(self, curr_time_str, expected_values, freque current_time = str_to_date(curr_time_str, DateFormat.FULL_ISO) self.timer.set_current_time(current_time) expected_series = PricesSeries(data=expected_values, index=self.tickers) - actual_series = self.data_handlers[frequency].get_last_available_price(self.tickers) + actual_series = self.data_providers[frequency].get_last_available_price(self.tickers) assert_series_equal(expected_series, actual_series, check_names=False) def _assert_last_prices_are_correct_with_end_date(self, curr_time_str, end_date, expected_values, frequency): @@ -662,13 +669,14 @@ def _assert_last_prices_are_correct_with_end_date(self, curr_time_str, end_date, end_date = str_to_date(end_date, DateFormat.FULL_ISO) self.timer.set_current_time(current_time) expected_series = PricesSeries(data=expected_values, index=self.tickers) - actual_series = self.data_handlers[frequency].get_last_available_price(self.tickers, end_time=end_date) + actual_series = self.data_providers[frequency].get_last_available_price(self.tickers, end_time=end_date) assert_series_equal(expected_series, actual_series, check_names=False) - def _assert_get_prices_are_correct(self, curr_time_str, start_date, end_date, tickers, fields, expected_result, frequency): + def _assert_get_prices_are_correct(self, curr_time_str, start_date, end_date, tickers, fields, expected_result, + frequency): current_time = str_to_date(curr_time_str, DateFormat.FULL_ISO) self.timer.set_current_time(current_time) - actual_prices = self.data_handlers[frequency].get_price(tickers, fields, start_date, end_date) + actual_prices = self.data_providers[frequency].get_price(tickers, fields, start_date, end_date) self.assertEqual(type(expected_result), type(actual_prices)) if isinstance(expected_result, QFSeries): @@ -682,7 +690,7 @@ def _assert_get_prices_are_correct(self, curr_time_str, start_date, end_date, ti else: self.assertEqual(expected_result, actual_prices) - def _create_price_provider_mock(self, tickers, frequency): + def _create_price_provider_mock(self, tickers, frequency, timer): if frequency == Frequency.MIN_1: dates = concatenate([ @@ -798,30 +806,5 @@ def _create_price_provider_mock(self, tickers, frequency): ] ) - def get_price(t, fields, start_date, end_date, frequency): - got_single_date = start_date + frequency.time_delta() > end_date - fields, got_single_field = convert_to_list(fields, PriceField) - t, got_single_ticker = convert_to_list(t, Ticker) - - return normalize_data_array(mock_data_array.loc[start_date:end_date, t, fields], t, fields, - got_single_date, got_single_ticker, got_single_field, use_prices_types=True) - - def get_last_available_price(t, freq, end_time: datetime = None): - open_prices = mock_data_array.loc[:, t, PriceField.Open].to_pandas() - open_prices.index = [ind for ind in open_prices.index] - close_prices = mock_data_array.loc[:, t, PriceField.Close].to_pandas() - close_prices.index = [ind + freq.time_delta() + RelativeDelta(microseconds=-1) for ind in close_prices.index] - prices = PricesDataFrame(concat([open_prices, close_prices])).sort_index().ffill() - - end_date = end_time + RelativeDelta(second=0, microsecond=0) - prices = prices.loc[:end_date + freq.time_delta() + RelativeDelta(microseconds=-1)] - return prices.iloc[-1] if not prices.empty else PricesSeries(index=t, data=nan) - - price_data_provider_mock = Mock(spec=DataProvider, frequency=frequency) - price_data_provider_mock.get_price.side_effect = get_price - price_data_provider_mock.get_last_available_price.side_effect = get_last_available_price - return price_data_provider_mock - - -if __name__ == '__main__': - unittest.main() + return PresetDataProvider(mock_data_array, datetime(2009, 12, 1), datetime(2009, 12, 31, 23, 59), frequency, + timer=timer) diff --git a/qf_lib/tests/unit_tests/backtesting/portfolio/test_portfolio.py b/qf_lib/tests/unit_tests/backtesting/portfolio/test_portfolio.py index 18e25490..5eb7b0df 100644 --- a/qf_lib/tests/unit_tests/backtesting/portfolio/test_portfolio.py +++ b/qf_lib/tests/unit_tests/backtesting/portfolio/test_portfolio.py @@ -18,7 +18,6 @@ from numpy.core.umath import sign from qf_lib.analysis.trade_analysis.trades_generator import TradesGenerator -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.portfolio.portfolio import Portfolio from qf_lib.backtesting.portfolio.transaction import Transaction from qf_lib.common.enums.security_type import SecurityType @@ -27,6 +26,7 @@ from qf_lib.common.utils.dateutils.timer import SettableTimer from qf_lib.containers.series.prices_series import PricesSeries from qf_lib.containers.series.qf_series import QFSeries +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_series_equal from qf_lib.tests.unit_tests.backtesting.portfolio.dummy_ticker import DummyTicker @@ -51,21 +51,22 @@ def setUpClass(cls): cls.trades_generator = TradesGenerator() def setUp(self) -> None: - self.data_handler_prices = None + self.data_provider_prices = None - def get_portfolio_and_data_handler(self): - data_handler = Mock(spec=DataHandler) - data_handler.get_last_available_price.side_effect = lambda tickers: self.data_handler_prices[tickers] \ + def get_portfolio_and_data_provider(self): + data_provider = Mock(spec=AbstractPriceDataProvider) + data_provider.get_last_available_price.side_effect = lambda tickers: self.data_provider_prices[tickers] \ if tickers else None timer = SettableTimer() timer.set_current_time(self.start_time) + data_provider.timer = timer - portfolio = Portfolio(data_handler, self.initial_cash, timer) - return portfolio, data_handler, timer + portfolio = Portfolio(data_provider, self.initial_cash) + return portfolio, data_provider def test_initial_cash(self): - portfolio, _, _ = self.get_portfolio_and_data_handler() + portfolio, _ = self.get_portfolio_and_data_provider() self.assertEqual(portfolio.initial_cash, self.initial_cash) self.assertEqual(portfolio.current_cash, self.initial_cash) self.assertEqual(portfolio.net_liquidation, self.initial_cash) @@ -76,7 +77,7 @@ def _cash_move(transaction: Transaction): return -1 * transaction.price * transaction.quantity * transaction.ticker.point_value - transaction.commission def test_transact_transaction_1(self): - portfolio, _, _ = self.get_portfolio_and_data_handler() + portfolio, _ = self.get_portfolio_and_data_provider() transaction = Transaction(self.random_time, self.ticker, quantity=50, price=100, commission=5) portfolio.transact_transaction(transaction) @@ -101,14 +102,14 @@ def test_transact_transaction_1(self): self.assertEqual(len(portfolio.open_positions_dict), 1) def test_transact_transaction_2(self): - portfolio, dh, _ = self.get_portfolio_and_data_handler() + portfolio, dh = self.get_portfolio_and_data_provider() # First transaction transaction_1 = Transaction(self.random_time, self.ticker, quantity=50, price=100, commission=5) portfolio.transact_transaction(transaction_1) # Set new prices - self.data_handler_prices = self.prices_series + self.data_provider_prices = self.prices_series portfolio.update() # Get the new price of the contract @@ -143,14 +144,14 @@ def test_transact_transaction_2(self): self.assertEqual(len(portfolio.open_positions_dict), 1) def test_transact_transaction_3(self): - portfolio, dh, _ = self.get_portfolio_and_data_handler() + portfolio, dh = self.get_portfolio_and_data_provider() # First transaction transaction_1 = Transaction(self.random_time, self.fut_ticker, quantity=50, price=200, commission=7) portfolio.transact_transaction(transaction_1) # Set new prices - self.data_handler_prices = self.prices_series + self.data_provider_prices = self.prices_series portfolio.update() # Get the new price of the contract @@ -193,13 +194,13 @@ def test_transact_transaction_3(self): def test_transact_transaction_close_position_2_transactions(self): for quantity in (-50, 50): - portfolio, dh, _ = self.get_portfolio_and_data_handler() + portfolio, dh = self.get_portfolio_and_data_provider() all_commissions = 0.0 transaction_1 = Transaction(self.random_time, self.fut_ticker, quantity=quantity, price=200, commission=7) portfolio.transact_transaction(transaction_1) all_commissions += transaction_1.commission - self.data_handler_prices = self.prices_series + self.data_provider_prices = self.prices_series portfolio.update() new_price = dh.get_last_available_price(self.fut_ticker) @@ -220,7 +221,7 @@ def test_transact_transaction_close_position_2_transactions(self): self.assertEqual(len(portfolio.open_positions_dict), 0) def test_transact_transaction_split_position(self): - portfolio, dh, _ = self.get_portfolio_and_data_handler() + portfolio, dh = self.get_portfolio_and_data_provider() # Transact two transaction, which will result in transactions splitting quantity_after_first_transaction = 50 @@ -234,7 +235,7 @@ def test_transact_transaction_split_position(self): portfolio.transact_transaction(transaction_1) # Set new prices - self.data_handler_prices = self.prices_series + self.data_provider_prices = self.prices_series new_price = dh.get_last_available_price(self.fut_ticker) # == 250 portfolio.update() @@ -261,14 +262,14 @@ def test_transact_transaction_split_position(self): self.assertEqual(len(portfolio.open_positions_dict), 1) def test_transact_transaction_split_and_close(self): - portfolio, dh, _ = self.get_portfolio_and_data_handler() + portfolio, dh = self.get_portfolio_and_data_provider() # Transact the initial transaction transactions = [] quantity = 50 # Set initial price for the given ticker - self.data_handler_prices = self.prices_down + self.data_provider_prices = self.prices_down price_1 = dh.get_last_available_price(self.fut_ticker) # == 210 initial_transaction = Transaction(self.random_time, self.fut_ticker, quantity=quantity, price=price_1, @@ -278,7 +279,7 @@ def test_transact_transaction_split_and_close(self): portfolio.update() # Change of price for the given ticker - self.data_handler_prices = self.prices_series + self.data_provider_prices = self.prices_series price_2 = dh.get_last_available_price(self.fut_ticker) # == 250 transaction_to_split = Transaction(self.random_time, self.fut_ticker, quantity=(-2) * quantity, @@ -292,7 +293,7 @@ def test_transact_transaction_split_and_close(self): trade_pnl -= transaction_to_split.commission * abs(initial_transaction.quantity / transaction_to_split.quantity) # Change of price for the given ticker - self.data_handler_prices = self.prices_series + self.data_provider_prices = self.prices_series price_3 = dh.get_last_available_price(self.fut_ticker) # == 270 closing_transaction = Transaction(self.random_time, self.fut_ticker, quantity=quantity, price=price_3, commission=5) @@ -307,7 +308,8 @@ def test_portfolio_eod_series(self): expected_portfolio_eod_series = PricesSeries() # Empty portfolio - portfolio, dh, timer = self.get_portfolio_and_data_handler() + portfolio, dh = self.get_portfolio_and_data_provider() + timer = dh.timer portfolio.update(record=True) expected_portfolio_eod_series[timer.time] = self.initial_cash @@ -315,7 +317,7 @@ def test_portfolio_eod_series(self): self._shift_timer_to_next_day(timer) transaction_1 = Transaction(timer.time, self.fut_ticker, quantity=50, price=250, commission=7) portfolio.transact_transaction(transaction_1) - self.data_handler_prices = self.prices_series + self.data_provider_prices = self.prices_series portfolio.update(record=True) position = portfolio.open_positions_dict[self.fut_ticker] @@ -327,7 +329,7 @@ def test_portfolio_eod_series(self): # Contract goes up in value self._shift_timer_to_next_day(timer) - self.data_handler_prices = self.prices_up + self.data_provider_prices = self.prices_up portfolio.update(record=True) price_2 = dh.get_last_available_price(self.fut_ticker) # == 270 @@ -339,7 +341,7 @@ def test_portfolio_eod_series(self): self._shift_timer_to_next_day(timer) transaction_2 = Transaction(timer.time, self.fut_ticker, quantity=-25, price=price_2, commission=19) portfolio.transact_transaction(transaction_2) - self.data_handler_prices = self.prices_up + self.data_provider_prices = self.prices_up portfolio.update(record=True) pnl = (transaction_2.price - price_2) * transaction_2.quantity * self.fut_ticker.point_value - \ @@ -349,7 +351,7 @@ def test_portfolio_eod_series(self): # Price goes down self._shift_timer_to_next_day(timer) - self.data_handler_prices = self.prices_down + self.data_provider_prices = self.prices_down portfolio.update(record=True) price_3 = dh.get_last_available_price(self.fut_ticker) # == 210 @@ -366,7 +368,8 @@ def _shift_timer_to_next_day(timer: SettableTimer): timer.set_current_time(new_time) def test_portfolio_leverage1(self): - portfolio, dh, timer = self.get_portfolio_and_data_handler() + portfolio, dh = self.get_portfolio_and_data_provider() + timer = dh.timer expected_leverage_series = QFSeries() nav = self.initial_cash @@ -383,7 +386,7 @@ def record_leverage(): self._shift_timer_to_next_day(timer) transaction_1 = Transaction(timer.now(), self.fut_ticker, quantity=50, price=250, commission=7) portfolio.transact_transaction(transaction_1) - self.data_handler_prices = self.prices_series + self.data_provider_prices = self.prices_series portfolio.update(record=True) position = portfolio.open_positions_dict[self.fut_ticker] @@ -395,7 +398,7 @@ def record_leverage(): # Contract goes up in value self._shift_timer_to_next_day(timer) - self.data_handler_prices = self.prices_up + self.data_provider_prices = self.prices_up portfolio.update(record=True) # Compute the leverage @@ -409,7 +412,7 @@ def record_leverage(): transaction_2 = Transaction(timer.time, self.fut_ticker, quantity=-30, price=position.current_price, commission=9) portfolio.transact_transaction(transaction_2) - self.data_handler_prices = self.prices_up + self.data_provider_prices = self.prices_up portfolio.update(record=True) # Compute the leverage @@ -419,7 +422,7 @@ def record_leverage(): # Price goes down self._shift_timer_to_next_day(timer) - self.data_handler_prices = self.prices_down + self.data_provider_prices = self.prices_down portfolio.update(record=True) pnl = self.fut_ticker.point_value * position.quantity() * (position.current_price - transaction_2.price) @@ -434,7 +437,8 @@ def test_portfolio_leverage2(self): expected_dates = [] # empty portfolio - portfolio, dh, timer = self.get_portfolio_and_data_handler() + portfolio, dh = self.get_portfolio_and_data_provider() + timer = dh.timer portfolio.update(record=True) expected_values.append(0) expected_dates.append(self.start_time) @@ -447,7 +451,7 @@ def test_portfolio_leverage2(self): portfolio.transact_transaction(Transaction(new_time, self.ticker, quantity, price, commission1)) timer.set_current_time(new_time) - self.data_handler_prices = self.prices_series + self.data_provider_prices = self.prices_series portfolio.update(record=True) gross_value = quantity * price @@ -459,7 +463,7 @@ def test_portfolio_leverage2(self): # contract goes up in value new_time = timer.time + RelativeDelta(days=1) timer.set_current_time(new_time) - self.data_handler_prices = self.prices_up + self.data_provider_prices = self.prices_up portfolio.update(record=True) gross_value = quantity * 130 @@ -475,7 +479,7 @@ def test_portfolio_leverage2(self): new_time = timer.time + RelativeDelta(days=1) portfolio.transact_transaction(Transaction(new_time, self.ticker, quantity, price, commission2)) timer.set_current_time(new_time) - self.data_handler_prices = self.prices_up + self.data_provider_prices = self.prices_up portfolio.update(record=True) gross_value = 200 * 130 @@ -486,7 +490,7 @@ def test_portfolio_leverage2(self): # price goes down new_time = timer.time + RelativeDelta(days=1) timer.set_current_time(new_time) - self.data_handler_prices = self.prices_down + self.data_provider_prices = self.prices_down portfolio.update(record=True) gross_value = 200 * 100 @@ -503,7 +507,8 @@ def test_portfolio_leverage2(self): def test_portfolio_history(self): # empty portfolio - portfolio, dh, timer = self.get_portfolio_and_data_handler() + portfolio, dh = self.get_portfolio_and_data_provider() + timer = dh.timer portfolio.update(record=True) # buy contract @@ -514,14 +519,14 @@ def test_portfolio_history(self): portfolio.transact_transaction(Transaction(new_time, self.fut_ticker, quantity, price, commission1)) timer.set_current_time(new_time) - self.data_handler_prices = self.prices_series + self.data_provider_prices = self.prices_series portfolio.update(record=True) # buy another instrument - price goes up portfolio.transact_transaction(Transaction(new_time, self.ticker, 20, 120, commission1)) new_time = timer.time + RelativeDelta(days=1) timer.set_current_time(new_time) - self.data_handler_prices = self.prices_up + self.data_provider_prices = self.prices_up portfolio.update(record=True) # sell part of the contract @@ -531,13 +536,13 @@ def test_portfolio_history(self): new_time = timer.time + RelativeDelta(days=1) portfolio.transact_transaction(Transaction(new_time, self.fut_ticker, quantity, price, commission2)) timer.set_current_time(new_time) - self.data_handler_prices = self.prices_up + self.data_provider_prices = self.prices_up portfolio.update(record=True) # price goes down new_time = timer.time + RelativeDelta(days=1) timer.set_current_time(new_time) - self.data_handler_prices = self.prices_down + self.data_provider_prices = self.prices_down portfolio.update(record=True) asset_history = portfolio.positions_history() diff --git a/qf_lib/tests/unit_tests/backtesting/portfolio/test_trades.py b/qf_lib/tests/unit_tests/backtesting/portfolio/test_trades.py index 7020974f..1c933cd8 100644 --- a/qf_lib/tests/unit_tests/backtesting/portfolio/test_trades.py +++ b/qf_lib/tests/unit_tests/backtesting/portfolio/test_trades.py @@ -32,7 +32,7 @@ def setUpClass(cls): cls.time = str_to_date('2020-01-01') def setUp(self): - self.portfolio = Portfolio(Mock(), 100000, Mock()) + self.portfolio = Portfolio(Mock(), 100000) self.trades_generator = TradesGenerator() def test_long_trade(self): diff --git a/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_market_on_open_execution_style.py b/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_market_on_open_execution_style.py index 33ea0bb9..bf7b9544 100644 --- a/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_market_on_open_execution_style.py +++ b/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_market_on_open_execution_style.py @@ -17,7 +17,6 @@ import pandas as pd -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent from qf_lib.backtesting.events.time_event.scheduler import Scheduler @@ -38,7 +37,7 @@ from qf_lib.common.utils.dateutils.string_to_date import str_to_date from qf_lib.common.utils.dateutils.timer import SettableTimer from qf_lib.containers.series.qf_series import QFSeries -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_lists_equal @@ -55,9 +54,8 @@ def setUp(self): self.msft_ticker = BloombergTicker("MSFT US Equity") - self.data_handler = Mock(spec=DataHandler) - self.data_handler.frequency = Frequency.DAILY - self.data_handler.data_provider = Mock(spec=DataProvider) + self.data_provider = Mock(spec=AbstractPriceDataProvider, timer=self.timer) + self.data_provider.set_frequency(Frequency.DAILY) self.scheduler = Mock(spec=Scheduler) @@ -66,8 +64,8 @@ def setUp(self): self.portfolio = Mock(spec=Portfolio) self.portfolio.open_positions_dict = {} - slippage_model = PriceBasedSlippage(0.0, self.data_handler) - self.exec_handler = SimulatedExecutionHandler(self.data_handler, self.timer, self.scheduler, self.monitor, + slippage_model = PriceBasedSlippage(0.0, self.data_provider) + self.exec_handler = SimulatedExecutionHandler(self.data_provider, self.scheduler, self.monitor, self.commission_model, self.portfolio, slippage_model, RelativeDelta(minutes=self.scheduling_time_delay)) @@ -232,9 +230,9 @@ def test_market_open_does_not_trade(self): self.monitor.record_transaction.assert_not_called() def _set_last_available_price(self, price): - self.data_handler.get_last_available_price.side_effect = lambda t: QFSeries([price], - index=pd.Index([self.msft_ticker])) + self.data_provider.get_last_available_price.side_effect = lambda t: QFSeries([price], + index=pd.Index([self.msft_ticker])) def _set_current_price(self, price): - self.data_handler.data_provider.get_price.side_effect = lambda a, b, c, d, e: \ + self.data_provider.get_price.side_effect = lambda a, b, c, d, e, look_ahead_bias: \ QFSeries(data=[price], index=[self.msft_ticker]) diff --git a/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_simulated_executor.py b/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_simulated_executor.py index 4d3620ea..b7acab73 100644 --- a/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_simulated_executor.py +++ b/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_simulated_executor.py @@ -26,7 +26,7 @@ from qf_lib.common.enums.frequency import Frequency from qf_lib.common.tickers.tickers import BloombergTicker from qf_lib.common.utils.dateutils.string_to_date import str_to_date -from qf_lib.common.utils.dateutils.timer import Timer +from qf_lib.common.utils.dateutils.timer import SettableTimer class TestSimulatedExecutor(unittest.TestCase): @@ -150,10 +150,10 @@ def mock_record_transaction(transaction: Transaction): monitor: AbstractMonitor = MagicMock() monitor.record_transaction.side_effect = mock_record_transaction - timer: Timer = MagicMock() - timer.now.return_value = self.backtest_date + timer = SettableTimer(self.backtest_date) + data_provider = MagicMock(timer=timer) order_id_generator = count(start=1) - simulated_executor = MarketOrdersExecutor(MagicMock(), monitor, MagicMock(), timer, order_id_generator, + simulated_executor = MarketOrdersExecutor(data_provider, monitor, MagicMock(), order_id_generator, commission_model, slippage_model, Frequency.DAILY) return simulated_executor diff --git a/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_slippage.py b/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_slippage.py index a96e8629..d11df1d5 100644 --- a/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_slippage.py +++ b/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_slippage.py @@ -72,7 +72,7 @@ def setUp(self): )] def _create_data_provider_mock(self): - def get_price(tickers, fields, start_date, end_date, _): + def get_price(tickers, fields, start_date, end_date, frequency, look_ahead_bias=False): prices_bar = [5.0, 10.0, 1.0, 4.0, 50] # Open, High, Low, Close, Volume dates_index = pd.date_range(start_date, end_date, freq='B') diff --git a/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_stop_loss_execution_style.py b/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_stop_loss_execution_style.py index ebee407f..8e65b863 100644 --- a/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_stop_loss_execution_style.py +++ b/qf_lib/tests/unit_tests/backtesting/simulated_execution_handler/test_stop_loss_execution_style.py @@ -17,7 +17,6 @@ import pandas as pd -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.events.time_event.periodic_event.intraday_bar_event import IntradayBarEvent from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent @@ -40,7 +39,6 @@ from qf_lib.common.utils.dateutils.timer import SettableTimer from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame from qf_lib.containers.series.qf_series import QFSeries -from qf_lib.data_providers.data_provider import DataProvider from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_lists_equal @@ -58,9 +56,7 @@ def setUp(self): self.msft_ticker = BloombergTicker(self.MSFT_TICKER_STR) self.timer = SettableTimer(initial_time=before_close) - - self.data_handler = Mock(spec=DataHandler) - self.data_handler.data_provider = Mock(spec=DataProvider) + self.data_provider = Mock(timer=self.timer) scheduler = Mock(spec=Scheduler) ScheduleOrderExecutionEvent.clear() @@ -73,8 +69,8 @@ def setUp(self): self.portfolio = Mock(spec=Portfolio) self.portfolio.open_positions_dict = {} - slippage_model = PriceBasedSlippage(0.0, self.data_handler) - self.exec_handler = SimulatedExecutionHandler(self.data_handler, self.timer, scheduler, self.monitor, + slippage_model = PriceBasedSlippage(0.0, self.data_provider) + self.exec_handler = SimulatedExecutionHandler(self.data_provider, scheduler, self.monitor, commission_model, self.portfolio, slippage_model, RelativeDelta(minutes=self.number_of_minutes)) @@ -235,8 +231,8 @@ def result(tickers): else: return QFSeries() - self.data_handler.get_last_available_price.side_effect = result + self.data_provider.get_last_available_price.side_effect = result def _set_bar_for_today(self, open_price, high_price, low_price, close_price, volume): - self.data_handler.get_price.side_effect = lambda tickers, fields, start_date, end_date, frequency: \ + self.data_provider.get_price.side_effect = lambda tickers, fields, start_date, end_date, frequency: \ QFDataFrame(index=tickers, columns=fields, data=[[open_price, high_price, low_price, close_price, volume]]) diff --git a/qf_lib/tests/unit_tests/backtesting/test_order_factory.py b/qf_lib/tests/unit_tests/backtesting/test_order_factory.py index acf6bec1..63318aec 100644 --- a/qf_lib/tests/unit_tests/backtesting/test_order_factory.py +++ b/qf_lib/tests/unit_tests/backtesting/test_order_factory.py @@ -17,7 +17,6 @@ from unittest.mock import Mock from qf_lib.backtesting.broker.broker import Broker -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.order.execution_style import MarketOrder, StopOrder from qf_lib.backtesting.order.order import Order from qf_lib.backtesting.order.order_factory import OrderFactory @@ -25,6 +24,7 @@ from qf_lib.backtesting.portfolio.position import Position from qf_lib.common.tickers.tickers import BloombergTicker, BinanceTicker from qf_lib.containers.series.qf_series import QFSeries +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider class TestOrderFactory(unittest.TestCase): @@ -47,11 +47,11 @@ def setUpClass(cls): broker.get_portfolio_value.return_value = cls.current_portfolio_value broker.get_positions.return_value = [position, crypto_position] - data_handler = Mock(spec=DataHandler) - data_handler.get_last_available_price.side_effect = lambda tickers, _: \ + data_provider = Mock(spec=AbstractPriceDataProvider) + data_provider.get_last_available_price.side_effect = lambda tickers, _: \ QFSeries([cls.share_price] * len(tickers), index=tickers) - cls.order_factory = OrderFactory(broker, data_handler) + cls.order_factory = OrderFactory(broker, data_provider) def test_order(self): quantity = 5 diff --git a/qf_lib/tests/unit_tests/backtesting/test_position_sizer.py b/qf_lib/tests/unit_tests/backtesting/test_position_sizer.py index 0ab2f205..c2b61867 100644 --- a/qf_lib/tests/unit_tests/backtesting/test_position_sizer.py +++ b/qf_lib/tests/unit_tests/backtesting/test_position_sizer.py @@ -23,7 +23,6 @@ from qf_lib.backtesting.signals.backtest_signals_register import BacktestSignalsRegister from qf_lib.backtesting.signals.signal import Signal from qf_lib.backtesting.broker.broker import Broker -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.order.execution_style import MarketOrder, StopOrder from qf_lib.backtesting.order.order import Order from qf_lib.backtesting.order.order_factory import OrderFactory @@ -64,21 +63,21 @@ def setUp(self) -> None: self.broker.get_positions.return_value = [position, crypto_position] self.broker.get_portfolio_value.return_value = self.portfolio_value - data_handler = Mock(spec=DataHandler, timer=self.timer) - data_handler.get_last_available_price.side_effect = lambda _: self.last_price - data_handler.get_price.return_value = self.volume + data_provider = Mock(timer=self.timer) + data_provider.get_last_available_price.side_effect = lambda _: self.last_price + data_provider.get_price.return_value = self.volume order_factory = self._mock_order_factory(self.initial_position, self.initial_allocation) - self.simple_position_sizer = SimplePositionSizer(self.broker, data_handler, order_factory, + self.simple_position_sizer = SimplePositionSizer(self.broker, data_provider, order_factory, BacktestSignalsRegister()) - self.initial_risk_position_sizer = InitialRiskPositionSizer(self.broker, data_handler, order_factory, + self.initial_risk_position_sizer = InitialRiskPositionSizer(self.broker, data_provider, order_factory, BacktestSignalsRegister(), self.initial_risk, self.max_target_percentage) - self.initial_risk_with_volume_position_sizer = InitialRiskWithVolumePositionSizer(self.broker, data_handler, + self.initial_risk_with_volume_position_sizer = InitialRiskWithVolumePositionSizer(self.broker, data_provider, order_factory, BacktestSignalsRegister(), self.initial_risk, self.max_target_percentage) diff --git a/qf_lib/tests/unit_tests/backtesting/test_volume_orders_filter.py b/qf_lib/tests/unit_tests/backtesting/test_volume_orders_filter.py index dcfe0032..9df051c7 100644 --- a/qf_lib/tests/unit_tests/backtesting/test_volume_orders_filter.py +++ b/qf_lib/tests/unit_tests/backtesting/test_volume_orders_filter.py @@ -16,8 +16,6 @@ import pandas as pd -from qf_lib.backtesting.data_handler.daily_data_handler import DailyDataHandler -from qf_lib.backtesting.data_handler.data_handler import DataHandler from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent from qf_lib.backtesting.order.execution_style import MarketOrder, StopOrder from qf_lib.backtesting.order.order import Order @@ -29,6 +27,7 @@ from qf_lib.common.utils.dateutils.string_to_date import str_to_date from qf_lib.common.utils.dateutils.timer import SettableTimer from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame +from qf_lib.data_providers.data_provider import DataProvider from qf_lib.data_providers.helpers import tickers_dict_to_data_array from qf_lib.data_providers.preset_data_provider import PresetDataProvider @@ -43,11 +42,11 @@ def setUpClass(cls) -> None: def test_volume_orders_filter__resize_orders(self): """Tests VolumeOrdersVerifier with orders exceeding the limit.""" - # Setup DataHandler and VolumeOrdersVerifier + # Setup DataProvider and VolumeOrdersVerifier volume_percentage_limit = 0.15 volume_value = 100.0 - data_handler = self._setup_data_handler(volume_value) - volume_orders_verifier = VolumeOrdersFilter(data_handler, volume_percentage_limit) + data_provider = self._setup_data_provider(volume_value) + volume_orders_verifier = VolumeOrdersFilter(data_provider, volume_percentage_limit) # Initialize a list of orders, which exceed the maximum volume limit buy_orders = [Order(self.ticker, 100, MarketOrder(), TimeInForce.GTC)] @@ -64,11 +63,11 @@ def test_volume_orders_filter__resize_orders(self): def test_volume_orders_filter__no_resize_orders(self): """Tests if VolumeOrdersVerifier does not change orders, which do not exceed the limit.""" - # Setup DataHandler and VolumeOrdersVerifier + # Setup DataProvider and VolumeOrdersVerifier volume_percentage_limit = 0.15 volume_value = 100.0 - data_handler = self._setup_data_handler(volume_value) - volume_orders_verifier = VolumeOrdersFilter(data_handler, volume_percentage_limit) + data_provider = self._setup_data_provider(volume_value) + volume_orders_verifier = VolumeOrdersFilter(data_provider, volume_percentage_limit) # Initialize a list of orders, which do not exceed the maximum volume limit max_quantity = int(volume_percentage_limit * volume_value) @@ -81,11 +80,11 @@ def test_volume_orders_filter__no_resize_orders(self): def test_volume_orders_filter__no_volume_data(self): """Tests if VolumeOrdersVerifier does not change orders, which do not have the volume data.""" - # Setup DataHandler and VolumeOrdersVerifier + # Setup DataProvider and VolumeOrdersVerifier volume_percentage_limit = 0.15 volume_value = None - data_handler = self._setup_data_handler(volume_value) - volume_orders_verifier = VolumeOrdersFilter(data_handler, volume_percentage_limit) + data_provider = self._setup_data_provider(volume_value) + volume_orders_verifier = VolumeOrdersFilter(data_provider, volume_percentage_limit) # Initialize a list of orders, which do not exceed the maximum volume limit orders = [Order(self.ticker, 100, MarketOrder(), TimeInForce.GTC), @@ -109,8 +108,8 @@ def test_volume_orders_filter__adjust_buy_stop_orders(self): volume_percentage_limit = 0.15 volume_value = 100.0 - data_handler = self._setup_data_handler(volume_value) - volume_orders_verifier = VolumeOrdersFilter(data_handler, volume_percentage_limit) + data_provider = self._setup_data_provider(volume_value) + volume_orders_verifier = VolumeOrdersFilter(data_provider, volume_percentage_limit) # Initialize a list of orders, which do not exceed the maximum volume limit buy_order = [Order(self.ticker, 100, MarketOrder(), TimeInForce.GTC), @@ -129,8 +128,8 @@ def test_volume_orders_filter__adjust_sell_stop_orders(self): volume_percentage_limit = 0.15 volume_value = 100.0 - data_handler = self._setup_data_handler(volume_value) - volume_orders_verifier = VolumeOrdersFilter(data_handler, volume_percentage_limit) + data_provider = self._setup_data_provider(volume_value) + volume_orders_verifier = VolumeOrdersFilter(data_provider, volume_percentage_limit) # Initialize a list of orders, which do not exceed the maximum volume limit sell_order = [Order(self.ticker, -100, MarketOrder(), TimeInForce.GTC), @@ -142,7 +141,7 @@ def test_volume_orders_filter__adjust_sell_stop_orders(self): Order(self.ticker, 115, StopOrder(1.0), TimeInForce.GTC)] self.assertCountEqual(new_orders, expected_sell_order) - def _setup_data_handler(self, volume_value: Optional[float]) -> DataHandler: + def _setup_data_provider(self, volume_value: Optional[float]) -> DataProvider: dates = pd.date_range(str_to_date("2019-12-01"), str_to_date("2020-01-30"), freq='D') prices_data_frame = QFDataFrame(data={PriceField.Volume: [volume_value] * len(dates)}, index=dates) @@ -151,7 +150,5 @@ def _setup_data_handler(self, volume_value: Optional[float]) -> DataHandler: self.ticker: prices_data_frame, }, [self.ticker], [PriceField.Volume]) - data_provider = PresetDataProvider(prices_data_array, dates[0], dates[-1], Frequency.DAILY) timer = SettableTimer(dates[-1]) - - return DailyDataHandler(data_provider, timer) + return PresetDataProvider(prices_data_array, dates[0], dates[-1], Frequency.DAILY, timer=timer) diff --git a/qf_lib/tests/unit_tests/containers/test_future_ticker.py b/qf_lib/tests/unit_tests/containers/test_future_ticker.py index c6bee916..b83110c4 100644 --- a/qf_lib/tests/unit_tests/containers/test_future_ticker.py +++ b/qf_lib/tests/unit_tests/containers/test_future_ticker.py @@ -70,10 +70,11 @@ def setUp(self): self.timer = SettableTimer(initial_time=str_to_date('2017-01-01')) settings = get_test_settings() self.bbg_provider = BloombergDataProvider(settings) + self.bbg_provider.set_timer(self.timer) def test_valid_ticker_1(self): future_ticker = CustomFutureTicker("Custom", "CT{} Custom", 1, 5, 500) - future_ticker.initialize_data_provider(self.timer, self.bbg_provider) + future_ticker.initialize_data_provider(self.bbg_provider) # '2017-12-15' is the official expiration date of CustomTicker:B, setting the days_before_exp_date equal to # 5 forces the expiration to occur on the 11th ('2017-12-15' - 5 days = '2017-12-10' is the last day of old @@ -91,7 +92,7 @@ def test_valid_ticker_2(self): # Test the 2nd contract instead of front one future_ticker = CustomFutureTicker("Custom", "CT{} Custom", 2, 5, 500) - future_ticker.initialize_data_provider(self.timer, self.bbg_provider) + future_ticker.initialize_data_provider(self.bbg_provider) self.timer.set_current_time(str_to_date('2017-12-05')) self.assertEqual(future_ticker.get_current_specific_ticker(), CustomTicker("C")) @@ -104,7 +105,7 @@ def test_valid_ticker_2(self): def test_valid_ticker_3(self): future_ticker = CustomFutureTicker("Custom", "CT{} Custom", 1, 45, 500) - future_ticker.initialize_data_provider(self.timer, self.bbg_provider) + future_ticker.initialize_data_provider(self.bbg_provider) self.timer.set_current_time(str_to_date('2017-11-28')) # '2017-11-28' + 45 days = '2018-01-12' - the front contract will be equal to CustomTicker:D @@ -119,7 +120,7 @@ def test_valid_ticker_3(self): def test_valid_ticker_4(self): future_ticker = CustomFutureTicker("Custom", "CT{} Custom", 2, 45, 500) - future_ticker.initialize_data_provider(self.timer, self.bbg_provider) + future_ticker.initialize_data_provider(self.bbg_provider) self.timer.set_current_time(str_to_date('2017-11-28')) # '2017-11-28' + 45 days = '2018-01-12' - the front contract will be equal to CustomTicker:D @@ -135,7 +136,7 @@ def test_valid_ticker_4(self): def test_set_expiration_hour__first_caching_before_exp_hour(self): """ Test set expiration hour when the first caching occurs on the expiration day, before expiration hour. """ future_ticker = CustomFutureTicker("Custom", "CT{} Custom", 1, 5, 500) - future_ticker.initialize_data_provider(self.timer, self.bbg_provider) + future_ticker.initialize_data_provider(self.bbg_provider) future_ticker.set_expiration_hour(hour=8, minute=10) self.timer.set_current_time(str_to_date('2017-12-11 00:00:00.0', DateFormat.FULL_ISO)) @@ -157,7 +158,7 @@ def test_set_expiration_hour__first_caching_after_exp_hour(self): """ Test set expiration hour when the first caching occurs a day before the expiration day, after expiration hour. """ future_ticker = CustomFutureTicker("Custom", "CT{} Custom", 1, 5, 500) - future_ticker.initialize_data_provider(self.timer, self.bbg_provider) + future_ticker.initialize_data_provider(self.bbg_provider) future_ticker.set_expiration_hour(hour=10, minute=10) self.timer.set_current_time(str_to_date('2017-12-10 19:00:00.0', DateFormat.FULL_ISO)) @@ -173,7 +174,7 @@ def test_set_expiration_hour__first_caching_at_exp_hour(self): """ Test set expiration hour when the first caching occurs a day before the expiration day, at expiration hour. """ future_ticker = CustomFutureTicker("Custom", "CT{} Custom", 1, 5, 500) - future_ticker.initialize_data_provider(self.timer, self.bbg_provider) + future_ticker.initialize_data_provider(self.bbg_provider) future_ticker.set_expiration_hour(hour=8, minute=10) self.timer.set_current_time(str_to_date('2017-12-11 08:10:00.0', DateFormat.FULL_ISO)) diff --git a/qf_lib/tests/unit_tests/containers/test_futures_chain.py b/qf_lib/tests/unit_tests/containers/test_futures_chain.py index 556c3a68..2205e9bc 100644 --- a/qf_lib/tests/unit_tests/containers/test_futures_chain.py +++ b/qf_lib/tests/unit_tests/containers/test_futures_chain.py @@ -16,6 +16,8 @@ from numpy import nan from pandas import date_range +from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent +from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent from qf_lib.common.enums.expiration_date_field import ExpirationDateField from qf_lib.common.enums.frequency import Frequency from qf_lib.common.enums.price_field import PriceField @@ -88,6 +90,9 @@ def setUp(self) -> None: ], ]) + MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 30, "second": 0, "microsecond": 0}) + MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0}) + def test_future_chains(self): """ Open prices are available on the expiration date and the Close prices are available on the day before the expiration day. """ @@ -145,7 +150,7 @@ def _assert_adjustment_is_consistent(self, data_provider: DataProvider): timer = SettableTimer() timer.set_current_time(self.expiration_date) - self.future_ticker.initialize_data_provider(timer, data_provider) + self.future_ticker.initialize_data_provider(data_provider) futures_chain_1 = FuturesChain(self.future_ticker, data_provider, FuturesAdjustmentMethod.BACK_ADJUSTED) prices = futures_chain_1.get_price(PriceField.ohlcv(), self.start_date, timer.now()) @@ -159,14 +164,14 @@ def _assert_adjustment_is_consistent(self, data_provider: DataProvider): def _assert_adjustment_difference_is_correct(self, data_provider: DataProvider, expected_difference: float): """ Computes the adjusted and non adjusted futures chain and checks if all price values (OHLC) are adjusted by the correct value. """ - timer = SettableTimer(self.end_date) - self.future_ticker.initialize_data_provider(timer, data_provider) + data_provider.timer.set_current_time(self.end_date) + self.future_ticker.initialize_data_provider(data_provider) adjusted_chain = FuturesChain(self.future_ticker, data_provider, FuturesAdjustmentMethod.BACK_ADJUSTED) - adjusted_prices = adjusted_chain.get_price(PriceField.ohlcv(), self.start_date, timer.now()) + adjusted_prices = adjusted_chain.get_price(PriceField.ohlcv(), self.start_date, data_provider.timer.now()) non_adjusted_chain = FuturesChain(self.future_ticker, data_provider, FuturesAdjustmentMethod.NTH_NEAREST) - non_adjusted_prices = non_adjusted_chain.get_price(PriceField.ohlcv(), self.start_date, timer.now()) + non_adjusted_prices = non_adjusted_chain.get_price(PriceField.ohlcv(), self.start_date, data_provider.timer.now()) fields_to_adjust = [PriceField.Open, PriceField.Close, PriceField.High, PriceField.Low] fields_without_adjustment = [PriceField.Volume] @@ -185,4 +190,5 @@ def _assert_adjustment_difference_is_correct(self, data_provider: DataProvider, non_adjusted_prices.loc[self.expiration_date:], check_names=False) def _mock_data_provider(self, data_array: QFDataArray): - return PresetDataProvider(data_array, self.start_date, self.end_date, Frequency.DAILY, self.exp_dates) + return PresetDataProvider(data_array, self.start_date, self.end_date, Frequency.DAILY, self.exp_dates, + timer=SettableTimer(self.end_date)) diff --git a/qf_lib/tests/unit_tests/data_providers/portara/test_portara_future_ticker.py b/qf_lib/tests/unit_tests/data_providers/portara/test_portara_future_ticker.py index 07aa3d21..f64bca36 100644 --- a/qf_lib/tests/unit_tests/data_providers/portara/test_portara_future_ticker.py +++ b/qf_lib/tests/unit_tests/data_providers/portara/test_portara_future_ticker.py @@ -17,6 +17,8 @@ from pandas import concat +from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent +from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent from qf_lib.common.enums.frequency import Frequency from qf_lib.common.enums.price_field import PriceField from qf_lib.common.enums.security_type import SecurityType @@ -45,24 +47,24 @@ def setUpClass(cls) -> None: def setUp(self) -> None: self.data_provider = PortaraDataProvider(self.futures_path, [self.future_ticker_1, self.future_ticker_2], PriceField.ohlcv(), self.start_date, self.end_date, Frequency.DAILY) + self.data_provider.set_timer(SettableTimer()) def test_get_current_specific_ticker(self): - timer = SettableTimer() - self.future_ticker_1.initialize_data_provider(timer, self.data_provider) + self.future_ticker_1.initialize_data_provider(self.data_provider) - timer.set_current_time(str_to_date("2021-03-18")) + self.data_provider.timer.set_current_time(str_to_date("2021-03-18")) specific_ticker = self.future_ticker_1.get_current_specific_ticker() self.assertEqual(specific_ticker, PortaraTicker("AB2021M", SecurityType.FUTURE, 1)) - timer.set_current_time(str_to_date("2021-06-14")) + self.data_provider.timer.set_current_time(str_to_date("2021-06-14")) specific_ticker = self.future_ticker_1.get_current_specific_ticker() self.assertEqual(specific_ticker, PortaraTicker("AB2021M", SecurityType.FUTURE, 1)) - timer.set_current_time(str_to_date("2021-06-15")) + self.data_provider.timer.set_current_time(str_to_date("2021-06-15")) specific_ticker = self.future_ticker_1.get_current_specific_ticker() self.assertEqual(specific_ticker, PortaraTicker("AB2021U", SecurityType.FUTURE, 1)) - timer.set_current_time(datetime(2021, 12, 14, 23, 59)) + self.data_provider.timer.set_current_time(datetime(2021, 12, 14, 23, 59)) specific_ticker = self.future_ticker_1.get_current_specific_ticker() self.assertEqual(specific_ticker, PortaraTicker("AB2021Z", SecurityType.FUTURE, 1)) @@ -81,16 +83,18 @@ def test_belongs_to_family(self): def test_designated_contracts(self): future_ticker = PortaraFutureTicker("", "AB{}", 1, 1, designated_contracts="MZ") - timer = SettableTimer() - future_ticker.initialize_data_provider(timer, self.data_provider) + future_ticker.initialize_data_provider(self.data_provider) - timer.set_current_time(str_to_date("2021-06-15")) + self.data_provider.timer.set_current_time(str_to_date("2021-06-15")) specific_ticker = future_ticker.get_current_specific_ticker() self.assertEqual(specific_ticker, PortaraTicker("AB2021Z", SecurityType.FUTURE, 1)) def test_futures_chain_without_adjustment(self): - timer = SettableTimer(self.end_date) - self.future_ticker_1.initialize_data_provider(timer, self.data_provider) + self.data_provider.timer.set_current_time(self.end_date) + self.future_ticker_1.initialize_data_provider(self.data_provider) + + MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 30, "second": 0, "microsecond": 0}) + MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0}) futures_chain = FuturesChain(self.future_ticker_1, self.data_provider, FuturesAdjustmentMethod.NTH_NEAREST) diff --git a/qf_lib/tests/unit_tests/data_providers/test_data_provider.py b/qf_lib/tests/unit_tests/data_providers/test_abstract_price_data_provider.py similarity index 73% rename from qf_lib/tests/unit_tests/data_providers/test_data_provider.py rename to qf_lib/tests/unit_tests/data_providers/test_abstract_price_data_provider.py index 490dfa19..a707aea7 100644 --- a/qf_lib/tests/unit_tests/data_providers/test_data_provider.py +++ b/qf_lib/tests/unit_tests/data_providers/test_abstract_price_data_provider.py @@ -17,87 +17,93 @@ from numpy import isnan, nan from pandas import date_range +from qf_lib.backtesting.events.time_event.regular_time_event.market_close_event import MarketCloseEvent +from qf_lib.backtesting.events.time_event.regular_time_event.market_open_event import MarketOpenEvent from qf_lib.common.enums.frequency import Frequency from qf_lib.common.enums.price_field import PriceField from qf_lib.common.tickers.tickers import BloombergTicker, Ticker from qf_lib.common.utils.dateutils.date_format import DateFormat from qf_lib.common.utils.dateutils.string_to_date import str_to_date +from qf_lib.common.utils.dateutils.timer import SettableTimer from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame from qf_lib.containers.qf_data_array import QFDataArray from qf_lib.containers.series.prices_series import PricesSeries -from qf_lib.data_providers.data_provider import DataProvider +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider from qf_lib.data_providers.helpers import normalize_data_array from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_series_equal, assert_dataframes_equal -class TestDataProvider(TestCase): +class TestAbstractPriceDataProvider(TestCase): @classmethod - @patch.multiple(DataProvider, __abstractmethods__=set()) + @patch.multiple(AbstractPriceDataProvider, __abstractmethods__=set()) def setUpClass(cls) -> None: cls.ticker_1 = BloombergTicker("Example Index") cls.ticker_2 = BloombergTicker("Example Comdty") - cls.data_provider = DataProvider() + cls.timer = SettableTimer() + cls.data_provider = AbstractPriceDataProvider(cls.timer) - def setUp(self) -> None: - self.current_time = None + MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0}) + MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 0, "second": 0, "microsecond": 0}) - datetime_patcher = patch('qf_lib.data_providers.data_provider.datetime') - datetime_mock = datetime_patcher.start() - datetime_mock.now.side_effect = lambda: self.current_time - self.addCleanup(datetime_patcher.stop) + def setUp(self) -> None: + get_history_patcher = patch.object(AbstractPriceDataProvider, 'get_history') + mocked_get_history = get_history_patcher.start() + mocked_get_history.side_effect = self._mock_get_history + self.addCleanup(get_history_patcher.stop) - get_price_patcher = patch.object(DataProvider, 'get_price') - mocked_get_price = get_price_patcher.start() - mocked_get_price.side_effect = self._mock_get_price - self.addCleanup(get_price_patcher.stop) + price_field_to_str_map_patcher = patch.object(AbstractPriceDataProvider, 'price_field_to_str_map') + mocked_price_field_to_str_map = price_field_to_str_map_patcher.start() + mocked_price_field_to_str_map.side_effect = self._mock_price_field_to_str_map + self.addCleanup(price_field_to_str_map_patcher.stop) def test_get_last_available_price__before_data_starts__daily(self): # Check if the correct output is returned in case if a single ticker is provided - self._assert_last_price_is_correct("2021-04-30 00:00:00.000000", self.ticker_1, nan, Frequency.DAILY) - self._assert_last_price_is_correct("2021-04-30 00:00:00.000000", self.ticker_2, nan, Frequency.DAILY) + self._assert_last_price_is_correct("2021-04-30 21:00:00.000000", self.ticker_1, nan, Frequency.DAILY) + self._assert_last_price_is_correct("2021-04-30 21:00:00.000000", self.ticker_2, nan, Frequency.DAILY) # Check if the correct output is returned in case if a list of tickers is provided - self._assert_last_prices_are_correct("2021-04-30 00:00:00.000000", [nan, nan], Frequency.DAILY) - self._assert_last_prices_are_correct("2021-05-06 00:00:00.000000", [25.0, 27.0], Frequency.DAILY) + self._assert_last_prices_are_correct("2021-04-30 21:00:00.000000", [nan, nan], Frequency.DAILY) + self._assert_last_prices_are_correct("2021-05-06 21:00:00.000000", [25.0, 27.0], Frequency.DAILY) def test_get_last_available_price__data_available_for_every_ticker__daily(self): # Check if the correct output is returned in case if a single ticker is provided - self._assert_last_price_is_correct("2021-05-01 00:00:00.000000", self.ticker_1, 26, Frequency.DAILY) - self._assert_last_price_is_correct("2021-05-01 00:00:00.000000", self.ticker_2, 28, Frequency.DAILY) + self._assert_last_price_is_correct("2021-05-01 21:00:00.000000", self.ticker_1, 26, Frequency.DAILY) + self._assert_last_price_is_correct("2021-05-01 21:00:00.000000", self.ticker_2, 28, Frequency.DAILY) - self._assert_last_price_is_correct("2021-05-05 00:00:00.000000", self.ticker_1, 25, Frequency.DAILY) - self._assert_last_price_is_correct("2021-05-05 00:00:00.000000", self.ticker_2, 27, Frequency.DAILY) + self._assert_last_price_is_correct("2021-05-05 21:00:00.000000", self.ticker_1, 25, Frequency.DAILY) + self._assert_last_price_is_correct("2021-05-05 21:00:00.000000", self.ticker_2, 27, Frequency.DAILY) # Check if the correct output is returned in case if a list of tickers is provided - self._assert_last_prices_are_correct("2021-05-01 00:00:00.000000", [26, 28], Frequency.DAILY) + self._assert_last_prices_are_correct("2021-05-01 21:00:00.000000", [26, 28], Frequency.DAILY) self._assert_last_prices_are_correct("2021-05-05 14:30:00.000000", [25.0, 27.0], Frequency.DAILY) def test_get_last_available_price__data_available_for_the_first_ticker__daily(self): # Check if the correct output is returned in case if a single ticker is provided - self._assert_last_price_is_correct("2021-05-03 00:00:00.000000", self.ticker_1, 32, Frequency.DAILY) - self._assert_last_price_is_correct("2021-05-03 00:00:00.000000", self.ticker_2, 30, Frequency.DAILY) + self._assert_last_price_is_correct("2021-05-03 21:00:00.000000", self.ticker_1, 32, Frequency.DAILY) + self._assert_last_price_is_correct("2021-05-03 21:00:00.000000", self.ticker_2, 30, Frequency.DAILY) - self._assert_last_price_is_correct("2021-05-04 00:00:00.000000", self.ticker_1, 31, Frequency.DAILY) - self._assert_last_price_is_correct("2021-05-04 00:00:00.000000", self.ticker_2, 30, Frequency.DAILY) + self._assert_last_price_is_correct("2021-05-04 21:00:00.000000", self.ticker_1, 31, Frequency.DAILY) + self._assert_last_price_is_correct("2021-05-04 21:00:00.000000", self.ticker_2, 30, Frequency.DAILY) # Check if the correct output is returned in case if a list of tickers is provided - self._assert_last_prices_are_correct("2021-05-03 13:30:00.000000", [32.0, 30.0], Frequency.DAILY) - self._assert_last_prices_are_correct("2021-05-04 13:30:00.000000", [31.0, 30.0], Frequency.DAILY) + self._assert_last_prices_are_correct("2021-05-03 21:30:00.000000", [32.0, 30.0], Frequency.DAILY) + self._assert_last_prices_are_correct("2021-05-04 21:30:00.000000", [31.0, 30.0], Frequency.DAILY) def test_get_last_available_price__data_available_for_the_second_ticker__daily(self): # Check if the correct output is returned in case if a single ticker is provided - self._assert_last_price_is_correct("2021-05-02 00:00:00.000000", self.ticker_1, 26, Frequency.DAILY) - self._assert_last_price_is_correct("2021-05-02 00:00:00.000000", self.ticker_2, 30, Frequency.DAILY) + self._assert_last_price_is_correct("2021-05-02 21:00:00.000000", self.ticker_1, 26, Frequency.DAILY) + self._assert_last_price_is_correct("2021-05-02 21:00:00.000000", self.ticker_2, 30, Frequency.DAILY) # Check if the correct output is returned in case if a list of tickers is provided - self._assert_last_prices_are_correct("2021-05-02 13:30:00.000000", [26.0, 30.0], Frequency.DAILY) + self._assert_last_prices_are_correct("2021-05-02 21:30:00.000000", [26.0, 30.0], Frequency.DAILY) def test_historical_price__negative_number_of_bars__daily(self): # Check if the correct output is returned in case if a single ticker is provided - self.current_time = str_to_date("2021-05-03 00:00:00.000000", DateFormat.FULL_ISO) + current_time = str_to_date("2021-05-03 21:00:00.000000", DateFormat.FULL_ISO) + self.timer.set_current_time(current_time) with self.assertRaises(AssertionError): self.data_provider.historical_price(self.ticker_1, PriceField.Open, 0, frequency=Frequency.DAILY) self.data_provider.historical_price(self.ticker_1, PriceField.Open, -2, frequency=Frequency.DAILY) @@ -105,7 +111,8 @@ def test_historical_price__negative_number_of_bars__daily(self): frequency=Frequency.DAILY) def test_historical_price__before_data_starts__daily(self): - self.current_time = str_to_date("2021-04-30 00:00:00.000000", DateFormat.FULL_ISO) + current_time = str_to_date("2021-04-30 21:00:00.000000", DateFormat.FULL_ISO) + self.timer.set_current_time(current_time) with self.assertRaises(ValueError): self.data_provider.historical_price(self.ticker_2, PriceField.Open, 2, frequency=Frequency.DAILY) @@ -113,7 +120,8 @@ def test_historical_price__before_data_starts__daily(self): frequency=Frequency.DAILY) def test_historical_price__single_ticker__single_field__daily(self): - self.current_time = str_to_date("2021-05-03 00:00:00.000000", DateFormat.FULL_ISO) + current_time = str_to_date("2021-05-03 21:00:00.000000", DateFormat.FULL_ISO) + self.timer.set_current_time(current_time) # Test when the current day does not have the open price actual_series = self.data_provider.historical_price(self.ticker_2, PriceField.Open, 2, @@ -128,7 +136,8 @@ def test_historical_price__single_ticker__single_field__daily(self): assert_series_equal(actual_series, expected_series, check_names=False) def test_historical_price__single_ticker__multiple_fields__daily(self): - self.current_time = str_to_date("2021-05-06 00:00:00.000000", DateFormat.FULL_ISO) + current_time = str_to_date("2021-05-06 21:00:00.000000", DateFormat.FULL_ISO) + self.timer.set_current_time(current_time) # Test when the current day does not have the open price actual_bars = self.data_provider.historical_price(self.ticker_2, PriceField.ohlcv(), 2, @@ -139,7 +148,8 @@ def test_historical_price__single_ticker__multiple_fields__daily(self): columns=PriceField.ohlcv()) assert_dataframes_equal(expected_bars, actual_bars, check_names=False) - self.current_time = str_to_date("2021-05-06 00:00:00.000000", DateFormat.FULL_ISO) + current_time = str_to_date("2021-05-06 21:00:00.000000", DateFormat.FULL_ISO) + self.timer.set_current_time(current_time) actual_bars = self.data_provider.historical_price(self.ticker_2, PriceField.ohlcv(), 3, frequency=Frequency.DAILY) @@ -156,7 +166,8 @@ def test_historical_price__single_ticker__multiple_fields__daily(self): self.data_provider.historical_price(self.ticker_2, PriceField.ohlcv(), 4, frequency=Frequency.DAILY) def test_historical_price__multiple_tickers__multiple_fields__daily(self): - self.current_time = str_to_date("2021-05-06 00:00:00.000000", DateFormat.FULL_ISO) + current_time = str_to_date("2021-05-06 21:00:00.000000", DateFormat.FULL_ISO) + self.timer.set_current_time(current_time) # Test when the current day does not have the open price actual_bars = self.data_provider.historical_price([self.ticker_1, self.ticker_2], PriceField.ohlcv(), 2, @@ -168,46 +179,58 @@ def test_historical_price__multiple_tickers__multiple_fields__daily(self): def test_historical_price__margin_adjustment__daily(self): # In case if we want only 1 historical bar and the last full bar was more than ~12 days ago, the adjustment of # the margin for the "number of days to go back" need to be performed - self.current_time = str_to_date("2021-05-18 00:00:00.000000", DateFormat.FULL_ISO) + current_time = str_to_date("2021-05-18 21:00:00.000000", DateFormat.FULL_ISO) + self.timer.set_current_time(current_time) + actual_bars = self.data_provider.historical_price(self.ticker_1, PriceField.ohlcv(), 1, frequency=Frequency.DAILY) expected_bars = PricesDataFrame(data=[[25.0, 25.1, 25.2, None, 25.3]], index=[str_to_date('2021-05-05')], columns=PriceField.ohlcv()) assert_dataframes_equal(actual_bars, expected_bars, check_names=False) - self.current_time = str_to_date("2021-05-27 00:00:00.000000", DateFormat.FULL_ISO) + current_time = str_to_date("2021-05-27 21:00:00.000000", DateFormat.FULL_ISO) + self.timer.set_current_time(current_time) + actual_bars = self.data_provider.historical_price(self.ticker_1, PriceField.ohlcv(), 1, frequency=Frequency.DAILY) assert_dataframes_equal(actual_bars, expected_bars, check_names=False) with self.assertRaises(ValueError): - self.current_time = str_to_date("2021-06-06 00:00:00.000000", DateFormat.FULL_ISO) + current_time = str_to_date("2021-06-06 21:00:00.000000", DateFormat.FULL_ISO) + self.timer.set_current_time(current_time) + self.data_provider.historical_price(self.ticker_1, PriceField.ohlcv(), 1, frequency=Frequency.DAILY) + def test_str_to_price_field_map(self): + self.assertCountEqual(self.data_provider.str_to_price_field_map(), { + 'Close': PriceField.Close, 'Open': PriceField.Open, 'Low': PriceField.Low, + 'High': PriceField.High, 'Volume': PriceField.Volume + }) + def _assert_last_prices_are_correct(self, curr_time_str, expected_values, frequency): current_time = str_to_date(curr_time_str, DateFormat.FULL_ISO) - self.current_time = current_time + self.timer.set_current_time(current_time) expected_series = PricesSeries(data=expected_values, index=[self.ticker_1, self.ticker_2]) actual_series = self.data_provider.get_last_available_price([self.ticker_1, self.ticker_2], frequency) assert_series_equal(expected_series, actual_series, check_names=False) def _assert_last_price_is_correct(self, curr_time_str, ticker, expected_value, frequency): current_time = str_to_date(curr_time_str, DateFormat.FULL_ISO) - self.current_time = current_time + self.timer.set_current_time(current_time) actual_value = self.data_provider.get_last_available_price(ticker, frequency) if isnan(expected_value): self.assertTrue(isnan(actual_value)) else: self.assertEqual(expected_value, actual_value) - def _mock_get_price(self, tickers, fields, start_date, end_date, frequency): + def _mock_get_history(self, tickers, fields, start_date, end_date, frequency, look_ahead_bias=False): tickers, got_single_ticker = convert_to_list(tickers, Ticker) - fields, got_single_field = convert_to_list(fields, PriceField) + fields, got_single_field = convert_to_list(fields, str) mock_daily_data = QFDataArray.create( dates=date_range(start='2021-05-01', end='2021-05-06', freq='D'), tickers=[self.ticker_1, self.ticker_2], - fields=PriceField.ohlcv(), + fields=["Open", "High", "Low", "Close", "Volume"], data=[ # 2021-05-01 [ @@ -251,7 +274,7 @@ def _mock_get_price(self, tickers, fields, start_date, end_date, frequency): mock_intraday_data = QFDataArray.create( dates=date_range(start='2021-05-01', end='2021-05-06', freq='D'), tickers=[self.ticker_1, self.ticker_2], - fields=PriceField.ohlcv(), + fields=["Open", "High", "Low", "Close", "Volume"], data=[ # 2021-05-01 [ @@ -294,4 +317,15 @@ def _mock_get_price(self, tickers, fields, start_date, end_date, frequency): data = mock_daily_data.loc[start_date:end_date, tickers, fields] if frequency == Frequency.DAILY else \ mock_intraday_data.loc[start_date:end_date, tickers, fields] - return normalize_data_array(data, tickers, fields, False, got_single_ticker, got_single_field, True) + + got_single_date = start_date == end_date if frequency == Frequency.DAILY else False + return normalize_data_array(data, tickers, fields, got_single_date, got_single_ticker, got_single_field, True) + + def _mock_price_field_to_str_map(self): + return { + PriceField.Close: "Close", + PriceField.Open: "Open", + PriceField.Low: "Low", + PriceField.High: "High", + PriceField.Volume: "Volume" + } diff --git a/qf_lib/tests/unit_tests/data_providers/test_futures_data_provider.py b/qf_lib/tests/unit_tests/data_providers/test_futures_data_provider.py new file mode 100644 index 00000000..d4295e3c --- /dev/null +++ b/qf_lib/tests/unit_tests/data_providers/test_futures_data_provider.py @@ -0,0 +1,167 @@ +# Copyright 2016-present CERN – European Organization for Nuclear Research +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from datetime import datetime +from typing import Optional +from unittest import TestCase +from unittest.mock import patch + +from qf_lib.common.enums.expiration_date_field import ExpirationDateField +from qf_lib.common.tickers.tickers import Ticker +from qf_lib.common.utils.miscellaneous.to_list_conversion import convert_to_list +from qf_lib.containers.dataframe.qf_dataframe import QFDataFrame +from qf_lib.containers.futures.future_tickers.future_ticker import FutureTicker +from qf_lib.data_providers.futures_data_provider import FuturesDataProvider +from qf_lib.tests.helpers.testing_tools.containers_comparison import assert_dataframes_equal + + +@patch.multiple(Ticker, __abstractmethods__=set()) +@patch.multiple(FutureTicker, __abstractmethods__=set()) +class TestFuturesDataProvider(TestCase): + + @classmethod + @patch.multiple(FuturesDataProvider, __abstractmethods__=set()) + def setUpClass(cls) -> None: + cls.data_provider = FuturesDataProvider() + + @patch.multiple(Ticker, __abstractmethods__=set()) + @patch.multiple(FutureTicker, __abstractmethods__=set()) + def setUp(self) -> None: + # Mock the abstractmethod expiration_date_field_str_map + expiration_date_field_str_map_patcher = patch.object(FuturesDataProvider, 'expiration_date_field_str_map') + mocked_expiration_date_field_str_map_patcher = expiration_date_field_str_map_patcher.start() + mocked_expiration_date_field_str_map_patcher.side_effect = self._mock_expiration_date_field_str_map_patcher + self.addCleanup(mocked_expiration_date_field_str_map_patcher.stop) + + # Mock the abstractmethod _get_futures_chain_dict + _get_futures_chain_dict_patcher = patch.object(FuturesDataProvider, '_get_futures_chain_dict') + mocked_get_futures_chain_dict_patcher = _get_futures_chain_dict_patcher.start() + mocked_get_futures_chain_dict_patcher.side_effect = self._mock__get_futures_chain_dict + self.addCleanup(mocked_get_futures_chain_dict_patcher.stop) + + def test_str_to_expiration_date_field_map(self): + actual_mapping = self.data_provider.str_to_expiration_date_field_map() + self.assertEqual(actual_mapping["LAST_TRADEABLE_DATE"], ExpirationDateField.LastTradeableDate) + self.assertEqual(actual_mapping["FIRST_NOTICE"], ExpirationDateField.FirstNotice) + + def test_get_futures_chain_tickers__single_ticker__all_dates_available(self): + futures_chain = self.data_provider.get_futures_chain_tickers( + FutureTicker("Corn", "C {} Comdty", 1, 5, 10, "HMUZ"), ExpirationDateField.all_dates() + ) + + expected_chain = { + FutureTicker("Corn", "C {} Comdty", 1, 5, 10, "HMUZ"): QFDataFrame( + index=[Ticker("C H1 Comdty", None, None), Ticker("C N1 Comdty", None, None), + Ticker("C M1 Comdty", None, None)], + data=[[datetime(2021, 3, 1), datetime(2021, 3, 10)], [datetime(2021, 5, 1), datetime(2021, 5, 10)], + [datetime(2021, 7, 1), datetime(2021, 7, 10)]], + columns=ExpirationDateField.all_dates()) + } + for key in futures_chain.keys(): + self.assertTrue(key in expected_chain.keys()) + assert_dataframes_equal(futures_chain[key], expected_chain[key]) + + def test_get_futures_chain_tickers__multiple_tickers__all_dates_available(self): + futures_chain = self.data_provider.get_futures_chain_tickers( + [FutureTicker("Corn", "C {} Comdty", 1, 5, 10, "HMUZ"), + FutureTicker("Wheat", "W {} Comdty", 1, 5, 10)], ExpirationDateField.all_dates(), + ) + + expected_chain = { + FutureTicker("Corn", "C {} Comdty", 1, 5, 10, "HMUZ"): QFDataFrame( + index=[Ticker("C H1 Comdty", None, None), Ticker("C N1 Comdty", None, None), + Ticker("C M1 Comdty", None, None)], + data=[[datetime(2021, 3, 1), datetime(2021, 3, 10)], [datetime(2021, 5, 1), datetime(2021, 5, 10)], + [datetime(2021, 7, 1), datetime(2021, 7, 10)]], + columns=ExpirationDateField.all_dates()), + FutureTicker("Wheat", "W {} Comdty", 1, 5, 10): QFDataFrame( + index=[Ticker("W H1 Comdty", None, None), Ticker("W N1 Comdty", None, None), + Ticker("W M1 Comdty", None, None)], + data=[[datetime(2022, 3, 1), datetime(2022, 3, 10)], [datetime(2022, 5, 1), datetime(2022, 5, 10)], + [datetime(2022, 7, 1), datetime(2022, 7, 10)]], + columns=ExpirationDateField.all_dates()), + } + for key in futures_chain.keys(): + self.assertTrue(key in expected_chain.keys()) + assert_dataframes_equal(futures_chain[key], expected_chain[key]) + + def test_get_futures_chain_tickers__single_ticker__test_single_date(self): + futures_chain = self.data_provider.get_futures_chain_tickers( + FutureTicker("Corn", "C {} Comdty", 1, 5, 10, "HMUZ"), ExpirationDateField.LastTradeableDate + ) + + expected_chain = { + FutureTicker("Corn", "C {} Comdty", 1, 5, 10, "HMUZ"): QFDataFrame( + index=[Ticker("C H1 Comdty", None, None), Ticker("C N1 Comdty", None, None), + Ticker("C M1 Comdty", None, None)], + data=[[datetime(2021, 3, 10)], [datetime(2021, 5, 10)], [datetime(2021, 7, 10)]], + columns=[ExpirationDateField.LastTradeableDate]), + FutureTicker("Wheat", "W {} Comdty", 1, 5, 5): QFDataFrame( + index=[Ticker("W H1 Comdty", None, None), Ticker("W N1 Comdty", None, None), + Ticker("W M1 Comdty", None, None)], + data=[[datetime(2022, 3, 10)], [datetime(2022, 5, 10)], [datetime(2022, 7, 10)]], + columns=[ExpirationDateField.LastTradeableDate]), + } + for key in futures_chain.keys(): + self.assertTrue(key in expected_chain.keys()) + assert_dataframes_equal(futures_chain[key], expected_chain[key]) + + def test_get_futures_chain_tickers__multiple_tickers__test_single_date(self): + futures_chain = self.data_provider.get_futures_chain_tickers( + FutureTicker("Corn", "C {} Comdty", 1, 5, 10, "HMUZ"), ExpirationDateField.LastTradeableDate + ) + + expected_chain = { + FutureTicker("Corn", "C {} Comdty", 1, 5, 10, "HMUZ"): QFDataFrame( + index=[Ticker("C H1 Comdty", None, None), Ticker("C N1 Comdty", None, None), + Ticker("C M1 Comdty", None, None)], + data=[[datetime(2021, 3, 10)], [datetime(2021, 5, 10)], [datetime(2021, 7, 10)]], + columns=[ExpirationDateField.LastTradeableDate]) + } + for key in futures_chain.keys(): + self.assertTrue(key in expected_chain.keys()) + assert_dataframes_equal(futures_chain[key], expected_chain[key]) + + @staticmethod + def _mock_expiration_date_field_str_map_patcher(_: Optional = None): + return { + ExpirationDateField.FirstNotice: "FIRST_NOTICE", + ExpirationDateField.LastTradeableDate: "LAST_TRADEABLE_DATE", + } + + @staticmethod + def _mock__get_futures_chain_dict(tickers, expiration_date_fields): + data = { + FutureTicker("Corn", "C {} Comdty", 1, 5, 10, "HMUZ"): + QFDataFrame({ + "FIRST_NOTICE": [datetime(2021, 3, 1), datetime(2021, 5, 1), datetime(2021, 7, 1)], + "LAST_TRADEABLE_DATE": [datetime(2021, 3, 10), datetime(2021, 5, 10), datetime(2021, 7, 10)] + }, index=[Ticker("C H1 Comdty", None, None), Ticker("C N1 Comdty", None, None), + Ticker("C M1 Comdty", None, None)]), + FutureTicker("Wheat", "W {} Comdty", 1, 5, 10): + QFDataFrame({ + "FIRST_NOTICE": [datetime(2022, 3, 1), datetime(2022, 5, 1), datetime(2022, 7, 1)], + "LAST_TRADEABLE_DATE": [datetime(2022, 3, 10), datetime(2022, 5, 10), datetime(2022, 7, 10)] + }, index=[Ticker("W H1 Comdty", None, None), Ticker("W N1 Comdty", None, None), + Ticker("W M1 Comdty", None, None)]), + } + + tickers, _ = convert_to_list(tickers, FutureTicker) + results = {} + for ticker in tickers: + container = data[ticker] + if isinstance(container, QFDataFrame): + container = container.loc[:, expiration_date_fields] + results[ticker] = container + + return results diff --git a/qf_lib/tests/unit_tests/data_providers/test_prefetching_data_provider.py b/qf_lib/tests/unit_tests/data_providers/test_prefetching_data_provider.py index d71aaed8..257c2087 100644 --- a/qf_lib/tests/unit_tests/data_providers/test_prefetching_data_provider.py +++ b/qf_lib/tests/unit_tests/data_providers/test_prefetching_data_provider.py @@ -27,8 +27,8 @@ from qf_lib.containers.dataframe.prices_dataframe import PricesDataFrame from qf_lib.containers.dimension_names import DATES, TICKERS, FIELDS from qf_lib.containers.qf_data_array import QFDataArray +from qf_lib.data_providers.abstract_price_data_provider import AbstractPriceDataProvider from qf_lib.data_providers.prefetching_data_provider import PrefetchingDataProvider -from qf_lib.data_providers.data_provider import DataProvider class TestPrefetchingDataProvider(unittest.TestCase): @@ -52,15 +52,14 @@ def setUp(self): self.data_provider, self.cached_tickers, self.cached_fields, self.start_date, self.end_date, self.frequency ) - def mock_data_provider(self) -> DataProvider: - data_provider = Mock(spec=DataProvider) + def mock_data_provider(self) -> AbstractPriceDataProvider: + data_provider = Mock(spec=AbstractPriceDataProvider) data_provider.get_price.return_value = QFDataArray.create( data=np.full((len(self.cached_dates_idx), len(self.cached_tickers), len(self.cached_fields)), 0), dates=self.cached_dates_idx, tickers=self.cached_tickers, fields=self.cached_fields ) - data_provider.get_futures_chain_tickers.return_value = dict() return data_provider def test_get_price_with_single_ticker(self):