Added HAPI identifier type (#114)

Added HAPI identifier type
Added parser tests with ISIN

qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_data_provider.py

@@ -297,7 +297,7 @@ def _get_futures_chain_dict(self, tickers: Union[BloombergFutureTicker, Sequence
         self._assert_is_connected()
 
         tickers, got_single_ticker = convert_to_list(tickers, BloombergFutureTicker)
-        expiration_date_fields, _ = convert_to_list(expiration_date_fields, ExpirationDateField)
+        expiration_date_fields, _ = convert_to_list(expiration_date_fields, str)
 
         active_ticker_string_to_future_ticker = {
             future_ticker.get_active_ticker(): future_ticker for future_ticker in tickers
@@ -387,10 +387,9 @@ def get_current_values(self, tickers: Union[BloombergTicker, Sequence[BloombergT
         self.logger.info(f'universe_id: {universe_id} fields_list_id: {fields_list_id} request_id: {request_id}')
 
         out_path = self._download_response(request_id)
-        data_frame = self.parser.get_current_values(out_path, field_to_type)
+        data_frame = self.parser.get_current_values(out_path, field_to_type, tickers_str_to_obj)
 
         # to keep the order of tickers and fields we reindex the data frame
-        data_frame.index = [tickers_str_to_obj.get(x, BloombergTicker.from_string(x)) for x in data_frame.index]
         data_frame = data_frame.reindex(index=tickers, columns=fields)
 
         # squeeze unused dimensions

qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_parser.py

@@ -14,7 +14,7 @@
 import gzip
 import re
 from io import StringIO
-from typing import Tuple, List, Dict
+from typing import Tuple, List, Dict, Optional
 
 from qf_lib.common.tickers.tickers import BloombergTicker
 from qf_lib.common.utils.logging.qf_parent_logger import qf_logger
@@ -54,7 +54,8 @@ def __init__(self):
         self.logger = qf_logger.getChild(self.__class__.__name__)
         self.type_converter = BloombergDataLicenseTypeConverter()
 
-    def get_current_values(self, filepath: str, field_to_type: Dict[str, str]) -> QFDataFrame:
+    def get_current_values(self, filepath: str, field_to_type: Dict[str, str],
+                           tickers_mapping: Optional[Dict[str, BloombergTicker]] = None) -> QFDataFrame:
         """
         Method to parse hapi response and extract dates (e.g. FUT_NOTICE_FIRST, LAST_TRADEABLE_DT) for tickers
 
@@ -64,19 +65,28 @@ def get_current_values(self, filepath: str, field_to_type: Dict[str, str]) -> QF
             The full filepath with downloaded response
         field_to_type: Dict[str, str]
             dictionary mapping requested, correct fields into their corresponding types
+        tickers_mapping: Optional[Dict[str, BloombergTicker]]
+            dictionary mapping string representations of tickers onto corresponding ticker objects
 
         Returns
         -------
         QFDataFrame
             QFDataFrame with current values
         """
+        tickers_mapping = tickers_mapping or {}
+        tickers_mapping = {
+            self._strip_identifier_name(ticker_str): ticker for ticker_str, ticker in tickers_mapping.items()
+        }
         column_names = ["Ticker", "Error code", "Num flds"]
         field_to_type = {**field_to_type, "Ticker": "String"}
         fields, content = self._get_fields_and_data_content(filepath, field_to_type, column_names, header_row=True)
 
-        return content.set_index("Ticker")[fields]
+        data_frame = content.set_index("Ticker")[fields]
+        data_frame.index = data_frame.index.map(lambda x: tickers_mapping.get(x, BloombergTicker.from_string(x)))
+        return data_frame
 
-    def get_history(self, filepath: str, field_to_type: Dict[str, str], tickers_mapping: Dict[str, BloombergTicker]) \
+    def get_history(self, filepath: str, field_to_type: Dict[str, str],
+                    tickers_mapping: Optional[Dict[str, BloombergTicker]] = None) \
             -> QFDataArray:
         """
         Method to parse hapi response and get history data
@@ -87,21 +97,25 @@ def get_history(self, filepath: str, field_to_type: Dict[str, str], tickers_mapp
             The full filepath with downloaded response
         field_to_type: Dict[str, str]
             dictionary mapping requested, correct fields into their corresponding types
-        tickers_mapping: Dict[str, BloombergTicker]
+        tickers_mapping: Optional[Dict[str, BloombergTicker]]
             dictionary mapping string representations of tickers onto corresponding ticker objects
 
         Returns
         -------
         QFDataArray
             QFDataArray with history data
         """
+        tickers_mapping = tickers_mapping or {}
+        tickers_mapping = {
+            self._strip_identifier_name(ticker_str): ticker for ticker_str, ticker in tickers_mapping.items()
+        }
         column_names = ["Ticker", "Error code", "Num flds", "Pricing Source", "Dates"]
         field_to_type = {**field_to_type, "Ticker": "String", "Error code": "String", "Num flds": "String",
                          "Pricing Source": "String", "Dates": "Date"}
         fields, content = self._get_fields_and_data_content(filepath, field_to_type, column_names)
 
         tickers_dict = {
-            tickers_mapping[ticker]: df.set_index("Dates")[fields].dropna(how="all")
+            tickers_mapping.get(ticker, BloombergTicker.from_string(ticker)): df.set_index("Dates")[fields].dropna(how="all")
             for ticker, df in content.groupby(by="Ticker")
         }
 
@@ -154,3 +168,10 @@ def _read_csv(content: str, column_names: List[str], delimiter: str = "|", heade
         records = [line for line in records if len(line) == len(column_names)]
         df = QFDataFrame.from_records(records, columns=column_names)
         return df.replace(r'^\s+$', np.nan, regex=True)
+
+    def _strip_identifier_name(self, identifier: str) -> str:
+        """
+        In case if the identifier contains identifiertype in its name (e.g. /bbgid/BBG000BDTBL9), remove the identifier
+        type and leave just the identifier name (in the given example - BBG000BDTBL9).
+        """
+        return identifier.split("/")[-1]

qf_lib/data_providers/bloomberg_beap_hapi/bloomberg_beap_hapi_universe_provider.py

@@ -64,8 +64,13 @@ def get_universe_url(self, universe_id: str, tickers: Union[str, Sequence[str]],
             URL address of created hapi universe
         """
         tickers, got_single_field = convert_to_list(tickers, str)
-        contains = [{'@type': 'Identifier', 'identifierType': 'TICKER', 'identifierValue': ticker} for ticker in
-                    tickers]
+        tickers_and_types = [self._get_indentifier_and_type(ticker) for ticker in tickers]
+        contains = [{'@type': 'Identifier', 'identifierType': identifier_type, 'identifierValue': identifier}
+                    for identifier_type, identifier in tickers_and_types if identifier]
+        if len(contains) == 0:
+            raise ValueError("No valid identifiers (tickers) were provided. Please refer to the logs and adjust your "
+                             "data request accordingly.")
+
         if fields_overrides:
             # noinspection PyTypeChecker
             contains[0]['fieldOverrides'] = [{
@@ -102,3 +107,35 @@ def get_universe_url(self, universe_id: str, tickers: Union[str, Sequence[str]],
             self.logger.info('Universe successfully created at %s', universe_url)
 
         return universe_url
+
+    def _get_indentifier_and_type(self, ticker) -> Tuple[Optional[str], Optional[str]]:
+        blp_hapi_compatibility_mapping = {
+            "ticker": "TICKER",
+            "cusip": "CUSIP",
+            "buid": "BB_UNIQUE",
+            "bbgid": "BB_GLOBAL",
+            "isin": "ISIN",
+            "wpk": "WPK",
+            "sedol1": "SEDOL",
+            "common": "COMMON_NUMBER",
+            "cins": "CINS",
+            "cats": "CATS"
+        }
+
+        ticker = f"/ticker/{ticker}" if ticker.count("/") == 0 else ticker
+        if ticker.count("/") != 2:
+            self.logger.error(f"Detected incorrect identifier: {ticker}. It will be removed from the data request.\n"
+                              f"In order to provide an identifier, which is not a ticker, please use "
+                              f"'/id_type/identifier' format, with id_type being one of the following: "
+                              f"{blp_hapi_compatibility_mapping.values()}")
+            return None, None
+
+        id_type, id = ticker.lstrip("/").split("/")
+        try:
+            return blp_hapi_compatibility_mapping[id_type.lower()], id
+        except KeyError:
+            self.logger.error(
+                f"Detected incorrect identifier type: {id_type.lower()}. The identifier will be removed from the "
+                f"data request.\n"
+                f"List of valid identifier types: {blp_hapi_compatibility_mapping.values()}")
+            return None, None

qf_lib/tests/unit_tests/data_providers/bloomberg_beap_hapi/test_bloomberg_beap_hapi_data_provider.py

@@ -33,14 +33,14 @@ def test_get_tickers_universe__invalid_date(self):
 
     def test_get_tickers_universe__valid_ticker(self):
         self.data_provider.parser.get_current_values.return_value = QFDataFrame.from_records(
-            [("SPX Index", ["Member1", "Member2"]), ], columns=["Ticker", "INDX_MEMBERS"]).set_index("Ticker")
+            [(BloombergTicker("SPX Index"), ["Member1", "Member2"]), ], columns=["Ticker", "INDX_MEMBERS"]).set_index("Ticker")
 
         universe = self.data_provider.get_tickers_universe(BloombergTicker("SPX Index"))
         self.assertCountEqual(universe, [BloombergTicker("Member1 Equity"), BloombergTicker("Member2 Equity")])
 
     def test_get_tickers_universe__invalid_ticker(self):
         self.data_provider.parser.get_current_values.return_value = QFDataFrame.from_records(
-            [("Invalid Index", []), ], columns=["Ticker", "INDX_MEMBERS"]).set_index("Ticker")
+            [(BloombergTicker("Invalid Index"), []), ], columns=["Ticker", "INDX_MEMBERS"]).set_index("Ticker")
 
         universe = self.data_provider.get_tickers_universe(BloombergTicker("Invalid Index"))
         self.assertCountEqual(universe, [])