Bitpanda changed fileformat

config.ini

@@ -28,4 +28,7 @@ LOG_LEVEL = DEBUG
 # If False, all airdrops will be taxed as `Einkünfte aus sonstigen Leistungen`.
 # Setting this config falsly will result in a wrong tax calculation.
 # Please inform yourself and help to resolve this issue by working on/with #115.
+# Some airdrops can be classified as gifts (Schenkung) or income (Einkünfte)
+# relatively savely. For those, there is a flag to signal either type.
+# See the AirdropGift and AirdropIncome classes in transaction.py and their usage in book.py
 ALL_AIRDROPS_ARE_GIFTS = True

src/book.py

@@ -59,6 +59,7 @@ def create_operation(
         coin: str,
         row: int,
         file_path: Path,
+        exported_price: Optional[decimal.Decimal] = None,
         remark: Optional[str] = None,
     ) -> tr.Operation:
 
@@ -77,7 +78,7 @@ def create_operation(
         if remark:
             kwargs["remarks"] = [remark]
 
-        op = Op(utc_time, platform, change, coin, [row], file_path, **kwargs)
+        op = Op(utc_time, platform, change, coin, [row], file_path, None, exported_price, **kwargs)
         assert isinstance(op, tr.Operation)
         return op
 
@@ -99,6 +100,7 @@ def append_operation(
         coin: str,
         row: int,
         file_path: Path,
+        exported_price: Optional[decimal.Decimal] = None,
         remark: Optional[str] = None,
     ) -> None:
         # Discard operations after the `TAX_YEAR`.
@@ -112,6 +114,7 @@ def append_operation(
                 coin,
                 row,
                 file_path,
+                exported_price,
                 remark=remark,
             )
 
@@ -263,7 +266,7 @@ def _read_binance(self, file_path: Path, version: int = 1) -> None:
                         )
 
                 self.append_operation(
-                    operation, utc_time, platform, change, coin, row, file_path, remark
+                    operation, utc_time, platform, change, coin, row, file_path, None, remark
                 )
 
     def _read_binance_v2(self, file_path: Path) -> None:
@@ -1017,6 +1020,11 @@ def _read_bitpanda(self, file_path: Path) -> None:
             "withdrawal": "Withdrawal",
             "buy": "Buy",
             "sell": "Sell",
+            "reward": "StakingInterest",
+            "staking": "Staking",
+            "staking_end": "StakingEnd",
+            "airdrop_gift": "AirdropGift",
+            "airdrop_income": "AirdropIncome",
         }
 
         with open(file_path, encoding="utf8") as f:
@@ -1041,6 +1049,7 @@ def _read_bitpanda(self, file_path: Path) -> None:
                 "Fee asset",
                 "Spread",
                 "Spread Currency",
+                "Tax Fiat",
             ]:
                 try:
                     line = next(reader)
@@ -1052,50 +1061,146 @@ def _read_bitpanda(self, file_path: Path) -> None:
                 _tx_id,
                 csv_utc_time,
                 operation,
-                _inout,
+                inout,
                 amount_fiat,
                 fiat,
                 amount_asset,
                 asset,
-                _asset_price,
+                asset_price,
                 asset_price_currency,
                 asset_class,
                 _product_id,
                 fee,
                 fee_currency,
                 _spread,
                 _spread_currency,
+                _tax_fiat,
             ) in reader:
                 row = reader.line_num
 
+                # Skip stocks for now!
+                # TODO: Handle Stocks?
+                if asset_class.startswith("Stock"):
+                    continue
+
                 # make RFC3339 timestamp ISO 8601 parseable
                 if csv_utc_time[-1] == "Z":
                     csv_utc_time = csv_utc_time[:-1] + "+00:00"
 
+                if asset_price != "-":
+                    exported_price = misc.force_decimal(asset_price)
+                else:
+                    exported_price = None
+
                 # timezone information is already taken care of with this
                 utc_time = datetime.datetime.fromisoformat(csv_utc_time)
 
                 # transfer ops seem to be akin to airdrops. In my case I got a
                 # CocaCola transfer, which I don't want to track. Would need to
                 # be implemented if need be.
                 if operation == "transfer":
-                    log.warning(
-                        f"'Transfer' operations are not "
-                        f"implemented, skipping row {row} of file {file_path}"
-                    )
-                    continue
+                    if asset == "BEST" and asset_class == "Cryptocurrency" and inout == "incoming":
+                        # BEST is awarded for trading activity and holding a portfolio at bitpanda
+                        # The BEST awards are listed as "transfer" but must be processed as Airdrop (non-taxable)
+                        operation = "airdrop_gift"
+                    elif (
+                        inout == "incoming"
+                        and asset == "ETHW"
+                        and asset_class == "Cryptocurrency"
+                        and utc_time.year == 2022
+                        and utc_time.month == 9
+                    ):
+                        # In September 2022 the ETH blockchain switched from proof of work to
+                        # proof of stake. This bore the potential for a hardfork and continuation
+                        # of the original PoW chain albeit with a drastically reduced hashrate.
+                        # Bitpanda considered listing the resulting token if there was still value
+                        # in trading the ETH token on the PoW fork and considered distributing airdrops
+                        # in that case. The resulting token would be traded using the ETHW handle.
+                        # See: https://blog.bitpanda.com/en/ethereum-merge-everything-you-need-know
+                        #
+                        # German law regarding this case is not entirely clear
+                        # (see https://www.winheller.com/bankrecht-finanzrecht/bitcointrading/bitcoinundsteuer/besteuerung-hardforks-ledger-splits.html).
+                        # TODO: This should actually copy the history from the original ETH history.
+                        log.warning(
+                            f"Airdrop of {asset} is likely a result of Ethereums switch "
+                            f"to PoS in September 2022. The legal status of taxation of fork "
+                            f"airdrops is unclear in Germany (at least). Also, the original "
+                            f"history should be copied, which is NOT YET IMPLEMENTED. "
+                            f"See https://blog.bitpanda.com/en/ethereum-merge-everything-you-need-know "
+                            f"for more information. "
+                            f"Please open an issue or PR if you know how to resolve this. "
+                            f"In row {row} in file {file_path}."
+                        )
+                        operation = "airdrop_gift"
+                    elif (
+                        inout == "incoming"
+                        and asset == "LUNC"
+                        and asset_class == "Fiat"
+                        and utc_time.year == 2022
+                        and utc_time.month == 5
+                    ):
+                        # In May 2022 the Terra (LUNA) blockchain crashed. In response, a new chain
+                        # Terra 2.0 (LUNA) was created. The new old chain is still tradeable as
+                        # Terra Classic (LUNC) and holders of LUNA before the crash received their
+                        # LUNC tokens as airdrop. This also applied to LUNA tokens held through
+                        # bitpanda crypto indices.
+                        # Source for bitpanda LUNC airdrop:
+                        # https://support.bitpanda.com/hc/en-us/articles/4995318011292-Terra-2-0-LUNA-Airdrop
+                        #
+                        # The German law regarding this case is not entirely clear:
+                        # https://www.winheller.com/bankrecht-finanzrecht/bitcointrading/bitcoinundsteuer/besteuerung-hardforks-ledger-splits.html
+                        # TODO: This should actually copy the history from the original LUNA history.
+                        log.warning(
+                            f"WARNING: Airdrop of {asset} is a result of the fork of the "
+                            f"LUNA blockchain in May 2022. The legal status of "
+                            f"taxation of hardfork results is not clear in German law. "
+                            f"Also, the date of procurement should be set to the date(s) "
+                            f"of procurement of the original coins, essentially copying "
+                            f"the history of the original chain, which is NOT YET IMPLEMENTED. "
+                            f"See https://support.bitpanda.com/hc/en-us/articles/4995318011292-Terra-2-0-LUNA-Airdrop "
+                            f"for more information. "
+                            f"Please open an issue or PR if you know how to resolve this. "
+                            f"In row {row} in file {file_path}."
+                        )
+                        # Rewrite this asset_class because "Fiat" clearly wrong.
+                        asset_class = "Cryptocurrency"
+                        operation = "airdrop_gift"
+                    elif (
+                        inout == "incoming"
+                        and asset_class == "Cryptocurrency"
+                        and asset != "BEST"
+                        and utc_time < datetime.datetime(2022, 6, 14, 0, 0, 0, 0, utc_time.tzinfo)
+                    ):
+                        # Bitpanda tagged incoming staking rewards as incoming transfer until June 14 2022
+                        # or a few days before that date. After that, staking rewards are correctly tagged as "reward".
+                        operation = "reward"
+                    else:
+                        log.warning(
+                            f"'Transfer' operations are not "
+                            f"implemented, skipping row {row} of file {file_path}"
+                        )
+                        continue
+
+                # remap tansfer(stake)
+                if operation == "transfer(stake)":
+                    if inout == "incoming":
+                        operation = "staking"
+                if operation == "transfer(unstake)":
+                    if inout == "outgoing":
+                        operation = "staking_end"
 
                 # fail for unknown ops
                 try:
                     operation = operation_mapping[operation]
                 except KeyError:
                     log.error(
-                        f"Unsupported operation '{operation}' "
+                        f"Unsupported operation '{operation}' for asset {asset} "
                         f"in row {row} of file {file_path}"
                     )
                     raise RuntimeError
 
-                if operation in ["Deposit", "Withdrawal"]:
+                # Handling Airdrops the same as Deposits and Withdrawals here. Otherwise, balance doesn't add up.
+                if operation in ["Deposit", "Withdrawal", "Airdrop", "AirdropGift", "AirdropIncome"]:
                     if asset_class == "Fiat":
                         change = misc.force_decimal(amount_fiat)
                         if fiat != asset:
@@ -1128,6 +1233,13 @@ def _read_bitpanda(self, file_path: Path) -> None:
                     # Calculated price
                     price_calc = change_fiat / change
                     set_price_db(platform, asset, config.FIAT, utc_time, price_calc)
+                elif operation in ["Staking", "StakingEnd", "StakingInterest"]:
+                    change = misc.force_decimal(amount_asset)
+                else:
+                    # If something slips through the if/elifs above, the change will be wrong!
+                    # That's why we have to raise an exception here!
+                    log.error(f"Failed to appropriately handle operation '{operation}' for {platform}!")
+                    raise RuntimeError
 
                 if change < 0:
                     log.error(
@@ -1136,8 +1248,11 @@ def _read_bitpanda(self, file_path: Path) -> None:
                     )
                     raise RuntimeError
 
+                # Asset price is added to operation as 'exported_price' because some asset prices
+                # can't be checked anymore (like BEST and ETHW, which are both not available using
+                # ONE TRADINGs (ex Bitpanda Pro) candlebars API.
                 self.append_operation(
-                    operation, utc_time, platform, change, asset, row, file_path
+                    operation, utc_time, platform, change, asset, row, file_path, exported_price
                 )
 
                 # add buy / sell operation for fiat currency
@@ -1418,6 +1533,7 @@ def detect_exchange(self, file_path: Path) -> Optional[str]:
                     "Fee asset",
                     "Spread",
                     "Spread Currency",
+                    "Tax Fiat",
                 ],
                 "custom_eur": [
                     "Type",

src/config.py

@@ -69,7 +69,7 @@
     PRINCIPLE = core.Principle.FIFO
     LOCAL_TIMEZONE = zoneinfo.ZoneInfo("CET")
     LOCAL_TIMEZONE_KEY = "MEZ"
-    locale_str = "de_DE"
+    locale_str = ["de_DE", "de_DE.utf8"] # try multiple german locales in order
 
 else:
     raise NotImplementedError(
@@ -79,4 +79,9 @@
 
 # Program specific constants.
 FIAT = FIAT_CLASS.name  # Convert to string.
-locale.setlocale(locale.LC_ALL, locale_str)
+for loc in locale_str:
+    try:
+        locale.setlocale(locale.LC_ALL, loc)
+    except:
+        continue
+    break

src/price_data.py

@@ -281,8 +281,11 @@ def _get_price_coinbase_pro(
     def _get_price_bitpanda(
         self, base_asset: str, utc_time: datetime.datetime, quote_asset: str
     ) -> decimal.Decimal:
+        # TODO: Do we want to get historic price data from ONE TRADING (ex Bitpanda Pro) or do we want something else?
         return self._get_price_bitpanda_pro(base_asset, utc_time, quote_asset)
 
+    # Bitpanda Pro is now ONE TRADING.
+    # TODO: Handle something different?
     @misc.delayed
     def _get_price_bitpanda_pro(
         self, base_asset: str, utc_time: datetime.datetime, quote_asset: str
@@ -304,7 +307,7 @@ def _get_price_bitpanda_pro(
         """
 
         baseurl = (
-            f"https://api.exchange.bitpanda.com/public/v1/"
+            f"https://api.onetrading.com/fast/v1/"
             f"candlesticks/{base_asset}_{quote_asset}"
         )
 
@@ -341,19 +344,21 @@ def _get_price_bitpanda_pro(
                 }
                 if num_offset:
                     log.debug(
-                        f"Calling Bitpanda API for {base_asset} / {quote_asset} price "
+                        f"Calling ONE TRADING (ex Bitpanda Pro) API for {base_asset} / {quote_asset} price "
                         f"for {t} minute timeframe ending at {end} "
                         f"(includes {window_offset} minutes offset)"
                     )
                 else:
                     log.debug(
-                        f"Calling Bitpanda API for {base_asset} / {quote_asset} price "
+                        f"Calling ONE TRADING (ex Bitpanda Pro) API for {base_asset} / {quote_asset} price "
                         f"for {t} minute timeframe ending at {end}"
                     )
                 r = requests.get(baseurl, params=params)
 
-                assert r.status_code == 200, "No valid response from Bitpanda API"
                 data = r.json()
+                if r.status_code == 400 and data["error"] == f"The requested market {base_asset}_{quote_asset} is not available.":
+                    raise ValueError(data["error"])
+                assert r.status_code == 200, f"No valid response from ONE TRADING (ex Bitpanda Pro) API\nError: {r.json()['error']}"
 
                 # exit loop if data is valid
                 if data:
@@ -377,11 +382,12 @@ def _get_price_bitpanda_pro(
             raise RuntimeError
 
         # this should never be triggered, but just in case assert received data
-        assert data, f"No valid price data for {base_asset} / {quote_asset} at {end}"
+        assert data["candlesticks"], f"No valid price data for {base_asset} / {quote_asset} at {end}"
+        data = data["candlesticks"]
 
-        # simply take the average of the latest data element
-        high = misc.force_decimal(data[-1]["high"])
-        low = misc.force_decimal(data[-1]["low"])
+        # simply take the average of the first data element
+        high = misc.force_decimal(data[0]["high"])
+        low = misc.force_decimal(data[0]["low"])
 
         # if spread is greater than 3%
         if (high - low) / high > 0.03:
@@ -604,7 +610,40 @@ def get_cost(
         reference_coin: str = config.FIAT,
     ) -> decimal.Decimal:
         op = op_sc if isinstance(op_sc, tr.Operation) else op_sc.op
-        price = self.get_price(op.platform, op.coin, op.utc_time, reference_coin)
+        try:
+            price = self.get_price(op.platform, op.coin, op.utc_time, reference_coin)
+        except ValueError as e:
+            log.warning(
+                f"The API didn't provide a valid response. Using the price from the csv file if possible.\n"
+                f"\t\tCoin: {op.coin} | Op: {type(op).__name__} | Platform: {op.platform} | Row: {op.line} | File: {op.file_path}\n"
+                f"\t\tCaught exception: {e}"
+            )
+            if op.platform == "bitpanda":
+                # LUNC, ETHW, BEST and maybe more are not available via ONE TRADING (ex Bitpanda Pro) API
+                # => use the price from the exported data.
+                if op.exported_price is not None:
+                    price = op.exported_price
+
+                # Fees paid with BEST don't have a value given in the exported data.
+                # The value also can't be queried from the ONE TRADING (ex Bitpanda Pro) API (anymore)
+                if op.coin == "BEST" and isinstance(op, tr.Fee):
+                    log.warning(
+                        f"Can't get price for '{type(op).__name__}' of {op.coin} on platform {op.platform} anymore.\n"
+                        f"A withdrawal of BEST on bitpanda is likely a deduction of fees. For now we'll assume a value of 0.\n"
+                        f"For accurately calculating fees, this needs to be fixed. PRs welcome!\n"
+                        f"(row {op.line} in {op.file_path}"
+                    )
+                    return 0
+            else:
+                log.warning(
+                    f"Could not get any price info for {type(op).__name__} {op.coin} on {op.platform}! "
+                    f"Row: {op.line} | File: {op.file_path}"
+                )
+                raise RuntimeError(e)
+
+        # This may fail if an exchange is queried for a non existant coin/fiat pair and the operation doesn't include an exported price.
+        assert price, f"Could not get a price for asset {op.coin} at {op.utc_time}"
+
         if isinstance(op_sc, tr.Operation):
             return price * op_sc.change
         if isinstance(op_sc, tr.SoldCoin):