Skip to content

Commit

Permalink
Merge pull request #107 from dev590t/main
Browse files Browse the repository at this point in the history 
Support JSON File
  • Loading branch information
@redstreet
redstreet authored Oct 6, 2024
2 parents 0cef611 + 85080cd commit be5abaa
Show file tree
Hide file tree
Showing 10 changed files with 242 additions and 150 deletions.
39 changes: 20 additions & 19 deletions beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,42 +14,43 @@ def custom_init(self):
self.filename_pattern_def = ".*History"
self.date_format = "%m/%d/%Y"
self.header_identifier = ""
self.column_labels_line = (
"Run Date,Action,Symbol,Description,Type,Quantity,Price ($),Commission ($),Fees ($),Accrued Interest ($),Amount ($),Cash Balance ($),Settlement Date"
)
self.column_labels_line = "Run Date,Action,Symbol,Description,Type,Quantity,Price ($),Commission ($),Fees ($),Accrued Interest ($),Amount ($),Cash Balance ($),Settlement Date"
self.header_map = {
"Run Date": "date",
"Action": "memo",
"Symbol": "security",
"Amount ($)": "amount",
"Settlement Date": "settleDate",
"Quantity": "units",
"Run Date": "date",
"Action": "memo",
"Symbol": "security",
"Amount ($)": "amount",
"Settlement Date": "settleDate",
"Quantity": "units",
"Accrued Interest ($)": "accrued_interest",
"Fees ($)": "fees",
"Commission ($)": "commission",
"Cash Balance ($)": "balance",
"Price ($)": "unit_price",
"Fees ($)": "fees",
"Commission ($)": "commission",
"Cash Balance ($)": "balance",
"Price ($)": "unit_price",
}
self.transaction_type_map = {
"DIVIDEND RECEIVED": "dividends",
"TRANSFERRED FROM": "cash",
"YOU BOUGHT": "buystock",
"YOU SOLD": "sellstock",
"DIVIDEND RECEIVED": "dividends",
"TRANSFERRED FROM": "cash",
"YOU BOUGHT": "buystock",
"YOU SOLD": "sellstock",
}
self.skip_transaction_types = []
# fmt: on

def deep_identify(self, file):
last_four = self.config.get("account_number", "")[-4:]
return re.match(self.header_identifier, file.head(), flags=re.DOTALL) and f"{last_four}" in file.name
return (
re.match(self.header_identifier, file.head(), flags=re.DOTALL)
and f"{last_four}" in file.name
)

def prepare_table(self, rdr):
for field in ["Action", "Symbol", "Description"]:
rdr = rdr.convert(field, lambda x: x.lstrip())

rdr = rdr.addfield("total", lambda x: x["Amount ($)"])
rdr = rdr.addfield("tradeDate", lambda x: x["Run Date"])
rdr = rdr.cutout('Type')
rdr = rdr.cutout("Type")
rdr = rdr.capture("Action", "(\\S+(?:\\s+\\S+)?)", ["type"], include_original=True)

# for field in ["memo"]:
Expand Down
113 changes: 71 additions & 42 deletions beancount_reds_importers/importers/ibkr/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,11 @@
"""

import datetime

from beancount.core.number import D

from beancount_reds_importers.libreader import xmlreader
from beancount_reds_importers.libtransactionbuilder import investments
from beancount.core.number import D


class DictToObject:
Expand All @@ -106,8 +108,8 @@ def __init__(self, dictionary):

# xml on left, ofx on right
ofx_type_map = {
'BUY': 'buystock',
'SELL': 'selltock',
"BUY": "buystock",
"SELL": "selltock",
}


Expand All @@ -119,92 +121,119 @@ def custom_init(self):
self.max_rounding_error = 0.04
self.filename_pattern_def = "ibkr"
self.custom_init_run = True
self.date_format = '%Y-%m-%d'
self.date_format = "%Y-%m-%d"
self.get_ticker_info = self.get_ticker_info_from_id

def deep_identify(self, file):
try:
if self.config.get('account_number', None):
if self.config.get("account_number", None):
# account number specific matching
return self.config['account_number'] == list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['accountId']
return (
self.config["account_number"]
== list(
self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"
)
)[0]["accountId"]
)
else:
# base check: simply ensure this looks like a valid IBKR Flex Query file
return list(self.get_xpath_elements("/FlexQueryResponse"))[0] is not None
except IndexError:
return False

def set_currency(self):
self.currency = list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['currency']
self.currency = list(
self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"
)
)[0]["currency"]

# fixup dates
def convert_date(self, d):
d = d.split(' ')[0]
d = d.split(" ")[0]
return datetime.datetime.strptime(d, self.date_format)

def xml_transfer_interpreter(self, xml_data):
# map, with ofx fields on the left and xml fields on the right
ofx_dict = {
'security': xml_data['isin'],
'tradeDate': self.convert_date(xml_data['dateTime']),
'units': D(xml_data['quantity']),
'memo': 'Transfer in kind',
'type': 'transfer',
"security": xml_data["isin"],
"tradeDate": self.convert_date(xml_data["dateTime"]),
"units": D(xml_data["quantity"]),
"memo": "Transfer in kind",
"type": "transfer",
}
return DictToObject(ofx_dict)

def xml_trade_interpreter(self, xml_data):
# map, with ofx fields on the left and xml fields on the right
ofx_dict = {
'security': xml_data['isin'],
'tradeDate': self.convert_date(xml_data['dateTime']),
'memo': xml_data['transactionType'],
'type': ofx_type_map[xml_data['buySell']],
'units': D(xml_data['quantity']),
'unit_price': D(xml_data['tradePrice']),
'commission': -1 * D(xml_data['ibCommission']),
'total': D(xml_data['netCash']),
"security": xml_data["isin"],
"tradeDate": self.convert_date(xml_data["dateTime"]),
"memo": xml_data["transactionType"],
"type": ofx_type_map[xml_data["buySell"]],
"units": D(xml_data["quantity"]),
"unit_price": D(xml_data["tradePrice"]),
"commission": -1 * D(xml_data["ibCommission"]),
"total": D(xml_data["netCash"]),
}
return DictToObject(ofx_dict)

def xml_cash_interpreter(self, xml_data):
# map, with ofx fields on the left and xml fields on the right
ofx_dict = {
'tradeDate': self.convert_date(xml_data['dateTime']),
'amount': D(xml_data['amount']),
'security': xml_data.get('isin', None),
'type': 'cash',
'memo': xml_data['type'],
"tradeDate": self.convert_date(xml_data["dateTime"]),
"amount": D(xml_data["amount"]),
"security": xml_data.get("isin", None),
"type": "cash",
"memo": xml_data["type"],
}

if xml_data['type'] == 'Dividends':
ofx_dict['type'] = 'dividends'
ofx_dict['total'] = ofx_dict['amount']
if xml_data["type"] == "Dividends":
ofx_dict["type"] = "dividends"
ofx_dict["total"] = ofx_dict["amount"]

return DictToObject(ofx_dict)

def get_transactions(self):
yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade',
xml_interpreter=self.xml_trade_interpreter)
yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction',
xml_interpreter=self.xml_cash_interpreter)
yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Transfers/Transfer',
xml_interpreter=self.xml_transfer_interpreter)
yield from self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade",
xml_interpreter=self.xml_trade_interpreter,
)
yield from self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction",
xml_interpreter=self.xml_cash_interpreter,
)
yield from self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/Transfers/Transfer",
xml_interpreter=self.xml_transfer_interpreter,
)

def get_balance_assertion_date(self):
ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0]
return self.convert_date(ac['toDate']).date()
ac = list(
self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency"
)
)[0]
return self.convert_date(ac["toDate"]).date()

def get_available_cash(self, settlement_fund_balance=0):
"""Assumes there's only one cash currency.
TODO: get investments transaction builder to accept date from get_available_cash
"""
ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0]
return D(ac['slbNetCash'])
ac = list(
self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency"
)
)[0]
return D(ac["slbNetCash"])

def get_balance_positions(self):
for pos in self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/OpenPositions/OpenPosition'):
for pos in self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/OpenPositions/OpenPosition"
):
balance = {
'security': pos['isin'],
'units': D(pos['position']),
"security": pos["isin"],
"units": D(pos["position"]),
}
yield DictToObject(balance)
32 changes: 16 additions & 16 deletions beancount_reds_importers/importers/ibkr/flexquery_download.py
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,32 @@
#!/usr/bin/env python3
"""IBKR Flex Query Downloader"""

import requests
import click
import requests


@click.command()
@click.argument('token', required=True)
@click.argument('query_id', required=True)
@click.argument("token", required=True)
@click.argument("query_id", required=True)
def flexquery_download(token, query_id):
url = "https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.SendRequest"

url = (
"https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.SendRequest"
)

# Request Flex Query
request_payload = {
"v": "3",
"t": token,
"q": query_id
}

request_payload = {"v": "3", "t": token, "q": query_id}

response = requests.post(url, data=request_payload)

if response.status_code == 200:
request_id = response.text.split("<ReferenceCode>")[1].split("</ReferenceCode>")[0]
# print(f"Request ID: {request_id}")

# Construct URL to get the query result
result_url = f"https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.GetStatement?q={request_id}&t={token}&v=3"

result_response = requests.get(result_url)

if result_response.status_code == 200:
print(result_response.text)
else:
Expand All @@ -37,5 +36,6 @@ def flexquery_download(token, query_id):
print(f"Failed to request the query. Status Code: {response.status_code}")
return None

if __name__ == '__main__':

if __name__ == "__main__":
flexquery_download()
2 changes: 1 addition & 1 deletion beancount_reds_importers/importers/vanguard/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def cleanup_memo(self, ot):
# some vanguard files have memos repeated like this:
# 'DIVIDEND REINVESTMENTDIVIDEND REINVESTMENT'
retval = ot.memo
if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2):]:
if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2) :]:
retval = ot.memo[: int(len(ot.memo) / 2)]
return retval

Expand Down
Loading

0 comments on commit be5abaa

Please sign in to comment.