From 80afa1b0222ae776826393fc53a230db73892a22 Mon Sep 17 00:00:00 2001 From: Red S Date: Fri, 15 Mar 2024 00:55:25 -0700 Subject: [PATCH 01/13] feat: add 'show_configured' in paycheck transaction builder This lists all entries in the paycheck which are being ignored by the current configuration. This is useful when new items appear on the paycheck and the configuration needs to be appended with it. --- .../libtransactionbuilder/paycheck.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index d6709fe..24efeb7 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -3,7 +3,7 @@ from beancount.core import data from beancount.core.number import D from beancount_reds_importers.libtransactionbuilder import banking - +from collections import defaultdict # paychecks are typically transaction with many (10-40) postings including several each of income, taxes, # pre-tax and post-tax deductions, transfers, reimbursements, etc. This importer enables importing a single @@ -66,15 +66,19 @@ def build_postings(self, entry): template = self.config['paycheck_template'] currency = self.config['currency'] total = 0 + template_missing = defaultdict(set) for section, table in self.alltables.items(): if section not in template: + template_missing[section] = set() continue for row in table.namedtuples(): # TODO: 'bank' is workday specific; move it there row_description = getattr(row, 'description', getattr(row, 'bank', None)) row_pattern = next(filter(lambda ts: row_description.startswith(ts), template[section]), None) - if row_pattern: + if not row_pattern: + template_missing[section].add(row_description) + else: accounts = template[section][row_pattern] accounts = [accounts] if not isinstance(accounts, list) else accounts for account in accounts: @@ -89,6 +93,14 @@ def build_postings(self, entry): total += amount if amount: data.create_simple_posting(entry, account, amount, currency) + + if self.config.get('show_unconfigured', False): + for section in template_missing: + print(section) + if template_missing[section]: + print(' ' + '\n '.join(i for i in template_missing[section])) + print() + if total != 0: data.create_simple_posting(entry, "TOTAL:NONZERO", total, currency) From 1c9adcd32f94ce4c21f9e1e70a9f65d7c905cd4a Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 23 Mar 2024 14:10:09 -0700 Subject: [PATCH 02/13] ci: fix conventionalcommits branch name so it runs on PRs --- .github/workflows/conventionalcommits.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conventionalcommits.yml b/.github/workflows/conventionalcommits.yml index b7fa7a1..add48c6 100644 --- a/.github/workflows/conventionalcommits.yml +++ b/.github/workflows/conventionalcommits.yml @@ -2,7 +2,7 @@ name: Conventional Commits on: pull_request: - branches: [ master ] + branches: [ main ] jobs: build: From b75c8e55bf0ca6a4b8e3c906a8fb5043c4910545 Mon Sep 17 00:00:00 2001 From: Rane Brown Date: Fri, 22 Mar 2024 11:36:01 -0600 Subject: [PATCH 03/13] feat: enforce formatting with ruff --- .github/workflows/pythonpackage.yml | 3 + .ruff.toml | 8 +- beancount_reds_importers/example/fund_info.py | 14 +- .../importers/alliant/__init__.py | 4 +- .../importers/ally/__init__.py | 4 +- .../importers/amazongc/__init__.py | 40 +- .../importers/amex/__init__.py | 4 +- .../importers/becu/__init__.py | 4 +- .../importers/capitalonebank/__init__.py | 4 +- .../importers/chase/__init__.py | 4 +- .../importers/citi/__init__.py | 4 +- .../importers/dcu/__init__.py | 17 +- .../importers/discover/__init__.py | 24 +- .../importers/discover/discover_ofx.py | 4 +- .../importers/etrade/__init__.py | 8 +- .../etrade/tests/etrade_qfx_brokerage_test.py | 54 ++- .../importers/fidelity/__init__.py | 18 +- .../importers/fidelity/fidelity_cma_csv.py | 46 ++- .../importers/morganstanley/__init__.py | 6 +- .../importers/schwab/schwab_csv_balances.py | 43 ++- .../importers/schwab/schwab_csv_brokerage.py | 105 ++--- .../importers/schwab/schwab_csv_checking.py | 52 +-- .../importers/schwab/schwab_csv_creditline.py | 7 +- .../importers/schwab/schwab_csv_positions.py | 31 +- .../importers/schwab/schwab_json_brokerage.py | 12 +- .../importers/schwab/schwab_ofx_bank_ofx.py | 4 +- .../importers/schwab/schwab_ofx_brokerage.py | 6 +- .../schwab_csv_brokerage_test.py | 56 ++- .../schwab_csv_checking_test.py | 7 +- .../importers/stanchart/scbbank.py | 64 ++-- .../importers/stanchart/scbcard.py | 72 ++-- .../importers/target/__init__.py | 4 +- .../importers/tdameritrade/__init__.py | 9 +- .../importers/techcubank/__init__.py | 4 +- .../unitedoverseas/tests/uobbank_test.py | 14 +- .../importers/unitedoverseas/uobbank.py | 46 ++- .../importers/unitedoverseas/uobcard.py | 60 +-- .../importers/unitedoverseas/uobsrs.py | 45 ++- .../importers/vanguard/__init__.py | 20 +- .../vanguard/vanguard_screenscrape.py | 82 ++-- .../importers/workday/__init__.py | 30 +- .../libreader/csv_multitable_reader.py | 24 +- .../libreader/csvreader.py | 54 ++- .../libreader/jsonreader.py | 9 +- .../libreader/ofxreader.py | 82 ++-- beancount_reds_importers/libreader/reader.py | 27 +- .../last_transaction_date_test.py | 12 +- .../ofx_date/ofx_date_test.py | 12 +- .../smart/smart_date_test.py | 10 +- .../libreader/tsvreader.py | 3 +- .../libreader/xlsreader.py | 10 +- .../libreader/xlsx_multitable_reader.py | 6 +- .../libreader/xlsxreader.py | 2 +- .../libtransactionbuilder/banking.py | 65 ++-- .../libtransactionbuilder/common.py | 66 +++- .../libtransactionbuilder/investments.py | 360 ++++++++++++------ .../libtransactionbuilder/paycheck.py | 54 ++- .../transactionbuilder.py | 19 +- .../util/bean_download.py | 107 ++++-- beancount_reds_importers/util/needs_update.py | 113 ++++-- .../util/ofx_summarize.py | 63 ++- setup.py | 64 ++-- 62 files changed, 1342 insertions(+), 863 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 172e3d6..af768a6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -34,3 +34,6 @@ jobs: - name: Test with pytest run: | pytest + - name: Check format with ruff + run: | + ruff format --check diff --git a/.ruff.toml b/.ruff.toml index 46121f3..03403de 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1 +1,7 @@ -line-length = 127 +line-length = 88 + +[format] +docstring-code-format = true +indent-style = "space" +line-ending = "lf" +quote-style = "double" diff --git a/beancount_reds_importers/example/fund_info.py b/beancount_reds_importers/example/fund_info.py index 807e647..65858d2 100755 --- a/beancount_reds_importers/example/fund_info.py +++ b/beancount_reds_importers/example/fund_info.py @@ -20,15 +20,15 @@ # mutual funds since those are brokerage specific. fund_data = [ - ('SCHF', '808524805', 'Schwab International Equity ETF'), - ('VGTEST', '012345678', 'Vanguard Test Fund'), - ('VMFXX', '922906300', 'Vanguard Federal Money Market Fund'), + ("SCHF", "808524805", "Schwab International Equity ETF"), + ("VGTEST", "012345678", "Vanguard Test Fund"), + ("VMFXX", "922906300", "Vanguard Federal Money Market Fund"), ] # list of money_market accounts. These will not be held at cost, and instead will use price conversions -money_market = ['VMFXX'] +money_market = ["VMFXX"] fund_info = { - 'fund_data': fund_data, - 'money_market': money_market, - } + "fund_data": fund_data, + "money_market": money_market, +} diff --git a/beancount_reds_importers/importers/alliant/__init__.py b/beancount_reds_importers/importers/alliant/__init__.py index a67681c..dd6c587 100644 --- a/beancount_reds_importers/importers/alliant/__init__.py +++ b/beancount_reds_importers/importers/alliant/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Alliant Credit Union' + IMPORTER_NAME = "Alliant Credit Union" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*alliant' + self.filename_pattern_def = ".*alliant" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/ally/__init__.py b/beancount_reds_importers/importers/ally/__init__.py index 3764d0f..08b1264 100644 --- a/beancount_reds_importers/importers/ally/__init__.py +++ b/beancount_reds_importers/importers/ally/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Ally Bank' + IMPORTER_NAME = "Ally Bank" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*transactions' + self.filename_pattern_def = ".*transactions" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/amazongc/__init__.py b/beancount_reds_importers/importers/amazongc/__init__.py index f4f486d..e1f5d3b 100644 --- a/beancount_reds_importers/importers/amazongc/__init__.py +++ b/beancount_reds_importers/importers/amazongc/__init__.py @@ -37,25 +37,25 @@ class Importer(importer.ImporterProtocol): def __init__(self, config): self.config = config - self.currency = self.config.get('currency', 'CURRENCY_NOT_CONFIGURED') - self.filename_pattern_def = 'amazon-gift-card.tsv' + self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED") + self.filename_pattern_def = "amazon-gift-card.tsv" def identify(self, file): return self.filename_pattern_def in file.name def file_name(self, file): - return '{}'.format(ntpath.basename(file.name)) + return "{}".format(ntpath.basename(file.name)) def file_account(self, _): - return self.config['main_account'] + return self.config["main_account"] def file_date(self, file): "Get the maximum date from the file." maxdate = datetime.date.min - for line in open(file.name, 'r').readlines()[1:]: - f = line.split('\t') + for line in open(file.name, "r").readlines()[1:]: + f = line.split("\t") f = [i.strip() for i in f] - date = datetime.datetime.strptime(f[0], '%B %d, %Y').date() + date = datetime.datetime.strptime(f[0], "%B %d, %Y").date() maxdate = max(date, maxdate) return maxdate @@ -65,18 +65,28 @@ def extract(self, file, existing_entries=None): new_entries = [] counter = itertools.count() - for line in open(file.name, 'r').readlines()[1:]: - f = line.split('\t') + for line in open(file.name, "r").readlines()[1:]: + f = line.split("\t") f = [i.strip() for i in f] - date = datetime.datetime.strptime(f[0], '%B %d, %Y').date() + date = datetime.datetime.strptime(f[0], "%B %d, %Y").date() description = f[1].encode("ascii", "ignore").decode() - number = D(f[2].replace('$', '')) + number = D(f[2].replace("$", "")) metadata = data.new_metadata(file.name, next(counter)) - entry = data.Transaction(metadata, date, self.FLAG, - None, description, data.EMPTY_SET, data.EMPTY_SET, []) - data.create_simple_posting(entry, config['main_account'], number, self.currency) - data.create_simple_posting(entry, config['target_account'], None, None) + entry = data.Transaction( + metadata, + date, + self.FLAG, + None, + description, + data.EMPTY_SET, + data.EMPTY_SET, + [], + ) + data.create_simple_posting( + entry, config["main_account"], number, self.currency + ) + data.create_simple_posting(entry, config["target_account"], None, None) new_entries.append(entry) return new_entries diff --git a/beancount_reds_importers/importers/amex/__init__.py b/beancount_reds_importers/importers/amex/__init__.py index 8ebae4e..086cb0a 100644 --- a/beancount_reds_importers/importers/amex/__init__.py +++ b/beancount_reds_importers/importers/amex/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'American Express' + IMPORTER_NAME = "American Express" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*amex' + self.filename_pattern_def = ".*amex" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/becu/__init__.py b/beancount_reds_importers/importers/becu/__init__.py index 3f978fe..a0a019a 100644 --- a/beancount_reds_importers/importers/becu/__init__.py +++ b/beancount_reds_importers/importers/becu/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'BECU' + IMPORTER_NAME = "BECU" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*becu' + self.filename_pattern_def = ".*becu" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/capitalonebank/__init__.py b/beancount_reds_importers/importers/capitalonebank/__init__.py index 43a93d2..48c1043 100644 --- a/beancount_reds_importers/importers/capitalonebank/__init__.py +++ b/beancount_reds_importers/importers/capitalonebank/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Capital One Bank' + IMPORTER_NAME = "Capital One Bank" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*360Checking' + self.filename_pattern_def = ".*360Checking" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/chase/__init__.py b/beancount_reds_importers/importers/chase/__init__.py index 4e70ebf..4a79edc 100644 --- a/beancount_reds_importers/importers/chase/__init__.py +++ b/beancount_reds_importers/importers/chase/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Chase' + IMPORTER_NAME = "Chase" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*[Cc]hase' + self.filename_pattern_def = ".*[Cc]hase" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/citi/__init__.py b/beancount_reds_importers/importers/citi/__init__.py index 4460ae2..c4d210f 100644 --- a/beancount_reds_importers/importers/citi/__init__.py +++ b/beancount_reds_importers/importers/citi/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Citi' + IMPORTER_NAME = "Citi" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*citi' + self.filename_pattern_def = ".*citi" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/dcu/__init__.py b/beancount_reds_importers/importers/dcu/__init__.py index d818a1b..6837ddf 100644 --- a/beancount_reds_importers/importers/dcu/__init__.py +++ b/beancount_reds_importers/importers/dcu/__init__.py @@ -14,19 +14,20 @@ def custom_init(self): self.header_identifier = "" self.column_labels_line = '"DATE","TRANSACTION TYPE","DESCRIPTION","AMOUNT","ID","MEMO","CURRENT BALANCE"' self.date_format = "%m/%d/%Y" + # fmt: off self.header_map = { - "DATE": "date", - "DESCRIPTION": "payee", - "MEMO": "memo", - "AMOUNT": "amount", - "CURRENT BALANCE": "balance", + "DATE": "date", + "DESCRIPTION": "payee", + "MEMO": "memo", + "AMOUNT": "amount", + "CURRENT BALANCE": "balance", "TRANSACTION TYPE": "type", } - self.transaction_type_map = { - "DEBIT": "transfer", - "CREDIT": "transfer", + "DEBIT": "transfer", + "CREDIT": "transfer", } + # fmt: on self.skip_transaction_types = [] def get_balance_statement(self, file=None): diff --git a/beancount_reds_importers/importers/discover/__init__.py b/beancount_reds_importers/importers/discover/__init__.py index a554e07..d06cdfc 100644 --- a/beancount_reds_importers/importers/discover/__init__.py +++ b/beancount_reds_importers/importers/discover/__init__.py @@ -1,4 +1,4 @@ -""" Discover credit card .csv importer.""" +"""Discover credit card .csv importer.""" from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking @@ -9,21 +9,23 @@ class Importer(csvreader.Importer, banking.Importer): def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'Discover.*' - self.header_identifier = 'Trans. Date,Post Date,Description,Amount,Category' - self.date_format = '%m/%d/%Y' + self.filename_pattern_def = "Discover.*" + self.header_identifier = "Trans. Date,Post Date,Description,Amount,Category" + self.date_format = "%m/%d/%Y" + # fmt: off self.header_map = { - "Category": 'payee', - "Description": 'memo', - "Trans. Date": 'date', - "Post Date": 'postDate', - "Amount": 'amount', - } + "Category": "payee", + "Description": "memo", + "Trans. Date": "date", + "Post Date": "postDate", + "Amount": "amount", + } + # fmt: on def skip_transaction(self, ot): return False def prepare_processed_table(self, rdr): # Need to invert numbers supplied by Discover - rdr = rdr.convert('amount', lambda x: -1 * x) + rdr = rdr.convert("amount", lambda x: -1 * x) return rdr diff --git a/beancount_reds_importers/importers/discover/discover_ofx.py b/beancount_reds_importers/importers/discover/discover_ofx.py index 0593a2e..abee824 100644 --- a/beancount_reds_importers/importers/discover/discover_ofx.py +++ b/beancount_reds_importers/importers/discover/discover_ofx.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Discover' + IMPORTER_NAME = "Discover" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*Discover' + self.filename_pattern_def = ".*Discover" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/etrade/__init__.py b/beancount_reds_importers/importers/etrade/__init__.py index b5eb96a..bc6d3d3 100644 --- a/beancount_reds_importers/importers/etrade/__init__.py +++ b/beancount_reds_importers/importers/etrade/__init__.py @@ -1,18 +1,18 @@ -""" ETrade Brokerage ofx importer.""" +"""ETrade Brokerage ofx importer.""" from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'ETrade Brokerage OFX' + IMPORTER_NAME = "ETrade Brokerage OFX" def custom_init(self): self.max_rounding_error = 0.11 - self.filename_pattern_def = '.*etrade' + self.filename_pattern_def = ".*etrade" self.get_ticker_info = self.get_ticker_info_from_id def skip_transaction(self, ot): - if 'JNL' in ot.memo: + if "JNL" in ot.memo: return True return False diff --git a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py index fefd32d..1ad8980 100644 --- a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py +++ b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py @@ -6,49 +6,45 @@ fund_data = [ - ('TSM', '874039100', 'Taiwan Semiconductor Mfg LTD'), - ('VISA', '92826C839', 'Visa Inc'), + ("TSM", "874039100", "Taiwan Semiconductor Mfg LTD"), + ("VISA", "92826C839", "Visa Inc"), ] # list of money_market accounts. These will not be held at cost, and instead will use price conversions -money_market = ['VMFXX'] +money_market = ["VMFXX"] fund_info = { - 'fund_data': fund_data, - 'money_market': money_market, - } + "fund_data": fund_data, + "money_market": money_market, +} def build_config(): acct = "Assets:Investments:Etrade" - root = 'Investments' - taxability = 'Taxable' - leaf = 'Etrade' - currency = 'USD' + root = "Investments" + taxability = "Taxable" + leaf = "Etrade" + currency = "USD" config = { - 'account_number' : '555555555', - 'main_account' : acct + ':{ticker}', - 'cash_account' : f'{acct}:{{currency}}', - 'transfer' : 'Assets:Zero-Sum-Accounts:Transfers:Bank-Account', - 'dividends' : f'Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}', - 'interest' : f'Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}', - 'cg' : f'Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}', - 'capgainsd_lt' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}', - 'capgainsd_st' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}', - 'fees' : f'Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}', - 'invexpense' : f'Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}', - 'rounding_error' : 'Equity:Rounding-Errors:Imports', - 'fund_info' : fund_info, - 'currency' : currency, + "account_number": "555555555", + "main_account": acct + ":{ticker}", + "cash_account": f"{acct}:{{currency}}", + "transfer": "Assets:Zero-Sum-Accounts:Transfers:Bank-Account", + "dividends": f"Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}", + "interest": f"Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}", + "cg": f"Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}", + "capgainsd_lt": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}", + "capgainsd_st": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}", + "fees": f"Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}", + "invexpense": f"Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}", + "rounding_error": "Equity:Rounding-Errors:Imports", + "fund_info": fund_info, + "currency": currency, } return config -@regtest.with_importer( - etrade.Importer( - build_config() - ) -) +@regtest.with_importer(etrade.Importer(build_config())) @regtest.with_testdir(path.dirname(__file__)) class TestEtradeQFX(regtest.ImporterTestBase): pass diff --git a/beancount_reds_importers/importers/fidelity/__init__.py b/beancount_reds_importers/importers/fidelity/__init__.py index d0438b7..3330376 100644 --- a/beancount_reds_importers/importers/fidelity/__init__.py +++ b/beancount_reds_importers/importers/fidelity/__init__.py @@ -6,27 +6,31 @@ class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Fidelity Net Benefits / Fidelity Investments OFX' + IMPORTER_NAME = "Fidelity Net Benefits / Fidelity Investments OFX" def custom_init(self): self.max_rounding_error = 0.18 - self.filename_pattern_def = '.*fidelity' + self.filename_pattern_def = ".*fidelity" self.get_ticker_info = self.get_ticker_info_from_id - self.get_payee = lambda ot: ot.memo.split(";", 1)[0] if ';' in ot.memo else ot.memo + self.get_payee = ( + lambda ot: ot.memo.split(";", 1)[0] if ";" in ot.memo else ot.memo + ) def security_narration(self, ot): ticker, ticker_long_name = self.get_ticker_info(ot.security) return f"[{ticker}]" def file_name(self, file): - return 'fidelity-{}-{}'.format(self.config['account_number'], ntpath.basename(file.name)) + return "fidelity-{}-{}".format( + self.config["account_number"], ntpath.basename(file.name) + ) def get_target_acct_custom(self, transaction, ticker=None): if transaction.memo.startswith("CONTRIBUTION"): - return self.config['transfer'] + return self.config["transfer"] if transaction.memo.startswith("FEES"): - return self.config['fees'] + return self.config["fees"] return None def get_available_cash(self, settlement_fund_balance=0): - return getattr(self.ofx_account.statement, 'available_cash', None) + return getattr(self.ofx_account.statement, "available_cash", None) diff --git a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py index c609e1c..e6abf7b 100644 --- a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py +++ b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py @@ -6,45 +6,49 @@ class Importer(banking.Importer, csvreader.Importer): - IMPORTER_NAME = 'Fidelity Cash Management Account' + IMPORTER_NAME = "Fidelity Cash Management Account" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*History' - self.date_format = '%m/%d/%Y' + self.filename_pattern_def = ".*History" + self.date_format = "%m/%d/%Y" header_s0 = ".*Run Date,Action,Symbol,Security Description,Security Type,Quantity,Price \\(\\$\\)," header_s1 = "Commission \\(\\$\\),Fees \\(\\$\\),Accrued Interest \\(\\$\\),Amount \\(\\$\\),Settlement Date" header_sum = header_s0 + header_s1 self.header_identifier = header_sum self.skip_head_rows = 5 self.skip_tail_rows = 16 + # fmt: off self.header_map = { - "Run Date": 'date', - "Action": 'description', - "Amount ($)": 'amount', - - "Settlement Date": 'settleDate', - "Accrued Interest ($)": 'accrued_interest', - "Fees ($)": 'fees', - "Security Type": 'security_type', - "Commission ($)": 'commission', - "Security Description": 'security_description', - "Symbol": 'security', - "Price ($)": 'unit_price', - } + "Run Date": "date", + "Action": "description", + "Amount ($)": "amount", + "Settlement Date": "settleDate", + "Accrued Interest ($)": "accrued_interest", + "Fees ($)": "fees", + "Security Type": "security_type", + "Commission ($)": "commission", + "Security Description": "security_description", + "Symbol": "security", + "Price ($)": "unit_price", + } + # fmt: on def deep_identify(self, file): return re.match(self.header_identifier, file.head(), flags=re.DOTALL) def prepare_raw_columns(self, rdr): - - for field in ['Action']: + for field in ["Action"]: rdr = rdr.convert(field, lambda x: x.lstrip()) - rdr = rdr.capture('Action', '(?:\\s)(?:\\w*)(.*)', ['memo'], include_original=True) - rdr = rdr.capture('Action', '(\\S+(?:\\s+\\S+)?)', ['payee'], include_original=True) + rdr = rdr.capture( + "Action", "(?:\\s)(?:\\w*)(.*)", ["memo"], include_original=True + ) + rdr = rdr.capture( + "Action", "(\\S+(?:\\s+\\S+)?)", ["payee"], include_original=True + ) - for field in ['memo', 'payee']: + for field in ["memo", "payee"]: rdr = rdr.convert(field, lambda x: x.lstrip()) return rdr diff --git a/beancount_reds_importers/importers/morganstanley/__init__.py b/beancount_reds_importers/importers/morganstanley/__init__.py index 7084e1d..5a859d8 100644 --- a/beancount_reds_importers/importers/morganstanley/__init__.py +++ b/beancount_reds_importers/importers/morganstanley/__init__.py @@ -1,13 +1,13 @@ -""" Morgan Stanley Investments ofx importer.""" +"""Morgan Stanley Investments ofx importer.""" from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Morgan Stanley Investments' + IMPORTER_NAME = "Morgan Stanley Investments" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*morganstanley' + self.filename_pattern_def = ".*morganstanley" self.get_ticker_info = self.get_ticker_info_from_id diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py index 5f52426..1ea5361 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py @@ -1,4 +1,4 @@ -""" Schwab csv importer.""" +"""Schwab csv importer.""" import datetime import re @@ -8,35 +8,38 @@ class Importer(investments.Importer, csv_multitable_reader.Importer): - IMPORTER_NAME = 'Schwab Brokerage Balances CSV' + IMPORTER_NAME = "Schwab Brokerage Balances CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Balances_' - self.header_identifier = 'Balances for account' + self.filename_pattern_def = ".*_Balances_" + self.header_identifier = "Balances for account" self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" + # fmt: off self.header_map = { - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - } + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + } + # fmt: on def prepare_table(self, rdr): return rdr def convert_columns(self, rdr): # fixup decimals - decimals = ['units'] + decimals = ["units"] for i in decimals: rdr = rdr.convert(i, D) # fixup currencies def remove_non_numeric(x): - return re.sub(r'[^0-9\.]', "", x) # noqa: W605 - currencies = ['unit_price'] + return re.sub(r"[^0-9\.]", "", x) # noqa: W605 + + currencies = ["unit_price"] for i in currencies: rdr = rdr.convert(i, remove_non_numeric) rdr = rdr.convert(i, D) @@ -51,18 +54,18 @@ def get_max_transaction_date(self): def prepare_tables(self): # first row has date - d = self.raw_rdr[0][0].rsplit(' ', 1)[1] + d = self.raw_rdr[0][0].rsplit(" ", 1)[1] self.date = datetime.datetime.strptime(d, self.date_format) for section, table in self.alltables.items(): - if section in self.config['section_headers']: + if section in self.config["section_headers"]: table = table.rename(self.header_map) table = self.convert_columns(table) - table = table.cut('memo', 'security', 'units', 'unit_price') - table = table.selectne('memo', '--') # we don't need total rows - table = table.addfield('date', self.date) + table = table.cut("memo", "security", "units", "unit_price") + table = table.selectne("memo", "--") # we don't need total rows + table = table.addfield("date", self.date) self.alltables[section] = table def get_balance_positions(self): - for section in self.config['section_headers']: + for section in self.config["section_headers"]: yield from self.alltables[section].namedtuples() diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index 01583fc..0f91fa9 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -1,4 +1,4 @@ -""" Schwab Brokerage .csv importer.""" +"""Schwab Brokerage .csv importer.""" import re from beancount_reds_importers.libreader import csvreader @@ -6,71 +6,76 @@ class Importer(csvreader.Importer, investments.Importer): - IMPORTER_NAME = 'Schwab Brokerage CSV' + IMPORTER_NAME = "Schwab Brokerage CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Transactions_' - self.header_identifier = '' + self.filename_pattern_def = ".*_Transactions_" + self.header_identifier = "" self.column_labels_line = '"Date","Action","Symbol","Description","Quantity","Price","Fees & Comm","Amount"' self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" self.get_payee = lambda ot: ot.Action + # fmt: off self.header_map = { - "Date": 'date', - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - "Amount": 'amount', - # "tradeDate": 'tradeDate', - # "total": 'total', - "Fees & Comm": 'fees', - } + "Date": "date", + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + "Amount": "amount", + # "tradeDate": "tradeDate", + # "total": "total", + "Fees & Comm": "fees", + } self.transaction_type_map = { - 'Bank Interest': 'income', - 'Bank Transfer': 'cash', - 'Buy': 'buystock', - 'Journaled Shares': 'buystock', # These are in-kind tranfers - 'Reinvestment Adj': 'buystock', - 'Div Adjustment': 'dividends', - 'Long Term Cap Gain Reinvest': 'capgainsd_lt', - 'Misc Credits': 'cash', - 'MoneyLink Deposit': 'cash', - 'MoneyLink Transfer': 'cash', - 'Pr Yr Div Reinvest': 'dividends', - 'Journal': 'cash', # These are transfers - 'Reinvest Dividend': 'dividends', - 'Qualified Dividend': 'dividends', - 'Cash Dividend': 'dividends', - 'Reinvest Shares': 'buystock', - 'Sell': 'sellstock', - 'Short Term Cap Gain Reinvest': 'capgainsd_st', - 'Wire Funds Received': 'cash', - 'Wire Received': 'cash', - 'Funds Received': 'cash', - 'Stock Split': 'cash', - 'Cash In Lieu': 'cash', - } + "Bank Interest": "income", + "Bank Transfer": "cash", + "Buy": "buystock", + "Journaled Shares": "buystock", # These are in-kind tranfers + "Reinvestment Adj": "buystock", + "Div Adjustment": "dividends", + "Long Term Cap Gain Reinvest": "capgainsd_lt", + "Misc Credits": "cash", + "MoneyLink Deposit": "cash", + "MoneyLink Transfer": "cash", + "Pr Yr Div Reinvest": "dividends", + "Journal": "cash", # These are transfers + "Reinvest Dividend": "dividends", + "Qualified Dividend": "dividends", + "Cash Dividend": "dividends", + "Reinvest Shares": "buystock", + "Sell": "sellstock", + "Short Term Cap Gain Reinvest": "capgainsd_st", + "Wire Funds Received": "cash", + "Wire Received": "cash", + "Funds Received": "cash", + "Stock Split": "cash", + "Cash In Lieu": "cash", + } + # fmt: on def deep_identify(self, file): - last_three = self.config.get('account_number', '')[-3:] - return re.match(self.header_identifier, file.head()) and f'XX{last_three}' in file.name + last_three = self.config.get("account_number", "")[-3:] + return ( + re.match(self.header_identifier, file.head()) + and f"XX{last_three}" in file.name + ) def skip_transaction(self, ot): - return ot.type in ['', 'Journal'] + return ot.type in ["", "Journal"] def prepare_table(self, rdr): - if '' in rdr.fieldnames(): - rdr = rdr.cutout('') # clean up last column + if "" in rdr.fieldnames(): + rdr = rdr.cutout("") # clean up last column def cleanup_date(d): """'11/16/2018 as of 11/15/2018' --> '11/16/2018'""" - return d.split(' ', 1)[0] + return d.split(" ", 1)[0] - rdr = rdr.convert('Date', cleanup_date) - rdr = rdr.addfield('tradeDate', lambda x: x['Date']) - rdr = rdr.addfield('total', lambda x: x['Amount']) - rdr = rdr.addfield('type', lambda x: x['Action']) + rdr = rdr.convert("Date", cleanup_date) + rdr = rdr.addfield("tradeDate", lambda x: x["Date"]) + rdr = rdr.addfield("total", lambda x: x["Amount"]) + rdr = rdr.addfield("type", lambda x: x["Action"]) return rdr diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py index 7a3072c..8284364 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py @@ -1,43 +1,47 @@ -""" Schwab Checking .csv importer.""" +"""Schwab Checking .csv importer.""" from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking class Importer(csvreader.Importer, banking.Importer): - IMPORTER_NAME = 'Schwab Checking account CSV' + IMPORTER_NAME = "Schwab Checking account CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Checking_Transactions_' - self.header_identifier = '' + self.filename_pattern_def = ".*_Checking_Transactions_" + self.header_identifier = "" self.column_labels_line = '"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' - self.date_format = '%m/%d/%Y' - self.skip_comments = '# ' + self.date_format = "%m/%d/%Y" + self.skip_comments = "# " + # fmt: off self.header_map = { - "Date": "date", - "Type": "type", - "CheckNumber": "checknum", - "Description": "payee", - "Withdrawal": "withdrawal", - "Deposit": "deposit", - "RunningBalance": "balance" + "Date": "date", + "Type": "type", + "CheckNumber": "checknum", + "Description": "payee", + "Withdrawal": "withdrawal", + "Deposit": "deposit", + "RunningBalance": "balance", } self.transaction_type_map = { - "INTADJUST": 'income', - "TRANSFER": 'transfer', - "ACH": 'transfer' + "INTADJUST": "income", + "TRANSFER": "transfer", + "ACH": "transfer", } - self.skip_transaction_types = ['Journal'] + # fmt: on + self.skip_transaction_types = ["Journal"] def deep_identify(self, file): - last_three = self.config.get('account_number', '')[-3:] - return self.column_labels_line in file.head() and f'XX{last_three}' in file.name + last_three = self.config.get("account_number", "")[-3:] + return self.column_labels_line in file.head() and f"XX{last_three}" in file.name def prepare_table(self, rdr): - rdr = rdr.addfield('amount', - lambda x: "-" + x['Withdrawal'] if x['Withdrawal'] != '' else x['Deposit']) - rdr = rdr.addfield('memo', lambda x: '') + rdr = rdr.addfield( + "amount", + lambda x: "-" + x["Withdrawal"] if x["Withdrawal"] != "" else x["Deposit"], + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def get_balance_statement(self, file=None): @@ -45,4 +49,6 @@ def get_balance_statement(self, file=None): date = self.get_balance_assertion_date() if date: - yield banking.Balance(date, self.rdr.namedtuples()[0].balance, self.currency) + yield banking.Balance( + date, self.rdr.namedtuples()[0].balance, self.currency + ) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py index 5f452de..6dc55b7 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py @@ -1,11 +1,12 @@ -""" Schwab Credit Line (eg: Pledged Asset Line) .csv importer.""" +"""Schwab Credit Line (eg: Pledged Asset Line) .csv importer.""" from beancount_reds_importers.importers.schwab import schwab_csv_checking + class Importer(schwab_csv_checking.Importer): - IMPORTER_NAME = 'Schwab Line of Credit CSV' + IMPORTER_NAME = "Schwab Line of Credit CSV" def custom_init(self): super().custom_init() - self.filename_pattern_def = '.*_Transactions_' + self.filename_pattern_def = ".*_Transactions_" self.column_labels_line = '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_positions.py b/beancount_reds_importers/importers/schwab/schwab_csv_positions.py index 8f3bd1a..4cb0980 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_positions.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_positions.py @@ -1,4 +1,4 @@ -""" Schwab CSV Positions importer. +"""Schwab CSV Positions importer. Note: Schwab "Positions" CSV is not the same as Schwab "Balances" CSV.""" @@ -14,30 +14,33 @@ class Importer(investments.Importer, csvreader.Importer): def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*-Positions-' + self.filename_pattern_def = ".*-Positions-" self.header_identifier = '["]+Positions for account' self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%Y/%m/%d' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%Y/%m/%d" + self.funds_db_txt = "funds_by_ticker" self.column_labels_line = '"Symbol","Description","Quantity","Price","Price Change %","Price Change $","Market Value","Day Change %","Day Change $","Cost Basis","Gain/Loss %","Gain/Loss $","Ratings","Reinvest Dividends?","Capital Gains?","% Of Account","Security Type"' # noqa: #501 + # fmt: off self.header_map = { - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - } + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + } + # fmt: on self.skip_transaction_types = [] def convert_columns(self, rdr): # fixup decimals - decimals = ['units'] + decimals = ["units"] for i in decimals: rdr = rdr.convert(i, D) # fixup currencies def remove_non_numeric(x): - return re.sub(r'[^0-9\.]', "", x) # noqa: W605 - currencies = ['unit_price'] + return re.sub(r"[^0-9\.]", "", x) # noqa: W605 + + currencies = ["unit_price"] for i in currencies: rdr = rdr.convert(i, remove_non_numeric) rdr = rdr.convert(i, D) @@ -53,7 +56,7 @@ def get_max_transaction_date(self): def prepare_raw_file(self, rdr): # first row has date - d = rdr[0][0].rsplit(' ', 1)[1] + d = rdr[0][0].rsplit(" ", 1)[1] self.date = datetime.datetime.strptime(d, self.date_format) return rdr @@ -68,5 +71,5 @@ def prepare_table(self, rdr): def get_balance_positions(self): for pos in self.rdr.namedtuples(): - if pos.memo != '--': + if pos.memo != "--": yield pos diff --git a/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py index 3c50ad1..4ad4504 100644 --- a/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py @@ -1,18 +1,18 @@ -""" Schwab Brokerage .csv importer.""" +"""Schwab Brokerage .csv importer.""" from beancount_reds_importers.libreader import jsonreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(jsonreader.Importer, investments.Importer): - IMPORTER_NAME = 'Schwab Brokerage JSON' + IMPORTER_NAME = "Schwab Brokerage JSON" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Transactions_' + self.filename_pattern_def = ".*_Transactions_" self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" def skip_transaction(self, ot): - return ot.type in ['', 'Journal', 'Journaled Shares'] + return ot.type in ["", "Journal", "Journaled Shares"] diff --git a/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py b/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py index fda1141..03df212 100644 --- a/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py +++ b/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Schwab Bank' + IMPORTER_NAME = "Schwab Bank" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*Checking_Transations' + self.filename_pattern_def = ".*Checking_Transations" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py index 3e28c73..af7aeb0 100644 --- a/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py @@ -1,13 +1,13 @@ -""" Schwab Brokerage ofx importer.""" +"""Schwab Brokerage ofx importer.""" from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Schwab Brokerage' + IMPORTER_NAME = "Schwab Brokerage" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*schwab' + self.filename_pattern_def = ".*schwab" self.get_ticker_info = self.get_ticker_info_from_id diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py index 546fca6..fa46fea 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py @@ -6,50 +6,46 @@ fund_data = [ - ('SWVXX', '123', 'SCHWAB VALUE ADVANTAGE MONEY INV'), - ('GIS', '456', 'GENERAL MILLS INC'), - ('BND', '789', 'Vanguard Total Bond Market Index Fund'), + ("SWVXX", "123", "SCHWAB VALUE ADVANTAGE MONEY INV"), + ("GIS", "456", "GENERAL MILLS INC"), + ("BND", "789", "Vanguard Total Bond Market Index Fund"), ] # list of money_market accounts. These will not be held at cost, and instead will use price conversions -money_market = ['VMFXX'] +money_market = ["VMFXX"] fund_info = { - 'fund_data': fund_data, - 'money_market': money_market, - } + "fund_data": fund_data, + "money_market": money_market, +} def build_config(): acct = "Assets:Investments:Schwab" - root = 'Investments' - taxability = 'Taxable' - leaf = 'Schwab' - currency = 'USD' + root = "Investments" + taxability = "Taxable" + leaf = "Schwab" + currency = "USD" config = { - 'account_number' : '9876', - 'main_account' : acct + ':{ticker}', - 'cash_account' : f'{acct}:{{currency}}', - 'transfer' : 'Assets:Zero-Sum-Accounts:Transfers:Bank-Account', - 'dividends' : f'Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}', - 'interest' : f'Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}', - 'cg' : f'Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}', - 'capgainsd_lt' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}', - 'capgainsd_st' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}', - 'fees' : f'Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}', - 'invexpense' : f'Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}', - 'rounding_error' : 'Equity:Rounding-Errors:Imports', - 'fund_info' : fund_info, - 'currency' : currency, + "account_number": "9876", + "main_account": acct + ":{ticker}", + "cash_account": f"{acct}:{{currency}}", + "transfer": "Assets:Zero-Sum-Accounts:Transfers:Bank-Account", + "dividends": f"Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}", + "interest": f"Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}", + "cg": f"Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}", + "capgainsd_lt": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}", + "capgainsd_st": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}", + "fees": f"Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}", + "invexpense": f"Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}", + "rounding_error": "Equity:Rounding-Errors:Imports", + "fund_info": fund_info, + "currency": currency, } return config -@regtest.with_importer( - schwab_csv_brokerage.Importer( - build_config() - ) -) +@regtest.with_importer(schwab_csv_brokerage.Importer(build_config())) @regtest.with_testdir(path.dirname(__file__)) class TestSchwabBrokerage(regtest.ImporterTestBase): pass diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py index b44919e..ff7813c 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py @@ -4,12 +4,13 @@ from beancount.ingest import regression_pytest as regtest from beancount_reds_importers.importers.schwab import schwab_csv_checking + @regtest.with_importer( schwab_csv_checking.Importer( { - 'account_number' : '1234', - 'main_account' : 'Assets:Banks:Schwab', - 'currency' : 'USD', + "account_number": "1234", + "main_account": "Assets:Banks:Schwab", + "currency": "USD", } ) ) diff --git a/beancount_reds_importers/importers/stanchart/scbbank.py b/beancount_reds_importers/importers/stanchart/scbbank.py index 6a73fb7..7c0e864 100644 --- a/beancount_reds_importers/importers/stanchart/scbbank.py +++ b/beancount_reds_importers/importers/stanchart/scbbank.py @@ -7,44 +7,54 @@ class Importer(csvreader.Importer, banking.Importer): - IMPORTER_NAME = 'SCB Banking Account CSV' + IMPORTER_NAME = "SCB Banking Account CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'AccountTransactions[0-9]*' - self.header_identifier = self.config.get('custom_header', 'Account transactions shown:') - self.column_labels_line = 'Date,Transaction,Currency,Deposit,Withdrawal,Running Balance,SGD Equivalent Balance' - self.balance_column_labels_line = 'Account Name,Account Number,Currency,Current Balance,Available Balance' - self.date_format = '%d/%m/%Y' + self.filename_pattern_def = "AccountTransactions[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "Account transactions shown:" + ) + self.column_labels_line = "Date,Transaction,Currency,Deposit,Withdrawal,Running Balance,SGD Equivalent Balance" + self.balance_column_labels_line = ( + "Account Name,Account Number,Currency,Current Balance,Available Balance" + ) + self.date_format = "%d/%m/%Y" self.skip_tail_rows = 0 - self.skip_comments = '# ' + self.skip_comments = "# " + # fmt: off self.header_map = { - "Date": "date", - "Transaction": "payee", - "Currency": "currency", - "Withdrawal": "withdrawal", - "Deposit": "deposit", - "Running Balance": "balance_running", - "SGD Equivalent Balance": "balance", + "Date": "date", + "Transaction": "payee", + "Currency": "currency", + "Withdrawal": "withdrawal", + "Deposit": "deposit", + "Running Balance": "balance_running", + "SGD Equivalent Balance": "balance", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return ( + re.match(self.header_identifier, file.head()) + and account_number in file.head() + ) # TODO: move into utils, since this is probably a common operation def prepare_table(self, rdr): - rdr = rdr.addfield('amount', - lambda x: "-" + x['Withdrawal'] if x['Withdrawal'] != '' else x['Deposit']) - rdr = rdr.addfield('memo', lambda x: '') + rdr = rdr.addfield( + "amount", + lambda x: "-" + x["Withdrawal"] if x["Withdrawal"] != "" else x["Deposit"], + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) return rdr @@ -54,18 +64,18 @@ def get_balance_statement(self, file=None): if date: rdr = self.read_raw(file) rdr = self.prepare_raw_file(rdr) - col_labels = self.balance_column_labels_line.split(',') + col_labels = self.balance_column_labels_line.split(",") rdr = self.extract_table_with_header(rdr, col_labels) - header_map = {k: k.replace(' ', '_') for k in col_labels} + header_map = {k: k.replace(" ", "_") for k in col_labels} rdr = rdr.rename(header_map) - while '' in rdr.header(): - rdr = rdr.cutout('') + while "" in rdr.header(): + rdr = rdr.cutout("") row = rdr.namedtuples()[0] amount = row.Current_Balance units, debitcredit = amount.split() - if debitcredit != 'CR': - units = '-' + units + if debitcredit != "CR": + units = "-" + units yield banking.Balance(date, D(units), row.Currency) diff --git a/beancount_reds_importers/importers/stanchart/scbcard.py b/beancount_reds_importers/importers/stanchart/scbcard.py index b4ab04d..eb0e053 100644 --- a/beancount_reds_importers/importers/stanchart/scbcard.py +++ b/beancount_reds_importers/importers/stanchart/scbcard.py @@ -7,71 +7,87 @@ class Importer(csvreader.Importer, banking.Importer): - IMPORTER_NAME = 'SCB Card CSV' + IMPORTER_NAME = "SCB Card CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'CardTransactions[0-9]*' - self.header_identifier = self.config.get('custom_header', 'PRIORITY BANKING VISA INFINITE CARD') - self.column_labels_line = 'Date,DESCRIPTION,Foreign Currency Amount,SGD Amount' - self.date_format = '%d/%m/%Y' + self.filename_pattern_def = "CardTransactions[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "PRIORITY BANKING VISA INFINITE CARD" + ) + self.column_labels_line = "Date,DESCRIPTION,Foreign Currency Amount,SGD Amount" + self.date_format = "%d/%m/%Y" self.skip_tail_rows = 6 - self.skip_comments = '# ' + self.skip_comments = "# " + # fmt: off self.header_map = { - "Date": "date", - "DESCRIPTION": "payee", + "Date": "date", + "DESCRIPTION": "payee", } + # fmt: on self.transaction_type_map = {} def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return ( + re.match(self.header_identifier, file.head()) + and account_number in file.head() + ) def skip_transaction(self, row): - return '[UNPOSTED]' in row.payee + return "[UNPOSTED]" in row.payee def prepare_table(self, rdr): - rdr = rdr.select(lambda r: 'UNPOSTED' not in r['DESCRIPTION']) + rdr = rdr.select(lambda r: "UNPOSTED" not in r["DESCRIPTION"]) # parse foreign_currency amount: "YEN 74,000" - if self.config.get('convert_currencies', False): + if self.config.get("convert_currencies", False): # Currency conversions won't work as expected since Beancount v2 # doesn't support adding @@ (total price conversions) via code. # See https://groups.google.com/g/beancount/c/nMvuoR4yOmM # This means the '@' generated by this code below needs to be replaced with an '@@' - rdr = rdr.capture('Foreign Currency Amount', '(.*) (.*)', - ['foreign_currency', 'foreign_amount'], - fill=' ', include_original=True) - rdr = rdr.cutout('Foreign Currency Amount') + rdr = rdr.capture( + "Foreign Currency Amount", + "(.*) (.*)", + ["foreign_currency", "foreign_amount"], + fill=" ", + include_original=True, + ) + rdr = rdr.cutout("Foreign Currency Amount") # parse SGD Amount: "SGD 141.02 CR" into a single amount column - rdr = rdr.capture('SGD Amount', '(.*) (.*) (.*)', ['currency', 'amount', 'crdr']) + rdr = rdr.capture( + "SGD Amount", "(.*) (.*) (.*)", ["currency", "amount", "crdr"] + ) # change DR into -ve. TODO: move this into csvreader or csvreader.utils - crdrdict = {'DR': '-', 'CR': ''} - rdr = rdr.convert('amount', lambda i, row: crdrdict[row.crdr] + i, pass_row=True) - - rdr = rdr.addfield('memo', lambda x: '') # TODO: make this non-mandatory in csvreader + crdrdict = {"DR": "-", "CR": ""} + rdr = rdr.convert( + "amount", lambda i, row: crdrdict[row.crdr] + i, pass_row=True + ) + + rdr = rdr.addfield( + "memo", lambda x: "" + ) # TODO: make this non-mandatory in csvreader return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" date = self.get_balance_assertion_date() if date: - balance_row = self.get_row_by_label(file, 'Current Balance') + balance_row = self.get_row_by_label(file, "Current Balance") currency, amount = balance_row[1], balance_row[2] units, debitcredit = amount.split() - if debitcredit != 'CR': - units = '-' + units + if debitcredit != "CR": + units = "-" + units yield banking.Balance(date, D(units), currency) diff --git a/beancount_reds_importers/importers/target/__init__.py b/beancount_reds_importers/importers/target/__init__.py index 1a83926..69b6cc7 100644 --- a/beancount_reds_importers/importers/target/__init__.py +++ b/beancount_reds_importers/importers/target/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Target Credit Card' + IMPORTER_NAME = "Target Credit Card" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = 'Transactions' + self.filename_pattern_def = "Transactions" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/tdameritrade/__init__.py b/beancount_reds_importers/importers/tdameritrade/__init__.py index fd19326..24e4ace 100644 --- a/beancount_reds_importers/importers/tdameritrade/__init__.py +++ b/beancount_reds_importers/importers/tdameritrade/__init__.py @@ -1,17 +1,16 @@ - from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'TDAmeritrade' + IMPORTER_NAME = "TDAmeritrade" def custom_init(self): super(Importer, self).custom_init() self.max_rounding_error = 0.07 - self.filename_pattern_def = '.*tdameritrade' + self.filename_pattern_def = ".*tdameritrade" self.get_ticker_info = self.get_ticker_info_from_id def get_ticker_info(self, security): - ticker = self.config['fund_info']['cusip_map'][security] - return ticker, '' + ticker = self.config["fund_info"]["cusip_map"][security] + return ticker, "" diff --git a/beancount_reds_importers/importers/techcubank/__init__.py b/beancount_reds_importers/importers/techcubank/__init__.py index 725ca1b..a70a185 100644 --- a/beancount_reds_importers/importers/techcubank/__init__.py +++ b/beancount_reds_importers/importers/techcubank/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Tech Credit Union' + IMPORTER_NAME = "Tech Credit Union" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*Accounts' + self.filename_pattern_def = ".*Accounts" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py index f1d115b..396e36e 100644 --- a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py +++ b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py @@ -4,12 +4,14 @@ @regtest.with_importer( - uobbank.Importer({ - 'main_account': 'Assets:Banks:UOB:UNIPLUS', - 'account_number': '1234567890', - 'currency': 'SGD', - 'rounding_error': 'Equity:Rounding-Errors:Imports', - }) + uobbank.Importer( + { + "main_account": "Assets:Banks:UOB:UNIPLUS", + "account_number": "1234567890", + "currency": "SGD", + "rounding_error": "Equity:Rounding-Errors:Imports", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestUOB(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/importers/unitedoverseas/uobbank.py b/beancount_reds_importers/importers/unitedoverseas/uobbank.py index b59fa09..cd9bb1c 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobbank.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobbank.py @@ -11,41 +11,53 @@ class Importer(xlsreader.Importer, banking.Importer): def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'ACC_TXN_History[0-9]*' - self.header_identifier = self.config.get('custom_header', 'United Overseas Bank Limited.*Account Type:Uniplus Account') - self.column_labels_line = 'Transaction Date,Transaction Description,Withdrawal,Deposit,Available Balance' - self.date_format = '%d %b %Y' + self.filename_pattern_def = "ACC_TXN_History[0-9]*" + self.header_identifier = self.config.get( + "custom_header", + "United Overseas Bank Limited.*Account Type:Uniplus Account", + ) + self.column_labels_line = "Transaction Date,Transaction Description,Withdrawal,Deposit,Available Balance" + self.date_format = "%d %b %Y" + # fmt: off self.header_map = { - 'Transaction Date': 'date', - 'Transaction Description': 'payee', - 'Available Balance': 'balance' + "Transaction Date": "date", + "Transaction Description": "payee", + "Available Balance": "balance", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return ( + re.match(self.header_identifier, file.head()) + and account_number in file.head() + ) # TODO: move these into utils, since this is probably a common operation def prepare_table(self, rdr): # Remove carriage returns in description - rdr = rdr.convert('Transaction Description', lambda x: x.replace('\n', ' ')) + rdr = rdr.convert("Transaction Description", lambda x: x.replace("\n", " ")) def Ds(x): return D(str(x)) - rdr = rdr.addfield('amount', - lambda x: -1 * Ds(x['Withdrawal']) if x['Withdrawal'] != 0 else Ds(x['Deposit'])) - rdr = rdr.addfield('memo', lambda x: '') + + rdr = rdr.addfield( + "amount", + lambda x: -1 * Ds(x["Withdrawal"]) + if x["Withdrawal"] != 0 + else Ds(x["Deposit"]), + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr @@ -55,6 +67,6 @@ def get_balance_statement(self, file=None): if date: row = self.rdr.namedtuples()[0] # Get currency from input file - currency = self.get_row_by_label(file, 'Account Number:')[2] + currency = self.get_row_by_label(file, "Account Number:")[2] yield banking.Balance(date, D(str(row.balance)), currency) diff --git a/beancount_reds_importers/importers/unitedoverseas/uobcard.py b/beancount_reds_importers/importers/unitedoverseas/uobcard.py index 2dfc6b7..a195ed3 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobcard.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobcard.py @@ -7,62 +7,68 @@ class Importer(xlsreader.Importer, banking.Importer): - IMPORTER_NAME = 'SCB Card CSV' + IMPORTER_NAME = "SCB Card CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '^CC_TXN_History[0-9]*' - self.header_identifier = self.config.get('custom_header', 'United Overseas Bank Limited.*Account Type:VISA SIGNATURE') - self.column_labels_line = 'Transaction Date,Posting Date,Description,Foreign Currency Type,Transaction Amount(Foreign),Local Currency Type,Transaction Amount(Local)' # noqa: E501 - self.date_format = '%d %b %Y' + self.filename_pattern_def = "^CC_TXN_History[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "United Overseas Bank Limited.*Account Type:VISA SIGNATURE" + ) + self.column_labels_line = "Transaction Date,Posting Date,Description,Foreign Currency Type,Transaction Amount(Foreign),Local Currency Type,Transaction Amount(Local)" # noqa: E501 + self.date_format = "%d %b %Y" # Remove _DISABLED below to include currency conversions. This won't work as expected since # Beancount v2 doesn't support adding @@ (total price conversions) via code. See # https://groups.google.com/g/beancount/c/nMvuoR4yOmM This means the '@' generated by this # code below needs to be replaced with an '@@' - foreign_currency = 'foreign_currency_DISABLED' - foreign_amount = 'foreign_amount_DISABLED' - if self.config.get('convert_currencies', False): - foreign_currency = 'foreign_currency' - foreign_amount = 'foreign_amount' + foreign_currency = "foreign_currency_DISABLED" + foreign_amount = "foreign_amount_DISABLED" + if self.config.get("convert_currencies", False): + foreign_currency = "foreign_currency" + foreign_amount = "foreign_amount" + # fmt: off self.header_map = { - 'Transaction Date': 'date', - 'Posting Date': 'date_posting', - 'Description': 'payee', - 'Foreign Currency Type': foreign_currency, - 'Transaction Amount(Foreign)': foreign_amount, - 'Local Currency Type': 'currency', - 'Transaction Amount(Local)': 'amount' + "Transaction Date": "date", + "Posting Date": "date_posting", + "Description": "payee", + "Foreign Currency Type": foreign_currency, + "Transaction Amount(Foreign)": foreign_amount, + "Local Currency Type": "currency", + "Transaction Amount(Local)": "amount", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return ( + re.match(self.header_identifier, file.head()) + and account_number in file.head() + ) # TODO: move into utils, since this is probably a common operation def prepare_table(self, rdr): # Remove carriage returns in description - rdr = rdr.convert('Description', lambda x: x.replace('\n', ' ')) - rdr = rdr.addfield('memo', lambda x: '') + rdr = rdr.convert("Description", lambda x: x.replace("\n", " ")) + rdr = rdr.addfield("memo", lambda x: "") # delete empty rows - rdr = rdr.select(lambda x: x['Transaction Date'] != '') + rdr = rdr.select(lambda x: x["Transaction Date"] != "") return rdr def prepare_processed_table(self, rdr): - return rdr.convert('amount', lambda x: -1 * D(str(x))) + return rdr.convert("amount", lambda x: -1 * D(str(x))) def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr @@ -70,6 +76,6 @@ def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" date = self.get_balance_assertion_date() if date: - balance_row = self.get_row_by_label(file, 'Statement Balance:') + balance_row = self.get_row_by_label(file, "Statement Balance:") units, currency = balance_row[1], balance_row[2] yield banking.Balance(date, -1 * D(str(units)), currency) diff --git a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py index 7a689dc..9793ae4 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py @@ -7,42 +7,55 @@ class Importer(xlsreader.Importer, banking.Importer): - IMPORTER_NAME = 'UOB SRS' + IMPORTER_NAME = "UOB SRS" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'SRS_TXN_History[0-9]*' - self.header_identifier = self.config.get('custom_header', 'United Overseas Bank Limited.*Account Type:SRS Account') - self.column_labels_line = 'Transaction Date,Transaction Description,Withdrawal,Deposit' - self.date_format = '%Y%m%d' + self.filename_pattern_def = "SRS_TXN_History[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "United Overseas Bank Limited.*Account Type:SRS Account" + ) + self.column_labels_line = ( + "Transaction Date,Transaction Description,Withdrawal,Deposit" + ) + self.date_format = "%Y%m%d" + # fmt: off self.header_map = { - 'Transaction Date': 'date', - 'Transaction Description': 'payee', + "Transaction Date": "date", + "Transaction Description": "payee", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return ( + re.match(self.header_identifier, file.head()) + and account_number in file.head() + ) def prepare_table(self, rdr): # Remove carriage returns in description - rdr = rdr.convert('Transaction Description', lambda x: x.replace('\n', ' ')) + rdr = rdr.convert("Transaction Description", lambda x: x.replace("\n", " ")) def Ds(x): return D(str(x)) - rdr = rdr.addfield('amount', - lambda x: -1 * Ds(x['Withdrawal']) if x['Withdrawal'] != '' else Ds(x['Deposit'])) - rdr = rdr.addfield('memo', lambda x: '') + + rdr = rdr.addfield( + "amount", + lambda x: -1 * Ds(x["Withdrawal"]) + if x["Withdrawal"] != "" + else Ds(x["Deposit"]), + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr diff --git a/beancount_reds_importers/importers/vanguard/__init__.py b/beancount_reds_importers/importers/vanguard/__init__.py index dc2d4f2..170e014 100644 --- a/beancount_reds_importers/importers/vanguard/__init__.py +++ b/beancount_reds_importers/importers/vanguard/__init__.py @@ -1,4 +1,4 @@ -""" Vanguard Brokerage ofx importer.""" +"""Vanguard Brokerage ofx importer.""" import ntpath from beancount_reds_importers.libreader import ofxreader @@ -6,7 +6,7 @@ class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Vanguard' + IMPORTER_NAME = "Vanguard" # Any memo in the source OFX that's in this set is not carried forward. # Vanguard sets memos that aren't very useful and would create noise in the @@ -17,7 +17,7 @@ class Importer(investments.Importer, ofxreader.Importer): def custom_init(self): self.max_rounding_error = 0.11 - self.filename_pattern_def = '.*OfxDownload' + self.filename_pattern_def = ".*OfxDownload" self.get_ticker_info = self.get_ticker_info_from_id self.get_payee = self.cleanup_memo @@ -31,21 +31,21 @@ def custom_init(self): self.price_cost_both_zero_handler = lambda *args: None def file_name(self, file): - return 'vanguard-all-{}'.format(ntpath.basename(file.name)) + return "vanguard-all-{}".format(ntpath.basename(file.name)) def get_target_acct_custom(self, transaction, ticker=None): - if 'LT CAP GAIN' in transaction.memo: - return self.config['capgainsd_lt'] - elif 'ST CAP GAIN' in transaction.memo: - return self.config['capgainsd_st'] + if "LT CAP GAIN" in transaction.memo: + return self.config["capgainsd_lt"] + elif "ST CAP GAIN" in transaction.memo: + return self.config["capgainsd_st"] return None def cleanup_memo(self, ot): # some vanguard files have memos repeated like this: # 'DIVIDEND REINVESTMENTDIVIDEND REINVESTMENT' retval = ot.memo - if ot.memo[:int(len(ot.memo)/2)] == ot.memo[int(len(ot.memo)/2):]: - retval = ot.memo[:int(len(ot.memo)/2)] + if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2) :]: + retval = ot.memo[: int(len(ot.memo) / 2)] return retval # For users to comment out in their local file if they so prefer diff --git a/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py b/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py index ecb8b81..02ab167 100644 --- a/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py +++ b/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py @@ -1,4 +1,4 @@ -""" Vanguard screenscrape importer. Unsettled trades are not available in Vanguard's qfx and need to be +"""Vanguard screenscrape importer. Unsettled trades are not available in Vanguard's qfx and need to be screenscrapped into a tsv""" from beancount_reds_importers.libreader import tsvreader @@ -6,54 +6,76 @@ class Importer(investments.Importer, tsvreader.Importer): - IMPORTER_NAME = 'Vanguard screenscrape tsv' + IMPORTER_NAME = "Vanguard screenscrape tsv" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*vanguardss.*' - self.header_identifier = '' + self.filename_pattern_def = ".*vanguardss.*" + self.header_identifier = "" self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" + # fmt: off self.header_map = { - "date": 'date', - "settledate": 'tradeDate', - "symbol": 'security', - "description": 'memo', - "action": 'type', - "quantity": 'units', - "price": 'unit_price', - "fees": 'fees', - "amount": 'amount', - "total": 'total', - } + "date": "date", + "settledate": "tradeDate", + "symbol": "security", + "description": "memo", + "action": "type", + "quantity": "units", + "price": "unit_price", + "fees": "fees", + "amount": "amount", + "total": "total", + } self.transaction_type_map = { - 'Buy': 'buystock', - 'Sell': 'sellstock', - } - self.skip_transaction_types = [''] + "Buy": "buystock", + "Sell": "sellstock", + } + # fmt: on + self.skip_transaction_types = [""] def prepare_table(self, rdr): def extract_numbers(x): - replacements = {'– ': '-', - '$': '', - ',': '', - 'Free': '0', - } + replacements = { + "– ": "-", + "$": "", + ",": "", + "Free": "0", + } for k, v in replacements.items(): x = x.replace(k, v) return x - header = ('date', 'settledate', 'symbol', 'description', 'quantity', 'price', 'fees', 'amount') + header = ( + "date", + "settledate", + "symbol", + "description", + "quantity", + "price", + "fees", + "amount", + ) rdr = rdr.pushheader(header) - rdr = rdr.addfield('action', lambda x: x['description'].rsplit(' ', 2)[1].strip()) + rdr = rdr.addfield( + "action", lambda x: x["description"].rsplit(" ", 2)[1].strip() + ) - for field in ["date", "settledate", "symbol", "description", "quantity", "price", "fees"]: + for field in [ + "date", + "settledate", + "symbol", + "description", + "quantity", + "price", + "fees", + ]: rdr = rdr.convert(field, lambda x: x.strip()) for field in ["quantity", "amount", "price", "fees"]: rdr = rdr.convert(field, extract_numbers) - rdr = rdr.addfield('total', lambda x: x['amount']) + rdr = rdr.addfield("total", lambda x: x["amount"]) return rdr diff --git a/beancount_reds_importers/importers/workday/__init__.py b/beancount_reds_importers/importers/workday/__init__.py index eb0e55d..a86ce64 100644 --- a/beancount_reds_importers/importers/workday/__init__.py +++ b/beancount_reds_importers/importers/workday/__init__.py @@ -1,4 +1,4 @@ -""" Workday paycheck importer.""" +"""Workday paycheck importer.""" import datetime from beancount_reds_importers.libreader import xlsx_multitable_reader @@ -15,33 +15,35 @@ class Importer(paycheck.Importer, xlsx_multitable_reader.Importer): - IMPORTER_NAME = 'Workday Paycheck' + IMPORTER_NAME = "Workday Paycheck" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Complete' - self.header_identifier = '- Complete' + self.config.get('custom_header', '') - self.date_format = '%m/%d/%Y' + self.filename_pattern_def = ".*_Complete" + self.header_identifier = "- Complete" + self.config.get("custom_header", "") + self.date_format = "%m/%d/%Y" self.skip_head_rows = 1 # TODO: need to be smarter about this, and skip only when needed self.skip_tail_rows = 0 - self.funds_db_txt = 'funds_by_ticker' + self.funds_db_txt = "funds_by_ticker" + # fmt: off self.header_map = { - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - } + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + } + # fmt: on def paycheck_date(self, input_file): self.read_file(input_file) - d = self.alltables['Payslip Information'].namedtuples()[0].check_date + d = self.alltables["Payslip Information"].namedtuples()[0].check_date self.date = datetime.datetime.strptime(d, self.date_format) return self.date.date() def prepare_tables(self): def valid_header_label(label): - return label.lower().replace(' ', '_') + return label.lower().replace(" ", "_") for section, table in self.alltables.items(): for header in table.header(): @@ -49,4 +51,4 @@ def valid_header_label(label): self.alltables[section] = table def build_metadata(self, file, metatype=None, data={}): - return {'filing_account': self.config['main_account']} + return {"filing_account": self.config["main_account"]} diff --git a/beancount_reds_importers/libreader/csv_multitable_reader.py b/beancount_reds_importers/libreader/csv_multitable_reader.py index 8a8f4ef..94a0433 100644 --- a/beancount_reds_importers/libreader/csv_multitable_reader.py +++ b/beancount_reds_importers/libreader/csv_multitable_reader.py @@ -59,27 +59,35 @@ def read_file(self, file): self.raw_rdr = rdr = self.read_raw(file) - rdr = rdr.skip(getattr(self, 'skip_head_rows', 0)) # chop unwanted file header rows - rdr = rdr.head(len(rdr) - getattr(self, 'skip_tail_rows', 0) - 1) # chop unwanted file footer rows + rdr = rdr.skip( + getattr(self, "skip_head_rows", 0) + ) # chop unwanted file header rows + rdr = rdr.head( + len(rdr) - getattr(self, "skip_tail_rows", 0) - 1 + ) # chop unwanted file footer rows # [0, 2, 10] <-- starts # [-1, 1, 9] <-- ends - table_starts = [i for (i, row) in enumerate(rdr) if self.is_section_title(row)] + [len(rdr)] - table_ends = [r-1 for r in table_starts][1:] + table_starts = [ + i for (i, row) in enumerate(rdr) if self.is_section_title(row) + ] + [len(rdr)] + table_ends = [r - 1 for r in table_starts][1:] table_indexes = zip(table_starts, table_ends) # build the dictionary of tables self.alltables = {} - for (s, e) in table_indexes: + for s, e in table_indexes: if s == e: continue - table = rdr.skip(s+1) # skip past start index and header row - table = table.head(e-s-1) # chop lines after table section data + table = rdr.skip(s + 1) # skip past start index and header row + table = table.head(e - s - 1) # chop lines after table section data self.alltables[rdr[s][0]] = table for section, table in self.alltables.items(): table = table.rowlenselect(0, complement=True) # clean up empty rows - table = table.cut(*[h for h in table.header() if h]) # clean up empty columns + table = table.cut( + *[h for h in table.header() if h] + ) # clean up empty columns self.alltables[section] = table self.prepare_tables() # to be overridden by importer diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 84a3719..69d8cdb 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -59,10 +59,10 @@ class Importer(reader.Reader, importer.ImporterProtocol): - FILE_EXTS = ['csv'] + FILE_EXTS = ["csv"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.reader_ready = self.deep_identify(file) if self.reader_ready: @@ -96,19 +96,26 @@ def prepare_processed_table(self, rdr): def convert_columns(self, rdr): # convert data in transaction types column - if 'type' in rdr.header(): - rdr = rdr.convert('type', self.transaction_type_map) + if "type" in rdr.header(): + rdr = rdr.convert("type", self.transaction_type_map) # fixup decimals - decimals = ['units'] + decimals = ["units"] for i in decimals: if i in rdr.header(): rdr = rdr.convert(i, D) # fixup currencies def remove_non_numeric(x): - return re.sub(r'[^0-9\.-]', "", str(x).strip()) # noqa: W605 - currencies = getattr(self, 'currency_fields', []) + ['unit_price', 'fees', 'total', 'amount', 'balance'] + return re.sub(r"[^0-9\.-]", "", str(x).strip()) # noqa: W605 + + currencies = getattr(self, "currency_fields", []) + [ + "unit_price", + "fees", + "total", + "amount", + "balance", + ] for i in currencies: if i in rdr.header(): rdr = rdr.convert(i, remove_non_numeric) @@ -117,7 +124,8 @@ def remove_non_numeric(x): # fixup dates def convert_date(d): return datetime.datetime.strptime(d, self.date_format) - dates = getattr(self, 'date_fields', []) + ['date', 'tradeDate', 'settleDate'] + + dates = getattr(self, "date_fields", []) + ["date", "tradeDate", "settleDate"] for i in dates: if i in rdr.header(): rdr = rdr.convert(i, convert_date) @@ -131,8 +139,8 @@ def skip_until_main_table(self, rdr, col_labels=None): """Skip csv lines until the header line is found.""" # TODO: convert this into an 'extract_table()' method that handles the tail as well if not col_labels: - if hasattr(self, 'column_labels_line'): - col_labels = self.column_labels_line.replace('"', '').split(',') + if hasattr(self, "column_labels_line"): + col_labels = self.column_labels_line.replace('"', "").split(",") else: return rdr skip = None @@ -151,8 +159,8 @@ def skip_until_main_table(self, rdr, col_labels=None): def extract_table_with_header(self, rdr, col_labels=None): rdr = self.skip_until_main_table(rdr, col_labels) nrows = len(rdr) - for (n, r) in enumerate(rdr): - if not r or all(i == '' for i in r): + for n, r in enumerate(rdr): + if not r or all(i == "" for i in r): # blank line, terminate nrows = n - 1 break @@ -170,18 +178,22 @@ def skip_until_row_contains(self, rdr, value): return rdr.rowslice(start, len(rdr)) def read_file(self, file): - if not getattr(self, 'file_read_done', False): + if not getattr(self, "file_read_done", False): # read file rdr = self.read_raw(file) rdr = self.prepare_raw_file(rdr) # extract main table - rdr = rdr.skip(getattr(self, 'skip_head_rows', 0)) # chop unwanted header rows - rdr = rdr.head(len(rdr) - getattr(self, 'skip_tail_rows', 0) - 1) # chop unwanted footer rows + rdr = rdr.skip( + getattr(self, "skip_head_rows", 0) + ) # chop unwanted header rows + rdr = rdr.head( + len(rdr) - getattr(self, "skip_tail_rows", 0) - 1 + ) # chop unwanted footer rows rdr = self.extract_table_with_header(rdr) - if hasattr(self, 'skip_comments'): + if hasattr(self, "skip_comments"): rdr = rdr.skipcomments(self.skip_comments) - rdr = rdr.rowslice(getattr(self, 'skip_data_rows', 0), None) + rdr = rdr.rowslice(getattr(self, "skip_data_rows", 0), None) rdr = self.prepare_table(rdr) # process table @@ -201,7 +213,7 @@ def get_transactions(self): # TOOD: custom, overridable def skip_transaction(self, row): - return getattr(row, 'type', 'NO_TYPE') in self.skip_transaction_types + return getattr(row, "type", "NO_TYPE") in self.skip_transaction_types def get_balance_assertion_date(self): """ @@ -220,8 +232,10 @@ def get_max_transaction_date(self): # TODO: clean this up. this probably suffices: # return max(ot.date for ot in self.get_transactions()).date() - date = max(ot.tradeDate if hasattr(ot, 'tradeDate') else ot.date - for ot in self.get_transactions()).date() + date = max( + ot.tradeDate if hasattr(ot, "tradeDate") else ot.date + for ot in self.get_transactions() + ).date() except Exception as err: print("ERROR: no end_date. SKIPPING input.") traceback.print_tb(err.__traceback__) diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py index 5ae1e82..7536b46 100644 --- a/beancount_reds_importers/libreader/jsonreader.py +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -1,4 +1,3 @@ - """JSON importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers. @@ -19,16 +18,18 @@ from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning import json + # import re import warnings + warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) class Importer(reader.Reader, importer.ImporterProtocol): - FILE_EXTS = ['json'] + FILE_EXTS = ["json"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.reader_ready = self.deep_identify(file) if self.reader_ready: @@ -61,8 +62,6 @@ def read_file(self, file): # total = transaction['Amount'] # ) - - # def get_transactions(self): # Transaction = namedtuple('Transaction', ['date', 'type', 'security', 'memo', 'unit_price', # 'units', 'fees', 'total']) diff --git a/beancount_reds_importers/libreader/ofxreader.py b/beancount_reds_importers/libreader/ofxreader.py index b40970e..f4590ac 100644 --- a/beancount_reds_importers/libreader/ofxreader.py +++ b/beancount_reds_importers/libreader/ofxreader.py @@ -8,14 +8,15 @@ from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning import warnings + warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) class Importer(reader.Reader, importer.ImporterProtocol): - FILE_EXTS = ['ofx', 'qfx'] + FILE_EXTS = ["ofx", "qfx"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.ofx_account = None self.reader_ready = False @@ -26,9 +27,10 @@ def initialize_reader(self, file): for acc in self.ofx.accounts: # account identifying info fieldname varies across institutions # self.acc_num_field can be overridden in self.custom_init() if needed - acc_num_field = getattr(self, 'account_number_field', 'account_id') - if self.match_account_number(getattr(acc, acc_num_field), - self.config['account_number']): + acc_num_field = getattr(self, "account_number_field", "account_id") + if self.match_account_number( + getattr(acc, acc_num_field), self.config["account_number"] + ): self.ofx_account = acc self.reader_ready = True if self.reader_ready: @@ -41,7 +43,7 @@ def match_account_number(self, file_account, config_account): def file_date(self, file): """Get the ending date of the statement.""" - if not getattr(self, 'ofx_account', None): + if not getattr(self, "ofx_account", None): self.initialize(file) try: return self.ofx_account.statement.end_date @@ -56,27 +58,27 @@ def get_transactions(self): yield from self.ofx_account.statement.transactions def get_balance_statement(self, file=None): - if not hasattr(self.ofx_account.statement, 'balance'): + if not hasattr(self.ofx_account.statement, "balance"): return [] date = self.get_balance_assertion_date() if date: - Balance = namedtuple('Balance', ['date', 'amount']) + Balance = namedtuple("Balance", ["date", "amount"]) yield Balance(date, self.ofx_account.statement.balance) def get_balance_positions(self): - if not hasattr(self.ofx_account.statement, 'positions'): + if not hasattr(self.ofx_account.statement, "positions"): return [] yield from self.ofx_account.statement.positions def get_available_cash(self, settlement_fund_balance=0): - available_cash = getattr(self.ofx_account.statement, 'available_cash', None) + available_cash = getattr(self.ofx_account.statement, "available_cash", None) if available_cash is not None: # Some institutions compute available_cash this way. For others, override this method # in the importer return available_cash - settlement_fund_balance return None - def get_ofx_end_date(self, field='end_date'): + def get_ofx_end_date(self, field="end_date"): end_date = getattr(self.ofx_account.statement, field, None) if end_date: @@ -86,7 +88,7 @@ def get_ofx_end_date(self, field='end_date'): return None def get_smart_date(self): - """ We want the latest date we can assert balance on. Let's consider all the dates we have: + """We want the latest date we can assert balance on. Let's consider all the dates we have: b--------e-------(s-2)----(s)----(d) - b: date of first transaction in this ofx file (end_date) @@ -105,28 +107,41 @@ def get_smart_date(self): have. """ - ofx_max_transation_date = self.get_ofx_end_date('end_date') - ofx_balance_date1 = self.get_ofx_end_date('available_balance_date') - ofx_balance_date2 = self.get_ofx_end_date('balance_date') + ofx_max_transation_date = self.get_ofx_end_date("end_date") + ofx_balance_date1 = self.get_ofx_end_date("available_balance_date") + ofx_balance_date2 = self.get_ofx_end_date("balance_date") max_transaction_date = self.get_max_transaction_date() if ofx_balance_date1: - ofx_balance_date1 -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2)) + ofx_balance_date1 -= datetime.timedelta( + days=self.config.get("balance_assertion_date_fudge", 2) + ) if ofx_balance_date2: - ofx_balance_date2 -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2)) - - dates = [ofx_max_transation_date, max_transaction_date, ofx_balance_date1, ofx_balance_date2] - if all(v is None for v in dates[:2]): # because ofx_balance_date appears even for closed accounts + ofx_balance_date2 -= datetime.timedelta( + days=self.config.get("balance_assertion_date_fudge", 2) + ) + + dates = [ + ofx_max_transation_date, + max_transaction_date, + ofx_balance_date1, + ofx_balance_date2, + ] + if all( + v is None for v in dates[:2] + ): # because ofx_balance_date appears even for closed accounts return None - def vd(x): return x if x else datetime.date.min + def vd(x): + return x if x else datetime.date.min + return_date = max(*[vd(x) for x in dates]) # print("Smart date computation. Dates were: ", dates) return return_date def get_balance_assertion_date(self): - """ Choices for the date of the generated balance assertion can be specified in + """Choices for the date of the generated balance assertion can be specified in self.config['balance_assertion_date_type'], which can be: - 'smart': smart date (default) - 'ofx_date': date specified in ofx file @@ -139,16 +154,20 @@ def get_balance_assertion_date(self): on the beginning of the assertion date. """ - date_type_map = {'smart': self.get_smart_date, - 'ofx_date': self.get_ofx_end_date, - 'last_transaction': self.get_max_transaction_date, - 'today': datetime.date.today} - date_type = self.config.get('balance_assertion_date_type', 'smart') + date_type_map = { + "smart": self.get_smart_date, + "ofx_date": self.get_ofx_end_date, + "last_transaction": self.get_max_transaction_date, + "today": datetime.date.today, + } + date_type = self.config.get("balance_assertion_date_type", "smart") return_date = date_type_map[date_type]() if not return_date: return None - return return_date + datetime.timedelta(days=1) # Next day, as defined by Beancount + return return_date + datetime.timedelta( + days=1 + ) # Next day, as defined by Beancount def get_max_transaction_date(self): """ @@ -160,9 +179,10 @@ def get_max_transaction_date(self): """ try: - - date = max(ot.tradeDate if hasattr(ot, 'tradeDate') else ot.date - for ot in self.get_transactions()).date() + date = max( + ot.tradeDate if hasattr(ot, "tradeDate") else ot.date + for ot in self.get_transactions() + ).date() except TypeError: return None except ValueError: diff --git a/beancount_reds_importers/libreader/reader.py b/beancount_reds_importers/libreader/reader.py index 9efc9b6..d45fcbb 100644 --- a/beancount_reds_importers/libreader/reader.py +++ b/beancount_reds_importers/libreader/reader.py @@ -5,9 +5,9 @@ import re -class Reader(): - FILE_EXTS = [''] - IMPORTER_NAME = 'NOT SET' +class Reader: + FILE_EXTS = [""] + IMPORTER_NAME = "NOT SET" def identify(self, file): # quick check to filter out files that are not the right format @@ -18,17 +18,19 @@ def identify(self, file): # print("No match on extension") return False self.custom_init() - self.filename_pattern = self.config.get('filename_pattern', self.filename_pattern_def) + self.filename_pattern = self.config.get( + "filename_pattern", self.filename_pattern_def + ) if not re.match(self.filename_pattern, path.basename(file.name)): # print("No match on filename_pattern", self.filename_pattern, path.basename(file.name)) return False - self.currency = self.config.get('currency', 'CURRENCY_NOT_CONFIGURED') + self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED") self.initialize_reader(file) # print("reader_ready:", self.reader_ready) return self.reader_ready def file_name(self, file): - return '{}'.format(ntpath.basename(file.name)) + return "{}".format(ntpath.basename(file.name)) def file_account(self, file): # Ugly hack to handle an interaction with smart_importer. See: @@ -36,17 +38,18 @@ def file_account(self, file): # https://github.com/beancount/smart_importer/issues/122 # https://github.com/beancount/smart_importer/issues/30 import inspect + curframe = inspect.currentframe() calframe = inspect.getouterframes(curframe, 2) - if any('predictor' in i.filename for i in calframe): - if 'smart_importer_hack' in self.config: - return self.config['smart_importer_hack'] + if any("predictor" in i.filename for i in calframe): + if "smart_importer_hack" in self.config: + return self.config["smart_importer_hack"] # Otherwise handle a typical bean-file call self.initialize(file) - if 'filing_account' in self.config: - return self.config['filing_account'] - return self.config['main_account'] + if "filing_account" in self.config: + return self.config["filing_account"] + return self.config["main_account"] def get_balance_statement(self, file=None): return [] diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py index 848e283..cadbc9f 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py @@ -4,11 +4,13 @@ @regtest.with_importer( - ally.Importer({ - "account_number": "23456", - "main_account": "Assets:Banks:Checking", - "balance_assertion_date_type": "last_transaction", - }) + ally.Importer( + { + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + "balance_assertion_date_type": "last_transaction", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestSmart(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py index 0a1a210..fb1b011 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py @@ -4,11 +4,13 @@ @regtest.with_importer( - ally.Importer({ - "account_number": "23456", - "main_account": "Assets:Banks:Checking", - "balance_assertion_date_type": "ofx_date", - }) + ally.Importer( + { + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + "balance_assertion_date_type": "ofx_date", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestSmart(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py index ab7cecd..80ab291 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py @@ -5,10 +5,12 @@ # default balance_assertion_date_type is "smart" @regtest.with_importer( - ally.Importer({ - "account_number": "23456", - "main_account": "Assets:Banks:Checking", - }) + ally.Importer( + { + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestSmart(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/libreader/tsvreader.py b/beancount_reds_importers/libreader/tsvreader.py index 4ab4560..05ba6a8 100644 --- a/beancount_reds_importers/libreader/tsvreader.py +++ b/beancount_reds_importers/libreader/tsvreader.py @@ -1,14 +1,13 @@ """tsv (tab separated values) importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" - from beancount.ingest import importer import petl as etl from beancount_reds_importers.libreader import csvreader class Importer(csvreader.Importer, importer.ImporterProtocol): - FILE_EXTS = ['tsv'] + FILE_EXTS = ["tsv"] def read_raw(self, file): return etl.fromtsv(file.name) diff --git a/beancount_reds_importers/libreader/xlsreader.py b/beancount_reds_importers/libreader/xlsreader.py index e2077a3..4174c37 100644 --- a/beancount_reds_importers/libreader/xlsreader.py +++ b/beancount_reds_importers/libreader/xlsreader.py @@ -8,19 +8,19 @@ class Importer(csvreader.Importer): - FILE_EXTS = ['xls'] + FILE_EXTS = ["xls"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.file_read_done = False self.reader_ready = False # TODO: this reads the entire file. Chop off after perhaps 2k or n lines rdr = self.read_raw(file) - header = '' + header = "" for r in rdr: - line = ''.join(str(x) for x in r) + line = "".join(str(x) for x in r) header += line # TODO @@ -33,4 +33,4 @@ def initialize_reader(self, file): def read_raw(self, file): # set logfile to ignore WARNING *** file size (92598) not 512 + multiple of sector size (512) - return etl.fromxls(file.name, logfile=open(devnull, 'w')) + return etl.fromxls(file.name, logfile=open(devnull, "w")) diff --git a/beancount_reds_importers/libreader/xlsx_multitable_reader.py b/beancount_reds_importers/libreader/xlsx_multitable_reader.py index 8274988..60455d0 100644 --- a/beancount_reds_importers/libreader/xlsx_multitable_reader.py +++ b/beancount_reds_importers/libreader/xlsx_multitable_reader.py @@ -13,10 +13,10 @@ class Importer(csv_multitable_reader.Importer): - FILE_EXTS = ['xlsx'] + FILE_EXTS = ["xlsx"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.file_read_done = False self.reader_ready = True @@ -43,4 +43,4 @@ def read_raw(self, file): def is_section_title(self, row): if len(row) == 1: return True - return all(i == '' or i is None for i in row[1:]) + return all(i == "" or i is None for i in row[1:]) diff --git a/beancount_reds_importers/libreader/xlsxreader.py b/beancount_reds_importers/libreader/xlsxreader.py index 06199d0..a3f374e 100644 --- a/beancount_reds_importers/libreader/xlsxreader.py +++ b/beancount_reds_importers/libreader/xlsxreader.py @@ -6,7 +6,7 @@ class Importer(xlsreader.Importer): - FILE_EXTS = ['xlsx'] + FILE_EXTS = ["xlsx"] def read_raw(self, file): rdr = etl.fromxlsx(file.name) diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index 21b0fa7..f505077 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -8,7 +8,7 @@ from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder -Balance = namedtuple('Balance', ['date', 'amount', 'currency']) +Balance = namedtuple("Balance", ["date", "amount", "currency"]) class Importer(importer.ImporterProtocol, transactionbuilder.TransactionBuilder): @@ -55,7 +55,7 @@ def match_account_number(self, file_account, config_account): def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*bank_specific_filename.*' + self.filename_pattern_def = ".*bank_specific_filename.*" self.custom_init_run = True # def get_target_acct(self, transaction): @@ -68,11 +68,11 @@ def fields_contain_data(ot, fields): def get_main_account(self, ot): """Can be overridden by importer""" - return self.config['main_account'] + return self.config["main_account"] def get_target_account(self, ot): """Can be overridden by importer""" - return self.config.get('target_account') + return self.config.get("target_account") # -------------------------------------------------------------------------------- @@ -82,10 +82,15 @@ def extract_balance(self, file, counter): for bal in self.get_balance_statement(file=file): if bal: metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='balance')) - balance_entry = data.Balance(metadata, bal.date, self.config['main_account'], - amount.Amount(bal.amount, self.get_currency(bal)), - None, None) + metadata.update(self.build_metadata(file, metatype="balance")) + balance_entry = data.Balance( + metadata, + bal.date, + self.config["main_account"], + amount.Amount(bal.amount, self.get_currency(bal)), + None, + None, + ) entries.append(balance_entry) return entries @@ -110,9 +115,11 @@ def extract(self, file, existing_entries=None): continue metadata = data.new_metadata(file.name, next(counter)) # metadata['type'] = ot.type # Optional metadata, useful for debugging #TODO - metadata.update(self.build_metadata(file, - metatype='transaction', - data={'transaction': ot})) + metadata.update( + self.build_metadata( + file, metatype="transaction", data={"transaction": ot} + ) + ) # description fields: With OFX, ot.payee tends to be the "main" description field, # while ot.memo is optional @@ -125,24 +132,32 @@ def extract(self, file, existing_entries=None): # Banking transactions might include foreign currency transactions. TODO: figure out # how ofx handles this and use the same interface for csv and other files entry = data.Transaction( - meta=metadata, - date=ot.date.date(), - flag=self.FLAG, - # payee and narration are switched. See the preceding note - payee=self.get_narration(ot), - narration=self.get_payee(ot), - tags=self.get_tags(ot), - links=data.EMPTY_SET, - postings=[]) + meta=metadata, + date=ot.date.date(), + flag=self.FLAG, + # payee and narration are switched. See the preceding note + payee=self.get_narration(ot), + narration=self.get_payee(ot), + tags=self.get_tags(ot), + links=data.EMPTY_SET, + postings=[], + ) main_account = self.get_main_account(ot) - if self.fields_contain_data(ot, ['foreign_amount', 'foreign_currency']): - common.create_simple_posting_with_price(entry, main_account, - ot.amount, self.get_currency(ot), - ot.foreign_amount, ot.foreign_currency) + if self.fields_contain_data(ot, ["foreign_amount", "foreign_currency"]): + common.create_simple_posting_with_price( + entry, + main_account, + ot.amount, + self.get_currency(ot), + ot.foreign_amount, + ot.foreign_currency, + ) else: - data.create_simple_posting(entry, main_account, ot.amount, self.get_currency(ot)) + data.create_simple_posting( + entry, main_account, ot.amount, self.get_currency(ot) + ) # smart_importer can fill this in if the importer doesn't override self.get_target_acct() target_acct = self.get_target_account(ot) diff --git a/beancount_reds_importers/libtransactionbuilder/common.py b/beancount_reds_importers/libtransactionbuilder/common.py index 7d1fa79..45bade3 100644 --- a/beancount_reds_importers/libtransactionbuilder/common.py +++ b/beancount_reds_importers/libtransactionbuilder/common.py @@ -9,31 +9,55 @@ class PriceCostBothZeroException(Exception): """Raised when the input value is too small""" + pass -def create_simple_posting_with_price(entry, account, - number, currency, - price_number, price_currency): - return create_simple_posting_with_cost_or_price(entry, account, - number, currency, - price_number=price_number, price_currency=price_currency) +def create_simple_posting_with_price( + entry, account, number, currency, price_number, price_currency +): + return create_simple_posting_with_cost_or_price( + entry, + account, + number, + currency, + price_number=price_number, + price_currency=price_currency, + ) -def create_simple_posting_with_cost(entry, account, - number, currency, - cost_number, cost_currency, price_cost_both_zero_handler=None): - return create_simple_posting_with_cost_or_price(entry, account, - number, currency, - cost_number=cost_number, cost_currency=cost_currency, - price_cost_both_zero_handler=price_cost_both_zero_handler) +def create_simple_posting_with_cost( + entry, + account, + number, + currency, + cost_number, + cost_currency, + price_cost_both_zero_handler=None, +): + return create_simple_posting_with_cost_or_price( + entry, + account, + number, + currency, + cost_number=cost_number, + cost_currency=cost_currency, + price_cost_both_zero_handler=price_cost_both_zero_handler, + ) -def create_simple_posting_with_cost_or_price(entry, account, - number, currency, - price_number=None, price_currency=None, - cost_number=None, cost_currency=None, costspec=None, - price_cost_both_zero_handler=None): +def create_simple_posting_with_cost_or_price( + entry, + account, + number, + currency, + price_number=None, + price_currency=None, + cost_number=None, + cost_currency=None, + costspec=None, + price_cost_both_zero_handler=None, +): """Create a simple posting on the entry, with a cost (for purchases) or price (for sell transactions). Args: @@ -59,7 +83,11 @@ def create_simple_posting_with_cost_or_price(entry, account, if price_cost_both_zero_handler: price_cost_both_zero_handler() else: - print("WARNING: Either price ({}) or cost ({}) must be specified ({})".format(price_number, cost_number, entry)) + print( + "WARNING: Either price ({}) or cost ({}) must be specified ({})".format( + price_number, cost_number, entry + ) + ) raise PriceCostBothZeroException # import pdb; pdb.set_trace() diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index f0b9646..22321dd 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -83,51 +83,62 @@ def initialize(self, file): self.initialize_reader(file) if self.reader_ready: - config_subst_vars = {'currency': self.currency, - # Leave the other values as is - 'ticker': '{ticker}', - 'source401k': '{source401k}', - } + config_subst_vars = { + "currency": self.currency, + # Leave the other values as is + "ticker": "{ticker}", + "source401k": "{source401k}", + } self.set_config_variables(config_subst_vars) - self.money_market_funds = self.config['fund_info']['money_market'] - self.fund_data = self.config['fund_info']['fund_data'] # [(ticker, id, long_name), ...] + self.money_market_funds = self.config["fund_info"]["money_market"] + self.fund_data = self.config["fund_info"][ + "fund_data" + ] # [(ticker, id, long_name), ...] self.funds_by_id = {i: (ticker, desc) for ticker, i, desc in self.fund_data} - self.funds_by_ticker = {ticker: (ticker, desc) for ticker, _, desc in self.fund_data} + self.funds_by_ticker = { + ticker: (ticker, desc) for ticker, _, desc in self.fund_data + } # Most ofx/csv files refer to funds by id (cusip/isin etc.) Some use tickers instead - self.funds_db = getattr(self, getattr(self, 'funds_db_txt', 'funds_by_id')) + self.funds_db = getattr(self, getattr(self, "funds_db_txt", "funds_by_id")) self.build_account_map() self.initialized = True def build_account_map(self): + # fmt: off # map transaction types to target posting accounts self.target_account_map = { - "buymf": self.config['cash_account'], - "sellmf": self.config['cash_account'], - "buystock": self.config['cash_account'], - "sellstock": self.config['cash_account'], - "buyother": self.config['cash_account'], - "sellother": self.config['cash_account'], - "buydebt": self.config['cash_account'], - "reinvest": self.config['dividends'], - "dividends": self.config['dividends'], - "capgainsd_lt": self.config['capgainsd_lt'], - "capgainsd_st": self.config['capgainsd_st'], - "income": self.config['interest'], - "fee": self.config['fees'], - "invexpense": self.config.get('invexpense', "ACCOUNT_NOT_CONFIGURED:INVEXPENSE"), + "buymf": self.config["cash_account"], + "sellmf": self.config["cash_account"], + "buystock": self.config["cash_account"], + "sellstock": self.config["cash_account"], + "buyother": self.config["cash_account"], + "sellother": self.config["cash_account"], + "buydebt": self.config["cash_account"], + "reinvest": self.config["dividends"], + "dividends": self.config["dividends"], + "capgainsd_lt": self.config["capgainsd_lt"], + "capgainsd_st": self.config["capgainsd_st"], + "income": self.config["interest"], + "fee": self.config["fees"], + "invexpense": self.config.get("invexpense", "ACCOUNT_NOT_CONFIGURED:INVEXPENSE"), } - - if 'transfer' in self.config: - self.target_account_map.update({ - "other": self.config['transfer'], - "credit": self.config['transfer'], - "debit": self.config['transfer'], - "transfer": self.config['transfer'], - "cash": self.config['transfer'], - "dep": self.config['transfer'], - }) + # fmt: on + + if "transfer" in self.config: + # fmt: off + self.target_account_map.update( + { + "other": self.config["transfer"], + "credit": self.config["transfer"], + "debit": self.config["transfer"], + "transfer": self.config["transfer"], + "cash": self.config["transfer"], + "dep": self.config["transfer"], + } + ) + # fmt: on def build_metadata(self, file, metatype=None, data={}): """This method is for importers to override. The overridden method can @@ -138,20 +149,24 @@ def build_metadata(self, file, metatype=None, data={}): def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*bank_specific_filename.*' + self.filename_pattern_def = ".*bank_specific_filename.*" self.custom_init_run = True def get_ticker_info(self, security_id): - return security_id, 'UNKNOWN' + return security_id, "UNKNOWN" def get_ticker_info_from_id(self, security_id): try: # isin might look like "US293409829" while the ofx use only a substring like "29340982" ticker = None try: # first try a full match, fall back to substring - ticker, ticker_long_name = [v for k, v in self.funds_db.items() if security_id == k][0] + ticker, ticker_long_name = [ + v for k, v in self.funds_db.items() if security_id == k + ][0] except IndexError: - ticker, ticker_long_name = [v for k, v in self.funds_db.items() if security_id in k][0] + ticker, ticker_long_name = [ + v for k, v in self.funds_db.items() if security_id in k + ][0] except IndexError: print(f"Error: fund info not found for {security_id}", file=sys.stderr) securities = self.get_security_list() @@ -189,8 +204,11 @@ def get_target_acct(self, transaction, ticker): target = self.get_target_acct_custom(transaction, ticker) if target: return target - if transaction.type == 'income' and getattr(transaction, 'income_type', None) == 'DIV': - return self.target_account_map.get('dividends', None) + if ( + transaction.type == "income" + and getattr(transaction, "income_type", None) == "DIV" + ): + return self.target_account_map.get("dividends", None) return self.target_account_map.get(transaction.type, None) def security_narration(self, ot): @@ -200,32 +218,35 @@ def security_narration(self, ot): def get_security_list(self): tickers = set() for ot in self.get_transactions(): - if hasattr(ot, 'security'): + if hasattr(ot, "security"): tickers.add(ot.security) return tickers def subst_acct_vars(self, raw_acct, ot, ticker): - """Resolve variables within an account like {ticker}. - """ + """Resolve variables within an account like {ticker}.""" ot = ot if ot else {} # inv401ksource is an ofx field that is 'PRETAX', 'AFTERTAX', etc. - kwargs = {'ticker': ticker, 'source401k': getattr(ot, 'inv401ksource', '').title()} + kwargs = { + "ticker": ticker, + "source401k": getattr(ot, "inv401ksource", "").title(), + } acct = raw_acct.format(**kwargs) return self.remove_empty_subaccounts(acct) # if 'inv401ksource' was unavailable def get_acct(self, acct, ot, ticker): - """Get an account from self.config, resolve variables, and return - """ + """Get an account from self.config, resolve variables, and return""" template = self.config.get(acct) if not template: - raise KeyError(f'{acct} not set in importer configuration. Config: {self.config}') + raise KeyError( + f"{acct} not set in importer configuration. Config: {self.config}" + ) return self.subst_acct_vars(template, ot, ticker) # extract() and supporting methods # -------------------------------------------------------------------------------- def generate_trade_entry(self, ot, file, counter): - """ Involves a commodity. One of: ['buymf', 'sellmf', 'buystock', 'sellstock', 'buyother', + """Involves a commodity. One of: ['buymf', 'sellmf', 'buystock', 'sellstock', 'buyother', 'sellother', 'reinvest']""" config = self.config @@ -234,9 +255,16 @@ def generate_trade_entry(self, ot, file, counter): # Build metadata metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='transaction_trade', data={'transaction': ot})) - if getattr(ot, 'settleDate', None) is not None and ot.settleDate != ot.tradeDate: - metadata['settlement_date'] = str(ot.settleDate.date()) + metadata.update( + self.build_metadata( + file, metatype="transaction_trade", data={"transaction": ot} + ) + ) + if ( + getattr(ot, "settleDate", None) is not None + and ot.settleDate != ot.tradeDate + ): + metadata["settlement_date"] = str(ot.settleDate.date()) narration = self.security_narration(ot) raw_target_acct = self.get_target_acct(ot, ticker) @@ -244,45 +272,75 @@ def generate_trade_entry(self, ot, file, counter): total = ot.total # special cases - if 'sell' in ot.type: + if "sell" in ot.type: units = -1 * abs(ot.units) if not is_money_market: - metadata['todo'] = 'TODO: this entry is incomplete until lots are selected (bean-doctor context )' # noqa: E501 - if ot.type in ['reinvest']: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI + metadata["todo"] = ( + "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 + ) + if ot.type in [ + "reinvest" + ]: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI ticker_val = ticker else: ticker_val = self.currency target_acct = self.subst_acct_vars(raw_target_acct, ot, ticker_val) # Build transaction entry - entry = data.Transaction(metadata, ot.tradeDate.date(), self.FLAG, - self.get_payee(ot), narration, - self.get_tags(ot), data.EMPTY_SET, []) + entry = data.Transaction( + metadata, + ot.tradeDate.date(), + self.FLAG, + self.get_payee(ot), + narration, + self.get_tags(ot), + data.EMPTY_SET, + [], + ) # Main posting(s): - main_acct = self.get_acct('main_account', ot, ticker) + main_acct = self.get_acct("main_account", ot, ticker) if is_money_market: # Use price conversions instead of holding these at cost - common.create_simple_posting_with_price(entry, main_acct, - units, ticker, ot.unit_price, self.currency) - elif 'sell' in ot.type: - common.create_simple_posting_with_cost_or_price(entry, main_acct, - units, ticker, price_number=ot.unit_price, - price_currency=self.currency, - costspec=CostSpec(None, None, None, None, None, None)) - cg_acct = self.get_acct('cg', ot, ticker) + common.create_simple_posting_with_price( + entry, main_acct, units, ticker, ot.unit_price, self.currency + ) + elif "sell" in ot.type: + common.create_simple_posting_with_cost_or_price( + entry, + main_acct, + units, + ticker, + price_number=ot.unit_price, + price_currency=self.currency, + costspec=CostSpec(None, None, None, None, None, None), + ) + cg_acct = self.get_acct("cg", ot, ticker) data.create_simple_posting(entry, cg_acct, None, None) else: # buy stock/fund - unit_price = getattr(ot, 'unit_price', 0) + unit_price = getattr(ot, "unit_price", 0) # annoyingly, vanguard reinvests have ot.unit_price set to zero. so manually compute it - if (hasattr(ot, 'security') and ot.security) and ot.units and not ot.unit_price: + if ( + (hasattr(ot, "security") and ot.security) + and ot.units + and not ot.unit_price + ): unit_price = round(abs(ot.total) / ot.units, 4) - common.create_simple_posting_with_cost(entry, main_acct, units, ticker, unit_price, - self.currency, self.price_cost_both_zero_handler) + common.create_simple_posting_with_cost( + entry, + main_acct, + units, + ticker, + unit_price, + self.currency, + self.price_cost_both_zero_handler, + ) # "Other" account posting reverser = 1 - if units > 0 and total > 0: # (ugly) hack for some brokerages with incorrect signs (TODO: remove) + if ( + units > 0 and total > 0 + ): # (ugly) hack for some brokerages with incorrect signs (TODO: remove) reverser = -1 data.create_simple_posting(entry, target_acct, reverser * total, self.currency) @@ -290,7 +348,8 @@ def generate_trade_entry(self, ot, file, counter): rounding_error = (reverser * total) + (ot.unit_price * units) if 0.0005 <= abs(rounding_error) <= self.max_rounding_error: data.create_simple_posting( - entry, config['rounding_error'], -1 * rounding_error, self.currency) + entry, config["rounding_error"], -1 * rounding_error, self.currency + ) # if abs(rounding_error) > self.max_rounding_error: # print("Transactions legs do not sum up! Difference: {}. Entry: {}, ot: {}".format( # rounding_error, entry, ot)) @@ -298,21 +357,34 @@ def generate_trade_entry(self, ot, file, counter): return entry def generate_transfer_entry(self, ot, file, counter): - """ Cash transactions, or in-kind transfers. One of: - [credit, debit, dep, transfer, income, dividends, capgainsd_lt, capgainsd_st, other]""" + """Cash transactions, or in-kind transfers. One of: + [credit, debit, dep, transfer, income, dividends, capgainsd_lt, capgainsd_st, other]""" config = self.config metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='transaction_transfer', data={'transaction': ot})) + metadata.update( + self.build_metadata( + file, metatype="transaction_transfer", data={"transaction": ot} + ) + ) ticker = None - date = getattr(ot, 'tradeDate', None) + date = getattr(ot, "tradeDate", None) if not date: date = ot.date date = date.date() try: - if ot.type in ['transfer']: + if ot.type in ["transfer"]: units = ot.units - elif ot.type in ['other', 'credit', 'debit', 'dep', 'cash', 'payment', 'check', 'xfer']: + elif ot.type in [ + "other", + "credit", + "debit", + "dep", + "cash", + "payment", + "check", + "xfer", + ]: units = ot.amount else: units = ot.total @@ -321,28 +393,48 @@ def generate_transfer_entry(self, ot, file, counter): # import pdb; pdb.set_trace() main_acct = None - if ot.type in ['income', 'dividends', 'capgainsd_lt', - 'capgainsd_st', 'transfer'] and (hasattr(ot, 'security') and ot.security): + if ot.type in [ + "income", + "dividends", + "capgainsd_lt", + "capgainsd_st", + "transfer", + ] and (hasattr(ot, "security") and ot.security): ticker, ticker_long_name = self.get_ticker_info(ot.security) narration = self.security_narration(ot) - main_acct = self.get_acct('main_account', ot, ticker) + main_acct = self.get_acct("main_account", ot, ticker) else: # cash transaction narration = ot.type ticker = self.currency - main_acct = config['cash_account'] + main_acct = config["cash_account"] # Build transaction entry - entry = data.Transaction(metadata, date, self.FLAG, - self.get_payee(ot), narration, - self.get_tags(ot), data.EMPTY_SET, []) + entry = data.Transaction( + metadata, + date, + self.FLAG, + self.get_payee(ot), + narration, + self.get_tags(ot), + data.EMPTY_SET, + [], + ) target_acct = self.get_target_acct(ot, ticker) if target_acct: target_acct = self.subst_acct_vars(target_acct, ot, ticker) # Build postings - if ot.type in ['income', 'dividends', 'capgainsd_st', 'capgainsd_lt', 'fee']: # cash - amount = ot.total if hasattr(ot, 'total') else ot.amount - data.create_simple_posting(entry, config['cash_account'], amount, self.currency) + if ot.type in [ + "income", + "dividends", + "capgainsd_st", + "capgainsd_lt", + "fee", + ]: # cash + amount = ot.total if hasattr(ot, "total") else ot.amount + data.create_simple_posting( + entry, config["cash_account"], amount, self.currency + ) data.create_simple_posting(entry, target_acct, -1 * amount, self.currency) else: data.create_simple_posting(entry, main_acct, units, ticker) @@ -371,14 +463,38 @@ def extract_transactions(self, file, counter): for ot in self.get_transactions(): if self.skip_transaction(ot): continue - if ot.type in ['buymf', 'sellmf', 'buystock', 'buydebt', 'sellstock', 'buyother', 'sellother', 'reinvest']: + if ot.type in [ + "buymf", + "sellmf", + "buystock", + "buydebt", + "sellstock", + "buyother", + "sellother", + "reinvest", + ]: entry = self.generate_trade_entry(ot, file, counter) - elif ot.type in ['other', 'credit', 'debit', 'transfer', 'xfer', 'dep', 'income', 'fee', - 'dividends', 'capgainsd_st', 'capgainsd_lt', 'cash', 'payment', 'check', 'invexpense']: + elif ot.type in [ + "other", + "credit", + "debit", + "transfer", + "xfer", + "dep", + "income", + "fee", + "dividends", + "capgainsd_st", + "capgainsd_lt", + "cash", + "payment", + "check", + "invexpense", + ]: entry = self.generate_transfer_entry(ot, file, counter) else: print("ERROR: unknown entry type:", ot.type) - raise Exception('Unknown entry type') + raise Exception("Unknown entry type") self.add_fee_postings(entry, ot) self.add_custom_postings(entry, ot) new_entries.append(entry) @@ -392,37 +508,57 @@ def extract_balances_and_prices(self, file, counter): for pos in self.get_balance_positions(): ticker, ticker_long_name = self.get_ticker_info(pos.security) metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='balance', data={'pos': pos})) + metadata.update( + self.build_metadata(file, metatype="balance", data={"pos": pos}) + ) # if there are no transactions, use the date in the source file for the balance. This gives us the # bonus of an updated, recent balance assertion bal_date = date if date else pos.date.date() - main_acct = self.get_acct('main_account', None, ticker) - balance_entry = data.Balance(metadata, bal_date, main_acct, - amount.Amount(pos.units, ticker), - None, None) + main_acct = self.get_acct("main_account", None, ticker) + balance_entry = data.Balance( + metadata, + bal_date, + main_acct, + amount.Amount(pos.units, ticker), + None, + None, + ) new_entries.append(balance_entry) if ticker in self.money_market_funds: settlement_fund_balance = pos.units # extract price info if available - if hasattr(pos, 'unit_price') and hasattr(pos, 'date'): + if hasattr(pos, "unit_price") and hasattr(pos, "date"): metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='price', data={'pos': pos})) - price_entry = data.Price(metadata, pos.date.date(), ticker, - amount.Amount(pos.unit_price, self.currency)) + metadata.update( + self.build_metadata(file, metatype="price", data={"pos": pos}) + ) + price_entry = data.Price( + metadata, + pos.date.date(), + ticker, + amount.Amount(pos.unit_price, self.currency), + ) new_entries.append(price_entry) # ----------------- available cash available_cash = self.get_available_cash(settlement_fund_balance) if available_cash is not None: metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='balance_cash')) + metadata.update(self.build_metadata(file, metatype="balance_cash")) try: - bal_date = date if date else self.file_date(file).date() # unavailable file_date raises AttributeError - balance_entry = data.Balance(metadata, bal_date, self.config['cash_account'], - amount.Amount(available_cash, self.currency), - None, None) + bal_date = ( + date if date else self.file_date(file).date() + ) # unavailable file_date raises AttributeError + balance_entry = data.Balance( + metadata, + bal_date, + self.config["cash_account"], + amount.Amount(available_cash, self.currency), + None, + None, + ) new_entries.append(balance_entry) except AttributeError: pass @@ -431,11 +567,15 @@ def extract_balances_and_prices(self, file, counter): def add_fee_postings(self, entry, ot): config = self.config - if hasattr(ot, 'fees') or hasattr(ot, 'commission'): - if getattr(ot, 'fees', 0) != 0: - data.create_simple_posting(entry, config['fees'], ot.fees, self.currency) - if getattr(ot, 'commission', 0) != 0: - data.create_simple_posting(entry, config['fees'], ot.commission, self.currency) + if hasattr(ot, "fees") or hasattr(ot, "commission"): + if getattr(ot, "fees", 0) != 0: + data.create_simple_posting( + entry, config["fees"], ot.fees, self.currency + ) + if getattr(ot, "commission", 0) != 0: + data.create_simple_posting( + entry, config["fees"], ot.commission, self.currency + ) def add_custom_postings(self, entry, ot): pass diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index 24efeb7..63c4232 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -50,10 +50,15 @@ # }, # } + def flip_if_needed(amount, account): - if amount >= 0 and any(account.startswith(prefix) for prefix in ['Income:', 'Equity:', 'Liabilities:']): + if amount >= 0 and any( + account.startswith(prefix) for prefix in ["Income:", "Equity:", "Liabilities:"] + ): amount *= -1 - if amount < 0 and any(account.startswith(prefix) for prefix in ['Expenses:', 'Assets:']): + if amount < 0 and any( + account.startswith(prefix) for prefix in ["Expenses:", "Assets:"] + ): amount *= -1 return amount @@ -63,8 +68,8 @@ def file_date(self, input_file): return self.paycheck_date(input_file) def build_postings(self, entry): - template = self.config['paycheck_template'] - currency = self.config['currency'] + template = self.config["paycheck_template"] + currency = self.config["currency"] total = 0 template_missing = defaultdict(set) @@ -74,16 +79,29 @@ def build_postings(self, entry): continue for row in table.namedtuples(): # TODO: 'bank' is workday specific; move it there - row_description = getattr(row, 'description', getattr(row, 'bank', None)) - row_pattern = next(filter(lambda ts: row_description.startswith(ts), template[section]), None) + row_description = getattr( + row, "description", getattr(row, "bank", None) + ) + row_pattern = next( + filter( + lambda ts: row_description.startswith(ts), template[section] + ), + None, + ) if not row_pattern: template_missing[section].add(row_description) else: accounts = template[section][row_pattern] - accounts = [accounts] if not isinstance(accounts, list) else accounts + accounts = ( + [accounts] if not isinstance(accounts, list) else accounts + ) for account in accounts: # TODO: 'amount_in_pay_group_currency' is workday specific; move it there - amount = getattr(row, 'amount', getattr(row, 'amount_in_pay_group_currency', None)) + amount = getattr( + row, + "amount", + getattr(row, "amount_in_pay_group_currency", None), + ) # import pdb; pdb.set_trace() if not amount: @@ -94,17 +112,17 @@ def build_postings(self, entry): if amount: data.create_simple_posting(entry, account, amount, currency) - if self.config.get('show_unconfigured', False): + if self.config.get("show_unconfigured", False): for section in template_missing: print(section) if template_missing[section]: - print(' ' + '\n '.join(i for i in template_missing[section])) + print(" " + "\n ".join(i for i in template_missing[section])) print() if total != 0: data.create_simple_posting(entry, "TOTAL:NONZERO", total, currency) - if self.config.get('sort_postings', True): + if self.config.get("sort_postings", True): postings = sorted(entry.postings) else: postings = entry.postings @@ -123,9 +141,17 @@ def extract(self, file, existing_entries=None): self.read_file(file) metadata = data.new_metadata(file.name, 0) - metadata.update(self.build_metadata(file, metatype='transaction')) - entry = data.Transaction(metadata, self.paycheck_date(file), self.FLAG, - None, config['desc'], self.get_tags(), data.EMPTY_SET, []) + metadata.update(self.build_metadata(file, metatype="transaction")) + entry = data.Transaction( + metadata, + self.paycheck_date(file), + self.FLAG, + None, + config["desc"], + self.get_tags(), + data.EMPTY_SET, + [], + ) entry = self.build_postings(entry) return [entry] diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index 56ea30b..f789e9b 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -4,7 +4,7 @@ from beancount.core import data -class TransactionBuilder(): +class TransactionBuilder: def skip_transaction(self, ot): """For custom importers to override""" return False @@ -16,7 +16,7 @@ def get_tags(self, ot=None): @staticmethod def remove_empty_subaccounts(acct): """Translates 'Assets:Foo::Bar' to 'Assets:Foo:Bar'.""" - return ':'.join(x for x in acct.split(':') if x) + return ":".join(x for x in acct.split(":") if x) def set_config_variables(self, substs): """ @@ -31,11 +31,16 @@ def set_config_variables(self, substs): 'source401k': '{source401k}', } """ - self.config = {k: v.format(**substs) if isinstance(v, str) else v for k, v in self.config.items()} + self.config = { + k: v.format(**substs) if isinstance(v, str) else v + for k, v in self.config.items() + } # Prevent the replacement fields from appearing in the output of # the file_account method - if 'filing_account' not in self.config: - kwargs = {k: '' for k in substs} - filing_account = self.config['main_account'].format(**kwargs) - self.config['filing_account'] = self.remove_empty_subaccounts(filing_account) + if "filing_account" not in self.config: + kwargs = {k: "" for k in substs} + filing_account = self.config["main_account"].format(**kwargs) + self.config["filing_account"] = self.remove_empty_subaccounts( + filing_account + ) diff --git a/beancount_reds_importers/util/bean_download.py b/beancount_reds_importers/util/bean_download.py index 9bf37ec..fd4d8fc 100755 --- a/beancount_reds_importers/util/bean_download.py +++ b/beancount_reds_importers/util/bean_download.py @@ -30,26 +30,31 @@ def readConfigFile(configfile): def get_sites(sites, t, config): - return [s for s in sites if config[s]['type'] == t] - - -@cli.command(aliases=['list']) -@click.option('-c', '--config-file', envvar='BEAN_DOWNLOAD_CONFIG', required=True, - help='Config file. The environment variable BEAN_DOWNLOAD_CONFIG can also be used to specify this', - type=click.Path(exists=True)) -@click.option('-s', '--sort', is_flag=True, help='Sort output') + return [s for s in sites if config[s]["type"] == t] + + +@cli.command(aliases=["list"]) +@click.option( + "-c", + "--config-file", + envvar="BEAN_DOWNLOAD_CONFIG", + required=True, + help="Config file. The environment variable BEAN_DOWNLOAD_CONFIG can also be used to specify this", + type=click.Path(exists=True), +) +@click.option("-s", "--sort", is_flag=True, help="Sort output") def list_institutions(config_file, sort): """List institutions (sites) currently configured.""" config = readConfigFile(config_file) all_sites = config.sections() - types = set([config[s]['type'] for s in all_sites]) + types = set([config[s]["type"] for s in all_sites]) for t in sorted(types): sites = get_sites(all_sites, t, config) if sort: sites = sorted(sites) name = f"{t} ({len(sites)})".ljust(14) - print(f"{name}:", end='') - print(*sites, sep=', ') + print(f"{name}:", end="") + print(*sites, sep=", ") print() @@ -57,31 +62,48 @@ def get_sites_and_sections(config_file): if config_file and os.path.exists(config_file): config = readConfigFile(config_file) all_sites = config.sections() - types = set([config[s]['type'] for s in all_sites]) + types = set([config[s]["type"] for s in all_sites]) return all_sites, types def complete_sites(ctx, param, incomplete): - config_file = ctx.params['config_file'] + config_file = ctx.params["config_file"] all_sites, _ = get_sites_and_sections(config_file) return [s for s in all_sites if s.startswith(incomplete)] def complete_site_types(ctx, param, incomplete): - config_file = ctx.params['config_file'] + config_file = ctx.params["config_file"] _, types = get_sites_and_sections(config_file) return [s for s in types if s.startswith(incomplete)] @cli.command() -@click.option('-c', '--config-file', envvar='BEAN_DOWNLOAD_CONFIG', required=True, help='Config file') -@click.option('-i', '--sites', '--institutions', help="Institutions to download (comma separated); unspecified means all", - default='', shell_complete=complete_sites) -@click.option('-t', '--site-types', '--institution-types', - help="Download all institutions of specified types (comma separated)", - default='', shell_complete=complete_site_types) -@click.option('--dry-run', is_flag=True, help="Do not actually download", default=False) -@click.option('--verbose', is_flag=True, help="Verbose", default=False) +@click.option( + "-c", + "--config-file", + envvar="BEAN_DOWNLOAD_CONFIG", + required=True, + help="Config file", +) +@click.option( + "-i", + "--sites", + "--institutions", + help="Institutions to download (comma separated); unspecified means all", + default="", + shell_complete=complete_sites, +) +@click.option( + "-t", + "--site-types", + "--institution-types", + help="Download all institutions of specified types (comma separated)", + default="", + shell_complete=complete_site_types, +) +@click.option("--dry-run", is_flag=True, help="Do not actually download", default=False) +@click.option("--verbose", is_flag=True, help="Verbose", default=False) def download(config_file, sites, site_types, dry_run, verbose): # noqa: C901 """Download statements for the specified institutions (sites).""" @@ -91,12 +113,14 @@ def pverbose(*args, **kwargs): config = readConfigFile(config_file) if sites: - sites = sites.split(',') + sites = sites.split(",") else: sites = config.sections() if site_types: - site_types = site_types.split(',') - sites_lists = [get_sites(sites, site_type, config) for site_type in site_types] + site_types = site_types.split(",") + sites_lists = [ + get_sites(sites, site_type, config) for site_type in site_types + ] sites = [j for i in sites_lists for j in i] errors = [] @@ -106,8 +130,8 @@ def pverbose(*args, **kwargs): print(f"Processing {numsites} institutions.") async def download_site(i, site): - tid = f'[{i+1}/{numsites} {site}]' - pverbose(f'{tid}: Begin') + tid = f"[{i+1}/{numsites} {site}]" + pverbose(f"{tid}: Begin") try: options = config[site] except KeyError: @@ -116,11 +140,11 @@ async def download_site(i, site): return # We support cmd and display, and type to filter - if 'display' in options: + if "display" in options: displays.append([site, f"{options['display']}"]) success.append(site) - if 'cmd' in options: - cmd = os.path.expandvars(options['cmd']) + if "cmd" in options: + cmd = os.path.expandvars(options["cmd"]) pverbose(f"{tid}: Executing: {cmd}") if dry_run: await asyncio.sleep(2) @@ -129,9 +153,8 @@ async def download_site(i, site): else: # https://docs.python.org/3.8/library/asyncio-subprocess.html#asyncio.create_subprocess_exec proc = await asyncio.create_subprocess_shell( - cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) + cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) stdout, stderr = await proc.communicate() if proc.returncode: @@ -150,26 +173,30 @@ async def perform_downloads(sites): if displays: print() displays = [[i + 1, *row] for i, row in enumerate(displays)] - click.secho(tabulate.tabulate(displays, - headers=["#", "Institution", "Instructions"], tablefmt="plain"), fg='blue') + click.secho( + tabulate.tabulate( + displays, headers=["#", "Institution", "Instructions"], tablefmt="plain" + ), + fg="blue", + ) print() s = len(sites) if success: print(f"{len(success)}/{s} sites succeeded: {', '.join(success)}") if errors: - click.secho(f"{len(errors)}/{s} sites failed: {', '.join(errors)}", fg='red') + click.secho(f"{len(errors)}/{s} sites failed: {', '.join(errors)}", fg="red") -@cli.command(aliases=['init']) +@cli.command(aliases=["init"]) def config_template(): """Output a template for download.cfg that you can then use to build your own.""" path = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(*[path, 'template.cfg'])) as f: + with open(os.path.join(*[path, "template.cfg"])) as f: for line in f: - print(line, end='') + print(line, end="") -if __name__ == '__main__': +if __name__ == "__main__": cli() diff --git a/beancount_reds_importers/util/needs_update.py b/beancount_reds_importers/util/needs_update.py index e93c1fc..0d1c9e5 100755 --- a/beancount_reds_importers/util/needs_update.py +++ b/beancount_reds_importers/util/needs_update.py @@ -11,33 +11,45 @@ import ast -tbl_options = {'tablefmt': 'simple'} +tbl_options = {"tablefmt": "simple"} def get_config(entries, args): """Get beancount config for the given plugin that can then be used on the command line""" global excluded_re, included_re - _extension_entries = [e for e in entries - if isinstance(e, Custom) and e.type == 'reds-importers'] - config_meta = {entry.values[0].value: - (entry.values[1].value if (len(entry.values) == 2) else None) - for entry in _extension_entries} - - config = {k: ast.literal_eval(v) for k, v in config_meta.items() if 'needs-updates' in k} - config = config.get('needs-updates', {}) - if args['all_accounts']: - config['included_account_pats'] = [] - config['excluded_account_pats'] = ['$-^'] - included_account_pats = config.get('included_account_pats', ['^Assets:', '^Liabilities:']) - excluded_account_pats = config.get('excluded_account_pats', ['$-^']) # exclude nothing by default - excluded_re = re.compile('|'.join(excluded_account_pats)) - included_re = re.compile('|'.join(included_account_pats)) + _extension_entries = [ + e for e in entries if isinstance(e, Custom) and e.type == "reds-importers" + ] + config_meta = { + entry.values[0].value: ( + entry.values[1].value if (len(entry.values) == 2) else None + ) + for entry in _extension_entries + } + + config = { + k: ast.literal_eval(v) for k, v in config_meta.items() if "needs-updates" in k + } + config = config.get("needs-updates", {}) + if args["all_accounts"]: + config["included_account_pats"] = [] + config["excluded_account_pats"] = ["$-^"] + included_account_pats = config.get( + "included_account_pats", ["^Assets:", "^Liabilities:"] + ) + excluded_account_pats = config.get( + "excluded_account_pats", ["$-^"] + ) # exclude nothing by default + excluded_re = re.compile("|".join(excluded_account_pats)) + included_re = re.compile("|".join(included_account_pats)) def is_interesting_account(account, closes): - return account not in closes and \ - included_re.match(account) and \ - not excluded_re.match(account) + return ( + account not in closes + and included_re.match(account) + and not excluded_re.match(account) + ) def handle_commodity_leaf_accounts(last_balance): @@ -49,9 +61,9 @@ def handle_commodity_leaf_accounts(last_balance): considered to be the latest date of a balance assertion on any child. """ d = {} - pat_ticker = re.compile(r'^[A-Z0-9]+$') + pat_ticker = re.compile(r"^[A-Z0-9]+$") for acc in last_balance: - parent, leaf = acc.rsplit(':', 1) + parent, leaf = acc.rsplit(":", 1) if pat_ticker.match(leaf): if parent in d: if d[parent].date < last_balance[acc].date: @@ -72,33 +84,48 @@ def accounts_with_no_balance_entries(entries, closes, last_balance): # Handle commodity leaf accounts accs_no_bal = [] - pat_ticker = re.compile(r'^[A-Z0-9]+$') + pat_ticker = re.compile(r"^[A-Z0-9]+$") def acc_or_parent(acc): - parent, leaf = acc.rsplit(':', 1) + parent, leaf = acc.rsplit(":", 1) if pat_ticker.match(leaf): return parent return acc + accs_no_bal = [acc_or_parent(i) for i in accs_no_bal_raw] # Remove accounts where one or more children do have a balance entry. Needed because of # commodity leaf accounts - accs_no_bal = [(i,) for i in set(accs_no_bal) if not any(j.startswith(i) for j in last_balance)] + accs_no_bal = [ + (i,) for i in set(accs_no_bal) if not any(j.startswith(i) for j in last_balance) + ] return accs_no_bal def pretty_print_table(not_updated_accounts, sort_by_date): field = 0 if sort_by_date else 1 - output = sorted([(v.date, k) for k, v in not_updated_accounts.items()], key=lambda x: x[field]) - headers = ['Last Updated', 'Account'] + output = sorted( + [(v.date, k) for k, v in not_updated_accounts.items()], key=lambda x: x[field] + ) + headers = ["Last Updated", "Account"] print(click.style(tabulate.tabulate(output, headers=headers, **tbl_options))) -@click.command("needs-update", context_settings={'show_default': True}) -@click.argument('beancount-file', type=click.Path(exists=True), envvar='BEANCOUNT_FILE') -@click.option('--recency', help='How many days ago should the last balance assertion be to be considered old', default=15) -@click.option('--sort-by-date', help='Sort output by date (instead of account name)', is_flag=True) -@click.option('--all-accounts', help='Show all account (ignore include/exclude in config)', is_flag=True) +@click.command("needs-update", context_settings={"show_default": True}) +@click.argument("beancount-file", type=click.Path(exists=True), envvar="BEANCOUNT_FILE") +@click.option( + "--recency", + help="How many days ago should the last balance assertion be to be considered old", + default=15, +) +@click.option( + "--sort-by-date", help="Sort output by date (instead of account name)", is_flag=True +) +@click.option( + "--all-accounts", + help="Show all account (ignore include/exclude in config)", + is_flag=True, +) def accounts_needing_updates(beancount_file, recency, sort_by_date, all_accounts): """ Show a list of accounts needing updates, and the date of the last update (which is defined as @@ -141,21 +168,33 @@ def accounts_needing_updates(beancount_file, recency, sort_by_date, all_accounts entries, _, _ = loader.load_file(beancount_file) get_config(entries, locals()) closes = [a.account for a in entries if isinstance(a, Close)] - balance_entries = [a for a in entries if isinstance(a, Balance) and - is_interesting_account(a.account, closes)] + balance_entries = [ + a + for a in entries + if isinstance(a, Balance) and is_interesting_account(a.account, closes) + ] last_balance = {v.account: v for v in balance_entries} d = handle_commodity_leaf_accounts(last_balance) # find accounts with balance assertions older than N days - need_updates = {acc: bal for acc, bal in d.items() if ((datetime.now().date() - d[acc].date).days > recency)} + need_updates = { + acc: bal + for acc, bal in d.items() + if ((datetime.now().date() - d[acc].date).days > recency) + } pretty_print_table(need_updates, sort_by_date) # If there are accounts with zero balance entries, print them accs_no_bal = accounts_with_no_balance_entries(entries, closes, last_balance) if accs_no_bal: - headers = ['Accounts without balance entries:'] - print(click.style('\n' + tabulate.tabulate(sorted(accs_no_bal), headers=headers, **tbl_options))) + headers = ["Accounts without balance entries:"] + print( + click.style( + "\n" + + tabulate.tabulate(sorted(accs_no_bal), headers=headers, **tbl_options) + ) + ) -if __name__ == '__main__': +if __name__ == "__main__": accounts_needing_updates() diff --git a/beancount_reds_importers/util/ofx_summarize.py b/beancount_reds_importers/util/ofx_summarize.py index e5ab2f6..541465b 100755 --- a/beancount_reds_importers/util/ofx_summarize.py +++ b/beancount_reds_importers/util/ofx_summarize.py @@ -8,10 +8,11 @@ from ofxparse import OfxParser from bs4.builder import XMLParsedAsHTMLWarning import warnings + warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) -def analyze(filename, ttype='dividends', pdb_explore=False): +def analyze(filename, ttype="dividends", pdb_explore=False): ts = defaultdict(list) ofx = OfxParser.parse(open(filename)) for acc in ofx.accounts: @@ -19,14 +20,21 @@ def analyze(filename, ttype='dividends', pdb_explore=False): ts[t.type].append(t) if pdb_explore: import pdb + pdb.set_trace() @click.command() -@click.argument('filename', type=click.Path(exists=True)) -@click.option('-n', '--num-transactions', default=5, help='Number of transactions to show') -@click.option('-e', '--pdb-explore', is_flag=True, help='Open a pdb shell to explore') -@click.option('--stats-only', is_flag=True, help='Print total number of transactions contained in the file, and quit') +@click.argument("filename", type=click.Path(exists=True)) +@click.option( + "-n", "--num-transactions", default=5, help="Number of transactions to show" +) +@click.option("-e", "--pdb-explore", is_flag=True, help="Open a pdb shell to explore") +@click.option( + "--stats-only", + is_flag=True, + help="Print total number of transactions contained in the file, and quit", +) def summarize(filename, pdb_explore, num_transactions, stats_only): # noqa: C901 """Quick and dirty way to summarize a .ofx file and peek inside it.""" if os.stat(filename).st_size == 0: @@ -45,45 +53,64 @@ def summarize(filename, pdb_explore, num_transactions, stats_only): # noqa: C90 sys.exit(0) print("Total number of accounts:", len(ofx.accounts)) for acc in ofx.accounts: - print('----------------') + print("----------------") try: - print("Account info: ", acc.account_type, acc.account_id, acc.institution.organization) + print( + "Account info: ", + acc.account_type, + acc.account_id, + acc.institution.organization, + ) except AttributeError: print("Account info: ", acc.account_type, acc.account_id) pass try: - print("Statement info: {} -- {}. Bal: {}".format(acc.statement.start_date, - acc.statement.end_date, acc.statement.balance)) + print( + "Statement info: {} -- {}. Bal: {}".format( + acc.statement.start_date, + acc.statement.end_date, + acc.statement.balance, + ) + ) except AttributeError: try: positions = [(p.units, p.security) for p in acc.statement.positions] - print("Statement info: {} -- {}. Bal: {}".format(acc.statement.start_date, - acc.statement.end_date, positions)) + print( + "Statement info: {} -- {}. Bal: {}".format( + acc.statement.start_date, acc.statement.end_date, positions + ) + ) except AttributeError: print("Statement info: UNABLE to get start_date and end_date") print("Types: ", set([t.type for t in acc.statement.transactions])) print() - txns = sorted(acc.statement.transactions, reverse=True, - key=lambda t: t.date if hasattr(t, 'date') else t.tradeDate) + txns = sorted( + acc.statement.transactions, + reverse=True, + key=lambda t: t.date if hasattr(t, "date") else t.tradeDate, + ) for t in txns[:num_transactions]: - date = t.date if hasattr(t, 'date') else t.tradeDate - description = t.payee + ' ' + t.memo if hasattr(t, 'payee') else t.memo - amount = t.amount if hasattr(t, 'amount') else t.total + date = t.date if hasattr(t, "date") else t.tradeDate + description = t.payee + " " + t.memo if hasattr(t, "payee") else t.memo + amount = t.amount if hasattr(t, "amount") else t.total print(date, t.type, description, amount) if pdb_explore: print("Hints:") print("- try dir(acc), dir(acc.statement.transactions)") - print("- try the 'interact' command to start an interactive python interpreter") + print( + "- try the 'interact' command to start an interactive python interpreter" + ) if len(ofx.accounts) > 1: print("- type 'c' to explore the next account in this file") import pdb + pdb.set_trace() print() print() -if __name__ == '__main__': +if __name__ == "__main__": summarize() diff --git a/setup.py b/setup.py index fd3debf..0926e9f 100644 --- a/setup.py +++ b/setup.py @@ -1,54 +1,54 @@ from os import path from setuptools import find_packages, setup -with open(path.join(path.dirname(__file__), 'README.md')) as readme: +with open(path.join(path.dirname(__file__), "README.md")) as readme: LONG_DESCRIPTION = readme.read() setup( - name='beancount_reds_importers', + name="beancount_reds_importers", use_scm_version=True, - setup_requires=['setuptools_scm'], - description='Importers for various institutions for Beancount', + setup_requires=["setuptools_scm"], + description="Importers for various institutions for Beancount", long_description=LONG_DESCRIPTION, - long_description_content_type='text/markdown', - url='https://github.com/redstreet/beancount_reds_importers', - author='Red Street', - author_email='redstreet@users.noreply.github.com', - keywords='importer ingestor beancount accounting', - license='GPL-3.0', + long_description_content_type="text/markdown", + url="https://github.com/redstreet/beancount_reds_importers", + author="Red Street", + author_email="redstreet@users.noreply.github.com", + keywords="importer ingestor beancount accounting", + license="GPL-3.0", packages=find_packages(), include_package_data=True, extras_require={ - 'dev': [ - 'ruff', + "dev": [ + "ruff", ] }, install_requires=[ - 'Click >= 7.0', - 'beancount >= 2.3.5', - 'click_aliases >= 1.0.1', - 'ofxparse >= 0.21', - 'openpyxl >= 3.0.9', - 'packaging >= 20.3', - 'petl >= 1.7.4', - 'tabulate >= 0.8.9', - 'tqdm >= 4.64.0', + "Click >= 7.0", + "beancount >= 2.3.5", + "click_aliases >= 1.0.1", + "ofxparse >= 0.21", + "openpyxl >= 3.0.9", + "packaging >= 20.3", + "petl >= 1.7.4", + "tabulate >= 0.8.9", + "tqdm >= 4.64.0", ], entry_points={ - 'console_scripts': [ - 'ofx-summarize = beancount_reds_importers.util.ofx_summarize:summarize', - 'bean-download = beancount_reds_importers.util.bean_download:cli', + "console_scripts": [ + "ofx-summarize = beancount_reds_importers.util.ofx_summarize:summarize", + "bean-download = beancount_reds_importers.util.bean_download:cli", ] }, zip_safe=False, classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Financial and Insurance Industry', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Natural Language :: English', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.7', - 'Topic :: Office/Business :: Financial :: Accounting', - 'Topic :: Office/Business :: Financial :: Investment', + "Development Status :: 4 - Beta", + "Intended Audience :: Financial and Insurance Industry", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Topic :: Office/Business :: Financial :: Accounting", + "Topic :: Office/Business :: Financial :: Investment", ], ) From 3d4a6e98bde62cd2726622ae8065f34963090cd3 Mon Sep 17 00:00:00 2001 From: Rane Brown Date: Sat, 23 Mar 2024 14:55:24 -0600 Subject: [PATCH 04/13] feat: format with isort, use pyproject.toml --- .github/workflows/pythonpackage.yml | 3 ++- beancount_reds_importers/example/my-smart.import | 3 ++- beancount_reds_importers/example/my.import | 4 +++- .../importers/ally/tests/ally_test.py | 2 ++ .../importers/amazongc/__init__.py | 3 ++- .../importers/capitalonebank/tests/capitalone_test.py | 2 ++ .../importers/dcu/tests/dcu_csv_test.py | 2 ++ .../etrade/tests/etrade_qfx_brokerage_test.py | 3 ++- .../importers/fidelity/__init__.py | 1 + .../importers/fidelity/fidelity_cma_csv.py | 3 ++- .../importers/schwab/schwab_csv_balances.py | 2 ++ .../importers/schwab/schwab_csv_brokerage.py | 1 + .../importers/schwab/schwab_csv_positions.py | 2 ++ .../schwab_csv_brokerage/schwab_csv_brokerage_test.py | 3 ++- .../schwab_csv_checking/schwab_csv_checking_test.py | 2 ++ .../importers/stanchart/scbbank.py | 6 ++++-- .../importers/stanchart/scbcard.py | 6 ++++-- .../importers/unitedoverseas/tests/uobbank_test.py | 2 ++ .../importers/unitedoverseas/uobbank.py | 6 ++++-- .../importers/unitedoverseas/uobcard.py | 6 ++++-- .../importers/unitedoverseas/uobsrs.py | 4 +++- .../importers/vanguard/__init__.py | 1 + .../importers/vanguard/tests/vanguard_test.py | 2 ++ beancount_reds_importers/importers/workday/__init__.py | 1 + beancount_reds_importers/libreader/csvreader.py | 8 +++++--- beancount_reds_importers/libreader/jsonreader.py | 10 ++++++---- beancount_reds_importers/libreader/ofxreader.py | 8 +++++--- beancount_reds_importers/libreader/reader.py | 2 +- .../last_transaction/last_transaction_date_test.py | 2 ++ .../balance_assertion_date/ofx_date/ofx_date_test.py | 2 ++ .../balance_assertion_date/smart/smart_date_test.py | 2 ++ beancount_reds_importers/libreader/tsvreader.py | 3 ++- beancount_reds_importers/libreader/xlsreader.py | 6 ++++-- .../libreader/xlsx_multitable_reader.py | 8 +++++--- beancount_reds_importers/libreader/xlsxreader.py | 1 + .../libtransactionbuilder/banking.py | 6 +++--- .../libtransactionbuilder/common.py | 3 +-- .../libtransactionbuilder/investments.py | 7 ++++--- .../libtransactionbuilder/paycheck.py | 4 +++- beancount_reds_importers/util/bean_download.py | 6 ++++-- beancount_reds_importers/util/needs_update.py | 8 ++++---- beancount_reds_importers/util/ofx_summarize.py | 7 ++++--- .ruff.toml => pyproject.toml | 6 +++++- setup.py | 2 ++ 44 files changed, 119 insertions(+), 52 deletions(-) rename .ruff.toml => pyproject.toml (64%) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index af768a6..0312cb2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -34,6 +34,7 @@ jobs: - name: Test with pytest run: | pytest - - name: Check format with ruff + - name: Check formatting is applied run: | ruff format --check + isort --profile black --check . diff --git a/beancount_reds_importers/example/my-smart.import b/beancount_reds_importers/example/my-smart.import index 66b62da..2ba0b7b 100644 --- a/beancount_reds_importers/example/my-smart.import +++ b/beancount_reds_importers/example/my-smart.import @@ -4,7 +4,8 @@ import sys from os import path -from smart_importer import apply_hooks, PredictPayees, PredictPostings +from smart_importer import PredictPayees, PredictPostings, apply_hooks + sys.path.insert(0, path.join(path.dirname(__file__))) from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/example/my.import b/beancount_reds_importers/example/my.import index 8cc410e..6eec804 100644 --- a/beancount_reds_importers/example/my.import +++ b/beancount_reds_importers/example/my.import @@ -6,9 +6,11 @@ from os import path sys.path.insert(0, path.join(path.dirname(__file__))) +from fund_info import * + from beancount_reds_importers.importers import vanguard from beancount_reds_importers.importers.schwab import schwab_csv_brokerage -from fund_info import * + # For a better solution for fund_info, see: https://reds-rants.netlify.app/personal-finance/tickers-and-identifiers/ # Setting this variable provides a list of importer instances. diff --git a/beancount_reds_importers/importers/ally/tests/ally_test.py b/beancount_reds_importers/importers/ally/tests/ally_test.py index 84b2bad..8f7e87f 100644 --- a/beancount_reds_importers/importers/ally/tests/ally_test.py +++ b/beancount_reds_importers/importers/ally/tests/ally_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/importers/amazongc/__init__.py b/beancount_reds_importers/importers/amazongc/__init__.py index e1f5d3b..e05e0b6 100644 --- a/beancount_reds_importers/importers/amazongc/__init__.py +++ b/beancount_reds_importers/importers/amazongc/__init__.py @@ -23,9 +23,10 @@ import datetime import itertools import ntpath + from beancount.core import data -from beancount.ingest import importer from beancount.core.number import D +from beancount.ingest import importer # account flow ingest source # ---------------------------------------------------- diff --git a/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py b/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py index 6ca6ac9..a7215c0 100644 --- a/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py +++ b/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import capitalonebank diff --git a/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py b/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py index dbadc22..c05e35c 100644 --- a/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py +++ b/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import dcu diff --git a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py index 1ad8980..656fa32 100644 --- a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py +++ b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py @@ -1,9 +1,10 @@ # flake8: noqa from os import path + from beancount.ingest import regression_pytest as regtest -from beancount_reds_importers.importers import etrade +from beancount_reds_importers.importers import etrade fund_data = [ ("TSM", "874039100", "Taiwan Semiconductor Mfg LTD"), diff --git a/beancount_reds_importers/importers/fidelity/__init__.py b/beancount_reds_importers/importers/fidelity/__init__.py index 3330376..d71ff9c 100644 --- a/beancount_reds_importers/importers/fidelity/__init__.py +++ b/beancount_reds_importers/importers/fidelity/__init__.py @@ -1,6 +1,7 @@ """Fidelity Net Benefits and Fidelity Investments OFX importer.""" import ntpath + from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments diff --git a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py index e6abf7b..7380253 100644 --- a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py +++ b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py @@ -1,8 +1,9 @@ """Fidelity CMA/checking csv importer for beancount.""" +import re + from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking -import re class Importer(banking.Importer, csvreader.Importer): diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py index 1ea5361..98fdc32 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py @@ -2,7 +2,9 @@ import datetime import re + from beancount.core.number import D + from beancount_reds_importers.libreader import csv_multitable_reader from beancount_reds_importers.libtransactionbuilder import investments diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index 0f91fa9..1c08ec8 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -1,6 +1,7 @@ """Schwab Brokerage .csv importer.""" import re + from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import investments diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_positions.py b/beancount_reds_importers/importers/schwab/schwab_csv_positions.py index 4cb0980..3e7de52 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_positions.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_positions.py @@ -4,7 +4,9 @@ import datetime import re + from beancount.core.number import D + from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import investments diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py index fa46fea..ea91049 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py @@ -1,9 +1,10 @@ # flake8: noqa from os import path + from beancount.ingest import regression_pytest as regtest -from beancount_reds_importers.importers.schwab import schwab_csv_brokerage +from beancount_reds_importers.importers.schwab import schwab_csv_brokerage fund_data = [ ("SWVXX", "123", "SCHWAB VALUE ADVANTAGE MONEY INV"), diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py index ff7813c..04e3bb7 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py @@ -1,7 +1,9 @@ # flake8: noqa from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers.schwab import schwab_csv_checking diff --git a/beancount_reds_importers/importers/stanchart/scbbank.py b/beancount_reds_importers/importers/stanchart/scbbank.py index 7c0e864..9d344bb 100644 --- a/beancount_reds_importers/importers/stanchart/scbbank.py +++ b/beancount_reds_importers/importers/stanchart/scbbank.py @@ -1,10 +1,12 @@ """SCB Banking .csv importer.""" -from beancount_reds_importers.libreader import csvreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import csvreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(csvreader.Importer, banking.Importer): IMPORTER_NAME = "SCB Banking Account CSV" diff --git a/beancount_reds_importers/importers/stanchart/scbcard.py b/beancount_reds_importers/importers/stanchart/scbcard.py index eb0e053..4dfc4da 100644 --- a/beancount_reds_importers/importers/stanchart/scbcard.py +++ b/beancount_reds_importers/importers/stanchart/scbcard.py @@ -1,10 +1,12 @@ """SCB Credit .csv importer.""" -from beancount_reds_importers.libreader import csvreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import csvreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(csvreader.Importer, banking.Importer): IMPORTER_NAME = "SCB Card CSV" diff --git a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py index 396e36e..f989eb4 100644 --- a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py +++ b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers.unitedoverseas import uobbank diff --git a/beancount_reds_importers/importers/unitedoverseas/uobbank.py b/beancount_reds_importers/importers/unitedoverseas/uobbank.py index cd9bb1c..44fb7e0 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobbank.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobbank.py @@ -1,10 +1,12 @@ """United Overseas Bank, Bank account .csv importer.""" -from beancount_reds_importers.libreader import xlsreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import xlsreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(xlsreader.Importer, banking.Importer): IMPORTER_NAME = __doc__ diff --git a/beancount_reds_importers/importers/unitedoverseas/uobcard.py b/beancount_reds_importers/importers/unitedoverseas/uobcard.py index a195ed3..e1f840a 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobcard.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobcard.py @@ -1,10 +1,12 @@ """SCB Credit .csv importer.""" -from beancount_reds_importers.libreader import xlsreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import xlsreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(xlsreader.Importer, banking.Importer): IMPORTER_NAME = "SCB Card CSV" diff --git a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py index 9793ae4..3e459ef 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py @@ -1,9 +1,11 @@ """UOB SRS importer.""" import re + +from beancount.core.number import D + from beancount_reds_importers.libreader import xlsreader from beancount_reds_importers.libtransactionbuilder import banking -from beancount.core.number import D class Importer(xlsreader.Importer, banking.Importer): diff --git a/beancount_reds_importers/importers/vanguard/__init__.py b/beancount_reds_importers/importers/vanguard/__init__.py index 170e014..76f48e9 100644 --- a/beancount_reds_importers/importers/vanguard/__init__.py +++ b/beancount_reds_importers/importers/vanguard/__init__.py @@ -1,6 +1,7 @@ """Vanguard Brokerage ofx importer.""" import ntpath + from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py b/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py index 379661b..8473b32 100644 --- a/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import vanguard diff --git a/beancount_reds_importers/importers/workday/__init__.py b/beancount_reds_importers/importers/workday/__init__.py index a86ce64..ff06c3f 100644 --- a/beancount_reds_importers/importers/workday/__init__.py +++ b/beancount_reds_importers/importers/workday/__init__.py @@ -1,6 +1,7 @@ """Workday paycheck importer.""" import datetime + from beancount_reds_importers.libreader import xlsx_multitable_reader from beancount_reds_importers.libtransactionbuilder import paycheck diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 69d8cdb..c6610ad 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -3,12 +3,14 @@ import datetime import re +import sys import traceback -from beancount.ingest import importer -from beancount.core.number import D + import petl as etl +from beancount.core.number import D +from beancount.ingest import importer + from beancount_reds_importers.libreader import reader -import sys # This csv reader uses petl to read a .csv into a table for maniupulation. The output of this reader is a list # of namedtuples corresponding roughly to ofx transactions. The following steps achieve this. When writing diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py index 7536b46..38ffc8f 100644 --- a/beancount_reds_importers/libreader/jsonreader.py +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -11,16 +11,18 @@ Until that happens, perhaps this file should be renamed to schwabjsonreader.py. """ +import json + +# import re +import warnings + # import datetime # import ofxparse # from collections import namedtuple from beancount.ingest import importer -from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning -import json -# import re -import warnings +from beancount_reds_importers.libreader import reader warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) diff --git a/beancount_reds_importers/libreader/ofxreader.py b/beancount_reds_importers/libreader/ofxreader.py index f4590ac..45fd204 100644 --- a/beancount_reds_importers/libreader/ofxreader.py +++ b/beancount_reds_importers/libreader/ofxreader.py @@ -2,12 +2,14 @@ beancount_reds_importers.""" import datetime -import ofxparse +import warnings from collections import namedtuple + +import ofxparse from beancount.ingest import importer -from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning -import warnings + +from beancount_reds_importers.libreader import reader warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) diff --git a/beancount_reds_importers/libreader/reader.py b/beancount_reds_importers/libreader/reader.py index d45fcbb..7309978 100644 --- a/beancount_reds_importers/libreader/reader.py +++ b/beancount_reds_importers/libreader/reader.py @@ -1,8 +1,8 @@ """Reader module base class for beancount_reds_importers. ofx, csv, etc. readers inherit this.""" import ntpath -from os import path import re +from os import path class Reader: diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py index cadbc9f..fb4b451 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py index fb1b011..b13ac82 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py index 80ab291..a194f50 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/libreader/tsvreader.py b/beancount_reds_importers/libreader/tsvreader.py index 05ba6a8..4f8fad8 100644 --- a/beancount_reds_importers/libreader/tsvreader.py +++ b/beancount_reds_importers/libreader/tsvreader.py @@ -1,8 +1,9 @@ """tsv (tab separated values) importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" -from beancount.ingest import importer import petl as etl +from beancount.ingest import importer + from beancount_reds_importers.libreader import csvreader diff --git a/beancount_reds_importers/libreader/xlsreader.py b/beancount_reds_importers/libreader/xlsreader.py index 4174c37..765357e 100644 --- a/beancount_reds_importers/libreader/xlsreader.py +++ b/beancount_reds_importers/libreader/xlsreader.py @@ -1,11 +1,13 @@ """xlsx importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" -import petl as etl import re -from beancount_reds_importers.libreader import csvreader from os import devnull +import petl as etl + +from beancount_reds_importers.libreader import csvreader + class Importer(csvreader.Importer): FILE_EXTS = ["xls"] diff --git a/beancount_reds_importers/libreader/xlsx_multitable_reader.py b/beancount_reds_importers/libreader/xlsx_multitable_reader.py index 60455d0..d9d12c0 100644 --- a/beancount_reds_importers/libreader/xlsx_multitable_reader.py +++ b/beancount_reds_importers/libreader/xlsx_multitable_reader.py @@ -1,11 +1,13 @@ """xlsx importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" -import petl as etl -from io import StringIO import csv -import openpyxl import warnings +from io import StringIO + +import openpyxl +import petl as etl + from beancount_reds_importers.libreader import csv_multitable_reader # This xlsx reader uses petl to read a .csv with multiple tables into a dictionary of petl tables. The section diff --git a/beancount_reds_importers/libreader/xlsxreader.py b/beancount_reds_importers/libreader/xlsxreader.py index a3f374e..7235766 100644 --- a/beancount_reds_importers/libreader/xlsxreader.py +++ b/beancount_reds_importers/libreader/xlsxreader.py @@ -2,6 +2,7 @@ beancount_reds_importers.""" import petl as etl + from beancount_reds_importers.libreader import xlsreader diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index f505077..abddbca 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -2,11 +2,11 @@ import itertools from collections import namedtuple -from beancount.core import data -from beancount.core import amount + +from beancount.core import amount, data from beancount.ingest import importer -from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder +from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder Balance = namedtuple("Balance", ["date", "amount", "currency"]) diff --git a/beancount_reds_importers/libtransactionbuilder/common.py b/beancount_reds_importers/libtransactionbuilder/common.py index 45bade3..c348bb2 100644 --- a/beancount_reds_importers/libtransactionbuilder/common.py +++ b/beancount_reds_importers/libtransactionbuilder/common.py @@ -2,9 +2,8 @@ from beancount.core import data from beancount.core.amount import Amount +from beancount.core.number import D, Decimal from beancount.core.position import Cost -from beancount.core.number import Decimal -from beancount.core.number import D class PriceCostBothZeroException(Exception): diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 22321dd..7dca20d 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -3,10 +3,11 @@ import itertools import sys -from beancount.core import data -from beancount.core import amount -from beancount.ingest import importer + +from beancount.core import amount, data from beancount.core.position import CostSpec +from beancount.ingest import importer + from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index 63c4232..b487b07 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -1,9 +1,11 @@ """Generic banking ofx importer for beancount.""" +from collections import defaultdict + from beancount.core import data from beancount.core.number import D + from beancount_reds_importers.libtransactionbuilder import banking -from collections import defaultdict # paychecks are typically transaction with many (10-40) postings including several each of income, taxes, # pre-tax and post-tax deductions, transfers, reimbursements, etc. This importer enables importing a single diff --git a/beancount_reds_importers/util/bean_download.py b/beancount_reds_importers/util/bean_download.py index fd4d8fc..fb05eb8 100755 --- a/beancount_reds_importers/util/bean_download.py +++ b/beancount_reds_importers/util/bean_download.py @@ -2,13 +2,15 @@ """Download account statements automatically when possible, or display a reminder of how to download them. Multi-threaded.""" -from click_aliases import ClickAliasedGroup import asyncio -import click import configparser import os + +import click import tabulate import tqdm +from click_aliases import ClickAliasedGroup + import beancount_reds_importers.util.needs_update as needs_update diff --git a/beancount_reds_importers/util/needs_update.py b/beancount_reds_importers/util/needs_update.py index 0d1c9e5..f602dac 100755 --- a/beancount_reds_importers/util/needs_update.py +++ b/beancount_reds_importers/util/needs_update.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 """Determine the list of accounts needing updates based on the last balance entry.""" -import click +import ast import re +from datetime import datetime + +import click import tabulate from beancount import loader from beancount.core import getters from beancount.core.data import Balance, Close, Custom -from datetime import datetime -import ast - tbl_options = {"tablefmt": "simple"} diff --git a/beancount_reds_importers/util/ofx_summarize.py b/beancount_reds_importers/util/ofx_summarize.py index 541465b..1bde0ad 100755 --- a/beancount_reds_importers/util/ofx_summarize.py +++ b/beancount_reds_importers/util/ofx_summarize.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 """Quick and dirty way to summarize a .ofx file and peek inside it.""" -import click import os import sys +import warnings from collections import defaultdict -from ofxparse import OfxParser + +import click from bs4.builder import XMLParsedAsHTMLWarning -import warnings +from ofxparse import OfxParser warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) diff --git a/.ruff.toml b/pyproject.toml similarity index 64% rename from .ruff.toml rename to pyproject.toml index 03403de..d2d873b 100644 --- a/.ruff.toml +++ b/pyproject.toml @@ -1,7 +1,11 @@ +[tool.ruff] line-length = 88 -[format] +[tool.ruff.format] docstring-code-format = true indent-style = "space" line-ending = "lf" quote-style = "double" + +[tool.isort] +profile = "black" diff --git a/setup.py b/setup.py index 0926e9f..217e140 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ from os import path + from setuptools import find_packages, setup with open(path.join(path.dirname(__file__), "README.md")) as readme: @@ -21,6 +22,7 @@ extras_require={ "dev": [ "ruff", + "isort", ] }, install_requires=[ From b5f2855d49438d7b6da67df77b5336d19a1689ae Mon Sep 17 00:00:00 2001 From: Rane Brown Date: Sat, 23 Mar 2024 15:08:03 -0600 Subject: [PATCH 05/13] docs: include setup and formatting steps --- CONTRIBUTING.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a33af04..2cdb81e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,5 @@ +# Contributing + Contributions welcome. Preferably: - include a test file. I realize this is sometimes a pain to create, but there is no way for me to test external contributions without test files @@ -18,3 +20,30 @@ Contributions welcome. Preferably:          ├── History_for_Account_X8YYYYYYY.csv          └── run_test.bash ``` + +## Setup + +Development setup would typically look something like this: + +```bash +# clone repo, cd to repo + +# create virtual environment +python3 -m venv venv + +# activate virtual environment +source venv/bin/activate + +# install dependencies +pip install -e .[dev] +``` + +## Formatting + +Prior to finalizing a pull request make sure to run the formatting tools and +commit any resulting changes. + +```bash +ruff format +isort --profile black . +``` From e94b3f1e3b37cb52ce83d37bd421005be1a40470 Mon Sep 17 00:00:00 2001 From: William Davies <36384318+william-davies@users.noreply.github.com> Date: Mon, 25 Mar 2024 20:37:47 +0000 Subject: [PATCH 06/13] docs: fix typo --- beancount_reds_importers/libreader/csvreader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index c6610ad..414d54c 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -12,7 +12,7 @@ from beancount_reds_importers.libreader import reader -# This csv reader uses petl to read a .csv into a table for maniupulation. The output of this reader is a list +# This csv reader uses petl to read a .csv into a table for manipulation. The output of this reader is a list # of namedtuples corresponding roughly to ofx transactions. The following steps achieve this. When writing # your own importer, you only should need to: # - override prepare_table() From 45fb8554a08742e758b89e2f542753513dd1c698 Mon Sep 17 00:00:00 2001 From: Red S Date: Wed, 3 Apr 2024 18:55:07 -0700 Subject: [PATCH 07/13] fix: one-file-per-account broke with smart_importer #97 --- .../importers/workday/__init__.py | 3 --- .../libtransactionbuilder/banking.py | 6 ------ .../libtransactionbuilder/investments.py | 12 +++--------- .../libtransactionbuilder/paycheck.py | 6 ------ .../libtransactionbuilder/transactionbuilder.py | 11 +++++++++++ 5 files changed, 14 insertions(+), 24 deletions(-) diff --git a/beancount_reds_importers/importers/workday/__init__.py b/beancount_reds_importers/importers/workday/__init__.py index ff06c3f..e8c4261 100644 --- a/beancount_reds_importers/importers/workday/__init__.py +++ b/beancount_reds_importers/importers/workday/__init__.py @@ -50,6 +50,3 @@ def valid_header_label(label): for header in table.header(): table = table.rename(header, valid_header_label(header)) self.alltables[section] = table - - def build_metadata(self, file, metatype=None, data={}): - return {"filing_account": self.config["main_account"]} diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index abddbca..b4816b9 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -43,12 +43,6 @@ def build_account_map(self): # } pass - def build_metadata(self, file, metatype=None, data={}): - """This method is for importers to override. The overridden method can - look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) - and the data dictionary to return additional metadata""" - return {} - def match_account_number(self, file_account, config_account): return file_account.endswith(config_account) diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 7dca20d..81b3466 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -141,12 +141,6 @@ def build_account_map(self): ) # fmt: on - def build_metadata(self, file, metatype=None, data={}): - """This method is for importers to override. The overridden method can - look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) - and the data dictionary to return additional metadata""" - return {} - def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 @@ -276,9 +270,9 @@ def generate_trade_entry(self, ot, file, counter): if "sell" in ot.type: units = -1 * abs(ot.units) if not is_money_market: - metadata["todo"] = ( - "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 - ) + metadata[ + "todo" + ] = "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 if ot.type in [ "reinvest" ]: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index b487b07..a7f26ea 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -131,12 +131,6 @@ def build_postings(self, entry): newentry = entry._replace(postings=postings) return newentry - def build_metadata(self, file, metatype=None, data={}): - """This method is for importers to override. The overridden method can - look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) - and the data dictionary to return additional metadata""" - return {} - def extract(self, file, existing_entries=None): self.initialize(file) config = self.config diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index f789e9b..07694eb 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -44,3 +44,14 @@ def set_config_variables(self, substs): self.config["filing_account"] = self.remove_empty_subaccounts( filing_account ) + + def build_metadata(self, file, metatype=None, data={}): + """This method is for importers to override. The overridden method can + look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) + and the data dictionary to return additional metadata""" + + # This 'filing_account' is read by a patch to bean-extract so it can output transactions to + # a file that corresponds with filing_account, when the one-file-per-account feature is + # used. + acct = self.config.get("filing_account", self.config.get("main_account", None)) + return {"filing_account": acct} From 7f387c36740361f87f5b5a5e0d12f3dc5ead4d9a Mon Sep 17 00:00:00 2001 From: Red S Date: Wed, 3 Apr 2024 19:15:19 -0700 Subject: [PATCH 08/13] fix: only emit filing account metadata if configured #97 --- beancount_reds_importers/importers/workday/__init__.py | 4 ++++ .../libtransactionbuilder/transactionbuilder.py | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/beancount_reds_importers/importers/workday/__init__.py b/beancount_reds_importers/importers/workday/__init__.py index e8c4261..2a1f08b 100644 --- a/beancount_reds_importers/importers/workday/__init__.py +++ b/beancount_reds_importers/importers/workday/__init__.py @@ -50,3 +50,7 @@ def valid_header_label(label): for header in table.header(): table = table.rename(header, valid_header_label(header)) self.alltables[section] = table + + def build_metadata(self, file, metatype=None, data={}): + acct = self.config.get("filing_account", self.config.get("main_account", None)) + return {"filing_account": acct} diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index 07694eb..c0082ec 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -53,5 +53,9 @@ def build_metadata(self, file, metatype=None, data={}): # This 'filing_account' is read by a patch to bean-extract so it can output transactions to # a file that corresponds with filing_account, when the one-file-per-account feature is # used. - acct = self.config.get("filing_account", self.config.get("main_account", None)) - return {"filing_account": acct} + if self.config.get("emit_filing_account_metadata"): + acct = self.config.get( + "filing_account", self.config.get("main_account", None) + ) + return {"filing_account": acct} + return {} From cc214e2a3831c6795b3d1baf8fe43ccef6241ef9 Mon Sep 17 00:00:00 2001 From: Red S Date: Wed, 3 Apr 2024 19:21:01 -0700 Subject: [PATCH 09/13] style: reformat to 99 col width (previously 88 col) - trying this out, and will revert/change based on how it works out - toml only in this commit. reformatting in the next commit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d2d873b..574e924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.ruff] -line-length = 88 +line-length = 99 [tool.ruff.format] docstring-code-format = true From 246427fe8c7635ea02ac10a43e6904953cca4f1d Mon Sep 17 00:00:00 2001 From: Red S Date: Wed, 3 Apr 2024 19:21:26 -0700 Subject: [PATCH 10/13] style: reformat to 99 col width (previously 88 col) --- .../importers/amazongc/__init__.py | 4 +- .../importers/dcu/__init__.py | 8 +-- .../importers/fidelity/__init__.py | 8 +-- .../importers/fidelity/fidelity_cma_csv.py | 12 ++--- .../importers/schwab/schwab_csv_brokerage.py | 9 ++-- .../importers/schwab/schwab_csv_checking.py | 4 +- .../importers/schwab/schwab_csv_creditline.py | 4 +- .../importers/stanchart/scbbank.py | 11 ++-- .../importers/stanchart/scbcard.py | 17 ++---- .../importers/unitedoverseas/uobbank.py | 13 ++--- .../importers/unitedoverseas/uobcard.py | 5 +- .../importers/unitedoverseas/uobsrs.py | 13 ++--- .../vanguard/vanguard_screenscrape.py | 4 +- .../libreader/csv_multitable_reader.py | 14 ++--- .../libreader/csvreader.py | 4 +- .../libreader/ofxreader.py | 4 +- beancount_reds_importers/libreader/reader.py | 4 +- .../libtransactionbuilder/banking.py | 8 +-- .../libtransactionbuilder/investments.py | 52 +++++-------------- .../libtransactionbuilder/paycheck.py | 16 ++---- .../transactionbuilder.py | 11 ++-- .../util/bean_download.py | 4 +- beancount_reds_importers/util/needs_update.py | 33 +++--------- .../util/ofx_summarize.py | 8 +-- 24 files changed, 78 insertions(+), 192 deletions(-) diff --git a/beancount_reds_importers/importers/amazongc/__init__.py b/beancount_reds_importers/importers/amazongc/__init__.py index e05e0b6..031e2f1 100644 --- a/beancount_reds_importers/importers/amazongc/__init__.py +++ b/beancount_reds_importers/importers/amazongc/__init__.py @@ -84,9 +84,7 @@ def extract(self, file, existing_entries=None): data.EMPTY_SET, [], ) - data.create_simple_posting( - entry, config["main_account"], number, self.currency - ) + data.create_simple_posting(entry, config["main_account"], number, self.currency) data.create_simple_posting(entry, config["target_account"], None, None) new_entries.append(entry) diff --git a/beancount_reds_importers/importers/dcu/__init__.py b/beancount_reds_importers/importers/dcu/__init__.py index 6837ddf..3a31104 100644 --- a/beancount_reds_importers/importers/dcu/__init__.py +++ b/beancount_reds_importers/importers/dcu/__init__.py @@ -12,7 +12,9 @@ def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = ".*Account_Transactions" self.header_identifier = "" - self.column_labels_line = '"DATE","TRANSACTION TYPE","DESCRIPTION","AMOUNT","ID","MEMO","CURRENT BALANCE"' + self.column_labels_line = ( + '"DATE","TRANSACTION TYPE","DESCRIPTION","AMOUNT","ID","MEMO","CURRENT BALANCE"' + ) self.date_format = "%m/%d/%Y" # fmt: off self.header_map = { @@ -35,6 +37,4 @@ def get_balance_statement(self, file=None): date = self.get_balance_assertion_date() if date: - yield banking.Balance( - date, self.rdr.namedtuples()[0].balance, self.currency - ) + yield banking.Balance(date, self.rdr.namedtuples()[0].balance, self.currency) diff --git a/beancount_reds_importers/importers/fidelity/__init__.py b/beancount_reds_importers/importers/fidelity/__init__.py index d71ff9c..96d869a 100644 --- a/beancount_reds_importers/importers/fidelity/__init__.py +++ b/beancount_reds_importers/importers/fidelity/__init__.py @@ -13,18 +13,14 @@ def custom_init(self): self.max_rounding_error = 0.18 self.filename_pattern_def = ".*fidelity" self.get_ticker_info = self.get_ticker_info_from_id - self.get_payee = ( - lambda ot: ot.memo.split(";", 1)[0] if ";" in ot.memo else ot.memo - ) + self.get_payee = lambda ot: ot.memo.split(";", 1)[0] if ";" in ot.memo else ot.memo def security_narration(self, ot): ticker, ticker_long_name = self.get_ticker_info(ot.security) return f"[{ticker}]" def file_name(self, file): - return "fidelity-{}-{}".format( - self.config["account_number"], ntpath.basename(file.name) - ) + return "fidelity-{}-{}".format(self.config["account_number"], ntpath.basename(file.name)) def get_target_acct_custom(self, transaction, ticker=None): if transaction.memo.startswith("CONTRIBUTION"): diff --git a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py index 7380253..d2b9076 100644 --- a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py +++ b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py @@ -13,7 +13,9 @@ def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = ".*History" self.date_format = "%m/%d/%Y" - header_s0 = ".*Run Date,Action,Symbol,Security Description,Security Type,Quantity,Price \\(\\$\\)," + header_s0 = ( + ".*Run Date,Action,Symbol,Security Description,Security Type,Quantity,Price \\(\\$\\)," + ) header_s1 = "Commission \\(\\$\\),Fees \\(\\$\\),Accrued Interest \\(\\$\\),Amount \\(\\$\\),Settlement Date" header_sum = header_s0 + header_s1 self.header_identifier = header_sum @@ -42,12 +44,8 @@ def prepare_raw_columns(self, rdr): for field in ["Action"]: rdr = rdr.convert(field, lambda x: x.lstrip()) - rdr = rdr.capture( - "Action", "(?:\\s)(?:\\w*)(.*)", ["memo"], include_original=True - ) - rdr = rdr.capture( - "Action", "(\\S+(?:\\s+\\S+)?)", ["payee"], include_original=True - ) + rdr = rdr.capture("Action", "(?:\\s)(?:\\w*)(.*)", ["memo"], include_original=True) + rdr = rdr.capture("Action", "(\\S+(?:\\s+\\S+)?)", ["payee"], include_original=True) for field in ["memo", "payee"]: rdr = rdr.convert(field, lambda x: x.lstrip()) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index 1c08ec8..ee01119 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -13,7 +13,9 @@ def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = ".*_Transactions_" self.header_identifier = "" - self.column_labels_line = '"Date","Action","Symbol","Description","Quantity","Price","Fees & Comm","Amount"' + self.column_labels_line = ( + '"Date","Action","Symbol","Description","Quantity","Price","Fees & Comm","Amount"' + ) self.get_ticker_info = self.get_ticker_info_from_id self.date_format = "%m/%d/%Y" self.funds_db_txt = "funds_by_ticker" @@ -59,10 +61,7 @@ def custom_init(self): def deep_identify(self, file): last_three = self.config.get("account_number", "")[-3:] - return ( - re.match(self.header_identifier, file.head()) - and f"XX{last_three}" in file.name - ) + return re.match(self.header_identifier, file.head()) and f"XX{last_three}" in file.name def skip_transaction(self, ot): return ot.type in ["", "Journal"] diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py index 8284364..a26db2c 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py @@ -49,6 +49,4 @@ def get_balance_statement(self, file=None): date = self.get_balance_assertion_date() if date: - yield banking.Balance( - date, self.rdr.namedtuples()[0].balance, self.currency - ) + yield banking.Balance(date, self.rdr.namedtuples()[0].balance, self.currency) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py index 6dc55b7..cac8038 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py @@ -9,4 +9,6 @@ class Importer(schwab_csv_checking.Importer): def custom_init(self): super().custom_init() self.filename_pattern_def = ".*_Transactions_" - self.column_labels_line = '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' + self.column_labels_line = ( + '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' + ) diff --git a/beancount_reds_importers/importers/stanchart/scbbank.py b/beancount_reds_importers/importers/stanchart/scbbank.py index 9d344bb..06f6319 100644 --- a/beancount_reds_importers/importers/stanchart/scbbank.py +++ b/beancount_reds_importers/importers/stanchart/scbbank.py @@ -14,10 +14,10 @@ class Importer(csvreader.Importer, banking.Importer): def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = "AccountTransactions[0-9]*" - self.header_identifier = self.config.get( - "custom_header", "Account transactions shown:" + self.header_identifier = self.config.get("custom_header", "Account transactions shown:") + self.column_labels_line = ( + "Date,Transaction,Currency,Deposit,Withdrawal,Running Balance,SGD Equivalent Balance" ) - self.column_labels_line = "Date,Transaction,Currency,Deposit,Withdrawal,Running Balance,SGD Equivalent Balance" self.balance_column_labels_line = ( "Account Name,Account Number,Currency,Current Balance,Available Balance" ) @@ -40,10 +40,7 @@ def custom_init(self): def deep_identify(self, file): account_number = self.config.get("account_number", "") - return ( - re.match(self.header_identifier, file.head()) - and account_number in file.head() - ) + return re.match(self.header_identifier, file.head()) and account_number in file.head() # TODO: move into utils, since this is probably a common operation def prepare_table(self, rdr): diff --git a/beancount_reds_importers/importers/stanchart/scbcard.py b/beancount_reds_importers/importers/stanchart/scbcard.py index 4dfc4da..7aba70b 100644 --- a/beancount_reds_importers/importers/stanchart/scbcard.py +++ b/beancount_reds_importers/importers/stanchart/scbcard.py @@ -31,10 +31,7 @@ def custom_init(self): def deep_identify(self, file): account_number = self.config.get("account_number", "") - return ( - re.match(self.header_identifier, file.head()) - and account_number in file.head() - ) + return re.match(self.header_identifier, file.head()) and account_number in file.head() def skip_transaction(self, row): return "[UNPOSTED]" in row.payee @@ -59,19 +56,13 @@ def prepare_table(self, rdr): rdr = rdr.cutout("Foreign Currency Amount") # parse SGD Amount: "SGD 141.02 CR" into a single amount column - rdr = rdr.capture( - "SGD Amount", "(.*) (.*) (.*)", ["currency", "amount", "crdr"] - ) + rdr = rdr.capture("SGD Amount", "(.*) (.*) (.*)", ["currency", "amount", "crdr"]) # change DR into -ve. TODO: move this into csvreader or csvreader.utils crdrdict = {"DR": "-", "CR": ""} - rdr = rdr.convert( - "amount", lambda i, row: crdrdict[row.crdr] + i, pass_row=True - ) + rdr = rdr.convert("amount", lambda i, row: crdrdict[row.crdr] + i, pass_row=True) - rdr = rdr.addfield( - "memo", lambda x: "" - ) # TODO: make this non-mandatory in csvreader + rdr = rdr.addfield("memo", lambda x: "") # TODO: make this non-mandatory in csvreader return rdr def prepare_raw_file(self, rdr): diff --git a/beancount_reds_importers/importers/unitedoverseas/uobbank.py b/beancount_reds_importers/importers/unitedoverseas/uobbank.py index 44fb7e0..753faf6 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobbank.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobbank.py @@ -18,7 +18,9 @@ def custom_init(self): "custom_header", "United Overseas Bank Limited.*Account Type:Uniplus Account", ) - self.column_labels_line = "Transaction Date,Transaction Description,Withdrawal,Deposit,Available Balance" + self.column_labels_line = ( + "Transaction Date,Transaction Description,Withdrawal,Deposit,Available Balance" + ) self.date_format = "%d %b %Y" # fmt: off self.header_map = { @@ -32,10 +34,7 @@ def custom_init(self): def deep_identify(self, file): account_number = self.config.get("account_number", "") - return ( - re.match(self.header_identifier, file.head()) - and account_number in file.head() - ) + return re.match(self.header_identifier, file.head()) and account_number in file.head() # TODO: move these into utils, since this is probably a common operation def prepare_table(self, rdr): @@ -47,9 +46,7 @@ def Ds(x): rdr = rdr.addfield( "amount", - lambda x: -1 * Ds(x["Withdrawal"]) - if x["Withdrawal"] != 0 - else Ds(x["Deposit"]), + lambda x: -1 * Ds(x["Withdrawal"]) if x["Withdrawal"] != 0 else Ds(x["Deposit"]), ) rdr = rdr.addfield("memo", lambda x: "") return rdr diff --git a/beancount_reds_importers/importers/unitedoverseas/uobcard.py b/beancount_reds_importers/importers/unitedoverseas/uobcard.py index e1f840a..5a50f2c 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobcard.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobcard.py @@ -47,10 +47,7 @@ def custom_init(self): def deep_identify(self, file): account_number = self.config.get("account_number", "") - return ( - re.match(self.header_identifier, file.head()) - and account_number in file.head() - ) + return re.match(self.header_identifier, file.head()) and account_number in file.head() # TODO: move into utils, since this is probably a common operation def prepare_table(self, rdr): diff --git a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py index 3e459ef..ac6a761 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py @@ -17,9 +17,7 @@ def custom_init(self): self.header_identifier = self.config.get( "custom_header", "United Overseas Bank Limited.*Account Type:SRS Account" ) - self.column_labels_line = ( - "Transaction Date,Transaction Description,Withdrawal,Deposit" - ) + self.column_labels_line = "Transaction Date,Transaction Description,Withdrawal,Deposit" self.date_format = "%Y%m%d" # fmt: off self.header_map = { @@ -32,10 +30,7 @@ def custom_init(self): def deep_identify(self, file): account_number = self.config.get("account_number", "") - return ( - re.match(self.header_identifier, file.head()) - and account_number in file.head() - ) + return re.match(self.header_identifier, file.head()) and account_number in file.head() def prepare_table(self, rdr): # Remove carriage returns in description @@ -46,9 +41,7 @@ def Ds(x): rdr = rdr.addfield( "amount", - lambda x: -1 * Ds(x["Withdrawal"]) - if x["Withdrawal"] != "" - else Ds(x["Deposit"]), + lambda x: -1 * Ds(x["Withdrawal"]) if x["Withdrawal"] != "" else Ds(x["Deposit"]), ) rdr = rdr.addfield("memo", lambda x: "") return rdr diff --git a/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py b/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py index 02ab167..75ba296 100644 --- a/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py +++ b/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py @@ -59,9 +59,7 @@ def extract_numbers(x): ) rdr = rdr.pushheader(header) - rdr = rdr.addfield( - "action", lambda x: x["description"].rsplit(" ", 2)[1].strip() - ) + rdr = rdr.addfield("action", lambda x: x["description"].rsplit(" ", 2)[1].strip()) for field in [ "date", diff --git a/beancount_reds_importers/libreader/csv_multitable_reader.py b/beancount_reds_importers/libreader/csv_multitable_reader.py index 94a0433..6cf320b 100644 --- a/beancount_reds_importers/libreader/csv_multitable_reader.py +++ b/beancount_reds_importers/libreader/csv_multitable_reader.py @@ -59,18 +59,16 @@ def read_file(self, file): self.raw_rdr = rdr = self.read_raw(file) - rdr = rdr.skip( - getattr(self, "skip_head_rows", 0) - ) # chop unwanted file header rows + rdr = rdr.skip(getattr(self, "skip_head_rows", 0)) # chop unwanted file header rows rdr = rdr.head( len(rdr) - getattr(self, "skip_tail_rows", 0) - 1 ) # chop unwanted file footer rows # [0, 2, 10] <-- starts # [-1, 1, 9] <-- ends - table_starts = [ - i for (i, row) in enumerate(rdr) if self.is_section_title(row) - ] + [len(rdr)] + table_starts = [i for (i, row) in enumerate(rdr) if self.is_section_title(row)] + [ + len(rdr) + ] table_ends = [r - 1 for r in table_starts][1:] table_indexes = zip(table_starts, table_ends) @@ -85,9 +83,7 @@ def read_file(self, file): for section, table in self.alltables.items(): table = table.rowlenselect(0, complement=True) # clean up empty rows - table = table.cut( - *[h for h in table.header() if h] - ) # clean up empty columns + table = table.cut(*[h for h in table.header() if h]) # clean up empty columns self.alltables[section] = table self.prepare_tables() # to be overridden by importer diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 414d54c..92f8611 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -186,9 +186,7 @@ def read_file(self, file): rdr = self.prepare_raw_file(rdr) # extract main table - rdr = rdr.skip( - getattr(self, "skip_head_rows", 0) - ) # chop unwanted header rows + rdr = rdr.skip(getattr(self, "skip_head_rows", 0)) # chop unwanted header rows rdr = rdr.head( len(rdr) - getattr(self, "skip_tail_rows", 0) - 1 ) # chop unwanted footer rows diff --git a/beancount_reds_importers/libreader/ofxreader.py b/beancount_reds_importers/libreader/ofxreader.py index 45fd204..d4b2a69 100644 --- a/beancount_reds_importers/libreader/ofxreader.py +++ b/beancount_reds_importers/libreader/ofxreader.py @@ -167,9 +167,7 @@ def get_balance_assertion_date(self): if not return_date: return None - return return_date + datetime.timedelta( - days=1 - ) # Next day, as defined by Beancount + return return_date + datetime.timedelta(days=1) # Next day, as defined by Beancount def get_max_transaction_date(self): """ diff --git a/beancount_reds_importers/libreader/reader.py b/beancount_reds_importers/libreader/reader.py index 7309978..f93aa2b 100644 --- a/beancount_reds_importers/libreader/reader.py +++ b/beancount_reds_importers/libreader/reader.py @@ -18,9 +18,7 @@ def identify(self, file): # print("No match on extension") return False self.custom_init() - self.filename_pattern = self.config.get( - "filename_pattern", self.filename_pattern_def - ) + self.filename_pattern = self.config.get("filename_pattern", self.filename_pattern_def) if not re.match(self.filename_pattern, path.basename(file.name)): # print("No match on filename_pattern", self.filename_pattern, path.basename(file.name)) return False diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index b4816b9..e3616ae 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -110,9 +110,7 @@ def extract(self, file, existing_entries=None): metadata = data.new_metadata(file.name, next(counter)) # metadata['type'] = ot.type # Optional metadata, useful for debugging #TODO metadata.update( - self.build_metadata( - file, metatype="transaction", data={"transaction": ot} - ) + self.build_metadata(file, metatype="transaction", data={"transaction": ot}) ) # description fields: With OFX, ot.payee tends to be the "main" description field, @@ -149,9 +147,7 @@ def extract(self, file, existing_entries=None): ot.foreign_currency, ) else: - data.create_simple_posting( - entry, main_account, ot.amount, self.get_currency(ot) - ) + data.create_simple_posting(entry, main_account, ot.amount, self.get_currency(ot)) # smart_importer can fill this in if the importer doesn't override self.get_target_acct() target_acct = self.get_target_account(ot) diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 81b3466..2bab989 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -96,9 +96,7 @@ def initialize(self, file): "fund_data" ] # [(ticker, id, long_name), ...] self.funds_by_id = {i: (ticker, desc) for ticker, i, desc in self.fund_data} - self.funds_by_ticker = { - ticker: (ticker, desc) for ticker, _, desc in self.fund_data - } + self.funds_by_ticker = {ticker: (ticker, desc) for ticker, _, desc in self.fund_data} # Most ofx/csv files refer to funds by id (cusip/isin etc.) Some use tickers instead self.funds_db = getattr(self, getattr(self, "funds_db_txt", "funds_by_id")) @@ -199,10 +197,7 @@ def get_target_acct(self, transaction, ticker): target = self.get_target_acct_custom(transaction, ticker) if target: return target - if ( - transaction.type == "income" - and getattr(transaction, "income_type", None) == "DIV" - ): + if transaction.type == "income" and getattr(transaction, "income_type", None) == "DIV": return self.target_account_map.get("dividends", None) return self.target_account_map.get(transaction.type, None) @@ -232,9 +227,7 @@ def get_acct(self, acct, ot, ticker): """Get an account from self.config, resolve variables, and return""" template = self.config.get(acct) if not template: - raise KeyError( - f"{acct} not set in importer configuration. Config: {self.config}" - ) + raise KeyError(f"{acct} not set in importer configuration. Config: {self.config}") return self.subst_acct_vars(template, ot, ticker) # extract() and supporting methods @@ -251,14 +244,9 @@ def generate_trade_entry(self, ot, file, counter): # Build metadata metadata = data.new_metadata(file.name, next(counter)) metadata.update( - self.build_metadata( - file, metatype="transaction_trade", data={"transaction": ot} - ) + self.build_metadata(file, metatype="transaction_trade", data={"transaction": ot}) ) - if ( - getattr(ot, "settleDate", None) is not None - and ot.settleDate != ot.tradeDate - ): + if getattr(ot, "settleDate", None) is not None and ot.settleDate != ot.tradeDate: metadata["settlement_date"] = str(ot.settleDate.date()) narration = self.security_narration(ot) @@ -315,11 +303,7 @@ def generate_trade_entry(self, ot, file, counter): else: # buy stock/fund unit_price = getattr(ot, "unit_price", 0) # annoyingly, vanguard reinvests have ot.unit_price set to zero. so manually compute it - if ( - (hasattr(ot, "security") and ot.security) - and ot.units - and not ot.unit_price - ): + if (hasattr(ot, "security") and ot.security) and ot.units and not ot.unit_price: unit_price = round(abs(ot.total) / ot.units, 4) common.create_simple_posting_with_cost( entry, @@ -357,9 +341,7 @@ def generate_transfer_entry(self, ot, file, counter): config = self.config metadata = data.new_metadata(file.name, next(counter)) metadata.update( - self.build_metadata( - file, metatype="transaction_transfer", data={"transaction": ot} - ) + self.build_metadata(file, metatype="transaction_transfer", data={"transaction": ot}) ) ticker = None date = getattr(ot, "tradeDate", None) @@ -427,9 +409,7 @@ def generate_transfer_entry(self, ot, file, counter): "fee", ]: # cash amount = ot.total if hasattr(ot, "total") else ot.amount - data.create_simple_posting( - entry, config["cash_account"], amount, self.currency - ) + data.create_simple_posting(entry, config["cash_account"], amount, self.currency) data.create_simple_posting(entry, target_acct, -1 * amount, self.currency) else: data.create_simple_posting(entry, main_acct, units, ticker) @@ -503,9 +483,7 @@ def extract_balances_and_prices(self, file, counter): for pos in self.get_balance_positions(): ticker, ticker_long_name = self.get_ticker_info(pos.security) metadata = data.new_metadata(file.name, next(counter)) - metadata.update( - self.build_metadata(file, metatype="balance", data={"pos": pos}) - ) + metadata.update(self.build_metadata(file, metatype="balance", data={"pos": pos})) # if there are no transactions, use the date in the source file for the balance. This gives us the # bonus of an updated, recent balance assertion @@ -526,9 +504,7 @@ def extract_balances_and_prices(self, file, counter): # extract price info if available if hasattr(pos, "unit_price") and hasattr(pos, "date"): metadata = data.new_metadata(file.name, next(counter)) - metadata.update( - self.build_metadata(file, metatype="price", data={"pos": pos}) - ) + metadata.update(self.build_metadata(file, metatype="price", data={"pos": pos})) price_entry = data.Price( metadata, pos.date.date(), @@ -564,13 +540,9 @@ def add_fee_postings(self, entry, ot): config = self.config if hasattr(ot, "fees") or hasattr(ot, "commission"): if getattr(ot, "fees", 0) != 0: - data.create_simple_posting( - entry, config["fees"], ot.fees, self.currency - ) + data.create_simple_posting(entry, config["fees"], ot.fees, self.currency) if getattr(ot, "commission", 0) != 0: - data.create_simple_posting( - entry, config["fees"], ot.commission, self.currency - ) + data.create_simple_posting(entry, config["fees"], ot.commission, self.currency) def add_custom_postings(self, entry, ot): pass diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index a7f26ea..a6b3b25 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -58,9 +58,7 @@ def flip_if_needed(amount, account): account.startswith(prefix) for prefix in ["Income:", "Equity:", "Liabilities:"] ): amount *= -1 - if amount < 0 and any( - account.startswith(prefix) for prefix in ["Expenses:", "Assets:"] - ): + if amount < 0 and any(account.startswith(prefix) for prefix in ["Expenses:", "Assets:"]): amount *= -1 return amount @@ -81,22 +79,16 @@ def build_postings(self, entry): continue for row in table.namedtuples(): # TODO: 'bank' is workday specific; move it there - row_description = getattr( - row, "description", getattr(row, "bank", None) - ) + row_description = getattr(row, "description", getattr(row, "bank", None)) row_pattern = next( - filter( - lambda ts: row_description.startswith(ts), template[section] - ), + filter(lambda ts: row_description.startswith(ts), template[section]), None, ) if not row_pattern: template_missing[section].add(row_description) else: accounts = template[section][row_pattern] - accounts = ( - [accounts] if not isinstance(accounts, list) else accounts - ) + accounts = [accounts] if not isinstance(accounts, list) else accounts for account in accounts: # TODO: 'amount_in_pay_group_currency' is workday specific; move it there amount = getattr( diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index c0082ec..288998b 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -32,8 +32,7 @@ def set_config_variables(self, substs): } """ self.config = { - k: v.format(**substs) if isinstance(v, str) else v - for k, v in self.config.items() + k: v.format(**substs) if isinstance(v, str) else v for k, v in self.config.items() } # Prevent the replacement fields from appearing in the output of @@ -41,9 +40,7 @@ def set_config_variables(self, substs): if "filing_account" not in self.config: kwargs = {k: "" for k in substs} filing_account = self.config["main_account"].format(**kwargs) - self.config["filing_account"] = self.remove_empty_subaccounts( - filing_account - ) + self.config["filing_account"] = self.remove_empty_subaccounts(filing_account) def build_metadata(self, file, metatype=None, data={}): """This method is for importers to override. The overridden method can @@ -54,8 +51,6 @@ def build_metadata(self, file, metatype=None, data={}): # a file that corresponds with filing_account, when the one-file-per-account feature is # used. if self.config.get("emit_filing_account_metadata"): - acct = self.config.get( - "filing_account", self.config.get("main_account", None) - ) + acct = self.config.get("filing_account", self.config.get("main_account", None)) return {"filing_account": acct} return {} diff --git a/beancount_reds_importers/util/bean_download.py b/beancount_reds_importers/util/bean_download.py index fb05eb8..2cfe3af 100755 --- a/beancount_reds_importers/util/bean_download.py +++ b/beancount_reds_importers/util/bean_download.py @@ -120,9 +120,7 @@ def pverbose(*args, **kwargs): sites = config.sections() if site_types: site_types = site_types.split(",") - sites_lists = [ - get_sites(sites, site_type, config) for site_type in site_types - ] + sites_lists = [get_sites(sites, site_type, config) for site_type in site_types] sites = [j for i in sites_lists for j in i] errors = [] diff --git a/beancount_reds_importers/util/needs_update.py b/beancount_reds_importers/util/needs_update.py index f602dac..ed61da7 100755 --- a/beancount_reds_importers/util/needs_update.py +++ b/beancount_reds_importers/util/needs_update.py @@ -21,22 +21,16 @@ def get_config(entries, args): e for e in entries if isinstance(e, Custom) and e.type == "reds-importers" ] config_meta = { - entry.values[0].value: ( - entry.values[1].value if (len(entry.values) == 2) else None - ) + entry.values[0].value: (entry.values[1].value if (len(entry.values) == 2) else None) for entry in _extension_entries } - config = { - k: ast.literal_eval(v) for k, v in config_meta.items() if "needs-updates" in k - } + config = {k: ast.literal_eval(v) for k, v in config_meta.items() if "needs-updates" in k} config = config.get("needs-updates", {}) if args["all_accounts"]: config["included_account_pats"] = [] config["excluded_account_pats"] = ["$-^"] - included_account_pats = config.get( - "included_account_pats", ["^Assets:", "^Liabilities:"] - ) + included_account_pats = config.get("included_account_pats", ["^Assets:", "^Liabilities:"]) excluded_account_pats = config.get( "excluded_account_pats", ["$-^"] ) # exclude nothing by default @@ -45,11 +39,7 @@ def get_config(entries, args): def is_interesting_account(account, closes): - return ( - account not in closes - and included_re.match(account) - and not excluded_re.match(account) - ) + return account not in closes and included_re.match(account) and not excluded_re.match(account) def handle_commodity_leaf_accounts(last_balance): @@ -104,9 +94,7 @@ def acc_or_parent(acc): def pretty_print_table(not_updated_accounts, sort_by_date): field = 0 if sort_by_date else 1 - output = sorted( - [(v.date, k) for k, v in not_updated_accounts.items()], key=lambda x: x[field] - ) + output = sorted([(v.date, k) for k, v in not_updated_accounts.items()], key=lambda x: x[field]) headers = ["Last Updated", "Account"] print(click.style(tabulate.tabulate(output, headers=headers, **tbl_options))) @@ -118,9 +106,7 @@ def pretty_print_table(not_updated_accounts, sort_by_date): help="How many days ago should the last balance assertion be to be considered old", default=15, ) -@click.option( - "--sort-by-date", help="Sort output by date (instead of account name)", is_flag=True -) +@click.option("--sort-by-date", help="Sort output by date (instead of account name)", is_flag=True) @click.option( "--all-accounts", help="Show all account (ignore include/exclude in config)", @@ -169,9 +155,7 @@ def accounts_needing_updates(beancount_file, recency, sort_by_date, all_accounts get_config(entries, locals()) closes = [a.account for a in entries if isinstance(a, Close)] balance_entries = [ - a - for a in entries - if isinstance(a, Balance) and is_interesting_account(a.account, closes) + a for a in entries if isinstance(a, Balance) and is_interesting_account(a.account, closes) ] last_balance = {v.account: v for v in balance_entries} d = handle_commodity_leaf_accounts(last_balance) @@ -190,8 +174,7 @@ def accounts_needing_updates(beancount_file, recency, sort_by_date, all_accounts headers = ["Accounts without balance entries:"] print( click.style( - "\n" - + tabulate.tabulate(sorted(accs_no_bal), headers=headers, **tbl_options) + "\n" + tabulate.tabulate(sorted(accs_no_bal), headers=headers, **tbl_options) ) ) diff --git a/beancount_reds_importers/util/ofx_summarize.py b/beancount_reds_importers/util/ofx_summarize.py index 1bde0ad..e83a9a7 100755 --- a/beancount_reds_importers/util/ofx_summarize.py +++ b/beancount_reds_importers/util/ofx_summarize.py @@ -27,9 +27,7 @@ def analyze(filename, ttype="dividends", pdb_explore=False): @click.command() @click.argument("filename", type=click.Path(exists=True)) -@click.option( - "-n", "--num-transactions", default=5, help="Number of transactions to show" -) +@click.option("-n", "--num-transactions", default=5, help="Number of transactions to show") @click.option("-e", "--pdb-explore", is_flag=True, help="Open a pdb shell to explore") @click.option( "--stats-only", @@ -101,9 +99,7 @@ def summarize(filename, pdb_explore, num_transactions, stats_only): # noqa: C90 if pdb_explore: print("Hints:") print("- try dir(acc), dir(acc.statement.transactions)") - print( - "- try the 'interact' command to start an interactive python interpreter" - ) + print("- try the 'interact' command to start an interactive python interpreter") if len(ofx.accounts) > 1: print("- type 'c' to explore the next account in this file") import pdb From 0db8f8ed3ae3fd85659db35fa0bf85fc7302043a Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 6 Apr 2024 19:13:43 -0700 Subject: [PATCH 11/13] feat: add_custom_postings to banking.py From request here: https://github.com/redstreet/reds-ramblings-comments/issues/21#issuecomment-2041211354 --- beancount_reds_importers/libtransactionbuilder/banking.py | 1 + beancount_reds_importers/libtransactionbuilder/investments.py | 3 --- .../libtransactionbuilder/transactionbuilder.py | 4 ++++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index e3616ae..c97791a 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -154,6 +154,7 @@ def extract(self, file, existing_entries=None): if target_acct: data.create_simple_posting(entry, target_acct, None, None) + self.add_custom_postings(entry, ot) new_entries.append(entry) new_entries += self.extract_balance(file, counter) diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 2bab989..f74a3ab 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -544,9 +544,6 @@ def add_fee_postings(self, entry, ot): if getattr(ot, "commission", 0) != 0: data.create_simple_posting(entry, config["fees"], ot.commission, self.currency) - def add_custom_postings(self, entry, ot): - pass - def extract_custom_entries(self, file, counter): """For custom importers to override""" return [] diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index 288998b..d9f9992 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -42,6 +42,10 @@ def set_config_variables(self, substs): filing_account = self.config["main_account"].format(**kwargs) self.config["filing_account"] = self.remove_empty_subaccounts(filing_account) + def add_custom_postings(self, entry, ot): + """This method is for importers to override. Add arbitrary posting to each entry.""" + pass + def build_metadata(self, file, metatype=None, data={}): """This method is for importers to override. The overridden method can look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) From e82d4c56323e009ae660b538c4705e97e419dbbb Mon Sep 17 00:00:00 2001 From: Ad Timmering Date: Sun, 7 Apr 2024 23:36:14 +0900 Subject: [PATCH 12/13] fix: add file encoding support to csvreader fixed support for importers on non-UTF8 (e.g. Shift-JIS) text files. Fixes #99. --- beancount_reds_importers/libreader/csvreader.py | 7 +++++-- .../libtransactionbuilder/investments.py | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 92f8611..0bd11af 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -74,7 +74,10 @@ def initialize_reader(self, file): # print(self.header_identifier, file.head()) def deep_identify(self, file): - return re.match(self.header_identifier, file.head()) + return re.match( + self.header_identifier, + file.head(encoding=getattr(self, "file_encoding", None)), + ) def file_date(self, file): "Get the maximum date from the file." @@ -135,7 +138,7 @@ def convert_date(d): return rdr def read_raw(self, file): - return etl.fromcsv(file.name) + return etl.fromcsv(file.name, encoding=getattr(self, "file_encoding", None)) def skip_until_main_table(self, rdr, col_labels=None): """Skip csv lines until the header line is found.""" diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index f74a3ab..12ecd63 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -258,9 +258,9 @@ def generate_trade_entry(self, ot, file, counter): if "sell" in ot.type: units = -1 * abs(ot.units) if not is_money_market: - metadata[ - "todo" - ] = "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 + metadata["todo"] = ( + "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 + ) if ot.type in [ "reinvest" ]: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI From cc6feaf7abc1331a337b2729c4b5c024e0c4c242 Mon Sep 17 00:00:00 2001 From: Red S Date: Wed, 10 Apr 2024 16:49:00 -0700 Subject: [PATCH 13/13] fix: schwab_csv_creditline balance sign --- .../importers/schwab/schwab_csv_creditline.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py index cac8038..be9e836 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py @@ -1,6 +1,7 @@ """Schwab Credit Line (eg: Pledged Asset Line) .csv importer.""" from beancount_reds_importers.importers.schwab import schwab_csv_checking +from beancount_reds_importers.libtransactionbuilder import banking class Importer(schwab_csv_checking.Importer): @@ -12,3 +13,9 @@ def custom_init(self): self.column_labels_line = ( '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' ) + + def get_balance_statement(self, file=None): + """Return the balance on the first and last dates""" + + for i in super().get_balance_statement(file): + yield banking.Balance(i.date, -1 * i.amount, i.currency)