diff --git a/README.md b/README.md index c95a200..035ec6e 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,18 @@ $ pytr login --phone_no +49123456789 --pin 1234 If no arguments are supplied pytr will look for them in the file `~/.pytr/credentials` (the first line must contain the phone number, the second line the pin). If the file doesn't exist pytr will ask for for the phone number and pin. +## Location and File names of the downloaded Documents +During the first run of the 'dl_docs' command a config file is created in the user home directory `/.pytr/file_destination_config.yaml`. +The file contains destination patterns which describe where the file should be located and how the file name should look like. If a event/document matches the defined pattern it will be located in that specific `path` with the specified `filename`. + +There are three mandatory patterns defined at the top: +* `default` - Defines only `filename` and is used for all other patterns if no "filename" is provided +* `unknown` - Defines `filename` and `path`, this is used when no match can be found for the event and the given document. +* `multiple_match` - If there are multiple matching patterns and the destination would be ambiguous, the document will be stored in the given `path` with the given `filename` + +The other pattern can be as you like but keep in mind that patterns `path` and `filenames` should result in unique document names. If you see something like this ` (some strange string)` the document path + name was not unique. + +> Its also possible to copy the configuration file from `~/pytr/config/file_destination_config__template.yaml` to `/.pytr/file_destination_config.yaml` and modify it before the first download to avoid that the download need to be performed a second time. ## Linting and Code Formatting diff --git a/pyproject.toml b/pyproject.toml index ffb567a..d6fb1dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,8 @@ dependencies = [ "shtab", "websockets>=10.1", "babel", + "PyYAML", + "importlib_resources" ] [project.scripts] diff --git a/pytr/api.py b/pytr/api.py index ae471c7..7eedae3 100644 --- a/pytr/api.py +++ b/pytr/api.py @@ -32,18 +32,13 @@ import ssl import requests import websockets + from ecdsa import NIST256p, SigningKey from ecdsa.util import sigencode_der from http.cookiejar import MozillaCookieJar from pytr.utils import get_logger - - -home = pathlib.Path.home() -BASE_DIR = home / ".pytr" -CREDENTIALS_FILE = BASE_DIR / "credentials" -KEY_FILE = BASE_DIR / "keyfile.pem" -COOKIES_FILE = BASE_DIR / "cookies.txt" +from pytr.app_path import CREDENTIALS_FILE, COOKIES_FILE, KEY_FILE class TradeRepublicApi: diff --git a/pytr/app_path.py b/pytr/app_path.py new file mode 100644 index 0000000..4a8b8eb --- /dev/null +++ b/pytr/app_path.py @@ -0,0 +1,10 @@ +import pathlib + +home = pathlib.Path.home() +BASE_DIR = home / ".pytr" + +CREDENTIALS_FILE = BASE_DIR / "credentials" +KEY_FILE = BASE_DIR / "keyfile.pem" +COOKIES_FILE = BASE_DIR / "cookies.txt" + +DESTINATION_CONFIG_FILE = BASE_DIR / "file_destination_config.yaml" diff --git a/pytr/config/__init__.py b/pytr/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pytr/config/file_destination_config__template.yaml b/pytr/config/file_destination_config__template.yaml new file mode 100644 index 0000000..29551f0 --- /dev/null +++ b/pytr/config/file_destination_config__template.yaml @@ -0,0 +1,236 @@ +destination: + ################################################################################################ + ## Default and fallback patterns used for the destination of a downloaded document. + ################################################################################################ + # valid for all blocks without explicit filename + default: + filename: "{iso_date}.{iso_time} {event_title}" # {event_title} = Wertpapier-/ETF-/Produkt-Name + + # if pattern not found, use this block + unknown: + path: "Unknown/{section_title}/" + filename: "{iso_date}.{iso_time} {event_type} - {event_subtitle} - {document_title} - {event_title}" + + # if pattern found multiple times, use this block + multiple_match: + path: "MultipleMatch/{section_title}/" + filename: "{iso_date}.{iso_time} {event_type} - {event_subtitle} - {document_title} - {event_title}" + ################################################################################################ + + ################################################################################################ + ## Specific patterns for the destination of a downloaded document. + ################################################################################################ + # stocks + stock_order_settlement: + pattern: [ + {event_type: "ORDER_EXECUTED", document_title: "Abrechnung(.\\d+)?"}, # mit limit verkauft + {event_type: "TRADE_INVOICE", document_title: "Abrechnung(.\\d+)?"}, # mit limit gekauft + {event_type: "STOCK_PERK_REFUNDED", document_title: "Abrechnung(.\\d+)?"}, # Aktiengeschenk + {event_type: "SHAREBOOKING", document_title: "Abrechnung(.\\d+)?"}, # Kapitalmassnahme + ] + path: "Stocks/Settlement/{iso_date_year}/" + + stock_order_cost_report: + pattern: [ + {event_type: "ORDER_CREATED", document_title: "Kosteninformation(.\\d+)?"}, # limit erstellt + {event_type: "ORDER_EXECUTED", document_title: "Kosteninformation(.\\d+)?"}, + {event_type: "TRADE_INVOICE", document_title: "Kosteninformation(.\\d+)?"}, + {event_type: "STOCK_PERK_REFUNDED", document_title: "Kosteninformation(.\\d+)?"}, # Aktiengeschenk + {event_type: "EX_POST_COST_REPORT"}, + ] + path: "Stocks/Cost report/{iso_date_year}/" + + stock_order_created: + pattern: [{event_type: "ORDER_CREATED", document_title: "Auftragsbestätigung(.\\d+)?"}, {event_type: "ORDER_EXECUTED", document_title: "Auftragsbestätigung(.\\d+)?"}, {event_type: "TRADE_INVOICE", document_title: "Auftragsbestätigung(.\\d+)?"}] + path: "Stocks/Order created/{iso_date_year}/" + + stock_order_canceled: + pattern: [{event_type: "TRADE_CANCELED"}, {event_type: "ORDER_CANCELED"}] + path: "Stocks/Order canceled/{iso_date_year}/" + + order_expired: + pattern: [{event_type: "ORDER_EXPIRED"}] + path: "Stocks/Order canceled/{iso_date_year}/" + filename: "{iso_date} {document_title}" + + # Kapitalmaßnahmen + stock_notice_1: + pattern: [{event_type: "EXERCISE"}, {event_type: "SHAREBOOKING", document_title: "Ausführungsanzeige(.\\d+)?"}] + path: "Stocks/Notice/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_subtitle} {event_title}" + + # split for better readability + stock_notice_2: + pattern: [{event_type: "CORPORATE_ACTION"}, {event_type: "SHAREBOOKING_CANCELED"}] + path: "Stocks/Notice/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_subtitle} {event_title}" + + # split for better readability + stock_notice_3: + pattern: [ + {event_type: "SHAREBOOKING_TRANSACTIONAL"}, + {event_type: "INSTRUCTION_CORPORATE_ACTION", document_title: "Kundenanschreiben(.\\d+)?"}, #Kapitalerhöhung gegen Bar + ] + path: "Stocks/Notice/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_subtitle} {event_title}" + + # split for better readability + stock_notice_4: + pattern: + [ + {event_type: "ssp_corporate_action_invoice_shares", event_subtitle: "Spin-off"}, + {event_type: "ssp_corporate_action_invoice_shares", event_subtitle: "Reverse Split"}, + {event_type: "ssp_corporate_action_invoice_shares", event_subtitle: "Aktiendividende"}, + {event_type: "ssp_corporate_action_invoice_shares", event_subtitle: "Zwischenvertrieb von Wertpapieren"}, + {event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Wechsel"}, + {event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Information"}, + {event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Aufruf von Zwischenpapieren"}, + ] + path: "Stocks/Notice/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_subtitle} - {event_title}" + + stock_report: + pattern: [{event_type: "QUARTERLY_REPORT", document_title: "Kontoauszug(.\\d+)?"}, {event_type: "QUARTERLY_REPORT", document_title: "Depotauszug(.\\d+)?"}, {event_type: "QUARTERLY_REPORT", document_title: "Cryptoauszug(.\\d+)?"}] + path: "Stocks/Report/{iso_date_year}/" + filename: "{iso_date} {document_title} {event_title}" + + # General Meetings + stock_general_meetings: + pattern: [{event_type: "GENERAL_MEETING", document_title: "Hauptversammlung"}, {event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Jährliche Hauptversammlung"}] + path: "Stocks/General Meetings/{iso_date_year}/" + + stock_general_meetings_multiple_files: + pattern: [{event_type: "GENERAL_MEETING", document_title: "Hauptversammlung \\d+"}] + path: "Stocks/General Meetings/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_title} - {document_title}" + + stock_special_meetings: + pattern: [{event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Außerordentliche oder spezielle Hauptversammlung"}] + path: "Stocks/General Meetings/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_subtitle}" + + # Savings plan + savings_plan: + pattern: [{event_type: "SAVINGS_PLAN_INVOICE_CREATED"}, {event_type: "SAVINGS_PLAN_EXECUTED"}, {event_type: "SAVINGS_PLAN_CANCELED"}] + path: "Stocks/Savings plan/{iso_date_year}/" + + # pre-determined tax base earning + stock_pre_earning_tax: + pattern: [{event_type: "PRE_DETERMINED_TAX_BASE_EARNING"}] + path: "Stocks/PreEarningTax/{iso_date_year}/" + + # Dividends + dividends_received: + pattern: [{event_type: "CREDIT", event_subtitle: "Dividende"}, {event_type: "CREDIT", event_subtitle: "Ausschüttung"}, {event_type: "CREDIT_CANCELED"}, {event_type: "ssp_corporate_action_invoice_cash"}] + path: "Dividends/{iso_date_year}/" + + # Dividends Corporate action election + dividends_election: + pattern: [{event_type: "INSTRUCTION_CORPORATE_ACTION", document_title: "Dividende Wahlweise(.\\d+)?"}, {event_type: "ssp_dividend_option_customer_instruction", event_subtitle: "Cash oder Aktie"}, {event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Dividende Wahlweise"}] + path: "Dividendelection/{iso_date_year}/" + + # bonds + bond_repayment: + pattern: [{event_type: "REPAYMENT"}] + path: "Bonds/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Repayment {event_title}" + + bond_interest: + pattern: [{event_type: "COUPON_PAYMENT"}] + path: "Bonds/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Interest {event_title}" + + # Saveback + saveback_enabled: + pattern: [{event_type: "benefits_saveback_execution", document_title: "Enabled(.\\d+)?"}] + path: "Saveback/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Enabled {event_title}" + + saveback_executed: + pattern: [{event_type: "benefits_saveback_execution", document_title: "Abrechnung Ausführung(.\\d+)?"}] + path: "Saveback/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Report {event_title}" + + saveback_cost_report: + pattern: [{event_type: "benefits_saveback_execution", document_title: "Kosteninformation(.\\d+)?"}] + path: "Saveback/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Cost report {event_title}" + + # Round up + roundup_enabled: + pattern: [{event_type: "benefits_spare_change_execution", document_title: "Enabled(.\\d+)?"}] # same files - multiple times at once + path: "Roundup/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Enabled {event_title}" + + roundup_executed: + pattern: [{event_type: "benefits_spare_change_execution", document_title: "Abrechnung Ausführung(.\\d+)?"}] + path: "Roundup/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Report {event_title}" + + roundup_cost_report: + pattern: [{event_type: "benefits_spare_change_execution", document_title: "Kosteninformation(.\\d+)?"}] + path: "Roundup/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Cost report {event_title}" + + # account + cash_interest: + pattern: [{event_type: "INTEREST_PAYOUT"}, {event_type: "INTEREST_PAYOUT_CREATED"}] + path: "Cash Interest/" + filename: "{iso_date} Report" + + cash_transfer_report: + pattern: [{event_type: "INCOMING_TRANSFER"}, {event_type: "PAYMENT_INBOUND_GOOGLE_PAY"}, {event_type: "PAYMENT_INBOUND_CREDIT_CARD"}] + path: "Cash Report/" + filename: "{iso_date}.{iso_time} {document_title}" # {event_title} = Personal name + + # annual tax report for account + account_tax_report: + pattern: [{event_type: "TAX_REFUND"}, {event_type: "TAX_ENGINE_ANNUAL_REPORT"}, {event_type: "YEAR_END_TAX_REPORT"}] + path: "Tax/" + filename: "{iso_date} {document_title}" + + account_tax_adjustment: + pattern: [{event_type: "ssp_tax_correction_invoice"}, {event_type: "TAX_CORRECTION"}] + path: "Tax/" + filename: "{iso_date} {event_title}" + + # common informations + notice_stocks: + pattern: [{event_type: "ORDER_CREATED", document_title: "Basisinformationsblatt(.\\d+)?"}] + path: "Notice/{iso_date_year}/" + filename: "{iso_date} {document_title} - {event_title}" + + notice_stocks2: + pattern: [{event_type: "TRADE_INVOICE", document_title: "Basisinformationsblatt(.\\d+)?"}, {event_type: "GESH_CORPORATE_ACTION", event_subtitle: "Unternehmensmeldung"}] + path: "Notice/{iso_date_year}/" + filename: "{iso_date} {event_subtitle} - {event_title}" + + notice_stocks3: + pattern: [{event_type: "CUSTOMER_CREATED"}] + path: "Notice/{iso_date_year}/" + filename: "{iso_date} {document_title}" + + notice_option_contract: + pattern: [{event_type: "ORDER_EXECUTED", document_title: "Basisinformationsblatt(.\\d+)?"}] + path: "Notice/{iso_date_year}/" + filename: "{iso_date} {document_title} Option" + + notice_multiple_documents: + pattern: [{event_type: "GESH_CORPORATE_ACTION_MULTIPLE_POSITIONS"}] #event_subtitle: Gesellschaftshinweis + path: "Notice/{iso_date_year}/" + filename: "{iso_date} {event_subtitle} - {event_title} - {document_title}" + + contract_documents: + pattern: [ + {event_type: "card_order_billed"}, # Bestellung Trade Republic Karte + {event_type: "DOCUMENTS_CREATED"}, # Basisinformationen über Wertpapiere + {event_type: "DOCUMENTS_ACCEPTED"}, # Rechtliche Dokumente: Kundenvereinbarung / Vorvertragliche Informationen / Datenschutzinformationen* / Widerrufsbelehrung* / *Crypto* / Risikohinweise + {event_type: "DOCUMENTS_CHANGED", section_title: "Dokumente"}, # Rechtliche Dokumente: Kundenvereinbarung + ] + path: "Contract/" + filename: "{iso_date} {event_title} - {document_title}" + + contract_documents_updated: + pattern: [{event_type: "DOCUMENTS_CHANGED", section_title: "Aktualisierte Dokumente"}] # aktualisierte Rechtliche Dokumente: Kundenvereinbarung + path: "Contract/" + filename: "{iso_date} {event_title} - {document_title} updated" diff --git a/pytr/dl.py b/pytr/dl.py index a935f2a..8c9fae2 100644 --- a/pytr/dl.py +++ b/pytr/dl.py @@ -1,14 +1,16 @@ -import re +import os from concurrent.futures import as_completed from pathlib import Path from requests_futures.sessions import FuturesSession +from datetime import datetime from pathvalidate import sanitize_filepath from pytr.utils import preview, get_logger from pytr.api import TradeRepublicError from pytr.timeline import Timeline +from pytr.file_destination_provider import FileDestinationProvider class DL: @@ -16,7 +18,6 @@ def __init__( self, tr, output_path, - filename_fmt, since_timestamp=0, history_file="pytr_history", max_workers=8, @@ -26,13 +27,12 @@ def __init__( """ tr: api object output_path: name of the directory where the downloaded files are saved - filename_fmt: format string to customize the file names since_timestamp: downloaded files since this date (unix timestamp) """ self.tr = tr self.output_path = Path(output_path) self.history_file = self.output_path / history_file - self.filename_fmt = filename_fmt + self.file_destination_provider = self.__get_file_destination_provider() self.since_timestamp = since_timestamp self.universal_filepath = universal_filepath self.sort_export = sort_export @@ -88,67 +88,50 @@ async def dl_loop(self): f"unmatched subscription of type '{subscription['type']}':\n{preview(response)}" ) - def dl_doc(self, doc, titleText, subtitleText, subfolder=None): + def dl_doc( + self, + doc, + event_type: str, + event_title: str, + event_subtitle: str, + section_title: str, + timestamp: datetime, + ): """ send asynchronous request, append future with filepath to self.futures """ doc_url = doc["action"]["payload"] - if subtitleText is None: - subtitleText = "" - - try: - date = doc["detail"] - iso_date = "-".join(date.split(".")[::-1]) - except KeyError: - date = "" - iso_date = "" + document_title = doc.get("title", "") doc_id = doc["id"] - # extract time from subtitleText - try: - time = re.findall("um (\\d+:\\d+) Uhr", subtitleText) - if time == []: - time = "" - else: - time = f" {time[0]}" - except TypeError: - time = "" - - if subfolder is not None: - directory = self.output_path / subfolder - else: - directory = self.output_path - - # If doc_type is something like 'Kosteninformation 2', then strip the 2 and save it in doc_type_num - doc_type = doc["title"].rsplit(" ") - if doc_type[-1].isnumeric() is True: - doc_type_num = f" {doc_type.pop()}" - else: - doc_type_num = "" - - doc_type = " ".join(doc_type) - titleText = titleText.replace("\n", "").replace("/", "-") - subtitleText = subtitleText.replace("\n", "").replace("/", "-") - - filename = self.filename_fmt.format( - iso_date=iso_date, - time=time, - title=titleText, - subtitle=subtitleText, - doc_num=doc_type_num, - id=doc_id, + variables = {} + variables["iso_date"] = timestamp.strftime("%Y-%m-%d") + variables["iso_date_year"] = timestamp.strftime("%Y") + variables["iso_date_month"] = timestamp.strftime("%m") + variables["iso_date_day"] = timestamp.strftime("%d") + variables["iso_time"] = timestamp.strftime("%H-%M") + + filepath = self.file_destination_provider.get_file_path( + event_type, + event_title, + event_subtitle, + section_title, + document_title, + variables, ) + # Just in case someone defines file names with extension + if filepath.endswith(".pdf") is True: + filepath = filepath[:-4] - filename_with_doc_id = filename + f" ({doc_id})" + filepath_with_doc_id = f"{filepath} ({doc_id})" - if doc_type in ["Kontoauszug", "Depotauszug"]: - filepath = directory / "Abschlüsse" / f"{filename}" / f"{doc_type}.pdf" - filepath_with_doc_id = ( - directory / "Abschlüsse" / f"{filename_with_doc_id}" / f"{doc_type}.pdf" - ) - else: - filepath = directory / doc_type / f"{filename}.pdf" - filepath_with_doc_id = directory / doc_type / f"{filename_with_doc_id}.pdf" + filepath = f"{filepath}.pdf" + filepath_with_doc_id = f"{filepath_with_doc_id}.pdf" + + filepath = Path(os.path.join(self.output_path, filepath)) + filepath_with_doc_id = Path( + os.path.join(self.output_path, filepath_with_doc_id) + ) if self.universal_filepath: filepath = sanitize_filepath(filepath, "_", "universal") @@ -224,3 +207,6 @@ def work_responses(self): if self.done == len(self.doc_urls): self.log.info("Done.") exit(0) + + def __get_file_destination_provider(self): + return FileDestinationProvider() diff --git a/pytr/file_destination_provider.py b/pytr/file_destination_provider.py new file mode 100644 index 0000000..b0e5cb6 --- /dev/null +++ b/pytr/file_destination_provider.py @@ -0,0 +1,241 @@ +import os +import re +import shutil +import pytr.config + +from importlib_resources import files + +from dataclasses import dataclass, fields +from typing import Optional +from yaml import safe_load +from pathlib import Path +from pytr.app_path import DESTINATION_CONFIG_FILE +from pytr.utils import get_logger + + +DEFAULT_CONFIG = "default" +UNKNOWN_CONFIG = "unknown" +MULTIPLE_MATCH_CONFIG = "multiple_match" + +TEMPLATE_FILE_NAME = "file_destination_config__template.yaml" + +# Invalid characters translation table, for cleaning up the variables before using them. +# This was done to avoid issues with for example 'event_subtitle: “Umtausch/Bezug”' which caused a directory which was unintentional. +INVALID_CHARS_TRANSLATION_TABLE = str.maketrans( + {'"': "", "?": "", "<": "", ">": "", "*": "", "|": "-", "/": "-", "\\": "-"} +) + + +class DefaultFormateValue(dict): + def __missing__(self, key): + return key.join("{}") + + +@dataclass +class DestinationConfig: + config_name: str + filename: str + path: Optional[str] = None + pattern: Optional[list] = None + + +@dataclass +class Pattern: + event_type: Optional[str] = None + event_title: Optional[str] = None + event_subtitle: Optional[str] = None + section_title: Optional[str] = None + document_title: Optional[str] = None + + +class FileDestinationProvider: + + def __init__(self): + """ + A provider for file path and file names based on the event type and other parameters. + """ + self._log = get_logger(__name__) + + config_file_path = Path(DESTINATION_CONFIG_FILE) + if config_file_path.is_file() == False: + self.__create_default_config(config_file_path) + + config_file = open(config_file_path, "r", encoding="utf8") + destination_config = safe_load(config_file) + + self.__validate_config(destination_config) + + destinations = destination_config["destination"] + + self._destination_configs: list[DestinationConfig] = [] + + for config_name in destinations: + if config_name == DEFAULT_CONFIG: + self._default_file_config = DestinationConfig( + DEFAULT_CONFIG, destinations[DEFAULT_CONFIG]["filename"] + ) + elif config_name == UNKNOWN_CONFIG: + self._unknown_file_config = DestinationConfig( + UNKNOWN_CONFIG, + destinations[UNKNOWN_CONFIG]["filename"], + destinations[UNKNOWN_CONFIG]["path"], + ) + elif config_name == MULTIPLE_MATCH_CONFIG: + self._multiple_match_file_config = DestinationConfig( + MULTIPLE_MATCH_CONFIG, + destinations[MULTIPLE_MATCH_CONFIG]["filename"], + destinations[MULTIPLE_MATCH_CONFIG]["path"], + ) + else: + patterns = self.__extract_pattern( + destinations[config_name].get("pattern", None) + ) + for pattern in patterns: + self._destination_configs.append( + DestinationConfig( + config_name, + destinations[config_name].get("filename", None), + destinations[config_name].get("path", None), + pattern, + ) + ) + + def get_file_path( + self, + event_type: str, + event_title: str, + event_subtitle: str, + section_title: str, + document_title: str, + variables: dict, + ) -> str: + """ + Get the file path based on the event type and other parameters. + + Parameters: + event_type (str): The event type + event_title (str): The event title + event_subtitle (str): The event subtitle + section_title (str): The section title + document_title (str): The document title + variables (dict): The variables->value dict to be used in the file path and file name format. + """ + + doc = Pattern( + event_type, event_title, event_subtitle, section_title, document_title + ) + + matching_configs = self._destination_configs.copy() + # create a dictionary that maps the field names to their values in the pattern instance + pattern_dict = { + field.name: getattr(doc, field.name) for field in fields(Pattern) + } + + # iterate over the dictionary to filter the matching_configs list and update the variables dictionary + for field_name, search_pattern in pattern_dict.items(): + if search_pattern is not None: + matching_configs = list( + filter( + lambda config: self.__is_matching_config( + config, field_name, search_pattern + ), + matching_configs, + ) + ) + variables[field_name] = search_pattern.translate( + INVALID_CHARS_TRANSLATION_TABLE + ).strip() + + if len(matching_configs) == 0: + self._log.debug( + f"No destination config found for the given parameters: event_type:{event_type}, event_title:{event_title},event_subtitle:{event_subtitle},section_title:{section_title},document_title:{document_title}" + ) + return self.__create_file_path(self._unknown_file_config, variables) + + if len(matching_configs) > 1: + self._log.debug( + f"Multiple Destination Patterns where found. Using 'multiple_match' config! Parameter: event_type:{event_type}, event_title:{event_title},event_subtitle:{event_subtitle},section_title:{section_title},document_title:{document_title}" + ) + return self.__create_file_path(self._multiple_match_file_config, variables) + + return self.__create_file_path(matching_configs[0], variables) + + @staticmethod + def __is_matching_config( + config: DestinationConfig, field_name: str, search_pattern: str + ) -> bool: + pattern = config.pattern + return getattr(pattern, field_name, None) is None or re.fullmatch( + getattr(pattern, field_name, None), search_pattern + ) + + def __create_file_path(self, config: DestinationConfig, variables: dict): + formate_variables = DefaultFormateValue(variables) + + path = config.path + filename = config.filename + if filename is None: + filename = self._default_file_config.filename + + return os.path.join(path, filename).format_map(formate_variables) + + def __extract_pattern(self, pattern_config: list) -> list: + patterns = [] + for pattern in pattern_config: + patterns.append( + Pattern( + pattern.get("event_type", None), + pattern.get("event_title", None), + pattern.get("event_subtitle", None), + pattern.get("section_title", None), + pattern.get("document_title", None), + ) + ) + + return patterns + + def __validate_config(self, destination_config: dict): + if "destination" not in destination_config: + raise ValueError("'destination' key not found in config file") + + destinations = destination_config["destination"] + + # Check if default config is present + if ( + DEFAULT_CONFIG not in destinations + or "filename" not in destinations[DEFAULT_CONFIG] + ): + raise ValueError( + "'default' config not found or filename is not present in 'default' config" + ) + + if ( + UNKNOWN_CONFIG not in destinations + or "filename" not in destinations[UNKNOWN_CONFIG] + or "path" not in destinations[UNKNOWN_CONFIG] + ): + raise ValueError( + "'unknown' config not found or filename/path is not present in 'unknown' config" + ) + + if ( + MULTIPLE_MATCH_CONFIG not in destinations + or "filename" not in destinations[MULTIPLE_MATCH_CONFIG] + or "path" not in destinations[MULTIPLE_MATCH_CONFIG] + ): + raise ValueError( + "'multiple_match' config not found or filename/path is not present in 'multiple_match' config" + ) + + for config_name in destinations: + if ( + config_name != DEFAULT_CONFIG + and "path" not in destinations[config_name] + ): + raise ValueError( + f"'{config_name}' has no path defined in destination config" + ) + + def __create_default_config(self, config_file_path: Path): + path = files(pytr.config).joinpath(TEMPLATE_FILE_NAME) + shutil.copyfile(path, config_file_path) diff --git a/pytr/main.py b/pytr/main.py index 8d80eda..d2b60e5 100644 --- a/pytr/main.py +++ b/pytr/main.py @@ -93,6 +93,9 @@ def formatter(prog): "Download all pdf documents from the timeline and sort them into folders." + " Also export account transactions (account_transactions.csv)" + " and JSON files with all events (events_with_documents.json and other_events.json" + + " The file and folder where the structure is saved is defined in a config file located in your home" + + " directory '/.pytr/file_destination_config.yaml'. This is created during the first dl_docs run." + + " Its also possible to provide the config upfront by creating the file manually (copy from git repo)." ) parser_dl_docs = parser_cmd.add_parser( "dl_docs", @@ -111,13 +114,6 @@ def formatter(prog): metavar="FORMAT_STRING", default="{iso_date}{time} {title}{doc_num}", ) - parser_dl_docs.add_argument( - "--last_days", - help="Number of last days to include (use 0 get all days)", - metavar="DAYS", - default=0, - type=int, - ) parser_dl_docs.add_argument( "--workers", help="Number of workers for parallel downloading", @@ -257,7 +253,6 @@ def main(): dl = DL( login(phone_no=args.phone_no, pin=args.pin, web=not args.applogin), args.output, - args.format, since_timestamp=since_timestamp, max_workers=args.workers, universal_filepath=args.universal, diff --git a/pytr/timeline.py b/pytr/timeline.py index 1200291..ce74508 100644 --- a/pytr/timeline.py +++ b/pytr/timeline.py @@ -156,14 +156,14 @@ async def process_timelineDetail(self, response, dl): except (ValueError, KeyError): timestamp = datetime.now().timestamp() if self.max_age_timestamp == 0 or self.max_age_timestamp < timestamp: - title = f"{doc['title']} - {event['title']}" - if event["eventType"] in [ - "ACCOUNT_TRANSFER_INCOMING", - "ACCOUNT_TRANSFER_OUTGOING", - "CREDIT", - ]: - title += f" - {event['subtitle']}" - dl.dl_doc(doc, title, doc.get("detail"), subfolder) + dl.dl_doc( + doc, + event["eventType"], + event["title"], + event["subtitle"], + section["title"], + datetime.fromisoformat(event["timestamp"][:19]), + ) if event["has_docs"]: self.events_with_docs.append(event)