diff --git a/.github/helper/install.sh b/.github/helper/install.sh index d2f86a02..cddfe747 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -32,7 +32,7 @@ bench get-app erpnext --branch version-14 bench get-app hrms --branch version-14 bench get-app banking "${GITHUB_WORKSPACE}" -bench start &> bench_run_logs.txt & +bench start &> bench_start.log & bench new-site --db-root-password root --admin-password admin test_site --install-app erpnext bench --site test_site install-app hrms bench --site test_site install-app banking diff --git a/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/bank_reconciliation_tool_beta.py b/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/bank_reconciliation_tool_beta.py index e4b592be..3f991caa 100644 --- a/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/bank_reconciliation_tool_beta.py +++ b/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/bank_reconciliation_tool_beta.py @@ -7,9 +7,9 @@ from frappe import _ from frappe.model.document import Document from frappe.query_builder.custom import ConstantColumn -from frappe.query_builder.functions import Coalesce, CustomFunction from frappe.utils import cint, flt, sbool from pypika.terms import Parameter +from frappe.query_builder.functions import Cast, Coalesce from erpnext import get_company_currency, get_default_cost_center from erpnext.accounts.doctype.bank_transaction.bank_transaction import ( @@ -17,11 +17,16 @@ get_total_allocated_amount, ) from erpnext.accounts.utils import get_account_currency +from banking.klarna_kosma_integration.doctype.bank_reconciliation_tool_beta.utils import ( + amount_rank_condition, + get_description_match_condition, + get_reference_field_map, + ref_equality_condition, +) + from pypika import Order MAX_QUERY_RESULTS = 150 -Instr = CustomFunction("INSTR", ["a", "b"]) -RegExpReplace = CustomFunction("REGEXP_REPLACE", ["a", "b", "c"]) class BankReconciliationToolBeta(Document): @@ -596,24 +601,29 @@ def get_matching_queries( include_unpaid = "unpaid_invoices" in document_types invoice_dt = "sales_invoice" if is_deposit else "purchase_invoice" invoice_queries_map = get_invoice_function_map(document_types, is_deposit) - kwargs = { - "exact_match": exact_match, - "exact_party_match": exact_party_match, - "currency": currency, - "description": transaction.description, - } + kwargs = frappe._dict( + exact_match=exact_match, + exact_party_match=exact_party_match, + currency=currency, + description=transaction.description, + reference_number=transaction.reference_number, + ) + reference_field_map = get_reference_field_map() if include_unpaid: - kwargs["company"] = company + kwargs.company = company for doctype, fn in invoice_queries_map.items(): frappe.has_permission(frappe.unscrub(doctype), throw=True) + kwargs.reference_field = reference_field_map.get(doctype, "name") + if doctype in ["sales_invoice", "purchase_invoice"]: + kwargs.include_only_returns = doctype != invoice_dt + elif kwargs.include_only_returns is not None: + # Remove the key when doctype == "expense_claim" + del kwargs.include_only_returns - if doctype != "expense_claim": - kwargs["include_only_returns"] = doctype != invoice_dt - else: - del kwargs["include_only_returns"] queries.append(fn(**kwargs)) elif fn := invoice_queries_map.get(invoice_dt): frappe.has_permission(frappe.unscrub(invoice_dt), throw=True) + kwargs.reference_field = reference_field_map.get(invoice_dt, "name") queries.append(fn(**kwargs)) if "loan_disbursement" in document_types and is_withdrawal: @@ -638,28 +648,27 @@ def get_bt_matching_query(exact_match, transaction, exact_party_match): # same bank account must have same company and currency bt = frappe.qb.DocType("Bank Transaction") field = "deposit" if transaction.withdrawal > 0.0 else "withdrawal" + amount_field = getattr(bt, field) - ref_rank = ( - frappe.qb.terms.Case() - .when(bt.reference_number == transaction.reference_number, 1) - .else_(0) - ) + ref_rank = ref_equality_condition(bt.reference_number, transaction.reference_number) unallocated_rank = ( frappe.qb.terms.Case() .when(bt.unallocated_amount == transaction.unallocated_amount, 1) .else_(0) ) - amount_equality = getattr(bt, field) == transaction.unallocated_amount - amount_rank = frappe.qb.terms.Case().when(amount_equality, 1).else_(0) + amount_rank = amount_rank_condition(amount_field, transaction.unallocated_amount) + amount_filter = ( + amount_field == transaction.unallocated_amount if exact_match else amount_field > 0.0 + ) - party_condition = ( + party_filter = ( (bt.party_type == transaction.party_type) & (bt.party == transaction.party) & bt.party.isnotnull() ) - party_rank = frappe.qb.terms.Case().when(party_condition, 1).else_(0) - amount_condition = amount_equality if exact_match else getattr(bt, field) > 0.0 + party_rank = frappe.qb.terms.Case().when(party_filter, 1).else_(0) + rank_expression = ref_rank + amount_rank + party_rank + unallocated_rank + 1 query = ( @@ -683,23 +692,20 @@ def get_bt_matching_query(exact_match, transaction, exact_party_match): .where(bt.status != "Reconciled") .where(bt.name != transaction.name) .where(bt.bank_account == transaction.bank_account) - .where(amount_condition) + .where(amount_filter) .where(bt.docstatus == 1) .orderby(rank_expression, order=Order.desc) .limit(MAX_QUERY_RESULTS) ) if exact_party_match: - query = query.where(party_condition) + query = query.where(party_filter) return str(query) def get_ld_matching_query(bank_account, exact_match, transaction): loan_disbursement = frappe.qb.DocType("Loan Disbursement") - matching_reference = loan_disbursement.reference_number == transaction.get( - "reference_number" - ) matching_party = loan_disbursement.applicant_type == transaction.get( "party_type" ) and loan_disbursement.applicant == transaction.get("party") @@ -710,7 +716,9 @@ def get_ld_matching_query(bank_account, exact_match, transaction): ) date_rank = frappe.qb.terms.Case().when(date_condition, 1).else_(0) - reference_rank = frappe.qb.terms.Case().when(matching_reference, 1).else_(0) + reference_rank = ref_equality_condition( + loan_disbursement.reference_number, transaction.reference_number + ) party_rank = frappe.qb.terms.Case().when(matching_party, 1).else_(0) rank_expression = reference_rank + party_rank + date_rank + 1 @@ -749,9 +757,6 @@ def get_ld_matching_query(bank_account, exact_match, transaction): def get_lr_matching_query(bank_account, exact_match, transaction): loan_repayment = frappe.qb.DocType("Loan Repayment") - matching_reference = loan_repayment.reference_number == transaction.get( - "reference_number" - ) matching_party = loan_repayment.applicant_type == transaction.get( "party_type" ) and loan_repayment.applicant == transaction.get("party") @@ -762,7 +767,9 @@ def get_lr_matching_query(bank_account, exact_match, transaction): ) date_rank = frappe.qb.terms.Case().when(date_condition, 1).else_(0) - reference_rank = frappe.qb.terms.Case().when(matching_reference, 1).else_(0) + reference_rank = ref_equality_condition( + loan_repayment.reference_number, transaction.reference_number + ) party_rank = frappe.qb.terms.Case().when(matching_party, 1).else_(0) rank_expression = reference_rank + party_rank + date_rank + 1 @@ -814,23 +821,25 @@ def get_pe_matching_query( exact_party_match, ): to_from = "to" if transaction.deposit > 0.0 else "from" - currency_field = f"paid_{to_from}_account_currency" payment_type = "Receive" if transaction.deposit > 0.0 else "Pay" pe = frappe.qb.DocType("Payment Entry") + currency_field = getattr(pe, f"paid_{to_from}_account_currency") - ref_condition = pe.reference_no == transaction.reference_number - ref_rank = frappe.qb.terms.Case().when(ref_condition, 1).else_(0) + ref_rank = ref_equality_condition(pe.reference_no, transaction.reference_number) - amount_equality = pe.paid_amount == transaction.unallocated_amount - amount_rank = frappe.qb.terms.Case().when(amount_equality, 1).else_(0) - amount_condition = amount_equality if exact_match else pe.paid_amount > 0.0 + amount_rank = amount_rank_condition(pe.paid_amount, transaction.unallocated_amount) + amount_filter = ( + pe.paid_amount == transaction.unallocated_amount + if exact_match + else pe.paid_amount > 0.0 + ) - party_condition = ( - (pe.party_type == transaction.party_type) - & (pe.party == transaction.party) - & pe.party.isnotnull() + party_filter = ( + (pe.party == transaction.party) + & (pe.party_type == transaction.party_type) + & (pe.party.isnotnull()) ) - party_rank = frappe.qb.terms.Case().when(party_condition, 1).else_(0) + party_rank = frappe.qb.terms.Case().when(party_filter, 1).else_(0) filter_by_date = pe.posting_date.between(from_date, to_date) if cint(filter_by_reference_date): @@ -854,7 +863,7 @@ def get_pe_matching_query( pe.party_name, pe.party_type, pe.posting_date, - getattr(pe, currency_field).as_("currency"), + currency_field.as_("currency"), ref_rank.as_("reference_number_match"), amount_rank.as_("amount_match"), party_rank.as_("party_match"), @@ -864,16 +873,16 @@ def get_pe_matching_query( .where(pe.payment_type.isin([payment_type, "Internal Transfer"])) .where(pe.clearance_date.isnull()) .where(getattr(pe, account_from_to) == Parameter("%(bank_account)s")) - .where(amount_condition) + .where(amount_filter) .where(filter_by_date) .orderby(rank_expression, order=Order.desc) .limit(MAX_QUERY_RESULTS) ) if frappe.flags.auto_reconcile_vouchers: - query = query.where(ref_condition) + query = query.where(pe.reference_no == transaction.reference_number) if exact_party_match: - query = query.where(party_condition) + query = query.where(party_filter) return str(query) @@ -891,16 +900,17 @@ def get_je_matching_query( # We have mapping at the bank level # So one bank could have both types of bank accounts like asset and liability # So cr_or_dr should be judged only on basis of withdrawal and deposit and not account type - cr_or_dr = "credit" if transaction.withdrawal > 0.0 else "debit" je = frappe.qb.DocType("Journal Entry") jea = frappe.qb.DocType("Journal Entry Account") - ref_condition = je.cheque_no == transaction.reference_number - ref_rank = frappe.qb.terms.Case().when(ref_condition, 1).else_(0) + cr_or_dr = "credit" if transaction.withdrawal > 0.0 else "debit" + amount_field = getattr(jea, f"{cr_or_dr}_in_account_currency") - amount_field = f"{cr_or_dr}_in_account_currency" - amount_equality = getattr(jea, amount_field) == transaction.unallocated_amount - amount_rank = frappe.qb.terms.Case().when(amount_equality, 1).else_(0) + ref_rank = ref_equality_condition(je.cheque_no, transaction.reference_number) + amount_rank = amount_rank_condition(amount_field, transaction.unallocated_amount) + amount_filter = ( + amount_field == transaction.unallocated_amount if exact_match else amount_field > 0.0 + ) filter_by_date = je.posting_date.between(from_date, to_date) if cint(filter_by_reference_date): @@ -919,7 +929,7 @@ def get_je_matching_query( rank_expression.as_("rank"), ConstantColumn("Journal Entry").as_("doctype"), je.name, - getattr(jea, amount_field).as_("paid_amount"), + amount_field.as_("paid_amount"), je.cheque_no.as_("reference_no"), je.cheque_date.as_("reference_date"), je.pay_to_recd_from.as_("party"), @@ -934,7 +944,7 @@ def get_je_matching_query( .where(je.voucher_type != "Opening Entry") .where(je.clearance_date.isnull()) .where(jea.account == Parameter("%(bank_account)s")) - .where(amount_equality if exact_match else getattr(jea, amount_field) > 0.0) + .where(amount_filter) .where(je.docstatus == 1) .where(filter_by_date) .orderby(rank_expression, order=Order.desc) @@ -942,33 +952,56 @@ def get_je_matching_query( ) if frappe.flags.auto_reconcile_vouchers: - query = query.where(ref_condition) + query = query.where(je.cheque_no == transaction.reference_number) return str(query) -def get_si_matching_query(exact_match, exact_party_match, currency, description): +def get_si_matching_query( + exact_match: bool, + exact_party_match: bool, + currency: str, + description: str, + reference_number: str, + reference_field: str = "name", +): """ Get matching sales invoices when they are also used as payment entries (POS). """ - si = frappe.qb.DocType("Sales Invoice") - sip = frappe.qb.DocType("Sales Invoice Payment") + si = frappe.qb.DocType("Sales Invoice").as_("si") + sip = frappe.qb.DocType("Sales Invoice Payment").as_("sip") - amount_equality = sip.amount == Parameter("%(amount)s") - amount_rank = frappe.qb.terms.Case().when(amount_equality, 1).else_(0) - amount_condition = amount_equality if exact_match else sip.amount != 0.0 + amount_rank = amount_rank_condition(sip.amount, Parameter("%(amount)s")) + amount_filter = ( + sip.amount == Parameter("%(amount)s") if exact_match else sip.amount != 0.0 + ) - party_condition = si.customer == Parameter("%(party)s") - party_rank = frappe.qb.terms.Case().when(party_condition, 1).else_(0) + party_filter = si.customer == Parameter("%(party)s") + party_rank = frappe.qb.terms.Case().when(party_filter, 1).else_(0) date_condition = si.posting_date == Parameter("%(date)s") date_rank = frappe.qb.terms.Case().when(date_condition, 1).else_(0) - description_match = get_description_match_condition( - description, si.name + # Check reference field equality with passed reference_no + reference_field_is_set = reference_field and reference_field != "name" + ref_rank = ( + ref_equality_condition(si[reference_field], reference_number) + if (reference_number and reference_field_is_set) + else Cast(0, "int") + ) + + # if ref field is configured (!= name), perform desc-name and desc-ref match + # otherwise (== name), then perform desc-name match once + name_match = get_description_match_condition(description, si, "name") + ref_match = ( + get_description_match_condition(description, si, reference_field) + if reference_field_is_set + else Cast(0, "int") ) - rank_expression = party_rank + amount_rank + date_rank + description_match + 1 + rank_expression = ( + ref_rank + party_rank + amount_rank + date_rank + name_match + ref_match + 1 + ) query = ( frappe.qb.from_(sip) @@ -979,7 +1012,7 @@ def get_si_matching_query(exact_match, exact_party_match, currency, description) ConstantColumn("Sales Invoice").as_("doctype"), si.name, sip.amount.as_("paid_amount"), - si.name.as_("reference_no"), + si[reference_field or "name"].as_("reference_no"), si.posting_date.as_("reference_date"), si.customer.as_("party"), ConstantColumn("Customer").as_("party_type"), @@ -988,39 +1021,61 @@ def get_si_matching_query(exact_match, exact_party_match, currency, description) party_rank.as_("party_match"), amount_rank.as_("amount_match"), date_rank.as_("date_match"), - description_match.as_("name_in_desc_match"), + name_match.as_("name_in_desc_match"), + ref_match.as_("ref_in_desc_match"), + ref_rank.as_("reference_number_match"), ) .where(si.docstatus == 1) .where(sip.clearance_date.isnull()) .where(sip.account == Parameter("%(bank_account)s")) - .where(amount_condition) + .where(amount_filter) .where(si.currency == currency) .orderby(rank_expression, order=Order.desc) .limit(MAX_QUERY_RESULTS) ) if exact_party_match: - query = query.where(party_condition) + query = query.where(party_filter) return str(query) def get_unpaid_si_matching_query( - exact_match, exact_party_match, currency, company, description, include_only_returns=False + exact_match: bool, + exact_party_match: bool, + currency: str, + company: str, + description: str, + reference_number: str, + include_only_returns: bool = False, + reference_field: str = "name", ): sales_invoice = frappe.qb.DocType("Sales Invoice") + party_filter = sales_invoice.customer == Parameter("%(party)s") + party_rank = frappe.qb.terms.Case().when(party_filter, 1).else_(0) - party_condition = sales_invoice.customer == Parameter("%(party)s") - party_match = frappe.qb.terms.Case().when(party_condition, 1).else_(0) + amount_rank = amount_rank_condition( + sales_invoice.outstanding_amount, Parameter("%(amount)s") + ) - outstanding_amount_condition = sales_invoice.outstanding_amount == Parameter( - "%(amount)s" + # Check reference field equality with common_filters.reference_no + reference_field_is_set = reference_field and reference_field != "name" + ref_rank = ( + ref_equality_condition(sales_invoice[reference_field], reference_number) + if (reference_number and reference_field_is_set) + else Cast(0, "int") ) - amount_match = frappe.qb.terms.Case().when(outstanding_amount_condition, 1).else_(0) - description_match = get_description_match_condition( - description, sales_invoice.name + + # if ref field is configured (!= name), perform desc-name and desc-ref match + # otherwise (== name), then perform desc-name match once + name_match = get_description_match_condition(description, sales_invoice, "name") + ref_match = ( + get_description_match_condition(description, sales_invoice, reference_field) + if reference_field_is_set + else Cast(0, "int") ) - rank_expression = party_match + amount_match + description_match + 1 + + rank_expression = ref_rank + party_rank + amount_rank + name_match + ref_match + 1 query = ( frappe.qb.from_(sales_invoice) @@ -1029,16 +1084,18 @@ def get_unpaid_si_matching_query( ConstantColumn("Sales Invoice").as_("doctype"), sales_invoice.name.as_("name"), sales_invoice.outstanding_amount.as_("paid_amount"), - sales_invoice.name.as_("reference_no"), + sales_invoice[reference_field or "name"].as_("reference_no"), sales_invoice.posting_date.as_("reference_date"), sales_invoice.customer.as_("party"), ConstantColumn("Customer").as_("party_type"), sales_invoice.customer_name.as_("party_name"), sales_invoice.posting_date, sales_invoice.currency, - party_match.as_("party_match"), - amount_match.as_("amount_match"), - description_match.as_("name_in_desc_match"), + party_rank.as_("party_match"), + amount_rank.as_("amount_match"), + name_match.as_("name_in_desc_match"), + (ref_match).as_("ref_in_desc_match"), + (ref_rank).as_("reference_number_match"), ) .where(sales_invoice.docstatus == 1) .where(sales_invoice.company == company) # because we do not have bank account check @@ -1051,27 +1108,37 @@ def get_unpaid_si_matching_query( if include_only_returns: query = query.where(sales_invoice.is_return == 1) if exact_match: - query = query.where(outstanding_amount_condition) + query = query.where(sales_invoice.outstanding_amount == Parameter("%(amount)s")) if exact_party_match: - query = query.where(party_condition) + query = query.where(party_filter) return str(query) -def get_pi_matching_query(exact_match, exact_party_match, currency, description): +def get_pi_matching_query( + exact_match: bool, + exact_party_match: bool, + currency: str, + description: str, + reference_number: str, + reference_field: str = "name", +): """ Get matching purchase invoice query when they are also used as payment entries (is_paid) """ purchase_invoice = frappe.qb.DocType("Purchase Invoice") - amount_equality = purchase_invoice.paid_amount == Parameter("%(amount)s") - amount_rank = frappe.qb.terms.Case().when(amount_equality, 1).else_(0) - amount_condition = ( - amount_equality if exact_match else purchase_invoice.paid_amount != 0.0 + amount_rank = amount_rank_condition( + purchase_invoice.paid_amount, Parameter("%(amount)s") + ) + amount_filter = ( + (purchase_invoice.paid_amount == Parameter("%(amount)s")) + if exact_match + else purchase_invoice.paid_amount != 0.0 ) - party_condition = purchase_invoice.supplier == Parameter("%(party)s") - party_rank = frappe.qb.terms.Case().when(party_condition, 1).else_(0) + party_filter = purchase_invoice.supplier == Parameter("%(party)s") + party_rank = frappe.qb.terms.Case().when(party_filter, 1).else_(0) # date of BT and paid PI could be the same (date of payment or the date of the bill) date_condition = Coalesce( @@ -1079,11 +1146,26 @@ def get_pi_matching_query(exact_match, exact_party_match, currency, description) ) == Parameter("%(date)s") date_rank = frappe.qb.terms.Case().when(date_condition, 1).else_(0) - description_match = get_description_match_condition( - description, purchase_invoice.name + # Check reference field equality with common_filters.reference_no + reference_field_is_set = reference_field and reference_field != "name" + ref_rank = ( + ref_equality_condition(purchase_invoice[reference_field], reference_number) + if (reference_number and reference_field_is_set) + else Cast(0, "int") + ) + + # if ref field is configured (!= name), perform desc-name and desc-ref match + # otherwise (== name), then perform desc-name match once + name_match = get_description_match_condition(description, purchase_invoice, "name") + ref_match = ( + get_description_match_condition(description, purchase_invoice, reference_field) + if reference_field_is_set + else Cast(0, "int") ) - rank_expression = party_rank + amount_rank + date_rank + description_match + 1 + rank_expression = ( + ref_rank + party_rank + amount_rank + date_rank + name_match + ref_match + 1 + ) query = ( frappe.qb.from_(purchase_invoice) @@ -1092,7 +1174,7 @@ def get_pi_matching_query(exact_match, exact_party_match, currency, description) ConstantColumn("Purchase Invoice").as_("doctype"), purchase_invoice.name, purchase_invoice.paid_amount, - purchase_invoice.bill_no.as_("reference_no"), + purchase_invoice[reference_field or "bill_no"].as_("reference_no"), purchase_invoice.bill_date.as_("reference_date"), purchase_invoice.supplier.as_("party"), ConstantColumn("Supplier").as_("party_type"), @@ -1102,40 +1184,62 @@ def get_pi_matching_query(exact_match, exact_party_match, currency, description) party_rank.as_("party_match"), amount_rank.as_("amount_match"), date_rank.as_("date_match"), - description_match.as_("name_in_desc_match"), + name_match.as_("name_in_desc_match"), + (ref_match).as_("ref_in_desc_match"), + (ref_rank).as_("reference_number_match"), ) .where(purchase_invoice.docstatus == 1) .where(purchase_invoice.is_paid == 1) .where(purchase_invoice.clearance_date.isnull()) .where(purchase_invoice.cash_bank_account == Parameter("%(bank_account)s")) - .where(amount_condition) + .where(amount_filter) .where(purchase_invoice.currency == currency) .orderby(rank_expression, order=Order.desc) .limit(MAX_QUERY_RESULTS) ) if exact_party_match: - query = query.where(party_condition) + query = query.where(party_filter) return str(query) def get_unpaid_pi_matching_query( - exact_match, exact_party_match, currency, company, description, include_only_returns=False + exact_match: bool, + exact_party_match: bool, + currency: str, + company: str, + description: str, + reference_number: str, + include_only_returns: bool = False, + reference_field: str = "name", ): purchase_invoice = frappe.qb.DocType("Purchase Invoice") + party_filter = purchase_invoice.supplier == Parameter("%(party)s") + party_match = frappe.qb.terms.Case().when(party_filter, 1).else_(0) - party_condition = purchase_invoice.supplier == Parameter("%(party)s") - party_match = frappe.qb.terms.Case().when(party_condition, 1).else_(0) + amount_rank = amount_rank_condition( + purchase_invoice.outstanding_amount, Parameter("%(amount)s") + ) - outstanding_amount_condition = purchase_invoice.outstanding_amount == Parameter( - "%(amount)s" + # Check reference field equality with common_filters.reference_no + reference_field_is_set = reference_field and reference_field != "name" + ref_rank = ( + ref_equality_condition(purchase_invoice[reference_field], reference_number) + if (reference_number and reference_field_is_set) + else Cast(0, "int") ) - amount_match = frappe.qb.terms.Case().when(outstanding_amount_condition, 1).else_(0) - description_match = get_description_match_condition( - description, purchase_invoice.name + + # if ref field is configured (!= name), perform desc-name and desc-ref match + # otherwise (== name), then perform desc-name match once + name_match = get_description_match_condition(description, purchase_invoice, "name") + ref_match = ( + get_description_match_condition(description, purchase_invoice, reference_field) + if reference_field_is_set + else Cast(0, "int") ) - rank_expression = party_match + amount_match + description_match + 1 + + rank_expression = ref_rank + party_match + amount_rank + name_match + ref_match + 1 # We skip date rank as the date of an unpaid bill is mostly # earlier than the date of the bank transaction @@ -1146,7 +1250,7 @@ def get_unpaid_pi_matching_query( ConstantColumn("Purchase Invoice").as_("doctype"), purchase_invoice.name.as_("name"), purchase_invoice.outstanding_amount.as_("paid_amount"), - purchase_invoice.bill_no.as_("reference_no"), + purchase_invoice[reference_field or "name"].as_("reference_no"), purchase_invoice.bill_date.as_("reference_date"), purchase_invoice.supplier.as_("party"), ConstantColumn("Supplier").as_("party_type"), @@ -1154,8 +1258,10 @@ def get_unpaid_pi_matching_query( purchase_invoice.posting_date, purchase_invoice.currency, party_match.as_("party_match"), - amount_match.as_("amount_match"), - description_match.as_("name_in_desc_match"), + amount_rank.as_("amount_match"), + name_match.as_("name_in_desc_match"), + ref_match.as_("ref_in_desc_match"), + ref_rank.as_("reference_number_match"), ) .where(purchase_invoice.docstatus == 1) .where(purchase_invoice.company == company) @@ -1169,22 +1275,30 @@ def get_unpaid_pi_matching_query( if include_only_returns: query = query.where(purchase_invoice.is_return == 1) if exact_match: - query = query.where(outstanding_amount_condition) + query = query.where(purchase_invoice.outstanding_amount == Parameter("%(amount)s")) if exact_party_match: - query = query.where(party_condition) + query = query.where(party_filter) return str(query) -def get_unpaid_ec_matching_query(exact_match, exact_party_match, currency, company, description): +def get_unpaid_ec_matching_query( + exact_match: bool, + exact_party_match: bool, + currency: str, + company: str, + description: str, + reference_number: str, + reference_field: str = "name", +): if currency != get_company_currency(company): # Expense claims are always in company currency return "" expense_claim = frappe.qb.DocType("Expense Claim") - party_condition = expense_claim.employee == Parameter("%(party)s") - party_match = frappe.qb.terms.Case().when(party_condition, 1).else_(0) + party_filter = expense_claim.employee == Parameter("%(party)s") + party_match = frappe.qb.terms.Case().when(party_filter, 1).else_(0) outstanding_amount = ( expense_claim.total_sanctioned_amount @@ -1192,13 +1306,27 @@ def get_unpaid_ec_matching_query(exact_match, exact_party_match, currency, compa - expense_claim.total_amount_reimbursed - expense_claim.total_advance_amount ) - outstanding_amount_condition = outstanding_amount == Parameter("%(amount)s") - amount_match = frappe.qb.terms.Case().when(outstanding_amount_condition, 1).else_(0) - description_match = get_description_match_condition( - description, expense_claim.name + + amount_rank = amount_rank_condition(outstanding_amount, Parameter("%(amount)s")) + + # Check reference field equality with common_filters.reference_no + reference_field_is_set = reference_field and reference_field != "name" + ref_rank = ( + ref_equality_condition(expense_claim[reference_field], reference_number) + if (reference_number and reference_field_is_set) + else Cast(0, "int") + ) + + # if ref field is configured (!= name), perform desc-name and desc-ref match + # otherwise (== name), then perform desc-name match once + name_match = get_description_match_condition(description, expense_claim, "name") + ref_match = ( + get_description_match_condition(description, expense_claim, reference_field) + if reference_field_is_set + else Cast(0, "int") ) - rank_expression = party_match + amount_match + description_match + 1 + rank_expression = ref_rank + party_match + amount_rank + name_match + ref_match + 1 query = ( frappe.qb.from_(expense_claim) @@ -1207,7 +1335,7 @@ def get_unpaid_ec_matching_query(exact_match, exact_party_match, currency, compa ConstantColumn("Expense Claim").as_("doctype"), expense_claim.name.as_("name"), outstanding_amount.as_("paid_amount"), - expense_claim.name.as_("reference_no"), + expense_claim[reference_field or "name"].as_("reference_no"), expense_claim.posting_date.as_("reference_date"), expense_claim.employee.as_("party"), ConstantColumn("Employee").as_("party_type"), @@ -1215,8 +1343,10 @@ def get_unpaid_ec_matching_query(exact_match, exact_party_match, currency, compa expense_claim.posting_date, ConstantColumn(currency).as_("currency"), party_match.as_("party_match"), - amount_match.as_("amount_match"), - description_match.as_("name_in_desc_match"), + amount_rank.as_("amount_match"), + name_match.as_("name_in_desc_match"), + ref_match.as_("ref_in_desc_match"), + ref_rank.as_("reference_number_match"), ) .where(expense_claim.docstatus == 1) .where(expense_claim.company == company) @@ -1227,9 +1357,9 @@ def get_unpaid_ec_matching_query(exact_match, exact_party_match, currency, compa ) if exact_match: - query = query.where(outstanding_amount_condition) + query = query.where(outstanding_amount == Parameter("%(amount)s")) if exact_party_match: - query = query.where(party_condition) + query = query.where(party_filter) return str(query) @@ -1260,24 +1390,3 @@ def get_invoice_function_map(document_types: list, is_deposit: bool): for doctype in order if (doctype in document_types and fn_map[doctype]) } - - -def get_description_match_condition(description: str, name_column): - """Get the description match condition for a document name. - - Args: - description: The bank transaction description to search in - name_column: The document name column to match against (e.g., expense_claim.name) - - Returns: - A query condition that will be 1 if the description contains the document number - and 0 otherwise. - """ - return ( - frappe.qb.terms.Case() - .when( - Instr(description or "", RegExpReplace(name_column, r"^[^0-9]*", "")) > 0, - 1, - ) - .else_(0) - ) diff --git a/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/test_bank_reconciliation_tool_beta.py b/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/test_bank_reconciliation_tool_beta.py index 0b5989f1..e8ce8377 100644 --- a/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/test_bank_reconciliation_tool_beta.py +++ b/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/test_bank_reconciliation_tool_beta.py @@ -3,9 +3,11 @@ import json import frappe +from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.utils import add_days, getdate from frappe.tests.utils import FrappeTestCase + from erpnext.accounts.test.accounts_mixin import AccountsTestMixin from erpnext.accounts.doctype.bank_transaction.test_bank_transaction import ( @@ -23,6 +25,7 @@ bulk_reconcile_vouchers, create_journal_entry_bts, create_payment_entry_bts, + get_linked_payments, ) from hrms.hr.doctype.expense_claim.test_expense_claim import make_expense_claim @@ -30,8 +33,13 @@ class TestBankReconciliationToolBeta(AccountsTestMixin, FrappeTestCase): @classmethod - def setUpClass(cls): + def setUpClass(cls) -> None: super().setUpClass() + + create_custom_field( + "Sales Invoice", dict(fieldname="custom_ref_no", label="Ref No", fieldtype="Data") + ) # commits to db internally + create_bank() cls.gl_account = create_gl_account("_Test Bank Reco Tool") cls.bank_account = create_bank_account(gl_account=cls.gl_account) @@ -40,6 +48,12 @@ def setUpClass(cls): cls.create_item( cls, item_name="Reco Item", company="_Test Company", warehouse="Finished Goods - _TC" ) + frappe.db.savepoint(save_point="bank_reco_beta_before_tests") + + def tearDown(self) -> None: + """Runs after each test.""" + # Make sure invoices are rolled back to not affect invoice count assertions + frappe.db.rollback(save_point="bank_reco_beta_before_tests") def test_unpaid_invoices_more_than_transaction(self): """ @@ -554,6 +568,102 @@ def test_multi_party_reconciliation(self): self.assertEqual(si.outstanding_amount, 0) self.assertEqual(si2.outstanding_amount, 100) + def test_configurable_reference_field(self): + """Test if configured reference field is considered.""" + settings = frappe.get_single("Banking Settings") + settings.append( + "reference_fields", {"document_type": "Sales Invoice", "field_name": "custom_ref_no"} + ) + settings.save() + + bt = create_bank_transaction( + date=getdate(), + deposit=300, + reference_no="ORD-WXL-03456", + bank_account=self.bank_account, + description="Payment for Order: ORD-WXL-03456 | 300 | Thank you", + ) + si = create_sales_invoice( + rate=300, + warehouse="Finished Goods - _TC", + customer=self.customer, + cost_center="Main - _TC", + item="Reco Item", + do_not_submit=True, + ) + si.custom_ref_no = "ORD-WXL-03456" + si.submit() + + si2 = create_sales_invoice( + rate=20, + warehouse="Finished Goods - _TC", + customer=self.customer, + cost_center="Main - _TC", + item="Reco Item", + do_not_submit=True, + ) + si2.custom_ref_no = "ORD-WXL-15467" + si2.submit() + + matched_vouchers = get_linked_payments( + bank_transaction_name=bt.name, + document_types=["sales_invoice", "unpaid_invoices"], + from_date=add_days(getdate(), -1), + to_date=add_days(getdate(), 1), + ) + first_match, second_match = matched_vouchers[0], matched_vouchers[1] + + # Get linked payments and check if the custom field value is present + self.assertEqual(len(matched_vouchers), 2) + self.assertEqual(first_match["reference_no"], si.custom_ref_no) + self.assertEqual(first_match["name"], si.name) + self.assertEqual(first_match["rank"], 4) + self.assertEqual(first_match["ref_in_desc_match"], 1) + self.assertEqual(first_match["reference_number_match"], 1) + self.assertEqual(second_match["ref_in_desc_match"], 0) + self.assertEqual(second_match["reference_number_match"], 0) + # Check if ranking across another SI is correct + self.assertEqual(second_match["reference_no"], si2.custom_ref_no) + self.assertEqual(second_match["name"], si2.name) + self.assertEqual(second_match["rank"], 1) + self.assertEqual(second_match["ref_in_desc_match"], 0) + + def test_no_configurable_reference_field(self): + """Test if Name is considered as the reference field if not configured.""" + bt = create_bank_transaction( + date=getdate(), + deposit=300, + reference_no="Test001", + bank_account=self.bank_account, + description="Payment for Order: ORD-WXL-03456 | 300 | Thank you", + ) + si = create_sales_invoice( + rate=300, + warehouse="Finished Goods - _TC", + customer=self.customer, + cost_center="Main - _TC", + item="Reco Item", + do_not_submit=True, + ) + si.custom_ref_no = "ORD-WXL-03456" + si.submit() + + matched_vouchers = get_linked_payments( + bank_transaction_name=bt.name, + document_types=["sales_invoice", "unpaid_invoices"], + from_date=add_days(getdate(), -1), + to_date=add_days(getdate(), 1), + ) + first_match = matched_vouchers[0] + + # Get linked payments and check if the custom field value is present + self.assertEqual(len(matched_vouchers), 1) + self.assertEqual(first_match["reference_no"], si.name) + self.assertEqual(first_match["name"], si.name) + self.assertEqual(first_match["rank"], 2) + self.assertEqual(first_match["amount_match"], 1) + self.assertEqual(first_match["ref_in_desc_match"], 0) + def get_pe_references(vouchers: list): return frappe.get_all( @@ -571,12 +681,14 @@ def create_bank_transaction( reference_no: str = None, reference_date: str = None, bank_account: str = None, + description: str = None, ): doc = frappe.get_doc( { "doctype": "Bank Transaction", "company": "_Test Company", - "description": "1512567 BG/000002918 OPSKATTUZWXXX AT776000000098709837 Herr G", + "description": description + or "1512567 BG/000002918 OPSKATTUZWXXX AT776000000098709837 Herr G", "date": date or frappe.utils.nowdate(), "deposit": deposit or 0.0, "withdrawal": withdrawal or 0.0, diff --git a/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/utils.py b/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/utils.py new file mode 100644 index 00000000..4905964e --- /dev/null +++ b/banking/klarna_kosma_integration/doctype/bank_reconciliation_tool_beta/utils.py @@ -0,0 +1,110 @@ +import frappe +from frappe import _ + +from pypika.queries import Table +from pypika.terms import Case, Field +from frappe.query_builder.functions import CustomFunction, Cast + +Instr = CustomFunction("INSTR", ["a", "b"]) +RegExpReplace = CustomFunction("REGEXP_REPLACE", ["a", "b", "c"]) + +# NOTE: +# Ranking min: 1 (nothing matches), max: 7 (everything matches) + +# Types of matches: +# amount_match: if amount in voucher EQ amount in bank statement +# party_match: if party in voucher EQ party in bank statement +# date_match: if date in voucher EQ date in bank statement +# reference_number_match: if ref in voucher EQ ref in bank statement +# name_in_desc_match: if name in voucher IN bank statement description +# ref_in_desc_match: if ref in voucher IN bank statement description + + +def amount_rank_condition(amount: Field, bank_amount: float) -> Case: + """Get the rank query for amount matching.""" + return frappe.qb.terms.Case().when(amount == bank_amount, 1).else_(0) + + +def ref_equality_condition(reference_no: Field, bank_reference_no: str) -> Case: + """Get the rank query for reference number matching.""" + if not bank_reference_no or bank_reference_no == "NOTPROVIDED": + # If bank reference number is not provided, then it is not a match + return Cast(0, "int") + + return frappe.qb.terms.Case().when(reference_no == bank_reference_no, 1).else_(0) + + +def get_description_match_condition( + description: str, table: Table, column_name: str = "name" +) -> Case: + """Get the description match condition for a column. + + Args: + description: The bank transaction description to search in + column_name: The document column to match against (e.g., expense_claim.name, purchase_invoice.bill_no) + + Returns: + A query condition that will be 1 if the description contains the document number + and 0 otherwise. + """ + if not description: + return Cast(0, "int") + + column_name = column_name or "name" + column = table[column_name] + # Perform replace if the column is the name, else the column value is ambiguous + # Eg. column_name = "custom_ref_no" and its value = "tuf5673i" should be untouched + if column_name == "name": + return ( + frappe.qb.terms.Case() + .when( + Instr(description, RegExpReplace(column, r"^[^0-9]*", "")) > 0, + 1, + ) + .else_(0) + ) + else: + return ( + frappe.qb.terms.Case() + .when( + column.notnull() & (column != "") & (Instr(description, column) > 0), + 1, + ) + .else_(0) + ) + + +def get_reference_field_map() -> dict: + """Get the reference field map for the document types from Banking Settings. + Returns: {"sales_invoice": "custom_field_name", ...} + """ + + def _validate_and_get_field(row: dict) -> str: + is_docfield = frappe.db.exists( + "DocField", {"fieldname": row.field_name, "parent": row.document_type} + ) + is_custom = frappe.db.exists( + "Custom Field", {"fieldname": row.field_name, "dt": row.document_type} + ) + if not (is_docfield or is_custom): + frappe.throw( + title=_("Invalid Field"), + msg=_( + "Field {} does not exist in {}. Please check the configuration in Banking Settings." + ).format(frappe.bold(row.field_name), frappe.bold(row.document_type)), + ) + + return row.field_name + + reference_fields = frappe.get_all( + "Banking Reference Mapping", + filters={ + "parent": "Banking Settings", + }, + fields=["document_type", "field_name"], + ) + + return { + frappe.scrub(row.document_type): _validate_and_get_field(row) + for row in reference_fields + } diff --git a/banking/klarna_kosma_integration/doctype/banking_reference_mapping/__init__.py b/banking/klarna_kosma_integration/doctype/banking_reference_mapping/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/banking/klarna_kosma_integration/doctype/banking_reference_mapping/banking_reference_mapping.json b/banking/klarna_kosma_integration/doctype/banking_reference_mapping/banking_reference_mapping.json new file mode 100644 index 00000000..ef7bd3ab --- /dev/null +++ b/banking/klarna_kosma_integration/doctype/banking_reference_mapping/banking_reference_mapping.json @@ -0,0 +1,41 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2024-11-20 16:26:57.613210", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "document_type", + "field_name" + ], + "fields": [ + { + "fieldname": "document_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Document Type", + "options": "Sales Invoice\nPurchase Invoice\nExpense Claim", + "reqd": 1 + }, + { + "fieldname": "field_name", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Field Name", + "reqd": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2024-11-20 17:12:39.739249", + "modified_by": "Administrator", + "module": "Klarna Kosma Integration", + "name": "Banking Reference Mapping", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/banking/klarna_kosma_integration/doctype/banking_reference_mapping/banking_reference_mapping.py b/banking/klarna_kosma_integration/doctype/banking_reference_mapping/banking_reference_mapping.py new file mode 100644 index 00000000..9e401288 --- /dev/null +++ b/banking/klarna_kosma_integration/doctype/banking_reference_mapping/banking_reference_mapping.py @@ -0,0 +1,9 @@ +# Copyright (c) 2024, ALYF GmbH and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class BankingReferenceMapping(Document): + pass diff --git a/banking/klarna_kosma_integration/doctype/banking_settings/banking_settings.js b/banking/klarna_kosma_integration/doctype/banking_settings/banking_settings.js index 80bad68b..846b415c 100644 --- a/banking/klarna_kosma_integration/doctype/banking_settings/banking_settings.js +++ b/banking/klarna_kosma_integration/doctype/banking_settings/banking_settings.js @@ -50,6 +50,10 @@ frappe.ui.form.on('Banking Settings', { "primary" ); } + + frm.doc.reference_fields.map((field) => { + set_field_options(frm, field.doctype, field.name); + }); }, refresh_banks: (frm) => { @@ -256,6 +260,45 @@ frappe.ui.form.on('Banking Settings', { }, }); +frappe.ui.form.on('Banking Reference Mapping', { + reference_fields_add: (frm, cdt, cdn) => { + set_field_options(frm, cdt, cdn); + }, + + document_type: (frm, cdt, cdn) => { + set_field_options(frm, cdt, cdn); + } +}); + +function set_field_options(frm, cdt, cdn) { + const doc = frappe.get_doc(cdt, cdn); + const document_type = doc.document_type || "Sales Invoice"; + + // set options for `field_name` + frappe.model.with_doctype(document_type, () => { + const meta = frappe.get_meta(document_type); + const fields = meta.fields.filter((field) => { + return ( + ["Link", "Data"].includes(field.fieldtype) + && field.is_virtual === 0 + ); + }); + + frm.fields_dict.reference_fields.grid.update_docfield_property( + "field_name", + "options", + fields.map((field) => { + return { + value: field.fieldname, + label: __(field.label), + } + }).sort((a, b) => a.label.localeCompare(b.label)) + ); + frm.refresh_field("reference_fields"); + }); +} + + class KlarnaKosmaConnect { constructor(opts) { Object.assign(this, opts); diff --git a/banking/klarna_kosma_integration/doctype/banking_settings/banking_settings.json b/banking/klarna_kosma_integration/doctype/banking_settings/banking_settings.json index 30813e4f..3d4cc657 100644 --- a/banking/klarna_kosma_integration/doctype/banking_settings/banking_settings.json +++ b/banking/klarna_kosma_integration/doctype/banking_settings/banking_settings.json @@ -20,7 +20,10 @@ "ebics_section", "enable_ebics", "fintech_licensee_name", - "fintech_license_key" + "fintech_license_key", + "bank_reconciliation_tab", + "advanced_section", + "reference_fields" ], "fields": [ { @@ -108,12 +111,29 @@ "fieldname": "enable_ebics", "fieldtype": "Check", "label": "Enable EBICS (New)" + }, + { + "fieldname": "bank_reconciliation_tab", + "fieldtype": "Tab Break", + "label": "Bank Reconciliation" + }, + { + "description": "