From 1d0542bf6bdf2ac71d7842ffcc9cf5b0f38917c5 Mon Sep 17 00:00:00 2001 From: Zhenzhen Zhao Date: Wed, 10 May 2023 14:53:36 +0800 Subject: [PATCH] feat(provider): support ICBC bank card bills (#88) * feat(provider): support icbc in provider package Signed-off-by: TripleZ * feat(analyser): support ICBC provider & ignore rules Signed-off-by: TripleZ * Update README Signed-off-by: TripleZ * chore: lint code Signed-off-by: TripleZ --------- Signed-off-by: TripleZ --- Makefile | 5 +- README.md | 94 +++++++++++++++++- example/icbc/config.yaml | 15 +++ example/icbc/example-icbc-output.beancount | 45 +++++++++ example/icbc/example-icbc-records.csv | 18 ++++ pkg/analyser/alipay/alipay.go | 14 ++- pkg/analyser/htsec/htsec.go | 11 ++- pkg/analyser/huobi/huobi.go | 11 ++- pkg/analyser/icbc/icbc.go | 106 +++++++++++++++++++++ pkg/analyser/interface.go | 6 +- pkg/analyser/wechat/wechat.go | 14 ++- pkg/compiler/beancount/compiler.go | 19 ++-- pkg/compiler/beancount/template.go | 2 +- pkg/config/config.go | 2 + pkg/consts/consts.go | 4 +- pkg/provider/alipay/alipay.go | 5 +- pkg/provider/alipay/config.go | 1 + pkg/provider/alipay/convert.go | 4 +- pkg/provider/htsec/config.go | 1 + pkg/provider/huobi/config.go | 1 + pkg/provider/icbc/config.go | 24 +++++ pkg/provider/icbc/convert.go | 72 ++++++++++++++ pkg/provider/icbc/icbc.go | 72 ++++++++++++++ pkg/provider/icbc/parse.go | 49 ++++++++++ pkg/provider/icbc/types.go | 40 ++++++++ pkg/provider/interface.go | 4 + pkg/provider/wechat/config.go | 1 + pkg/provider/wechat/convert.go | 4 +- test/icbc-test.sh | 30 ++++++ 29 files changed, 642 insertions(+), 32 deletions(-) create mode 100644 example/icbc/config.yaml create mode 100644 example/icbc/example-icbc-output.beancount create mode 100644 example/icbc/example-icbc-records.csv create mode 100644 pkg/analyser/icbc/icbc.go create mode 100644 pkg/provider/icbc/config.go create mode 100644 pkg/provider/icbc/convert.go create mode 100644 pkg/provider/icbc/icbc.go create mode 100644 pkg/provider/icbc/parse.go create mode 100644 pkg/provider/icbc/types.go create mode 100644 test/icbc-test.sh diff --git a/Makefile b/Makefile index 5ec663c..22d217a 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ clean: ## Clean all the temporary files @rm -rf ./double-entry-generator @rm -rf ./wasm-dist -test: test-go test-alipay test-wechat test-huobi test-htsec ## Run all tests +test: test-go test-alipay test-wechat test-huobi test-htsec test-icbc ## Run all tests test-go: ## Run Golang tests @go test ./... @@ -107,6 +107,9 @@ test-huobi: ## Run tests for huobi provider test-htsec: ## Run tests for htsec provider @$(SHELL) ./test/htsec-test.sh +test-icbc: ## Run tests for ICBC provider + @$(SHELL) ./test/icbc-test.sh + format: ## Format code @gofmt -s -w pkg @goimports -w pkg diff --git a/README.md b/README.md index e23b9cc..7559243 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ - 微信 - 火币-币币交易 - 海通证券 +- 中国工商银行 目前记账语言支持： @@ -20,6 +21,8 @@ alipay beancount alipay wechat wechat huobi huobi + htsec htsec + icbc icbc ``` ## 安装 @@ -95,13 +98,23 @@ double-entry-generator translate \ ./example/htsec/example-htsec-records.xlsx ``` +### 中国工商银行 + +```bash +double-entry-generator translate \ + --config ./example/icbc/config.yaml \ + --provider icbc \ + --output ./example/icbc/example-icbc-output.beancount \ + ./example/icbc/example-icbc-records.csv +``` + ## 账单下载与格式问题 ### 支付宝 #### 下载方式 -`v1.0.0` 及以上的版本请参考[此文章](https://blog.triplez.cn/posts/bills-export-methods/#支付宝)获取支付宝账单。 +`v1.0.0` 及以上的版本请参考[此文章](https://blog.triplez.cn/posts/bills-export-methods/#%e6%94%af%e4%bb%98%e5%ae%9d)获取支付宝账单。 `v0.2.0` 及以下版本请使用此方式获取账单：登录 PC 支付宝后，访问 [这里](https://consumeprod.alipay.com/record/standard.htm)，选择时间区间，下拉到页面底端，点击下载查询结果。注意：请下载查询结果，而非[收支明细](https://cshall.alipay.com/lab/help_detail.htm?help_id=212688)。 @@ -117,7 +130,7 @@ double-entry-generator translate \ #### 下载方式 -微信支付的下载方式[见此](https://blog.triplez.cn/posts/bills-export-methods/#微信支付)。 +微信支付的下载方式[见此](https://blog.triplez.cn/posts/bills-export-methods/#%e5%be%ae%e4%bf%a1%e6%94%af%e4%bb%98)。 #### 格式示例 @@ -153,6 +166,17 @@ double-entry-generator translate \ 转换后的结果示例：[exmaple-htsec-output.beancount](./example/htsec/example-htsec-output.beancount). +### 中国工商银行 + +#### 下载方式 + +中国工商银行账单的下载方式[见此](https://blog.triplez.cn/posts/bills-export-methods/#%e4%b8%ad%e5%9b%bd%e5%b7%a5%e5%95%86%e9%93%b6%e8%a1%8c)。 + +#### 格式示例 + +[example-icbc-records.csv](./example/icbc/example-icbc-records.csv) + +转换后的结果示例：[exmaple-icbc-output.beancount](./example/icbc/example-icbc-output.beancount). ## 配置 @@ -260,6 +284,10 @@ alipay: 在单条规则中可以使用 `fullMatch` 来设置字符匹配规则，`true` 表示使用完全匹配(full match)，`false` 表示使用包含匹配(partial match)，不设置该项则默认使用包含匹配。 +在单条规则中可以使用 `tag` 来设置流水的 [Tag](https://beancount.github.io/docs/beancount_language_syntax.html#tags)，使用 `sep` 作为分隔符。 + +在单条规则中可以使用 `ignore` 来设置是否忽略匹配上该规则的交易，`true` 表示忽略匹配上该规则的交易，`fasle` 则为不忽略，缺省为 `false` 。 + 匹配成功则使用规则中定义的 `targetAccount` 、 `methodAccount` 等账户覆盖默认定义账户。 规则匹配的顺序是：从 `rules` 配置中的第一条开始匹配，如果匹配成功仍继续匹配。也就是后面的规则优先级要**高于**前面的规则。 @@ -386,6 +414,8 @@ wechat: 在单条规则中可以使用 `tag` 来设置流水的 [Tag](https://beancount.github.io/docs/beancount_language_syntax.html#tags)，使用 `sep` 作为分隔符。 +在单条规则中可以使用 `ignore` 来设置是否忽略匹配上该规则的交易，`true` 表示忽略匹配上该规则的交易，`fasle` 则为不忽略，缺省为 `false` 。 + 匹配成功则使用规则中定义的 `targetAccount` 、 `methodAccount` 等账户覆盖默认定义账户。 规则匹配的顺序是：从 `rules` 配置中的第一条开始匹配，如果匹配成功仍继续匹配。也就是后面的规则优先级要**高于**前面的规则。 @@ -446,6 +476,8 @@ huobi: 在单条规则中可以使用 `fullMatch` 来设置字符匹配规则，`true` 表示使用完全匹配(full match)，`false` 表示使用包含匹配(partial match)，不设置该项则默认使用包含匹配。 +在单条规则中可以使用 `ignore` 来设置是否忽略匹配上该规则的交易，`true` 表示忽略匹配上该规则的交易，`fasle` 则为不忽略，缺省为 `false` 。 + 匹配成功则使用规则中定义的 `cashAccount`, `positionAccount`, `commissionAccount` 和 `pnlAccount` 覆盖默认定义。 规则匹配的顺序是：从 `rules` 配置中的第一条开始匹配，如果匹配成功仍继续匹配。也就是后面的规则优先级要**高于**前面的规则。 @@ -501,6 +533,8 @@ htsec: 在单条规则中可以使用 `fullMatch` 来设置字符匹配规则，`true` 表示使用完全匹配(full match)，`false` 表示使用包含匹配(partial match)，不设置该项则默认使用包含匹配。 +在单条规则中可以使用 `ignore` 来设置是否忽略匹配上该规则的交易，`true` 表示忽略匹配上该规则的交易，`fasle` 则为不忽略，缺省为 `false` 。 + 匹配成功则使用规则中定义的 `cashAccount`, `positionAccount`, `commissionAccount` 和 `pnlAccount` 覆盖默认定义。 规则匹配的顺序是：从 `rules` 配置中的第一条开始匹配，如果匹配成功仍继续匹配。也就是后面的规则优先级要**高于**前面的规则。 @@ -512,6 +546,62 @@ htsec: - `defaultPnlAccount` 是默认损益账户。 - `defaultCurrency` 是默认货币。 +### 中国工商银行 + +
+ + 中国工商银行配置文件示例 + + +```yaml +defaultMinusAccount: Assets:FIXME +defaultPlusAccount: Expenses:FIXME +defaultCashAccount: Liabilities:Bank:CN:ICBC +defaultCurrency: CNY +title: 测试 +icbc: + rules: + - peer: 财付通,支付宝 + ignore: true + - peer: 广东联合电子收费股份 + targetAccount: Expenses:Transport:Highway + - txType: 人民币自动转帐还款 + targetAccount: Assets:Bank:CN:ICBC:Savings + - peer: XX旗舰店 + targetAccount: Expenses:Joy +``` + +

+ +`defaultMinusAccount`, `defaultPlusAccount`, `defaultCashAccount` 和 `defaultCurrency` 是全局的必填默认值。其中 `defaultMinusAccount` 是默认金额减少的账户，`defaultPlusAccount` 是默认金额增加的账户， `defaultCashAccount` 是该配置中默认使用的银行卡账户（等同于支付宝/微信中的 `methodAccount` ）。 `defaultCurrency` 是默认货币。 + +`icbc` 是中国工商银行相关的配置。它提供基于规则的匹配。可以指定： +- `peer`（交易对方）的完全/包含匹配。 +- `type`（收/支）的完全/包含匹配。 +- `txType`（交易类型）的完全/包含匹配。 + +在单条规则中可以使用分隔符 `sep` 填写多个关键字，在同一对象中，每个关键字之间是或的关系。 + +在单条规则中可以使用 `fullMatch` 来设置字符匹配规则，`true` 表示使用完全匹配(full match)，`false` 表示使用包含匹配(partial match)，不设置该项则默认使用包含匹配。 + +在单条规则中可以使用 `tag` 来设置流水的 [Tag](https://beancount.github.io/docs/beancount_language_syntax.html#tags)，使用 `sep` 作为分隔符。 + +在单条规则中可以使用 `ignore` 来设置是否忽略匹配上该规则的交易，`true` 表示忽略匹配上该规则的交易，`fasle` 则为不忽略，缺省为 `false` 。 + +匹配成功则使用规则中定义的 `targetAccount` 账户覆盖默认定义账户。 + +规则匹配的顺序是：从 `rules` 配置中的第一条开始匹配，如果匹配成功仍继续匹配。也就是后面的规则优先级要**高于**前面的规则。 + +中国工商银行账单中的记账金额中存在收入/支出之分，通过这个机制就可以判断银行卡账户在交易中的正负关系。如支付宝配置类似，匹配成功则使用规则中定义的 `targetAccount` 和全局值 `defaultCashAccount` ，并通过确认该笔交易是收入还是支出，决定 `targetAccount` 和 `defaultCashAccount` 的正负关系，来覆盖默认定义的增减账户。 + +`targetAccount` 与 `defaultCashAccount` 的增减账户关系如下表： + +|收/支|defaultCashAccount|targetAccount| +|----|----|----| +|收入|plusAccount|minusAccount| +|支出|minusAccount|plusAccount| + + ## Special Thanks - [dilfish/atb](https://github.com/dilfish/atb) convert alipay bill to beancount version diff --git a/example/icbc/config.yaml b/example/icbc/config.yaml new file mode 100644 index 0000000..36f2a52 --- /dev/null +++ b/example/icbc/config.yaml @@ -0,0 +1,15 @@ +defaultMinusAccount: Assets:FIXME +defaultPlusAccount: Expenses:FIXME +defaultCashAccount: Liabilities:Bank:CN:ICBC +defaultCurrency: CNY +title: 测试 +icbc: + rules: + - peer: 财付通,支付宝 + ignore: true + - peer: 广东联合电子收费股份 + targetAccount: Expenses:Transport:Highway + - txType: 人民币自动转帐还款 + targetAccount: Assets:Bank:CN:ICBC:Savings + - peer: XX旗舰店 + targetAccount: Expenses:Joy diff --git a/example/icbc/example-icbc-output.beancount b/example/icbc/example-icbc-output.beancount new file mode 100644 index 0000000..214c0fe --- /dev/null +++ b/example/icbc/example-icbc-output.beancount @@ -0,0 +1,45 @@ +option "title" "测试" +option "operating_currency" "CNY" + +1970-01-01 open Assets:Bank:CN:ICBC:Savings +1970-01-01 open Assets:FIXME +1970-01-01 open Expenses:FIXME +1970-01-01 open Expenses:Joy +1970-01-01 open Expenses:Transport:Highway + +2023-03-20 * "广东联合电子收费股份" + balances: "-16042.73" + currency: "人民币" + source: "中国工商银行" + txType: "********************" + type: "支出" + Expenses:Transport:Highway 29.45 CNY + Liabilities:Bank:CN:ICBC -29.45 CNY + +2023-04-22 * "XX旗舰店" + balances: "-6086.11" + currency: "人民币" + source: "中国工商银行" + txType: "银联在线支付" + type: "支出" + Expenses:Joy 0.01 CNY + Liabilities:Bank:CN:ICBC -0.01 CNY + +2023-04-25 * "XX分行银行卡中心" + balances: "-1971.63" + currency: "人民币" + source: "中国工商银行" + txType: "人民币自动转帐还款" + type: "收入" + Liabilities:Bank:CN:ICBC 4621.01 CNY + Assets:Bank:CN:ICBC:Savings -4621.01 CNY + +2023-05-04 * "广东联合电子收费股份" + balances: "-3684.62" + currency: "人民币" + source: "中国工商银行" + txType: "ETC" + type: "支出" + Expenses:Transport:Highway 12.35 CNY + Liabilities:Bank:CN:ICBC -12.35 CNY + diff --git a/example/icbc/example-icbc-records.csv b/example/icbc/example-icbc-records.csv new file mode 100644 index 0000000..e813956 --- /dev/null +++ b/example/icbc/example-icbc-records.csv @@ -0,0 +1,18 @@ +明细查询文件下载 + +卡号: 1234****5678,"卡别名: " + +子账户类别: + +交易日期,记账日期,摘要,交易场所,交易国家或地区简称,交易金额(收入),交易金额(支出),交易币种,记账金额(收入),记账金额(支出),记账币种,余额,对方户名 +2023-05-05 ,2023-05-05 ,"消费 ","财付通-拼多多平台商户 ","CHN "," ","18.00 ",人民币 ," ","18.00 ",人民币 ,"-3,728.62 "," ", +2023-05-05 ,2023-05-05 ,"消费 ","支付宝-金拱门(中国)有限公司 ","CHN "," ","26.00 ",人民币 ," ","26.00 ",人民币 ,"-3,710.62 "," ", +2023-05-04 ,2023-05-04 ,"ETC ","广东联合电子收费股份 ","CHN "," ","12.35 ",人民币 ," ","12.35 ",人民币 ,"-3,684.62 "," ", +2023-05-02 ,2023-05-02 ,"消费 ","支付宝-北京百度网讯科技有限公司 ","CHN "," ","17.32 ",人民币 ," ","17.32 ",人民币 ,"-3,648.27 "," ", +2023-04-25 ,2023-04-25 ,"人民币自动转帐还款 ",XX分行银行卡中心 ,"CHN ","4,621.01 "," ",人民币 ,"4,621.01 "," ",人民币 ,"-1,971.63 "," ", +2023-04-22 ,2023-04-22 ,"消费 ","财付通-美团平台商户 ","CHN "," ","238.00 ",人民币 ," ","238.00 ",人民币 ,"-6,324.11 "," ", +2023-04-22 ,2023-04-22 ,"银联在线支付 ",XX旗舰店 ,"CHN "," ","0.01 ",人民币 ," ","0.01 ",人民币 ,"-6,086.11 "," ", +2023-03-20 ,2023-03-20 ,"消费 ",财付通-餐馆 ,"CHN "," ","22.00 ",人民币 ," ","22.00 ",人民币 ,"-16,064.73 "," ", +2023-03-20 ,2023-03-20 ,"******************** ","广东联合电子收费股份 ","CHN "," ","29.45 ",人民币 ," ","29.45 ",人民币 ,"-16,042.73 "," ", + +合计金额,,,,,"1234.56 ","6543.21 ", ,"1234.56 ","6543.21 ", \ No newline at end of file diff --git a/pkg/analyser/alipay/alipay.go b/pkg/analyser/alipay/alipay.go index 66ad54b..02a4b6a 100644 --- a/pkg/analyser/alipay/alipay.go +++ b/pkg/analyser/alipay/alipay.go @@ -38,10 +38,11 @@ func (a Alipay) GetAllCandidateAccounts(cfg *config.Config) map[string]bool { } // GetAccounts returns minus and plus account. -func (a Alipay) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (string, string, map[ir.Account]string, []string) { +func (a Alipay) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (bool, string, string, map[ir.Account]string, []string) { + ignore := false if cfg.Alipay == nil || len(cfg.Alipay.Rules) == 0 { - return cfg.DefaultMinusAccount, cfg.DefaultPlusAccount, nil, nil + return ignore, cfg.DefaultMinusAccount, cfg.DefaultPlusAccount, nil, nil } resMinus := cfg.DefaultMinusAccount resPlus := cfg.DefaultPlusAccount @@ -91,6 +92,11 @@ func (a Alipay) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, prov } if match { + if r.Ignore { + ignore = true + break + } + // Support multiple matches, like one rule matches the // minus account, the other rule matches the plus account. if r.TargetAccount != nil { @@ -121,7 +127,7 @@ func (a Alipay) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, prov } if strings.HasPrefix(o.Item, "退款-") && ir.TypeRecv != o.Type { - return resPlus, resMinus, extraAccounts, tags + return ignore, resPlus, resMinus, extraAccounts, tags } - return resMinus, resPlus, extraAccounts, tags + return ignore, resMinus, resPlus, extraAccounts, tags } diff --git a/pkg/analyser/htsec/htsec.go b/pkg/analyser/htsec/htsec.go index 88caf3a..9322749 100644 --- a/pkg/analyser/htsec/htsec.go +++ b/pkg/analyser/htsec/htsec.go @@ -40,9 +40,10 @@ func (h Htsec) GetAllCandidateAccounts(cfg *config.Config) map[string]bool { return uniqMap } -func (h Htsec) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (string, string, map[ir.Account]string, []string) { +func (h Htsec) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (bool, string, string, map[ir.Account]string, []string) { + ignore := false if cfg.Htsec == nil || len(cfg.Htsec.Rules) == 0 { - return "", "", map[ir.Account]string{ + return ignore, "", "", map[ir.Account]string{ ir.CashAccount: cfg.DefaultCashAccount, ir.PositionAccount: cfg.DefaultPositionAccount, ir.CommissionAccount: cfg.DefaultCommissionAccount, @@ -89,6 +90,10 @@ func (h Htsec) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provi } if match { + if r.Ignore { + ignore = true + break + } if r.CashAccount != nil { cashAccount = *r.CashAccount } @@ -105,7 +110,7 @@ func (h Htsec) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provi } } - return "", "", map[ir.Account]string{ + return ignore, "", "", map[ir.Account]string{ ir.CashAccount: cashAccount, ir.PositionAccount: positionAccount, ir.CommissionAccount: commissionAccount, diff --git a/pkg/analyser/huobi/huobi.go b/pkg/analyser/huobi/huobi.go index f679053..545ab3c 100644 --- a/pkg/analyser/huobi/huobi.go +++ b/pkg/analyser/huobi/huobi.go @@ -40,9 +40,10 @@ func (h Huobi) GetAllCandidateAccounts(cfg *config.Config) map[string]bool { return uniqMap } -func (h Huobi) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (string, string, map[ir.Account]string, []string) { +func (h Huobi) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (bool, string, string, map[ir.Account]string, []string) { + ignore := false if cfg.Huobi == nil || len(cfg.Huobi.Rules) == 0 { - return "", "", map[ir.Account]string{ + return ignore, "", "", map[ir.Account]string{ ir.CashAccount: cfg.DefaultCashAccount, ir.PositionAccount: cfg.DefaultPositionAccount, ir.CommissionAccount: cfg.DefaultCommissionAccount, @@ -91,6 +92,10 @@ func (h Huobi) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provi } } if match { + if r.Ignore { + ignore = true + break + } if r.CashAccount != nil { cashAccount = *r.CashAccount } @@ -106,7 +111,7 @@ func (h Huobi) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provi } } - return "", "", map[ir.Account]string{ + return ignore, "", "", map[ir.Account]string{ ir.CashAccount: cashAccount, ir.PositionAccount: positionAccount, ir.CommissionAccount: commissionAccount, diff --git a/pkg/analyser/icbc/icbc.go b/pkg/analyser/icbc/icbc.go new file mode 100644 index 0000000..a228ea0 --- /dev/null +++ b/pkg/analyser/icbc/icbc.go @@ -0,0 +1,106 @@ +package icbc + +import ( + "strings" + + "github.com/deb-sig/double-entry-generator/pkg/config" + "github.com/deb-sig/double-entry-generator/pkg/ir" + "github.com/deb-sig/double-entry-generator/pkg/util" +) + +type Icbc struct { +} + +// GetAllCandidateAccounts returns all accounts defined in config. +func (i Icbc) GetAllCandidateAccounts(cfg *config.Config) map[string]bool { + // uniqMap will be used to create the concepts. + uniqMap := make(map[string]bool) + + if cfg.Icbc == nil || len(cfg.Icbc.Rules) == 0 { + return uniqMap + } + + for _, r := range cfg.Icbc.Rules { + if r.MethodAccount != nil { + uniqMap[*r.MethodAccount] = true + } + if r.TargetAccount != nil { + uniqMap[*r.TargetAccount] = true + } + if r.CommissionAccount != nil { + uniqMap[*r.CommissionAccount] = true + } + } + uniqMap[cfg.DefaultPlusAccount] = true + uniqMap[cfg.DefaultMinusAccount] = true + return uniqMap +} + +// GetAccountsAndTags GetAccounts returns minus and plus account. +func (i Icbc) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (bool, string, string, map[ir.Account]string, []string) { + ignore := false + + if cfg.Icbc == nil || len(cfg.Icbc.Rules) == 0 { + return ignore, cfg.DefaultMinusAccount, cfg.DefaultPlusAccount, nil, nil + } + + var tags = make([]string, 0) + resMinus := cfg.DefaultMinusAccount + resPlus := cfg.DefaultPlusAccount + cashAccount := cfg.DefaultCashAccount + + //var err error + for _, r := range cfg.Icbc.Rules { + match := true + // get separator + sep := "," + if r.Separator != nil { + sep = *r.Separator + } + + matchFunc := util.SplitFindContains + if r.FullMatch { + matchFunc = util.SplitFindEquals + } + + if r.Peer != nil { + match = matchFunc(*r.Peer, o.Peer, sep, match) + } + if r.Type != nil { + match = matchFunc(*r.Type, string(o.Type), sep, match) + } + if r.TxType != nil { + match = matchFunc(*r.TxType, o.TxTypeOriginal, sep, match) + } + + if match { + if r.Ignore { + ignore = true + break + } + // Support multiple matches, like one rule matches the minus account, the other rule matches the plus account. + if r.TargetAccount != nil { + if o.Type == ir.TypeRecv { + resMinus = *r.TargetAccount + } else { + resPlus = *r.TargetAccount + } + } + + // method account (bank card account) + if o.Type == ir.TypeRecv { + resPlus = cashAccount + } else { + resMinus = cashAccount + } + + if r.Tag != nil { + tags = strings.Split(*r.Tag, sep) + } + + } + + } + + return ignore, resMinus, resPlus, nil, tags +} diff --git a/pkg/analyser/interface.go b/pkg/analyser/interface.go index f5ef787..1012dde 100644 --- a/pkg/analyser/interface.go +++ b/pkg/analyser/interface.go @@ -3,6 +3,8 @@ package analyser import ( "fmt" + "github.com/deb-sig/double-entry-generator/pkg/analyser/icbc" + "github.com/deb-sig/double-entry-generator/pkg/analyser/alipay" "github.com/deb-sig/double-entry-generator/pkg/analyser/htsec" "github.com/deb-sig/double-entry-generator/pkg/analyser/huobi" @@ -15,7 +17,7 @@ import ( // Interface is the interface of analyser. type Interface interface { GetAllCandidateAccounts(cfg *config.Config) map[string]bool - GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (string, string, map[ir.Account]string, []string) + GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (bool, string, string, map[ir.Account]string, []string) } // New creates a new analyser. @@ -29,6 +31,8 @@ func New(providerName string) (Interface, error) { return huobi.Huobi{}, nil case consts.ProviderHtsec: return htsec.Htsec{}, nil + case consts.ProviderIcbc: + return icbc.Icbc{}, nil default: return nil, fmt.Errorf("Fail to create the analyser for the given name %s", providerName) } diff --git a/pkg/analyser/wechat/wechat.go b/pkg/analyser/wechat/wechat.go index 9a8f611..b9db72c 100644 --- a/pkg/analyser/wechat/wechat.go +++ b/pkg/analyser/wechat/wechat.go @@ -38,11 +38,12 @@ func (w Wechat) GetAllCandidateAccounts(cfg *config.Config) map[string]bool { } // GetAccounts returns minus and plus account. -func (w Wechat) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (string, string, map[ir.Account]string, []string) { +func (w Wechat) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, provider string) (bool, string, string, map[ir.Account]string, []string) { var resCommission string var tags = make([]string, 0) + ignore := false - // check this tx whether has commission + // check this tx whether it has commission if o.Commission != 0 { if cfg.DefaultCommissionAccount == "" { log.Fatalf("Found a tx with commission, but not setting the `defaultCommissionAccount` in config file!") @@ -52,7 +53,7 @@ func (w Wechat) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, prov } if cfg.Wechat == nil || len(cfg.Wechat.Rules) == 0 { - return cfg.DefaultMinusAccount, cfg.DefaultPlusAccount, map[ir.Account]string{ + return ignore, cfg.DefaultMinusAccount, cfg.DefaultPlusAccount, map[ir.Account]string{ ir.CommissionAccount: resCommission, }, nil } @@ -103,6 +104,11 @@ func (w Wechat) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, prov } if match { + if r.Ignore { + ignore = true + break + } + // Support multiple matches, like one rule matches the minus accout, the other rule matches the plus account. if r.TargetAccount != nil { if o.Type == ir.TypeRecv { @@ -130,7 +136,7 @@ func (w Wechat) GetAccountsAndTags(o *ir.Order, cfg *config.Config, target, prov } - return resMinus, resPlus, map[ir.Account]string{ + return ignore, resMinus, resPlus, map[ir.Account]string{ ir.CommissionAccount: resCommission, }, tags } diff --git a/pkg/compiler/beancount/compiler.go b/pkg/compiler/beancount/compiler.go index 019c1f7..cb1c0ad 100644 --- a/pkg/compiler/beancount/compiler.go +++ b/pkg/compiler/beancount/compiler.go @@ -85,15 +85,22 @@ func (b *BeanCount) initTemplates() error { func (b *BeanCount) Compile() error { log.SetPrefix("[Compiler-BeanCount] ") log.Printf("Getting the expected account for the bills") - for index, o := range b.IR.Orders { + var orders []ir.Order + for _, o := range b.IR.Orders { // Get the expected accounts according to the configuration. - minusAccount, plusAccount, extraAccounts, tags := b.GetAccountsAndTags(&o, b.Config, b.Provider, b.Target) - b.IR.Orders[index].MinusAccount = minusAccount - b.IR.Orders[index].PlusAccount = plusAccount - b.IR.Orders[index].ExtraAccounts = extraAccounts - b.IR.Orders[index].Tags = tags + ignore, minusAccount, plusAccount, extraAccounts, tags := b.GetAccountsAndTags(&o, b.Config, b.Provider, b.Target) + if ignore { + continue + } + o.MinusAccount = minusAccount + o.PlusAccount = plusAccount + o.ExtraAccounts = extraAccounts + o.Tags = tags + orders = append(orders, o) } + b.IR.Orders = orders + outputWriter, err := writer.GetWriter(b.Output) if err != nil { return fmt.Errorf("can't get output writer, err: %v", err) diff --git a/pkg/compiler/beancount/template.go b/pkg/compiler/beancount/template.go index ac02bb9..c4b67ef 100644 --- a/pkg/compiler/beancount/template.go +++ b/pkg/compiler/beancount/template.go @@ -6,7 +6,7 @@ import ( ) // 普通账单的模版（消费账） -var normalOrder = `{{ .PayTime.Format "2006-01-02" }} * "{{ EscapeString .Peer }}" "{{ EscapeString .Item }}"{{ range .Tags }} #{{ . }}{{ end }}{{ if .Note }} ; {{ .Note }}{{ end }} +var normalOrder = `{{ .PayTime.Format "2006-01-02" }} * "{{ EscapeString .Peer }}" {{- if .Item }} "{{ EscapeString .Item }}"{{ end }}{{ range .Tags }} #{{ . }}{{ end }}{{ if .Note }} ; {{ .Note }}{{ end }} {{- range $key, $value := .Metadata }}{{ if $value }}{{ printf "\n" }} {{ $key }}: "{{ $value }}"{{end}}{{end}} {{ .PlusAccount }} {{ .Money | printf "%.2f" }} {{ .Currency }} {{ .MinusAccount }} -{{ .Money | printf "%.2f" }} {{ .Currency }} diff --git a/pkg/config/config.go b/pkg/config/config.go index d972ded..3a6c678 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -4,6 +4,7 @@ import ( "github.com/deb-sig/double-entry-generator/pkg/provider/alipay" "github.com/deb-sig/double-entry-generator/pkg/provider/htsec" "github.com/deb-sig/double-entry-generator/pkg/provider/huobi" + "github.com/deb-sig/double-entry-generator/pkg/provider/icbc" "github.com/deb-sig/double-entry-generator/pkg/provider/wechat" ) @@ -21,4 +22,5 @@ type Config struct { Wechat *wechat.Config `yaml:"wechat,omitempty"` Huobi *huobi.Config `yaml:"huobi,omitempty"` Htsec *htsec.Config `yaml:"htsec,omitempty"` + Icbc *icbc.Config `yaml:"icbc,omitempty"` } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index cd9239d..c87f576 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -26,6 +26,8 @@ const ( ProviderWechat = "wechat" // ProviderHuobi is the name for huobi provider. ProviderHuobi = "huobi" - // ProviderHtsec is the name for htsec provider + // ProviderHtsec is the name for htsec provider. ProviderHtsec = "htsec" + //ProviderIcbc is the name for ICBC provider. + ProviderIcbc = "icbc" ) diff --git a/pkg/provider/alipay/alipay.go b/pkg/provider/alipay/alipay.go index bd4071f..48f80bb 100644 --- a/pkg/provider/alipay/alipay.go +++ b/pkg/provider/alipay/alipay.go @@ -19,11 +19,12 @@ package alipay import ( "encoding/csv" "fmt" - "github.com/deb-sig/double-entry-generator/pkg/io/reader" - "github.com/deb-sig/double-entry-generator/pkg/ir" "io" "log" "strings" + + "github.com/deb-sig/double-entry-generator/pkg/io/reader" + "github.com/deb-sig/double-entry-generator/pkg/ir" ) // Alipay is the provider for alipay. diff --git a/pkg/provider/alipay/config.go b/pkg/provider/alipay/config.go index 50f2020..e43dda6 100644 --- a/pkg/provider/alipay/config.go +++ b/pkg/provider/alipay/config.go @@ -36,4 +36,5 @@ type Rule struct { PnlAccount *string `mapstructure:"pnlAccount,omitempty"` FullMatch bool `mapstructure:"fullMatch,omitempty"` Tags *string `mapstructure:"tags,omitempty"` + Ignore bool `mapstructure:"ignore,omitempty"` // default: false } diff --git a/pkg/provider/alipay/convert.go b/pkg/provider/alipay/convert.go index 657599d..260f516 100644 --- a/pkg/provider/alipay/convert.go +++ b/pkg/provider/alipay/convert.go @@ -17,7 +17,7 @@ func (a *Alipay) convertToIR() *ir.IR { PayTime: o.PayTime, Money: o.Money, OrderID: &o.DealNo, - Type: conevertType(o.Type), + Type: convertType(o.Type), TypeOriginal: o.TypeOriginal, } irO.Metadata = getMetadata(o) @@ -29,7 +29,7 @@ func (a *Alipay) convertToIR() *ir.IR { return i } -func conevertType(t Type) ir.Type { +func convertType(t Type) ir.Type { switch t { case TypeSend: return ir.TypeSend diff --git a/pkg/provider/htsec/config.go b/pkg/provider/htsec/config.go index c91cb3b..774f6d0 100644 --- a/pkg/provider/htsec/config.go +++ b/pkg/provider/htsec/config.go @@ -16,4 +16,5 @@ type Rule struct { CommissionAccount *string `mapstructure:"commissionAccount,omitempty"` PnlAccount *string `mapstructure:"pnlAccount,omitempty"` FullMatch bool `mapstructure:"fullMatch,omitempty"` + Ignore bool `mapstructure:"ignore,omitempty"` // default: false } diff --git a/pkg/provider/huobi/config.go b/pkg/provider/huobi/config.go index 4b688ba..23456f8 100644 --- a/pkg/provider/huobi/config.go +++ b/pkg/provider/huobi/config.go @@ -17,4 +17,5 @@ type Rule struct { CommissionAccount *string `mapstructure:"commissionAccount,omitempty"` PnlAccount *string `mapstructure:"pnlAccount,omitempty"` FullMatch bool `mapstructure:"fullMatch,omitempty"` + Ignore bool `mapstructure:"ignore,omitempty"` // default: false } diff --git a/pkg/provider/icbc/config.go b/pkg/provider/icbc/config.go new file mode 100644 index 0000000..85decea --- /dev/null +++ b/pkg/provider/icbc/config.go @@ -0,0 +1,24 @@ +package icbc + +// Config is the configuration for Alipay. +type Config struct { + Rules []Rule `mapstructure:"rules,omitempty"` +} + +// Rule is the type for match rules. +type Rule struct { + Peer *string `mapstructure:"peer,omitempty"` + Item *string `mapstructure:"item,omitempty"` + Type *string `mapstructure:"type,omitempty"` + TxType *string `mapstructure:"txType,omitempty"` + Separator *string `mapstructure:"sep,omitempty"` // default: , + Method *string `mapstructure:"method,omitempty"` + Time *string `mapstructure:"time,omitempty"` + TimestampRange *string `mapstructure:"timestamp_range,omitempty"` + MethodAccount *string `mapstructure:"methodAccount,omitempty"` + TargetAccount *string `mapstructure:"targetAccount,omitempty"` + CommissionAccount *string `mapstructure:"commissionAccount,omitempty"` + FullMatch bool `mapstructure:"fullMatch,omitempty"` // default: false + Tag *string `mapstructure:"tag,omitempty"` + Ignore bool `mapstructure:"ignore,omitempty"` // default: false +} diff --git a/pkg/provider/icbc/convert.go b/pkg/provider/icbc/convert.go new file mode 100644 index 0000000..90c72f2 --- /dev/null +++ b/pkg/provider/icbc/convert.go @@ -0,0 +1,72 @@ +package icbc + +import ( + "strconv" + + "github.com/deb-sig/double-entry-generator/pkg/ir" +) + +// convertToIR convert ICBC bills to IR. +func (icbc *Icbc) convertToIR() *ir.IR { + i := ir.New() + for _, o := range icbc.Orders { + irO := ir.Order{ + Peer: o.Peer, + Money: o.Money, + PayTime: o.PayTime, + Type: convertType(o.Type), + TxTypeOriginal: o.TxTypeOriginal, + } + irO.Metadata = getMetadata(o) + i.Orders = append(i.Orders, irO) + } + return i +} + +func convertType(t OrderType) ir.Type { + switch t { + case OrderTypeSend: + return ir.TypeSend + case OrderTypeRecv: + return ir.TypeRecv + default: + return ir.TypeUnknown + } +} + +// getMetadata get the metadata (e.g. status, method, category and so on.) +// from order. +func getMetadata(o Order) map[string]string { + // FIXME(TripleZ): hard-coded, bad pattern + source := "中国工商银行" + var txTypeOriginal, guessedType, currency, balances, peerAccount string + + if o.TxTypeOriginal != "" { + txTypeOriginal = o.TxTypeOriginal + } + + if o.Type != "" { + guessedType = string(o.Type) + } + + if o.Currency != "" { + currency = o.Currency + } + + if o.Balances != 0 { + balances = strconv.FormatFloat(o.Balances, 'G', -1, 64) + } + + if o.PeerAccountName != "" { + peerAccount = o.PeerAccountName + } + + return map[string]string{ + "source": source, + "txType": txTypeOriginal, + "type": guessedType, + "currency": currency, + "balances": balances, + "peerAccount": peerAccount, + } +} diff --git a/pkg/provider/icbc/icbc.go b/pkg/provider/icbc/icbc.go new file mode 100644 index 0000000..3551425 --- /dev/null +++ b/pkg/provider/icbc/icbc.go @@ -0,0 +1,72 @@ +package icbc + +import ( + "encoding/csv" + "fmt" + "io" + "log" + + "github.com/deb-sig/double-entry-generator/pkg/io/reader" + "github.com/deb-sig/double-entry-generator/pkg/ir" +) + +// Icbc is the provider for Icbc. +type Icbc struct { + Statistics Statistics `json:"statistics,omitempty"` + LineNum int `json:"line_num,omitempty"` + Orders []Order `json:"orders,omitempty"` +} + +// New creates a new wechat provider. +func New() *Icbc { + return &Icbc{ + Statistics: Statistics{}, + LineNum: 0, + Orders: make([]Order, 0), + } +} + +// Translate translates the alipay bill records to IR. +func (icbc *Icbc) Translate(filename string) (*ir.IR, error) { + log.SetPrefix("[Provider-Icbc] ") + + billReader, err := reader.GetReader(filename) + if err != nil { + return nil, fmt.Errorf("can't get bill reader, err: %v", err) + } + + csvReader := csv.NewReader(billReader) + csvReader.LazyQuotes = true + // If FieldsPerRecord is negative, no check is made and records + // may have a variable number of fields. + csvReader.FieldsPerRecord = -1 + + for { + line, err := csvReader.Read() + + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + + icbc.LineNum++ + if icbc.LineNum <= 5 { + // The first 5 non-empty lines are useless for us. + continue + } + + if line[0] == "合计金额" { + // ignore the last line + break + } + + err = icbc.translateToOrders(line) + if err != nil { + return nil, fmt.Errorf("Failed to translate bill: line %d: %v", + icbc.LineNum, err) + } + } + log.Printf("Finished to parse the file %s", filename) + return icbc.convertToIR(), nil +} diff --git a/pkg/provider/icbc/parse.go b/pkg/provider/icbc/parse.go new file mode 100644 index 0000000..8dc3281 --- /dev/null +++ b/pkg/provider/icbc/parse.go @@ -0,0 +1,49 @@ +package icbc + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +// translateToOrders translates csv file to []Order. +func (icbc *Icbc) translateToOrders(array []string) error { + for idx, a := range array { + a = strings.Trim(a, " ") + a = strings.Trim(a, "\t") + array[idx] = a + } + var bill Order + var err error + bill.PayTime, err = time.Parse(localTimeFmt, strings.TrimSpace(array[1])+" +0800 CST") + if err != nil { + return fmt.Errorf("parse create time %s error: %v", array[1], err) + } + + bill.TxTypeOriginal = strings.TrimSpace(array[2]) + bill.Peer = strings.TrimSpace(array[3]) + bill.Region = strings.TrimSpace(array[4]) + + a8 := strings.ReplaceAll(strings.TrimSpace(array[8]), ",", "") + a9 := strings.ReplaceAll(strings.TrimSpace(array[9]), ",", "") + if a8 == "" && a9 == "" { + bill.Type = OrderTypeUnknown + } else if a9 == "" { + bill.Type = OrderTypeRecv + bill.Money, err = strconv.ParseFloat(a8, 64) + } else { + bill.Type = OrderTypeSend + bill.Money, err = strconv.ParseFloat(a9, 64) + } + if err != nil { + return fmt.Errorf("parse money %s error: %v", array[5], err) + } + + bill.Currency = strings.TrimSpace(array[10]) + bill.Balances, _ = strconv.ParseFloat(strings.ReplaceAll(strings.TrimSpace(array[11]), ",", ""), 64) + bill.PeerAccountName = strings.TrimSpace(array[12]) + + icbc.Orders = append(icbc.Orders, bill) + return nil +} diff --git a/pkg/provider/icbc/types.go b/pkg/provider/icbc/types.go new file mode 100644 index 0000000..3132df9 --- /dev/null +++ b/pkg/provider/icbc/types.go @@ -0,0 +1,40 @@ +package icbc + +import "time" + +// Statistics is the Statistics of the bill file. +type Statistics struct { + UserID string `json:"user_id,omitempty"` + Username string `json:"username,omitempty"` + ParsedItems int `json:"parsed_items,omitempty"` + Start time.Time `json:"start,omitempty"` + End time.Time `json:"end,omitempty"` + TotalInRecords int `json:"total_in_records,omitempty"` + TotalInMoney float64 `json:"total_in_money,omitempty"` + TotalOutRecords int `json:"total_out_records,omitempty"` + TotalOutMoney float64 `json:"total_out_money,omitempty"` +} + +// Order is the single order. +type Order struct { + PayTime time.Time // 记账日期 + TxTypeOriginal string // 摘要 + Peer string // 交易场所 + Region string // 交易国家或地区简称 + Money float64 // 记账金额 (收入/支出) + Type OrderType // 收/支 (数据中无该列，推测而来) + Currency string // 记账币种 + Balances float64 // 余额 + PeerAccountName string // 对方户名 +} + +// localTimeFmt set time format to utc+8 +const localTimeFmt = "2006-01-02 +0800 CST" + +type OrderType string + +const ( + OrderTypeSend OrderType = "支出" + OrderTypeRecv OrderType = "收入" + OrderTypeUnknown OrderType = "Unknown" +) diff --git a/pkg/provider/interface.go b/pkg/provider/interface.go index 3432e1d..cfb1a73 100644 --- a/pkg/provider/interface.go +++ b/pkg/provider/interface.go @@ -19,6 +19,8 @@ package provider import ( "fmt" + "github.com/deb-sig/double-entry-generator/pkg/provider/icbc" + "github.com/deb-sig/double-entry-generator/pkg/consts" "github.com/deb-sig/double-entry-generator/pkg/ir" "github.com/deb-sig/double-entry-generator/pkg/provider/alipay" @@ -43,6 +45,8 @@ func New(name string) (Interface, error) { return huobi.New(), nil case consts.ProviderHtsec: return htsec.New(), nil + case consts.ProviderIcbc: + return icbc.New(), nil default: return nil, fmt.Errorf("Fail to create the provider for the given name %s", name) } diff --git a/pkg/provider/wechat/config.go b/pkg/provider/wechat/config.go index be374c5..044d6ef 100644 --- a/pkg/provider/wechat/config.go +++ b/pkg/provider/wechat/config.go @@ -36,4 +36,5 @@ type Rule struct { CommissionAccount *string `mapstructure:"commissionAccount,omitempty"` FullMatch bool `mapstructure:"fullMatch,omitempty"` Tag *string `mapstructure:"tag,omitempty"` + Ignore bool `mapstructure:"ignore,omitempty"` // default: false } diff --git a/pkg/provider/wechat/convert.go b/pkg/provider/wechat/convert.go index 8599b2a..3b2e85c 100644 --- a/pkg/provider/wechat/convert.go +++ b/pkg/provider/wechat/convert.go @@ -14,7 +14,7 @@ func (w *Wechat) convertToIR() *ir.IR { PayTime: o.PayTime, Money: o.Money, OrderID: &o.OrderID, - Type: conevertType(o.Type), + Type: convertType(o.Type), TypeOriginal: o.TypeOriginal, TxTypeOriginal: o.TxTypeOriginal, Method: o.Method, @@ -29,7 +29,7 @@ func (w *Wechat) convertToIR() *ir.IR { return i } -func conevertType(t OrderType) ir.Type { +func convertType(t OrderType) ir.Type { switch t { case OrderTypeSend: return ir.TypeSend diff --git a/test/icbc-test.sh b/test/icbc-test.sh new file mode 100644 index 0000000..241a1f3 --- /dev/null +++ b/test/icbc-test.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# E2E test for icbc provider. + +# set -x # debug +set -eo errexit + +TEST_DIR=`dirname "$(realpath $0)"` +ROOT_DIR="$TEST_DIR/.." + +make -f "$ROOT_DIR/Makefile" build +mkdir -p "$ROOT_DIR/test/output" + +# generate icbc bills output in beancount format +"$ROOT_DIR/bin/double-entry-generator" translate \ + --provider icbc \ + --config "$ROOT_DIR/example/icbc/config.yaml" \ + --output "$ROOT_DIR/test/output/test-icbc-output.beancount" \ + "$ROOT_DIR/example/icbc/example-icbc-records.csv" + +diff -u --color \ + "$ROOT_DIR/example/icbc/example-icbc-output.beancount" \ + "$ROOT_DIR/test/output/test-icbc-output.beancount" + +if [ $? -ne 0 ]; then + echo "[FAIL] ICBC provider output is different from expected output." + exit 1 +fi + +echo "[PASS] All ICBC provider tests!"