From f801945e0ff2d16fbfe2377f34ad391849d3da64 Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 16:06:27 +0800 Subject: [PATCH 01/13] add alert action --- .github/workflows/alerting.yml | 36 +++++++++++++ cmd/alert/main.go | 92 ++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 .github/workflows/alerting.yml create mode 100644 cmd/alert/main.go diff --git a/.github/workflows/alerting.yml b/.github/workflows/alerting.yml new file mode 100644 index 00000000..0cb5c00c --- /dev/null +++ b/.github/workflows/alerting.yml @@ -0,0 +1,36 @@ +name: Integration Test +on: + schedule: + - cron: '1 */1 * * *' + workflow_dispatch: + +jobs: + notification: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.21' + + - name: check + run: | + cd ./cmd/alert + go build + ./alert --l1contract $ES_NODE_CONTRACT_ADDRESS --htmlbody ./body.html + + - name: Send notification email + uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{ secrets.SMTP_USERNAME }} + password: ${{ secrets.SMTP_PASSWORD }} + subject: EthStorage Alert! + to: molaokp@gmail.com + from: ${{ secrets.SMTP_USERNAME }} + body: ./cmd/alert/body.html + if: steps.check.outcome == 'failure' \ No newline at end of file diff --git a/cmd/alert/main.go b/cmd/alert/main.go new file mode 100644 index 00000000..16f7d5e6 --- /dev/null +++ b/cmd/alert/main.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethstorage/go-ethstorage/ethstorage/eth" + "github.com/ethstorage/go-ethstorage/ethstorage/miner" +) + +const ( + emailFormat = "

Ethstorage Alert!

%s
" + noMinedBlockAlertContent = "

No blocks mined in last 24 hours. Last mined block %d, last mined time: %v.

" +) + +var ( + rpcURLFlag = flag.String("rpcurl", "http://88.99.30.186:8545", "L1 RPC URL") + l1ContractFlag = flag.String("l1contract", "0x804C520d3c084C805E37A35E90057Ac32831F96f", "Storage contract address on l1") + bodyFileFlag = flag.String("htmlbody", "body.html", "Alert email html body file") +) + +func main() { + flag.Parse() + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(3), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + + var ( + contract = common.HexToAddress(*l1ContractFlag) + logger = log.New("app", "Dashboard") + ) + + client, err := eth.Dial(*rpcURLFlag, contract, 12, logger) + if err != nil { + log.Crit("Failed to create L1 source", "err", err) + } + + res, content, err := checkLastMinedBlock(client, contract, logger) + if err != nil { + log.Crit("Failed to create L1 source", "err", err) + } + + if res { + writeHtmlFile(fmt.Sprintf(emailFormat, content)) + os.Exit(1) + } +} + +func checkLastMinedBlock(client *eth.PollingClient, contract common.Address, logger log.Logger) (bool, string, error) { + var ( + ctx = context.Background() + api = miner.NewL1MiningAPI(client, nil, logger) + err error + ) + + for i := 0; i < 3; i++ { + info, e := api.GetMiningInfo(ctx, contract, 0) + if e != nil { + time.Sleep(time.Minute) + log.Error("Get mining info fail", "error", e) + err = e + continue + } + + lastMinedTime := time.Unix(int64(info.LastMineTime), 0) + logger.Info("", "last mined time", lastMinedTime, "mined block", info.BlockMined) + targetTime := time.Now().Add(-1 * time.Hour) + if targetTime.After(lastMinedTime) { + content := fmt.Sprintf(noMinedBlockAlertContent, info.BlockMined, lastMinedTime) + return true, content, nil + } + return false, "", nil + } + + return false, "", err +} + +func writeHtmlFile(content string) { + file, err := os.Create(*bodyFileFlag) + if err != nil { + log.Crit("Create html file fail", "error", err.Error()) + } + defer file.Close() + + _, err = file.WriteString(content) + if err != nil { + log.Crit("Write html file fail", "error", err.Error()) + } +} From d2b2264bb8d7a1d2e3598a9bb59c1432648bde9c Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 16:08:01 +0800 Subject: [PATCH 02/13] resolve --- cmd/alert/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/alert/main.go b/cmd/alert/main.go index 16f7d5e6..207088d9 100644 --- a/cmd/alert/main.go +++ b/cmd/alert/main.go @@ -30,7 +30,7 @@ func main() { var ( contract = common.HexToAddress(*l1ContractFlag) - logger = log.New("app", "Dashboard") + logger = log.New("app", "alert") ) client, err := eth.Dial(*rpcURLFlag, contract, 12, logger) From 8196ec025f95623fcae8d1c5762aab04f618c2b2 Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 16:10:24 +0800 Subject: [PATCH 03/13] update action name --- .github/workflows/alerting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/alerting.yml b/.github/workflows/alerting.yml index 0cb5c00c..1ba4f686 100644 --- a/.github/workflows/alerting.yml +++ b/.github/workflows/alerting.yml @@ -1,4 +1,4 @@ -name: Integration Test +name: Alerting on: schedule: - cron: '1 */1 * * *' From 03ef6242a05e13f07cb23de79789e53ea85a335a Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 16:18:40 +0800 Subject: [PATCH 04/13] add fetch fail alert --- cmd/alert/main.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/alert/main.go b/cmd/alert/main.go index 207088d9..8085392f 100644 --- a/cmd/alert/main.go +++ b/cmd/alert/main.go @@ -16,6 +16,7 @@ import ( const ( emailFormat = "

Ethstorage Alert!

%s
" noMinedBlockAlertContent = "

No blocks mined in last 24 hours. Last mined block %d, last mined time: %v.

" + errorContent = "

Check alert fail with error: %s

" ) var ( @@ -38,12 +39,11 @@ func main() { log.Crit("Failed to create L1 source", "err", err) } - res, content, err := checkLastMinedBlock(client, contract, logger) + needAlert, content, err := checkLastMinedBlock(client, contract, logger) if err != nil { - log.Crit("Failed to create L1 source", "err", err) - } - - if res { + writeHtmlFile(fmt.Sprintf(emailFormat, fmt.Sprintf(errorContent, err.Error()))) + os.Exit(1) + } else if needAlert { writeHtmlFile(fmt.Sprintf(emailFormat, content)) os.Exit(1) } From e2423183090e45f6f8006388c7255a405910fbec Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 16:25:04 +0800 Subject: [PATCH 05/13] add push --- .github/workflows/alerting.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/alerting.yml b/.github/workflows/alerting.yml index 1ba4f686..81f24b55 100644 --- a/.github/workflows/alerting.yml +++ b/.github/workflows/alerting.yml @@ -1,5 +1,6 @@ name: Alerting on: + push: schedule: - cron: '1 */1 * * *' workflow_dispatch: From 36035aae2001c94d19d7698bcff30009dc822c4d Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 16:28:54 +0800 Subject: [PATCH 06/13] bug fix --- .github/workflows/alerting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/alerting.yml b/.github/workflows/alerting.yml index 81f24b55..c43c0d55 100644 --- a/.github/workflows/alerting.yml +++ b/.github/workflows/alerting.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - notification: + alert: runs-on: ubuntu-latest steps: @@ -21,7 +21,7 @@ jobs: run: | cd ./cmd/alert go build - ./alert --l1contract $ES_NODE_CONTRACT_ADDRESS --htmlbody ./body.html + ./alert --htmlbody ./body.html - name: Send notification email uses: dawidd6/action-send-mail@v3 From fe2f892ddc684fa04e594181947b828883611df2 Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 16:35:31 +0800 Subject: [PATCH 07/13] refactor --- cmd/alert/main.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/cmd/alert/main.go b/cmd/alert/main.go index 8085392f..dee26408 100644 --- a/cmd/alert/main.go +++ b/cmd/alert/main.go @@ -39,17 +39,14 @@ func main() { log.Crit("Failed to create L1 source", "err", err) } - needAlert, content, err := checkLastMinedBlock(client, contract, logger) - if err != nil { - writeHtmlFile(fmt.Sprintf(emailFormat, fmt.Sprintf(errorContent, err.Error()))) - os.Exit(1) - } else if needAlert { + needAlert, content := checkLastMinedBlock(client, contract, logger) + if needAlert { writeHtmlFile(fmt.Sprintf(emailFormat, content)) os.Exit(1) } } -func checkLastMinedBlock(client *eth.PollingClient, contract common.Address, logger log.Logger) (bool, string, error) { +func checkLastMinedBlock(client *eth.PollingClient, contract common.Address, logger log.Logger) (bool, string) { var ( ctx = context.Background() api = miner.NewL1MiningAPI(client, nil, logger) @@ -70,12 +67,12 @@ func checkLastMinedBlock(client *eth.PollingClient, contract common.Address, log targetTime := time.Now().Add(-1 * time.Hour) if targetTime.After(lastMinedTime) { content := fmt.Sprintf(noMinedBlockAlertContent, info.BlockMined, lastMinedTime) - return true, content, nil + return true, content } - return false, "", nil + return false, "" } - return false, "", err + return true, fmt.Sprintf(errorContent, err.Error()) } func writeHtmlFile(content string) { From 20a00a2820ed5a52b19b4f35f9ce25c0e235f614 Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 17:27:18 +0800 Subject: [PATCH 08/13] update --- .github/workflows/alerting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/alerting.yml b/.github/workflows/alerting.yml index c43c0d55..4f918ea7 100644 --- a/.github/workflows/alerting.yml +++ b/.github/workflows/alerting.yml @@ -24,6 +24,7 @@ jobs: ./alert --htmlbody ./body.html - name: Send notification email + if: failure() uses: dawidd6/action-send-mail@v3 with: server_address: smtp.gmail.com @@ -33,5 +34,4 @@ jobs: subject: EthStorage Alert! to: molaokp@gmail.com from: ${{ secrets.SMTP_USERNAME }} - body: ./cmd/alert/body.html - if: steps.check.outcome == 'failure' \ No newline at end of file + body: ./cmd/alert/body.html \ No newline at end of file From db2a4bbda3d2f8704bffadafc188c7b6242ec205 Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 17:30:18 +0800 Subject: [PATCH 09/13] trigger action fail for test --- cmd/alert/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/alert/main.go b/cmd/alert/main.go index dee26408..125d0b74 100644 --- a/cmd/alert/main.go +++ b/cmd/alert/main.go @@ -64,7 +64,7 @@ func checkLastMinedBlock(client *eth.PollingClient, contract common.Address, log lastMinedTime := time.Unix(int64(info.LastMineTime), 0) logger.Info("", "last mined time", lastMinedTime, "mined block", info.BlockMined) - targetTime := time.Now().Add(-1 * time.Hour) + targetTime := time.Now().Add(-1 * time.Minute) if targetTime.After(lastMinedTime) { content := fmt.Sprintf(noMinedBlockAlertContent, info.BlockMined, lastMinedTime) return true, content From 31145b13db47de8711fcb29357d1c3413e5cbe7b Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 17:34:26 +0800 Subject: [PATCH 10/13] fix email content --- .github/workflows/alerting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/alerting.yml b/.github/workflows/alerting.yml index 4f918ea7..bded2ed9 100644 --- a/.github/workflows/alerting.yml +++ b/.github/workflows/alerting.yml @@ -34,4 +34,4 @@ jobs: subject: EthStorage Alert! to: molaokp@gmail.com from: ${{ secrets.SMTP_USERNAME }} - body: ./cmd/alert/body.html \ No newline at end of file + html_body: ./cmd/alert/body.html \ No newline at end of file From ff833d4825352fffebdf5af19aeab436341de08a Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 17:39:32 +0800 Subject: [PATCH 11/13] fix email content --- .github/workflows/alerting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/alerting.yml b/.github/workflows/alerting.yml index bded2ed9..fa9cc521 100644 --- a/.github/workflows/alerting.yml +++ b/.github/workflows/alerting.yml @@ -34,4 +34,4 @@ jobs: subject: EthStorage Alert! to: molaokp@gmail.com from: ${{ secrets.SMTP_USERNAME }} - html_body: ./cmd/alert/body.html \ No newline at end of file + html_body: file://./cmd/alert/body.html \ No newline at end of file From 62cc2e5f867f4a709ca8f21ef04763ff6440e7ea Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 17:46:47 +0800 Subject: [PATCH 12/13] test --- .github/workflows/alerting.yml | 2 +- cmd/alert/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/alerting.yml b/.github/workflows/alerting.yml index fa9cc521..12d8468f 100644 --- a/.github/workflows/alerting.yml +++ b/.github/workflows/alerting.yml @@ -32,6 +32,6 @@ jobs: username: ${{ secrets.SMTP_USERNAME }} password: ${{ secrets.SMTP_PASSWORD }} subject: EthStorage Alert! - to: molaokp@gmail.com + to: molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org from: ${{ secrets.SMTP_USERNAME }} html_body: file://./cmd/alert/body.html \ No newline at end of file diff --git a/cmd/alert/main.go b/cmd/alert/main.go index 125d0b74..647d7137 100644 --- a/cmd/alert/main.go +++ b/cmd/alert/main.go @@ -15,7 +15,7 @@ import ( const ( emailFormat = "

Ethstorage Alert!

%s
" - noMinedBlockAlertContent = "

No blocks mined in last 24 hours. Last mined block %d, last mined time: %v.

" + noMinedBlockAlertContent = "

No blocks mined in last 24 hours. Last mined block: %d; last mined time: %v.

" errorContent = "

Check alert fail with error: %s

" ) From 8c740d3738d912eeede68e46372dfdea280ccf69 Mon Sep 17 00:00:00 2001 From: pingke Date: Wed, 30 Oct 2024 17:56:10 +0800 Subject: [PATCH 13/13] add l1 source to email content --- .github/workflows/alerting.yml | 1 - cmd/alert/main.go | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/alerting.yml b/.github/workflows/alerting.yml index 12d8468f..7ab4b77a 100644 --- a/.github/workflows/alerting.yml +++ b/.github/workflows/alerting.yml @@ -1,6 +1,5 @@ name: Alerting on: - push: schedule: - cron: '1 */1 * * *' workflow_dispatch: diff --git a/cmd/alert/main.go b/cmd/alert/main.go index 647d7137..8afb5e48 100644 --- a/cmd/alert/main.go +++ b/cmd/alert/main.go @@ -15,7 +15,7 @@ import ( const ( emailFormat = "

Ethstorage Alert!

%s
" - noMinedBlockAlertContent = "

No blocks mined in last 24 hours. Last mined block: %d; last mined time: %v.

" + noMinedBlockAlertContent = "

No blocks mined in last 24 hours. Last mined block: %d; last mined time: %v; l1 source: %s.

" errorContent = "

Check alert fail with error: %s

" ) @@ -64,9 +64,9 @@ func checkLastMinedBlock(client *eth.PollingClient, contract common.Address, log lastMinedTime := time.Unix(int64(info.LastMineTime), 0) logger.Info("", "last mined time", lastMinedTime, "mined block", info.BlockMined) - targetTime := time.Now().Add(-1 * time.Minute) + targetTime := time.Now().Add(-24 * time.Hour) if targetTime.After(lastMinedTime) { - content := fmt.Sprintf(noMinedBlockAlertContent, info.BlockMined, lastMinedTime) + content := fmt.Sprintf(noMinedBlockAlertContent, info.BlockMined, lastMinedTime, *rpcURLFlag) return true, content } return false, ""