Skip to content

Commit f8af672

Browse files
ping-keping-ke
authored
Customize emails for each alert rule. (#433)
* custom email to for checker * fix
1 parent 5ceb7cc commit f8af672

6 files changed

Lines changed: 94 additions & 32 deletions

File tree

‎.github/workflows/alerting.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,6 @@ jobs:
3131
username: ${{ secrets.SMTP_USERNAME }}
3232
password: ${{ secrets.SMTP_PASSWORD }}
3333
subject: EthStorage Alert!
34-
to: molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org
34+
to: ${{ env.EMAIL_LIST }}
3535
from: ${{ secrets.SMTP_USERNAME }}
3636
html_body: file://./cmd/alert/body.html

‎cmd/alert/es_last_mined_block_checker.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,29 +18,30 @@ const (
1818

1919
type ESLastMinedBlockChecker struct {
2020
Name string `json:"name"`
21+
EmailTo string `json:"email-to"`
2122
Contract common.Address `json:"contract"`
2223
RPC string `json:"rpc"`
2324
}
2425

25-
func newESLastMinedBlockChecker(params map[string]string) (*ESLastMinedBlockChecker, error) {
26+
func newESLastMinedBlockChecker(params map[string]string, emailTo string) (*ESLastMinedBlockChecker, error) {
2627
name, contract, rpc := params["name"], params["contract"], params["rpc"]
2728
if name == "" || contract == "" || rpc == "" {
2829
return nil, errors.New("invalid params to load ESLastMinedBlockChecker")
2930
}
3031

3132
return &ESLastMinedBlockChecker{
3233
Name: name,
34+
EmailTo: emailTo,
3335
Contract: common.HexToAddress(contract),
3436
RPC: rpc,
3537
}, nil
3638
}
3739

38-
func (c *ESLastMinedBlockChecker) Check(lg log.Logger) (bool, string) {
39-
40+
func (c *ESLastMinedBlockChecker) Check(lg log.Logger) (bool, string, string) {
4041
client, err := eth.Dial(c.RPC, c.Contract, 12, lg)
4142
if err != nil {
4243
lg.Error("Failed to create source", "alert", c.Name, "err", err)
43-
return true, fmt.Sprintf(errorContent, c.Name, err.Error())
44+
return true, fmt.Sprintf(errorContent, c.Name, err.Error()), c.EmailTo
4445
}
4546
var (
4647
ctx = context.Background()
@@ -60,10 +61,10 @@ func (c *ESLastMinedBlockChecker) Check(lg log.Logger) (bool, string) {
6061
targetTime := time.Now().Add(-24 * time.Hour)
6162
if targetTime.After(lastMinedTime) {
6263
content := fmt.Sprintf(noMinedBlockAlertContent, c.Name, info.BlockMined, lastMinedTime, c.RPC)
63-
return true, content
64+
return true, content, c.EmailTo
6465
}
65-
return false, ""
66+
return false, "", c.EmailTo
6667
}
6768

68-
return true, fmt.Sprintf(errorContent, c.Name, err.Error())
69+
return true, fmt.Sprintf(errorContent, c.Name, err.Error()), c.EmailTo
6970
}

‎cmd/alert/ether_last_block_checker.go‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,28 +15,30 @@ const (
1515
)
1616

1717
type LastBlockChecker struct {
18-
Name string `json:"name"`
19-
RPC string `json:"rpc"`
18+
Name string `json:"name"`
19+
EmailTo string `json:"email-to"`
20+
RPC string `json:"rpc"`
2021
}
2122

22-
func newLastBlockChecker(params map[string]string) (*LastBlockChecker, error) {
23+
func newLastBlockChecker(params map[string]string, emailTo string) (*LastBlockChecker, error) {
2324
name, rpc := params["name"], params["rpc"]
2425
if name == "" || rpc == "" {
2526
return nil, errors.New("invalid params to load LastBlockChecker")
2627
}
2728

2829
return &LastBlockChecker{
29-
Name: name,
30-
RPC: rpc,
30+
Name: name,
31+
EmailTo: emailTo,
32+
RPC: rpc,
3133
}, nil
3234
}
3335

34-
func (c *LastBlockChecker) Check(lg log.Logger) (bool, string) {
36+
func (c *LastBlockChecker) Check(lg log.Logger) (bool, string, string) {
3537
ctx := context.Background()
3638
client, err := ethclient.Dial(c.RPC)
3739
if err != nil {
3840
lg.Error("Failed to create L1 source", "alert", c.Name, "err", err)
39-
return true, fmt.Sprintf(errorContent, c.Name, err.Error())
41+
return true, fmt.Sprintf(errorContent, c.Name, err.Error()), c.EmailTo
4042
}
4143

4244
for i := 0; i < 3; i++ {
@@ -53,10 +55,10 @@ func (c *LastBlockChecker) Check(lg log.Logger) (bool, string) {
5355
targetTime := time.Now().Add(-10 * time.Minute)
5456
if targetTime.After(lastMinedTime) {
5557
content := fmt.Sprintf(noBlockIn10MinutesAlertContent, c.Name, header.Number, lastMinedTime, c.RPC)
56-
return true, content
58+
return true, content, c.EmailTo
5759
}
58-
return false, ""
60+
return false, "", c.EmailTo
5961
}
6062

61-
return true, fmt.Sprintf(errorContent, c.Name, err.Error())
63+
return true, fmt.Sprintf(errorContent, c.Name, err.Error()), c.EmailTo
6264
}

‎cmd/alert/main.go‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"flag"
66
"fmt"
77
"os"
8+
"strings"
89

910
"github.com/ethereum/go-ethereum/log"
1011
)
@@ -20,11 +21,12 @@ var (
2021
)
2122

2223
type IChecker interface {
23-
Check(lg log.Logger) (bool, string)
24+
Check(lg log.Logger) (bool, string, string)
2425
}
2526

2627
type AlertConfig struct {
2728
AlertType string `json:"alert-type"`
29+
EmailTo string `json:"email-to"`
2830
Params map[string]string `json:"params"`
2931
}
3032

@@ -45,19 +47,19 @@ func LoadConfig(ruleFile string, lg log.Logger) []IChecker {
4547
for _, alert := range alerts {
4648
switch {
4749
case alert.AlertType == "ESLastMinedBlockChecker":
48-
checker, err := newESLastMinedBlockChecker(alert.Params)
50+
checker, err := newESLastMinedBlockChecker(alert.Params, alert.EmailTo)
4951
if err != nil {
5052
lg.Crit("Failed to load ESLastMinedBlockChecker", "params", alert.Params, "err", err)
5153
}
5254
checkers = append(checkers, checker)
5355
case alert.AlertType == "LastBlockChecker":
54-
checker, err := newLastBlockChecker(alert.Params)
56+
checker, err := newLastBlockChecker(alert.Params, alert.EmailTo)
5557
if err != nil {
5658
lg.Crit("Failed to load LastBlockChecker", "params", alert.Params, "err", err)
5759
}
5860
checkers = append(checkers, checker)
5961
case alert.AlertType == "WebsiteOnlineChecker":
60-
checker, err := newWebsiteOnlineChecker(alert.Params)
62+
checker, err := newWebsiteOnlineChecker(alert.Params, alert.EmailTo)
6163
if err != nil {
6264
lg.Crit("Failed to load WebsiteOnlineChecker", "params", alert.Params, "err", err)
6365
}
@@ -78,20 +80,23 @@ func main() {
7880
lg = log.New("app", "alert")
7981
needAlert = false
8082
contents = ""
83+
emailList = make(map[string]struct{})
8184
)
8285

8386
checkers := LoadConfig(*ruleFileFlag, lg)
8487

8588
for _, checker := range checkers {
86-
res, content := checker.Check(lg)
89+
res, content, emailTo := checker.Check(lg)
8790
if res {
8891
needAlert = res
8992
contents = contents + content + "\n"
93+
addToEmailList(emailList, emailTo)
9094
}
9195
}
9296

9397
if needAlert {
9498
writeHtmlFile(fmt.Sprintf(emailFormat, contents), lg)
99+
writeToGitHubEnv(emailList, lg)
95100
os.Exit(1)
96101
}
97102
}
@@ -108,3 +113,41 @@ func writeHtmlFile(content string, lg log.Logger) {
108113
lg.Crit("Write html file fail", "error", err.Error())
109114
}
110115
}
116+
117+
func addToEmailList(emailList map[string]struct{}, emailToString string) {
118+
emails := strings.Split(emailToString, ",")
119+
120+
for _, email := range emails {
121+
emailList[email] = struct{}{}
122+
}
123+
}
124+
125+
func writeToGitHubEnv(emailList map[string]struct{}, lg log.Logger) {
126+
if len(emailList) == 0 {
127+
lg.Error("The email list shoud not be empty")
128+
return
129+
}
130+
131+
emails := make([]string, 0, len(emailList))
132+
for email := range emailList {
133+
emails = append(emails, email)
134+
}
135+
emailTo := strings.Join(emails, ",")
136+
137+
githubEnv := os.Getenv("GITHUB_ENV")
138+
if githubEnv == "" {
139+
lg.Error("GITHUB_ENV not set, are you running inside GitHub Actions?", "email list", emailTo)
140+
return
141+
}
142+
143+
f, err := os.OpenFile(githubEnv, os.O_APPEND|os.O_WRONLY, 0600)
144+
if err != nil {
145+
lg.Crit("Open GITHUB_ENV fail", "error", err.Error())
146+
}
147+
defer f.Close()
148+
149+
_, err = f.WriteString(fmt.Sprintf("EMAIL_LIST=%s\n", emailTo))
150+
if err != nil {
151+
lg.Crit("Write GITHUB_ENV fail", "error", err.Error())
152+
}
153+
}

‎cmd/alert/rules.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
[
22
{
33
"alert-type":"ESLastMinedBlockChecker",
4+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
45
"params":{
56
"name":"Sepolia Last Mined Block Alert",
67
"contract":"0x804C520d3c084C805E37A35E90057Ac32831F96f",
@@ -9,6 +10,7 @@
910
},
1011
{
1112
"alert-type":"ESLastMinedBlockChecker",
13+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
1214
"params":{
1315
"name":"L2 Last Mined Block Alert",
1416
"contract":"0x64003adbdf3014f7E38FC6BE752EB047b95da89A",
@@ -17,6 +19,7 @@
1719
},
1820
{
1921
"alert-type":"WebsiteOnlineChecker",
22+
"email-to": "molaokp@gmail.com,pingke@quarkchain.org",
2023
"params":{
2124
"name":"grafana.ethstorage.io online Alert",
2225
"url":"https://grafana.ethstorage.io/"
@@ -25,76 +28,87 @@
2528

2629
{
2730
"alert-type":"LastBlockChecker",
31+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
2832
"params":{
2933
"name":"Quarkchain Shard 0 Last Block Alert",
3034
"rpc":"https://mainnet-s0-ethapi.quarkchain.io"
3135
}
3236
},
3337
{
3438
"alert-type":"LastBlockChecker",
39+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
3540
"params":{
3641
"name":"Quarkchain Shard 1 Last Block Alert",
3742
"rpc":"https://mainnet-s1-ethapi.quarkchain.io"
3843
}
3944
},
4045
{
4146
"alert-type":"LastBlockChecker",
47+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
4248
"params":{
4349
"name":"Quarkchain Shard 2 Last Block Alert",
4450
"rpc":"https://mainnet-s2-ethapi.quarkchain.io"
4551
}
4652
},
4753
{
4854
"alert-type":"LastBlockChecker",
55+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
4956
"params":{
5057
"name":"Quarkchain Shard 3 Last Block Alert",
5158
"rpc":"https://mainnet-s3-ethapi.quarkchain.io"
5259
}
5360
},
5461
{
5562
"alert-type":"LastBlockChecker",
63+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
5664
"params":{
5765
"name":"Quarkchain Shard 4 Last Block Alert",
5866
"rpc":"http://eth-jrpc.mainnet.quarkchain.io:39004"
5967
}
6068
},
6169
{
6270
"alert-type":"LastBlockChecker",
71+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
6372
"params":{
6473
"name":"Quarkchain Shard 5 Last Block Alert",
6574
"rpc":"http://eth-jrpc.mainnet.quarkchain.io:39005"
6675
}
6776
},
6877
{
6978
"alert-type":"LastBlockChecker",
79+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
7080
"params":{
7181
"name":"Quarkchain Shard 6 Last Block Alert",
7282
"rpc":"http://eth-jrpc.mainnet.quarkchain.io:39006"
7383
}
7484
},
7585
{
7686
"alert-type":"LastBlockChecker",
87+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
7788
"params":{
7889
"name":"Quarkchain Shard 7 Last Block Alert",
7990
"rpc":"http://eth-jrpc.mainnet.quarkchain.io:39007"
8091
}
8192
},
8293
{
8394
"alert-type":"WebsiteOnlineChecker",
95+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
8496
"params":{
8597
"name":"Quarkchain explorer online Alert",
8698
"url":"https://mainnet.quarkchain.io/"
8799
}
88100
},
89101
{
90102
"alert-type":"WebsiteOnlineChecker",
103+
"email-to": "molaokp@gmail.com,lundeng@quarkchain.org,qzhu@quarkchain.org,pingke@quarkchain.org,limingpeng@quarkchain.org",
91104
"params":{
92105
"name":"quarkchain.io online Alert",
93106
"url":"https://quarkchain.io/"
94107
}
95108
},
96109
{
97110
"alert-type":"WebsiteOnlineChecker",
111+
"email-to": "molaokp@gmail.com,pingke@quarkchain.org",
98112
"params":{
99113
"name":"grafana.quarkchain.io online Alert",
100114
"url":"https://grafana.quarkchain.io/"

‎cmd/alert/site_online_checker.go‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,39 +14,41 @@ const (
1414
)
1515

1616
type WebsiteOnlineChecker struct {
17-
Name string `json:"name"`
18-
URL string `json:"url"`
17+
Name string `json:"name"`
18+
EmailTo string `json:"email-to"`
19+
URL string `json:"url"`
1920
}
2021

21-
func newWebsiteOnlineChecker(params map[string]string) (*WebsiteOnlineChecker, error) {
22+
func newWebsiteOnlineChecker(params map[string]string, emailTo string) (*WebsiteOnlineChecker, error) {
2223
name, url := params["name"], params["url"]
2324
if name == "" || url == "" {
2425
return nil, errors.New("invalid params to load WebsiteOnlineChecker")
2526
}
2627

2728
return &WebsiteOnlineChecker{
28-
Name: name,
29-
URL: url,
29+
Name: name,
30+
EmailTo: emailTo,
31+
URL: url,
3032
}, nil
3133
}
3234

33-
func (c *WebsiteOnlineChecker) Check(lg log.Logger) (bool, string) {
35+
func (c *WebsiteOnlineChecker) Check(lg log.Logger) (bool, string, string) {
3436
client := http.Client{
3537
Timeout: 10 * time.Second,
3638
}
3739

3840
resp, err := client.Get(c.URL)
3941
if err != nil {
4042
lg.Error("Failed to request web site", "alert", c.Name, "url", c.URL, "err", err)
41-
return true, fmt.Sprintf(errorContent, c.Name, err.Error())
43+
return true, fmt.Sprintf(errorContent, c.Name, err.Error()), c.EmailTo
4244
}
4345
defer resp.Body.Close()
4446

4547
lg.Info("Check last block", "alert", c.Name, "url", c.URL, "status", resp.StatusCode)
4648
// Check the HTTP status code
4749
if resp.StatusCode == http.StatusOK {
48-
return false, ""
50+
return false, "", c.EmailTo
4951
} else {
50-
return true, fmt.Sprintf(websiteOfflineAlertContent, c.Name, c.URL)
52+
return true, fmt.Sprintf(websiteOfflineAlertContent, c.Name, c.URL), c.EmailTo
5153
}
5254
}

0 commit comments

Comments
 (0)