diff --git a/td2/alert.go b/td2/alert.go index a9295cc..3b18846 100644 --- a/td2/alert.go +++ b/td2/alert.go @@ -27,12 +27,15 @@ type alertMsg struct { slk bool wh bool - severity string - resolved bool - chain string - message string - uniqueId string - key string + severity string + resolved bool + chain string + chainName string + valoperAddress string + valconsAddress string + message string + uniqueId string + key string tgChannel string tgKey string @@ -456,10 +459,13 @@ func notifyWebhook(msg *alertMsg) (err error) { alert := WebhookAlert{ Status: status, Labels: map[string]string{ - "alertname": msg.uniqueId, - "chain": msg.chain, - "severity": msg.severity, - "source": "tenderduty", + "alertname": msg.uniqueId, + "chain": msg.chain, + "chain_name": msg.chainName, + "valoper_address": msg.valoperAddress, + "valcons_address": msg.valconsAddress, + "severity": msg.severity, + "source": "tenderduty", }, Annotations: map[string]string{ "summary": msg.message, @@ -521,41 +527,53 @@ func getAlarms(chain string) string { } // alert creates a universal alert and pushes it to the alertChan to be delivered to appropriate services -func (c *Config) alert(chainName, message, severity string, resolved bool, id *string) { +func (c *Config) alert(configName, message, severity string, resolved bool, id *string) { if id == nil { return } c.chainsMux.RLock() + cc := c.Chains[configName] + if cc == nil { + c.chainsMux.RUnlock() + return + } + valcons := "" + if cc.valInfo != nil { + valcons = cc.valInfo.Valcons + } a := &alertMsg{ - pd: boolVal(c.DefaultAlertConfig.Pagerduty.Enabled) && boolVal(c.Chains[chainName].Alerts.Pagerduty.Enabled), - disc: boolVal(c.DefaultAlertConfig.Discord.Enabled) && boolVal(c.Chains[chainName].Alerts.Discord.Enabled), - tg: boolVal(c.DefaultAlertConfig.Telegram.Enabled) && boolVal(c.Chains[chainName].Alerts.Telegram.Enabled), - slk: boolVal(c.DefaultAlertConfig.Slack.Enabled) && boolVal(c.Chains[chainName].Alerts.Slack.Enabled), - wh: boolVal(c.DefaultAlertConfig.Webhook.Enabled) && boolVal(c.Chains[chainName].Alerts.Webhook.Enabled), - severity: severity, - resolved: resolved, - chain: fmt.Sprintf("%s (%s)", chainName, c.Chains[chainName].ChainId), - message: message, - uniqueId: *id, - key: c.Chains[chainName].Alerts.Pagerduty.ApiKey, - tgChannel: c.Chains[chainName].Alerts.Telegram.Channel, - tgKey: c.Chains[chainName].Alerts.Telegram.ApiKey, - tgMentions: strings.Join(c.Chains[chainName].Alerts.Telegram.Mentions, " "), - discHook: c.Chains[chainName].Alerts.Discord.Webhook, - discMentions: strings.Join(c.Chains[chainName].Alerts.Discord.Mentions, " "), - slkHook: c.Chains[chainName].Alerts.Slack.Webhook, - whURL: c.Chains[chainName].Alerts.Webhook.URL, - alertConfig: &c.Chains[chainName].Alerts, + pd: boolVal(c.DefaultAlertConfig.Pagerduty.Enabled) && boolVal(cc.Alerts.Pagerduty.Enabled), + disc: boolVal(c.DefaultAlertConfig.Discord.Enabled) && boolVal(cc.Alerts.Discord.Enabled), + tg: boolVal(c.DefaultAlertConfig.Telegram.Enabled) && boolVal(cc.Alerts.Telegram.Enabled), + slk: boolVal(c.DefaultAlertConfig.Slack.Enabled) && boolVal(cc.Alerts.Slack.Enabled), + wh: boolVal(c.DefaultAlertConfig.Webhook.Enabled) && boolVal(cc.Alerts.Webhook.Enabled), + severity: severity, + resolved: resolved, + chain: fmt.Sprintf("%s (%s)", configName, cc.ChainId), + chainName: cc.ChainName, + valoperAddress: cc.ValAddress, + valconsAddress: valcons, + message: message, + uniqueId: *id, + key: cc.Alerts.Pagerduty.ApiKey, + tgChannel: cc.Alerts.Telegram.Channel, + tgKey: cc.Alerts.Telegram.ApiKey, + tgMentions: strings.Join(cc.Alerts.Telegram.Mentions, " "), + discHook: cc.Alerts.Discord.Webhook, + discMentions: strings.Join(cc.Alerts.Discord.Mentions, " "), + slkHook: cc.Alerts.Slack.Webhook, + whURL: cc.Alerts.Webhook.URL, + alertConfig: &cc.Alerts, } c.alertChan <- a c.chainsMux.RUnlock() alarms.notifyMux.Lock() defer alarms.notifyMux.Unlock() - if alarms.AllAlarms[chainName] == nil { - alarms.AllAlarms[chainName] = make(map[string]alertMsgCache) + if alarms.AllAlarms[configName] == nil { + alarms.AllAlarms[configName] = make(map[string]alertMsgCache) } - if resolved && !alarms.AllAlarms[chainName][*id].SentTime.IsZero() { - delete(alarms.AllAlarms[chainName], *id) + if resolved && !alarms.AllAlarms[configName][*id].SentTime.IsZero() { + delete(alarms.AllAlarms[configName], *id) return } else if resolved { return @@ -564,7 +582,7 @@ func (c *Config) alert(chainName, message, severity string, resolved bool, id *s Message: message, SentTime: time.Now(), } - alarms.AllAlarms[chainName][*id] = cache + alarms.AllAlarms[configName][*id] = cache } func evaluateConsecutiveBlocksMissedAlert(cc *ChainConfig) (bool, bool) { diff --git a/td2/alert_test.go b/td2/alert_test.go index 1010036..5b8d2d1 100644 --- a/td2/alert_test.go +++ b/td2/alert_test.go @@ -1,9 +1,10 @@ package tenderduty import ( + "bytes" "fmt" + "io" "net/http" - "net/http/httptest" "reflect" "sync" "testing" @@ -12,6 +13,12 @@ import ( gov "github.com/cosmos/cosmos-sdk/x/gov/types" ) +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} + // Helper function to create test config with minimal required fields func createTestConfig() *Config { falseBool := false @@ -441,12 +448,20 @@ func TestNotifySlack(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if tt.msg.slk { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(tt.serverResponse) - })) - defer server.Close() - tt.msg.slkHook = server.URL + originalTransport := http.DefaultTransport + // Avoid binding a local port; simulate Slack responses via a mock transport. + http.DefaultTransport = roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: tt.serverResponse, + Body: io.NopCloser(bytes.NewBuffer(nil)), + Header: make(http.Header), + Request: req, + }, nil + }) + t.Cleanup(func() { + http.DefaultTransport = originalTransport + }) + tt.msg.slkHook = "http://slack.test.local/" } err := notifySlack(tt.msg) @@ -646,13 +661,21 @@ func TestNotifyWebhook(t *testing.T) { serverCalled := false if tt.msg.wh { - // Create test server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + originalTransport := http.DefaultTransport + // Avoid binding a local port; simulate webhook responses via a mock transport. + http.DefaultTransport = roundTripperFunc(func(req *http.Request) (*http.Response, error) { serverCalled = true - w.WriteHeader(tt.serverResponse) - })) - defer server.Close() - tt.msg.whURL = server.URL + return &http.Response{ + StatusCode: tt.serverResponse, + Body: io.NopCloser(bytes.NewBuffer(nil)), + Header: make(http.Header), + Request: req, + }, nil + }) + t.Cleanup(func() { + http.DefaultTransport = originalTransport + }) + tt.msg.whURL = "http://webhook.test.local/" } err := notifyWebhook(tt.msg) @@ -683,6 +706,7 @@ func TestConfigAlert(t *testing.T) { "test-chain": { ChainId: "test-chain-1", ValAddress: "testval123", + valInfo: &ValInfo{Valcons: "testvalcons"}, Alerts: AlertConfig{ Pagerduty: PDConfig{ Enabled: &[]bool{true}[0], @@ -1939,10 +1963,10 @@ func TestEvaluateStakeChangeAlert(t *testing.T) { testAlarms.AllAlarms = make(map[string]map[string]alertMsgCache) cc := &ChainConfig{ - name: "test-chain", - ChainId: "test-chain-1", - ValAddress: "testval123", - valInfo: tt.valInfo, + name: "test-chain", + ChainId: "test-chain-1", + ValAddress: "testval123", + valInfo: tt.valInfo, lastValInfo: tt.lastValInfo, Alerts: AlertConfig{ StakeChangeIncreaseThreshold: &[]float64{0.15}[0],