Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 53 additions & 35 deletions td2/alert.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,15 @@ type alertMsg struct {
slk bool
wh bool

severity string
resolved bool
chain string
message string
uniqueId string
key string
severity string
resolved bool
chain string
chainName string
valoperAddress string
valconsAddress string
message string
uniqueId string
key string

tgChannel string
tgKey string
Expand Down Expand Up @@ -456,10 +459,13 @@ func notifyWebhook(msg *alertMsg) (err error) {
alert := WebhookAlert{
Status: status,
Labels: map[string]string{
"alertname": msg.uniqueId,
"chain": msg.chain,
"severity": msg.severity,
"source": "tenderduty",
"alertname": msg.uniqueId,
"chain": msg.chain,
"chain_name": msg.chainName,
"valoper_address": msg.valoperAddress,
"valcons_address": msg.valconsAddress,
"severity": msg.severity,
"source": "tenderduty",
},
Annotations: map[string]string{
"summary": msg.message,
Expand Down Expand Up @@ -521,41 +527,53 @@ func getAlarms(chain string) string {
}

// alert creates a universal alert and pushes it to the alertChan to be delivered to appropriate services
func (c *Config) alert(chainName, message, severity string, resolved bool, id *string) {
func (c *Config) alert(configName, message, severity string, resolved bool, id *string) {
if id == nil {
return
}
c.chainsMux.RLock()
cc := c.Chains[configName]
if cc == nil {
c.chainsMux.RUnlock()
return
}
valcons := ""
if cc.valInfo != nil {
valcons = cc.valInfo.Valcons
}
a := &alertMsg{
pd: boolVal(c.DefaultAlertConfig.Pagerduty.Enabled) && boolVal(c.Chains[chainName].Alerts.Pagerduty.Enabled),
disc: boolVal(c.DefaultAlertConfig.Discord.Enabled) && boolVal(c.Chains[chainName].Alerts.Discord.Enabled),
tg: boolVal(c.DefaultAlertConfig.Telegram.Enabled) && boolVal(c.Chains[chainName].Alerts.Telegram.Enabled),
slk: boolVal(c.DefaultAlertConfig.Slack.Enabled) && boolVal(c.Chains[chainName].Alerts.Slack.Enabled),
wh: boolVal(c.DefaultAlertConfig.Webhook.Enabled) && boolVal(c.Chains[chainName].Alerts.Webhook.Enabled),
severity: severity,
resolved: resolved,
chain: fmt.Sprintf("%s (%s)", chainName, c.Chains[chainName].ChainId),
message: message,
uniqueId: *id,
key: c.Chains[chainName].Alerts.Pagerduty.ApiKey,
tgChannel: c.Chains[chainName].Alerts.Telegram.Channel,
tgKey: c.Chains[chainName].Alerts.Telegram.ApiKey,
tgMentions: strings.Join(c.Chains[chainName].Alerts.Telegram.Mentions, " "),
discHook: c.Chains[chainName].Alerts.Discord.Webhook,
discMentions: strings.Join(c.Chains[chainName].Alerts.Discord.Mentions, " "),
slkHook: c.Chains[chainName].Alerts.Slack.Webhook,
whURL: c.Chains[chainName].Alerts.Webhook.URL,
alertConfig: &c.Chains[chainName].Alerts,
pd: boolVal(c.DefaultAlertConfig.Pagerduty.Enabled) && boolVal(cc.Alerts.Pagerduty.Enabled),
disc: boolVal(c.DefaultAlertConfig.Discord.Enabled) && boolVal(cc.Alerts.Discord.Enabled),
tg: boolVal(c.DefaultAlertConfig.Telegram.Enabled) && boolVal(cc.Alerts.Telegram.Enabled),
slk: boolVal(c.DefaultAlertConfig.Slack.Enabled) && boolVal(cc.Alerts.Slack.Enabled),
wh: boolVal(c.DefaultAlertConfig.Webhook.Enabled) && boolVal(cc.Alerts.Webhook.Enabled),
severity: severity,
resolved: resolved,
chain: fmt.Sprintf("%s (%s)", configName, cc.ChainId),
chainName: cc.ChainName,
valoperAddress: cc.ValAddress,
valconsAddress: valcons,
message: message,
uniqueId: *id,
key: cc.Alerts.Pagerduty.ApiKey,
tgChannel: cc.Alerts.Telegram.Channel,
tgKey: cc.Alerts.Telegram.ApiKey,
tgMentions: strings.Join(cc.Alerts.Telegram.Mentions, " "),
discHook: cc.Alerts.Discord.Webhook,
discMentions: strings.Join(cc.Alerts.Discord.Mentions, " "),
slkHook: cc.Alerts.Slack.Webhook,
whURL: cc.Alerts.Webhook.URL,
alertConfig: &cc.Alerts,
}
c.alertChan <- a
c.chainsMux.RUnlock()
alarms.notifyMux.Lock()
defer alarms.notifyMux.Unlock()
if alarms.AllAlarms[chainName] == nil {
alarms.AllAlarms[chainName] = make(map[string]alertMsgCache)
if alarms.AllAlarms[configName] == nil {
alarms.AllAlarms[configName] = make(map[string]alertMsgCache)
}
if resolved && !alarms.AllAlarms[chainName][*id].SentTime.IsZero() {
delete(alarms.AllAlarms[chainName], *id)
if resolved && !alarms.AllAlarms[configName][*id].SentTime.IsZero() {
delete(alarms.AllAlarms[configName], *id)
return
} else if resolved {
return
Expand All @@ -564,7 +582,7 @@ func (c *Config) alert(chainName, message, severity string, resolved bool, id *s
Message: message,
SentTime: time.Now(),
}
alarms.AllAlarms[chainName][*id] = cache
alarms.AllAlarms[configName][*id] = cache
}

func evaluateConsecutiveBlocksMissedAlert(cc *ChainConfig) (bool, bool) {
Expand Down
58 changes: 41 additions & 17 deletions td2/alert_test.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
package tenderduty

import (
"bytes"
"fmt"
"io"
"net/http"
"net/http/httptest"
"reflect"
"sync"
"testing"
Expand All @@ -12,6 +13,12 @@ import (
gov "github.com/cosmos/cosmos-sdk/x/gov/types"
)

type roundTripperFunc func(*http.Request) (*http.Response, error)

func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return fn(req)
}

// Helper function to create test config with minimal required fields
func createTestConfig() *Config {
falseBool := false
Expand Down Expand Up @@ -441,12 +448,20 @@ func TestNotifySlack(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.msg.slk {
// Create test server
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(tt.serverResponse)
}))
defer server.Close()
tt.msg.slkHook = server.URL
originalTransport := http.DefaultTransport
// Avoid binding a local port; simulate Slack responses via a mock transport.
http.DefaultTransport = roundTripperFunc(func(req *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: tt.serverResponse,
Body: io.NopCloser(bytes.NewBuffer(nil)),
Header: make(http.Header),
Request: req,
}, nil
})
t.Cleanup(func() {
http.DefaultTransport = originalTransport
})
tt.msg.slkHook = "http://slack.test.local/"
}

err := notifySlack(tt.msg)
Expand Down Expand Up @@ -646,13 +661,21 @@ func TestNotifyWebhook(t *testing.T) {
serverCalled := false

if tt.msg.wh {
// Create test server
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
originalTransport := http.DefaultTransport
// Avoid binding a local port; simulate webhook responses via a mock transport.
http.DefaultTransport = roundTripperFunc(func(req *http.Request) (*http.Response, error) {
serverCalled = true
w.WriteHeader(tt.serverResponse)
}))
defer server.Close()
tt.msg.whURL = server.URL
return &http.Response{
StatusCode: tt.serverResponse,
Body: io.NopCloser(bytes.NewBuffer(nil)),
Header: make(http.Header),
Request: req,
}, nil
})
t.Cleanup(func() {
http.DefaultTransport = originalTransport
})
tt.msg.whURL = "http://webhook.test.local/"
}

err := notifyWebhook(tt.msg)
Expand Down Expand Up @@ -683,6 +706,7 @@ func TestConfigAlert(t *testing.T) {
"test-chain": {
ChainId: "test-chain-1",
ValAddress: "testval123",
valInfo: &ValInfo{Valcons: "testvalcons"},
Alerts: AlertConfig{
Pagerduty: PDConfig{
Enabled: &[]bool{true}[0],
Expand Down Expand Up @@ -1939,10 +1963,10 @@ func TestEvaluateStakeChangeAlert(t *testing.T) {
testAlarms.AllAlarms = make(map[string]map[string]alertMsgCache)

cc := &ChainConfig{
name: "test-chain",
ChainId: "test-chain-1",
ValAddress: "testval123",
valInfo: tt.valInfo,
name: "test-chain",
ChainId: "test-chain-1",
ValAddress: "testval123",
valInfo: tt.valInfo,
lastValInfo: tt.lastValInfo,
Alerts: AlertConfig{
StakeChangeIncreaseThreshold: &[]float64{0.15}[0],
Expand Down
Loading