diff --git a/README.md b/README.md index 8ef9bfeb..579f3a1a 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,33 @@ If you want to enable Slack, provide either a webhook URL or a bot token with ch | `alert.slack.channel` | Required when using token. For webhooks, used by legacy webhooks to send messages to specific channel instead of default one | | `alert.slack.title` | Customized title in slack message | | `alert.slack.text` | Customized text in slack message | +| `alert.slack.routes` | Rules to route alerts to extra channels based on pod labels (token mode only) | + +With a bot token you can route alerts to different channels based on pod labels. +Each route lists label regexes (all must match; unanchored, use `^...$` to match +the full value) and the channels to notify. All matching routes apply. Set +`includeDefault: true` on a route to also send matching events to the default +`channel`. Events matching no route, and messages without labels (e.g. PVC +alerts), go to the default `channel`. + +```yaml +alert: + slack: + token: xoxb-... + channel: "#alerts-default" + routes: + - labels: + app: "^payments-.*" + tier: "backend" + channels: + - "#payments-alerts" + - "#backend-oncall" + includeDefault: true # also notify #alerts-default + - labels: + team: "platform" + channels: + - "#platform-alerts" +``` #### Discord diff --git a/alertmanager/slack/route.go b/alertmanager/slack/route.go new file mode 100644 index 00000000..8105e347 --- /dev/null +++ b/alertmanager/slack/route.go @@ -0,0 +1,81 @@ +package slack + +import ( + "fmt" + "regexp" + + "gopkg.in/yaml.v3" +) + +// route sends events matching all label regexes to a set of channels +type route struct { + labels map[string]*regexp.Regexp + channels []string + + // includeDefault also sends matching events to the default channel + includeDefault bool +} + +// matches reports whether every label regex in the route matches the +// corresponding event label. Regexes are unanchored; use ^...$ to match +// the full value. +func (r *route) matches(labels map[string]string) bool { + for key, re := range r.labels { + value, ok := labels[key] + if !ok || !re.MatchString(value) { + return false + } + } + return true +} + +type rawRoute struct { + Labels map[string]string `yaml:"labels"` + Channels []string `yaml:"channels"` + IncludeDefault bool `yaml:"includeDefault"` +} + +// parseRoutes converts the untyped routes config into compiled routes +func parseRoutes(raw interface{}) ([]route, error) { + if raw == nil { + return nil, nil + } + + b, err := yaml.Marshal(raw) + if err != nil { + return nil, fmt.Errorf("invalid routes config: %w", err) + } + + var rawRoutes []rawRoute + if err := yaml.Unmarshal(b, &rawRoutes); err != nil { + return nil, fmt.Errorf("invalid routes config: %w", err) + } + + routes := make([]route, 0, len(rawRoutes)) + for i, rr := range rawRoutes { + if len(rr.Labels) == 0 { + return nil, fmt.Errorf("route %d has no labels", i) + } + if len(rr.Channels) == 0 { + return nil, fmt.Errorf("route %d has no channels", i) + } + + labels := make(map[string]*regexp.Regexp, len(rr.Labels)) + for key, pattern := range rr.Labels { + re, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf( + "route %d has invalid regex for label %q: %w", i, key, err) + } + labels[key] = re + } + + routes = append(routes, route{ + labels: labels, + channels: rr.Channels, + includeDefault: rr.IncludeDefault, + }) + } + + return routes, nil +} diff --git a/alertmanager/slack/slack.go b/alertmanager/slack/slack.go index 1f45c94c..6e19d72b 100644 --- a/alertmanager/slack/slack.go +++ b/alertmanager/slack/slack.go @@ -1,6 +1,7 @@ package slack import ( + "errors" "fmt" "os" "strings" @@ -35,6 +36,15 @@ type Slack struct { // client is used for token-based messaging client *slackClient.Client + + // postMessage posts to a channel using the token-based client + postMessage func( + channelID string, + options ...slackClient.MsgOption) (string, string, error) + + // routes send events to additional channels based on their labels. + // Only supported with token-based authentication. + routes []route } // NewSlack returns new Slack instance @@ -62,14 +72,24 @@ func NewSlack(config map[string]interface{}, appCfg *config.App) *Slack { logrus.Warnf("initializing slack with token requires channel") return nil } + + routes, err := parseRoutes(config["routes"]) + if err != nil { + logrus.Errorf("initializing slack failed: %s", err) + return nil + } + logrus.Infof("initializing slack with token for channel: %s", channel) + client := slackClient.New(token) return &Slack{ - token: token, - channel: channel, - title: title, - text: text, - client: slackClient.New(token), - appCfg: appCfg, + token: token, + channel: channel, + title: title, + text: text, + client: client, + postMessage: client.PostMessage, + routes: routes, + appCfg: appCfg, } } @@ -79,6 +99,10 @@ func NewSlack(config map[string]interface{}, appCfg *config.App) *Slack { return nil } + if config["routes"] != nil { + logrus.Warnf("slack routes require token mode; ignoring routes") + } + logrus.Infof("initializing slack with webhook url: %s", webhook) return &Slack{ @@ -156,17 +180,46 @@ func (s *Slack) SendEvent(ev *event.Event) error { Blocks: &slackClient.Blocks{ BlockSet: append(blocks, markdownSection(constant.Footer)), }, - }) + }, s.channelsForEvent(ev)) } // SendMessage sends text message to the provider func (s *Slack) SendMessage(msg string) error { return s.sendAPI(&slackClient.WebhookMessage{ Text: msg, - }) + }, nil) +} + +// channelsForEvent returns channels of all routes matching event labels, +// deduplicated in first-seen order, or the default channel if none match +func (s *Slack) channelsForEvent(ev *event.Event) []string { + var channels []string + seen := map[string]struct{}{} + add := func(ch string) { + if _, ok := seen[ch]; !ok { + seen[ch] = struct{}{} + channels = append(channels, ch) + } + } + for _, r := range s.routes { + if !r.matches(ev.Labels) { + continue + } + for _, ch := range r.channels { + add(ch) + } + if r.includeDefault { + add(s.channel) + } + } + if len(channels) == 0 { + return []string{s.channel} + } + return channels } -func (s *Slack) sendAPI(msg *slackClient.WebhookMessage) error { +func (s *Slack) sendAPI( + msg *slackClient.WebhookMessage, channels []string) error { // Use token-based API if client is configured if s.client != nil { options := []slackClient.MsgOption{} @@ -176,8 +229,17 @@ func (s *Slack) sendAPI(msg *slackClient.WebhookMessage) error { if len(msg.Text) > 0 { options = append(options, slackClient.MsgOptionText(msg.Text, false)) } - _, _, err := s.client.PostMessage(s.channel, options...) - return err + + if len(channels) == 0 { + channels = []string{s.channel} + } + var errs []error + for _, ch := range channels { + if _, _, err := s.postMessage(ch, options...); err != nil { + errs = append(errs, fmt.Errorf("slack channel %q: %w", ch, err)) + } + } + return errors.Join(errs...) } // Fall back to webhook diff --git a/alertmanager/slack/slack_test.go b/alertmanager/slack/slack_test.go index baabc606..c15a25e6 100644 --- a/alertmanager/slack/slack_test.go +++ b/alertmanager/slack/slack_test.go @@ -1,6 +1,7 @@ package slack import ( + "errors" "os" "testing" @@ -60,6 +61,280 @@ func TestSlackTokenPrecedence(t *testing.T) { assert.Empty(s.webhook) } +func TestSlackTokenWithRoutes(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{ + "app": "^payments-.*", + }, + "channels": []interface{}{"#payments-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + assert.Len(s.routes, 1) + assert.Equal([]string{"#payments-alerts"}, s.routes[0].channels) +} + +func TestSlackRoutesInvalidRegex(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"app": "[invalid"}, + "channels": []interface{}{"#payments-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.Nil(s) +} + +func TestSlackRoutesEmptyLabels(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "channels": []interface{}{"#payments-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.Nil(s) +} + +func TestSlackRoutesEmptyChannels(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"app": "payments"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.Nil(s) +} + +func TestSlackRoutesIgnoredInWebhookMode(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "webhook": "testtest", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"app": "payments"}, + "channels": []interface{}{"#payments-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + assert.Empty(s.routes) +} + +func TestChannelsForEvent(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{ + "app": "^payments-.*", + "tier": "backend", + }, + "channels": []interface{}{"#payments-alerts", "#backend-oncall"}, + }, + map[string]interface{}{ + "labels": map[string]interface{}{"tier": "backend"}, + "channels": []interface{}{"#backend-oncall", "#backend-all"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + + // all labels must match (AND); both rules match here, channels + // are deduplicated in first-seen order + assert.Equal( + []string{"#payments-alerts", "#backend-oncall", "#backend-all"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "app": "payments-api", + "tier": "backend", + }})) + + // only second rule matches when app label doesn't match + assert.Equal( + []string{"#backend-oncall", "#backend-all"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "app": "billing-api", + "tier": "backend", + }})) + + // missing label key means no match; fall back to default channel + assert.Equal( + []string{"#alerts"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "app": "payments-api", + }})) + + // no labels at all falls back to default channel + assert.Equal( + []string{"#alerts"}, + s.channelsForEvent(&event.Event{})) +} + +func TestChannelsForEventIncludeDefault(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"team": "edr"}, + "channels": []interface{}{"#edr-alerts"}, + "includeDefault": true, + }, + map[string]interface{}{ + "labels": map[string]interface{}{"team": "platform"}, + "channels": []interface{}{"#platform-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + + // includeDefault adds the default channel after the route channels + assert.Equal( + []string{"#edr-alerts", "#alerts"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "team": "edr", + }})) + + // without includeDefault only the route channels are used + assert.Equal( + []string{"#platform-alerts"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "team": "platform", + }})) + + // default channel listed explicitly is not duplicated + s.routes[0].channels = []string{"#edr-alerts", "#alerts"} + assert.Equal( + []string{"#edr-alerts", "#alerts"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "team": "edr", + }})) +} + +func TestSendEventRouting(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"team": "platform"}, + "channels": []interface{}{"#platform-alerts", "#platform-oncall"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + + var posted []string + s.postMessage = func( + ch string, opts ...slackClient.MsgOption) (string, string, error) { + posted = append(posted, ch) + return "", "", nil + } + + assert.Nil(s.SendEvent(&event.Event{ + PodName: "test-pod", + Labels: map[string]string{"team": "platform"}, + })) + assert.Equal([]string{"#platform-alerts", "#platform-oncall"}, posted) + + posted = nil + assert.Nil(s.SendEvent(&event.Event{ + PodName: "test-pod", + Labels: map[string]string{"team": "payments"}, + })) + assert.Equal([]string{"#alerts"}, posted) +} + +func TestSendMessageTokenDefaultChannel(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"team": "platform"}, + "channels": []interface{}{"#platform-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + + var posted []string + s.postMessage = func( + ch string, opts ...slackClient.MsgOption) (string, string, error) { + posted = append(posted, ch) + return "", "", nil + } + + assert.Nil(s.SendMessage("test")) + assert.Equal([]string{"#alerts"}, posted) +} + +func TestSendEventRoutingPostError(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"team": "platform"}, + "channels": []interface{}{"#failing", "#working"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + + var posted []string + s.postMessage = func( + ch string, opts ...slackClient.MsgOption) (string, string, error) { + posted = append(posted, ch) + if ch == "#failing" { + return "", "", errors.New("channel_not_found") + } + return "", "", nil + } + + err := s.SendEvent(&event.Event{ + PodName: "test-pod", + Labels: map[string]string{"team": "platform"}, + }) + assert.Error(err) + assert.Contains(err.Error(), "#failing") + // failing channel must not block the remaining channels + assert.Equal([]string{"#failing", "#working"}, posted) +} + func TestSlack(t *testing.T) { assert := assert.New(t) diff --git a/deploy/config.yaml b/deploy/config.yaml index b3dc7b8b..c12f854c 100644 --- a/deploy/config.yaml +++ b/deploy/config.yaml @@ -15,6 +15,14 @@ data: alert: slack: webhook: + # or use a bot token with optional label-based channel routing: + # token: + # channel: "#alerts-default" + # routes: + # - labels: + # app: "^payments-.*" + # channels: + # - "#payments-alerts" pagerduty: integrationKey: discord: